From a74bfdad124a0dfb4c14581bbec217750f5d09ed Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Wed, 20 Oct 2021 13:32:44 +1000 Subject: [PATCH] Fix: Create temp file in empty dirs for S3 log dir Since S3 buckets does not have an underlying file system to view empty directories, an temp file has to be created when referencing the directory. This allows empty folders to be viewed through cloudfront, without returning a 404 error. Create temp file in the format: "_%d%m%Y_%H%M%S", would help with future clean or debugging. Issue-ID: RELENG-4016 Change-Id: I0a635cd32da860deaada0ac0cc15bf1aa5fc2d4a Signed-off-by: Anil Belur --- lftools/deploy.py | 12 +++++++++++- .../notes/fix-s3-logs-empty-directory-3f7041cb40dfd2fa.yaml | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fix-s3-logs-empty-directory-3f7041cb40dfd2fa.yaml diff --git a/lftools/deploy.py b/lftools/deploy.py index 6a7d34db..0a34c671 100755 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -11,6 +11,7 @@ """Library of functions for deploying artifacts to Nexus.""" import concurrent.futures +import datetime import errno import glob import gzip @@ -268,7 +269,6 @@ def copy_archives(workspace, pattern=None): log.debug("Files found: {}".format(paths)) no_dups_paths = _remove_duplicates_and_sort(paths) - for src in no_dups_paths: if len(os.path.basename(src)) > 255: log.warn("Filename {} is over 255 characters. Skipping...".format(os.path.basename(src))) @@ -286,6 +286,16 @@ def copy_archives(workspace, pattern=None): else: log.info("Not copying directories: {}.".format(src)) + # Create a temp file to handle empty dirs in AWS S3 buckets. + if os.environ.get("S3_BUCKET") is not None: + now = datetime.datetime.now() + p = now.strftime("_%d%m%Y_%H%M%S_") + for dirpath, dirnames, files in os.walk(dest_dir): + if not files: + fd, tmp = tempfile.mkstemp(prefix=p, dir=dirpath) + os.close(fd) + log.debug("temp file created in dir: {}.".format(dirpath)) + def deploy_archives(nexus_url, nexus_path, workspace, pattern=None): """Archive files to a Nexus site repository named logs. diff --git a/releasenotes/notes/fix-s3-logs-empty-directory-3f7041cb40dfd2fa.yaml b/releasenotes/notes/fix-s3-logs-empty-directory-3f7041cb40dfd2fa.yaml new file mode 100644 index 00000000..e6d98fb3 --- /dev/null +++ b/releasenotes/notes/fix-s3-logs-empty-directory-3f7041cb40dfd2fa.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Create an temp file for logs shipping with AWS S3 Buckets. + + Since S3 buckets does not have an underlying file system, to view empty + directories, an temp file has to be created when referencing the directory. + This allows empty folders to be viewed through cloudfront, without which + a 404 error is returned. -- 2.16.6