Add S3 log shipping subcommand

Add new click command for logs-s3.

Issue-Id: RELENG-2379, RELENG-2380
Signed-off-by: Houa Yang <hyang@contractor.linuxfoundation.org>
Change-Id: I4295272cd903c9cc33962f2ee54667c28c3132e0

diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py
index 9e749ab..ff01576 100644 (file)
--- a/lftools/cli/deploy.py
+++ b/lftools/cli/deploy.py
@@ -137,10 +137,26 @@ def logs(ctx, nexus_url, nexus_path, build_url):
     log.info("Logs upload complete.")
 
 
+@click.command(name="s3")
+@click.argument("s3_bucket", envvar="S3_BUCKET")
+@click.argument("s3_path")
+@click.argument("build-url", envvar="BUILD_URL")
+@click.argument("workspace", envvar="WORKSPACE")
+@click.option("-p", "--pattern", multiple=True)
+@click.pass_context
+def s3(ctx, s3_bucket, s3_path, build_url, workspace, pattern):
+    """Deploy logs and archives to a S3 bucket."""
+    if not pattern:
+        pattern = None
+        deploy_sys.deploy_s3(s3_bucket, s3_path, build_url, workspace, pattern)
+    log.info("Logs upload to S3 complete.")
+
+
 @click.command(name="maven-file")
 @click.argument("nexus-url", envvar="NEXUS_URL")
 @click.argument("repo-id", envvar="REPO_ID")
 @click.argument("file-name", envvar="FILE_NAME")
+
 # Maven Config
 @click.option("-b", "--maven-bin", envvar="MAVEN_BIN", help="Path of maven binary.")
 @click.option("-gs", "--global-settings", envvar="GLOBAL_SETTINGS_FILE", help="Global settings file.")
@@ -311,3 +327,4 @@ deploy.add_command(nexus_stage)
 deploy.add_command(nexus_stage_repo_close)
 deploy.add_command(nexus_stage_repo_create)
 deploy.add_command(nexus_zip)
+deploy.add_command(s3)

diff --git a/lftools/deploy.py b/lftools/deploy.py
index fa6bd36..15d181c 100755 (executable)
--- a/lftools/deploy.py
+++ b/lftools/deploy.py
@@ -17,7 +17,7 @@ import gzip
 import io
 import logging
 import math
-from multiprocessing import cpu_count
+import mimetypes
 import os
 import re
 import shutil
@@ -26,11 +26,14 @@ import sys
 import tempfile
 import zipfile
 
+import boto3
+from botocore.exceptions import ClientError
 from defusedxml.minidom import parseString
 import requests
 import six
 
 log = logging.getLogger(__name__)
+logging.getLogger("botocore").setLevel(logging.CRITICAL)
 
 
 def _compress_text(dir):
@@ -403,6 +406,169 @@ def deploy_logs(nexus_url, nexus_path, build_url):
     shutil.rmtree(work_dir)
 
 
+def deploy_s3(s3_bucket, s3_path, build_url, workspace, pattern=None):
+    """Add logs and archives to temp directory to be shipped to S3 bucket.
+
+    Fetches logs and system information and pushes them and archives to S3
+    for log archiving.
+
+    Requires the s3 bucket to exist.
+
+    Parameters:
+
+        :s3_bucket: Name of S3 bucket. Eg: lf-project-date
+        :s3_path: Path on S3 bucket place the logs and archives. Eg:
+            $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
+        :build_url: URL of the Jenkins build. Jenkins typically provides this
+                    via the $BUILD_URL environment variable.
+        :workspace: Directory in which to search, typically in Jenkins this is
+            $WORKSPACE
+        :pattern: Space-separated list of Globstar patterns of files to
+            archive. (optional)
+    """
+
+    def _upload_to_s3(file):
+        extra_args = {"ContentType": "text/plain"}
+        text_plain_extra_args = {"ContentType": "text/plain", "ContentEncoding": mimetypes.guess_type(file)[1]}
+        app_xml_extra_args = {"ContentType": "application/xml'", "ContentEncoding": mimetypes.guess_type(file)[1]}
+        if file == "_tmpfile":
+            for dir in (logs_dir, silo_dir, jenkins_node_dir):
+                try:
+                    s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(dir, file))
+                except ClientError as e:
+                    log.error(e)
+                    return False
+                return True
+        if mimetypes.guess_type(file)[0] is None and mimetypes.guess_type(file)[1] is None:
+            try:
+                s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args)
+            except ClientError as e:
+                log.error(e)
+                return False
+            return True
+        elif mimetypes.guess_type(file)[0] is None or mimetypes.guess_type(file)[0] in "text/plain":
+            extra_args = text_plain_extra_args
+            try:
+                s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args)
+            except ClientError as e:
+                log.error(e)
+                return False
+            return True
+        elif mimetypes.guess_type(file)[0] in "application/xml":
+            extra_args = app_xml_extra_args
+            try:
+                s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args)
+            except ClientError as e:
+                log.error(e)
+                return False
+            return True
+        else:
+            try:
+                s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args)
+            except ClientError as e:
+                log.error(e)
+                return False
+            return True
+
+    previous_dir = os.getcwd()
+    work_dir = tempfile.mkdtemp(prefix="lftools-dl.")
+    os.chdir(work_dir)
+    s3_bucket = s3_bucket.lower()
+    s3 = boto3.resource("s3")
+    logs_dir = s3_path.split("/")[0] + "/"
+    silo_dir = s3_path.split("/")[1] + "/"
+    jenkins_node_dir = logs_dir + silo_dir + s3_path.split("/")[2] + "/"
+
+    log.debug("work_dir: {}".format(work_dir))
+
+    # Copy archive files to tmp dir
+    copy_archives(workspace, pattern)
+
+    # Create build logs
+    build_details = open("_build-details.log", "w+")
+    build_details.write("build-url: {}".format(build_url))
+
+    with open("_sys-info.log", "w+") as sysinfo_log:
+        sys_cmds = []
+
+        log.debug("Platform: {}".format(sys.platform))
+        if sys.platform == "linux" or sys.platform == "linux2":
+            sys_cmds = [
+                ["uname", "-a"],
+                ["lscpu"],
+                ["nproc"],
+                ["df", "-h"],
+                ["free", "-m"],
+                ["ip", "addr"],
+                ["sar", "-b", "-r", "-n", "DEV"],
+                ["sar", "-P", "ALL"],
+            ]
+
+        for c in sys_cmds:
+            try:
+                output = subprocess.check_output(c).decode("utf-8")
+            except FileNotFoundError:
+                log.debug("Command not found: {}".format(c))
+                continue
+
+            output = "---> {}:\n{}\n".format(" ".join(c), output)
+            sysinfo_log.write(output)
+            log.info(output)
+
+    build_details.close()
+    sysinfo_log.close()
+
+    # Magic string used to trim console logs at the appropriate level during wget
+    MAGIC_STRING = "-----END_OF_BUILD-----"
+    log.info(MAGIC_STRING)
+
+    resp = requests.get("{}/consoleText".format(_format_url(build_url)))
+    with open("console.log", "w+", encoding="utf-8") as f:
+        f.write(six.text_type(resp.content.decode("utf-8").split(MAGIC_STRING)[0]))
+        f.close()
+
+    resp = requests.get("{}/timestamps?time=HH:mm:ss&appendLog".format(_format_url(build_url)))
+    with open("console-timestamp.log", "w+", encoding="utf-8") as f:
+        f.write(six.text_type(resp.content.decode("utf-8").split(MAGIC_STRING)[0]))
+        f.close()
+
+    # Create _tmpfile
+    """ Because s3 does not have a filesystem, this file is uploaded to generate/update the
+        index.html file in the top level "directories". """
+    open("_tmpfile", "a").close()
+
+    # Compress tmp directory
+    _compress_text(work_dir)
+
+    # Create file list to upload
+    file_list = []
+    files = glob.glob("**/*", recursive=True)
+    for file in files:
+        if os.path.isfile(file):
+            file_list.append(file)
+
+    log.info("#######################################################")
+    log.info("Deploying files from {} to {}/{}".format(work_dir, s3_bucket, s3_path))
+
+    # Perform s3 upload
+    for file in file_list:
+        log.info("Attempting to upload file {}".format(file))
+        if _upload_to_s3(file):
+            log.info("Successfully uploaded {}".format(file))
+        else:
+            log.error("FAILURE: Uploading {} failed".format(file))
+
+    log.info("Finished deploying from {} to {}/{}".format(work_dir, s3_bucket, s3_path))
+    log.info("#######################################################")
+
+    # Cleanup
+    s3.Object(s3_bucket, "{}{}".format(logs_dir, "_tmpfile")).delete()
+    s3.Object(s3_bucket, "{}{}".format(silo_dir, "_tmpfile")).delete()
+    s3.Object(s3_bucket, "{}{}".format(jenkins_node_dir, "_tmpfile")).delete()
+    os.chdir(previous_dir)
+    # shutil.rmtree(work_dir)
+
+
 def deploy_nexus_zip(nexus_url, nexus_repo, nexus_path, zip_file):
     """"Deploy zip file containing artifacts to Nexus using requests.
 

diff --git a/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml b/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml
new file mode 100644 (file)
index 0000000..f9eadec
--- /dev/null
+++ b/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Support log shipping logs to AWS S3 buckets.
+
+    A conditional statement is provided so that when a
+    s3 bucket name is provided, it checks to see if that
+    bucket exists, and if it exists, uploads the logs.

diff --git a/requirements.txt b/requirements.txt
index d1adbda..d0c6f95 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,8 @@ appdirs==1.4.3
 aspy.yaml==1.3.0
 attrs==19.3.0
 beautifulsoup4==4.8.2
+boto3==1.11.6
+botocore==1.14.6
 bs4==0.0.1
 certifi==2019.11.28
 cfgv==2.0.1