From: Houa Yang Date: Tue, 24 Sep 2019 20:51:50 +0000 (-0500) Subject: Add S3 log shipping subcommand X-Git-Tag: v0.33.0~1 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F28%2F61828%2F98;p=releng%2Flftools.git Add S3 log shipping subcommand Add new click command for logs-s3. Issue-Id: RELENG-2379, RELENG-2380 Signed-off-by: Houa Yang Change-Id: I4295272cd903c9cc33962f2ee54667c28c3132e0 --- diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index 9e749abf..ff01576d 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -137,10 +137,26 @@ def logs(ctx, nexus_url, nexus_path, build_url): log.info("Logs upload complete.") +@click.command(name="s3") +@click.argument("s3_bucket", envvar="S3_BUCKET") +@click.argument("s3_path") +@click.argument("build-url", envvar="BUILD_URL") +@click.argument("workspace", envvar="WORKSPACE") +@click.option("-p", "--pattern", multiple=True) +@click.pass_context +def s3(ctx, s3_bucket, s3_path, build_url, workspace, pattern): + """Deploy logs and archives to a S3 bucket.""" + if not pattern: + pattern = None + deploy_sys.deploy_s3(s3_bucket, s3_path, build_url, workspace, pattern) + log.info("Logs upload to S3 complete.") + + @click.command(name="maven-file") @click.argument("nexus-url", envvar="NEXUS_URL") @click.argument("repo-id", envvar="REPO_ID") @click.argument("file-name", envvar="FILE_NAME") + # Maven Config @click.option("-b", "--maven-bin", envvar="MAVEN_BIN", help="Path of maven binary.") @click.option("-gs", "--global-settings", envvar="GLOBAL_SETTINGS_FILE", help="Global settings file.") @@ -311,3 +327,4 @@ deploy.add_command(nexus_stage) deploy.add_command(nexus_stage_repo_close) deploy.add_command(nexus_stage_repo_create) deploy.add_command(nexus_zip) +deploy.add_command(s3) diff --git a/lftools/deploy.py b/lftools/deploy.py index fa6bd362..15d181c2 100755 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -17,7 +17,7 @@ import gzip import io import logging import math -from multiprocessing import cpu_count +import mimetypes import os import re import shutil @@ -26,11 +26,14 @@ import sys import tempfile import zipfile +import boto3 +from botocore.exceptions import ClientError from defusedxml.minidom import parseString import requests import six log = logging.getLogger(__name__) +logging.getLogger("botocore").setLevel(logging.CRITICAL) def _compress_text(dir): @@ -403,6 +406,169 @@ def deploy_logs(nexus_url, nexus_path, build_url): shutil.rmtree(work_dir) +def deploy_s3(s3_bucket, s3_path, build_url, workspace, pattern=None): + """Add logs and archives to temp directory to be shipped to S3 bucket. + + Fetches logs and system information and pushes them and archives to S3 + for log archiving. + + Requires the s3 bucket to exist. + + Parameters: + + :s3_bucket: Name of S3 bucket. Eg: lf-project-date + :s3_path: Path on S3 bucket place the logs and archives. Eg: + $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + :build_url: URL of the Jenkins build. Jenkins typically provides this + via the $BUILD_URL environment variable. + :workspace: Directory in which to search, typically in Jenkins this is + $WORKSPACE + :pattern: Space-separated list of Globstar patterns of files to + archive. (optional) + """ + + def _upload_to_s3(file): + extra_args = {"ContentType": "text/plain"} + text_plain_extra_args = {"ContentType": "text/plain", "ContentEncoding": mimetypes.guess_type(file)[1]} + app_xml_extra_args = {"ContentType": "application/xml'", "ContentEncoding": mimetypes.guess_type(file)[1]} + if file == "_tmpfile": + for dir in (logs_dir, silo_dir, jenkins_node_dir): + try: + s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(dir, file)) + except ClientError as e: + log.error(e) + return False + return True + if mimetypes.guess_type(file)[0] is None and mimetypes.guess_type(file)[1] is None: + try: + s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args) + except ClientError as e: + log.error(e) + return False + return True + elif mimetypes.guess_type(file)[0] is None or mimetypes.guess_type(file)[0] in "text/plain": + extra_args = text_plain_extra_args + try: + s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args) + except ClientError as e: + log.error(e) + return False + return True + elif mimetypes.guess_type(file)[0] in "application/xml": + extra_args = app_xml_extra_args + try: + s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args) + except ClientError as e: + log.error(e) + return False + return True + else: + try: + s3.Bucket(s3_bucket).upload_file(file, "{}{}".format(s3_path, file), ExtraArgs=extra_args) + except ClientError as e: + log.error(e) + return False + return True + + previous_dir = os.getcwd() + work_dir = tempfile.mkdtemp(prefix="lftools-dl.") + os.chdir(work_dir) + s3_bucket = s3_bucket.lower() + s3 = boto3.resource("s3") + logs_dir = s3_path.split("/")[0] + "/" + silo_dir = s3_path.split("/")[1] + "/" + jenkins_node_dir = logs_dir + silo_dir + s3_path.split("/")[2] + "/" + + log.debug("work_dir: {}".format(work_dir)) + + # Copy archive files to tmp dir + copy_archives(workspace, pattern) + + # Create build logs + build_details = open("_build-details.log", "w+") + build_details.write("build-url: {}".format(build_url)) + + with open("_sys-info.log", "w+") as sysinfo_log: + sys_cmds = [] + + log.debug("Platform: {}".format(sys.platform)) + if sys.platform == "linux" or sys.platform == "linux2": + sys_cmds = [ + ["uname", "-a"], + ["lscpu"], + ["nproc"], + ["df", "-h"], + ["free", "-m"], + ["ip", "addr"], + ["sar", "-b", "-r", "-n", "DEV"], + ["sar", "-P", "ALL"], + ] + + for c in sys_cmds: + try: + output = subprocess.check_output(c).decode("utf-8") + except FileNotFoundError: + log.debug("Command not found: {}".format(c)) + continue + + output = "---> {}:\n{}\n".format(" ".join(c), output) + sysinfo_log.write(output) + log.info(output) + + build_details.close() + sysinfo_log.close() + + # Magic string used to trim console logs at the appropriate level during wget + MAGIC_STRING = "-----END_OF_BUILD-----" + log.info(MAGIC_STRING) + + resp = requests.get("{}/consoleText".format(_format_url(build_url))) + with open("console.log", "w+", encoding="utf-8") as f: + f.write(six.text_type(resp.content.decode("utf-8").split(MAGIC_STRING)[0])) + f.close() + + resp = requests.get("{}/timestamps?time=HH:mm:ss&appendLog".format(_format_url(build_url))) + with open("console-timestamp.log", "w+", encoding="utf-8") as f: + f.write(six.text_type(resp.content.decode("utf-8").split(MAGIC_STRING)[0])) + f.close() + + # Create _tmpfile + """ Because s3 does not have a filesystem, this file is uploaded to generate/update the + index.html file in the top level "directories". """ + open("_tmpfile", "a").close() + + # Compress tmp directory + _compress_text(work_dir) + + # Create file list to upload + file_list = [] + files = glob.glob("**/*", recursive=True) + for file in files: + if os.path.isfile(file): + file_list.append(file) + + log.info("#######################################################") + log.info("Deploying files from {} to {}/{}".format(work_dir, s3_bucket, s3_path)) + + # Perform s3 upload + for file in file_list: + log.info("Attempting to upload file {}".format(file)) + if _upload_to_s3(file): + log.info("Successfully uploaded {}".format(file)) + else: + log.error("FAILURE: Uploading {} failed".format(file)) + + log.info("Finished deploying from {} to {}/{}".format(work_dir, s3_bucket, s3_path)) + log.info("#######################################################") + + # Cleanup + s3.Object(s3_bucket, "{}{}".format(logs_dir, "_tmpfile")).delete() + s3.Object(s3_bucket, "{}{}".format(silo_dir, "_tmpfile")).delete() + s3.Object(s3_bucket, "{}{}".format(jenkins_node_dir, "_tmpfile")).delete() + os.chdir(previous_dir) + # shutil.rmtree(work_dir) + + def deploy_nexus_zip(nexus_url, nexus_repo, nexus_path, zip_file): """"Deploy zip file containing artifacts to Nexus using requests. diff --git a/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml b/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml new file mode 100644 index 00000000..f9eadecf --- /dev/null +++ b/releasenotes/notes/add-s3-to-log-shipping-1fad234f538c13f4.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Support log shipping logs to AWS S3 buckets. + + A conditional statement is provided so that when a + s3 bucket name is provided, it checks to see if that + bucket exists, and if it exists, uploads the logs. diff --git a/requirements.txt b/requirements.txt index d1adbda2..d0c6f95a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,8 @@ appdirs==1.4.3 aspy.yaml==1.3.0 attrs==19.3.0 beautifulsoup4==4.8.2 +boto3==1.11.6 +botocore==1.14.6 bs4==0.0.1 certifi==2019.11.28 cfgv==2.0.1