From: Thanh Ha Date: Thu, 1 Nov 2018 15:52:48 +0000 (-0400) Subject: Refactor deploy-archives to Python X-Git-Tag: v0.19.0~17 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F72%2F13272%2F10;p=releng%2Flftools.git Refactor deploy-archives to Python CLI wise the function was reworked a bit to allow multiple (-p pattern) calls for better usability. Issue: RELENG-1376 Change-Id: I232a284c785b413129f8386fc7e799019da1f754 Signed-off-by: Thanh Ha --- diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index 31b9f9b0..43129dc0 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -12,18 +12,29 @@ __author__ = 'Thanh Ha' +import logging import subprocess import sys import click +from requests.exceptions import HTTPError import lftools.deploy as deploy_sys +log = logging.getLogger(__name__) + @click.group() @click.pass_context def deploy(ctx): - """Deploy files to a Nexus sites repository.""" + """Deploy files to a Nexus sites repository. + + Deploy commands use ~/.netrc for authentication. This file should be + pre-configured with an entry for the Nexus server. Eg. + + \b + machine nexus.opendaylight.org login logs_user password logs_password + """ pass @@ -31,7 +42,7 @@ def deploy(ctx): @click.argument('nexus-url', envvar='NEXUS_URL') @click.argument('nexus-path', envvar='NEXUS_PATH') @click.argument('workspace', envvar='WORKSPACE') -@click.option('-p', '--pattern', default='') +@click.option('-p', '--pattern', multiple=True) @click.pass_context def archives(ctx, nexus_url, nexus_path, workspace, pattern): """Archive files to a Nexus site repository. @@ -42,12 +53,19 @@ def archives(ctx, nexus_url, nexus_path, workspace, pattern): 1) globstar pattern provided by the user. 2) $WORKSPACE/archives directory provided by the user. - To use this script the Nexus server must have a site repository configured - with the name "logs" as this is a hardcoded path. Also this script uses - ~/.netrc for it's authentication which must be provided. + To use this command the Nexus server must have a site repository configured + with the name "logs" as this is a hardcoded log path. """ - status = subprocess.call(['deploy', 'archives', nexus_url, nexus_path, workspace, pattern]) - sys.exit(status) + if not pattern: + pattern = None + + try: + deploy_sys.deploy_archives(nexus_url, nexus_path, workspace, pattern) + except HTTPError as e: + log.error(str(e)) + sys.exit(1) + + log.info('Archives upload complete.') @click.command(name='copy-archives') diff --git a/lftools/deploy.py b/lftools/deploy.py index 786e527c..eceb398b 100644 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -9,11 +9,13 @@ ############################################################################## """Library of functions for deploying artifacts to Nexus.""" +import gzip import logging import os import re import shutil import sys +import tempfile import glob2 # Switch to glob when Python < 3.5 support is dropped import requests @@ -60,6 +62,11 @@ def copy_archives(workspace, pattern=None): The best way to use this function is to cd into the directory you wish to store the files first before calling the function. + This function provides 2 ways to archive files: + + 1) copy $WORKSPACE/archives directory + 2) copy globstar pattern + :params: :arg str pattern: Space-separated list of Unix style glob patterns. @@ -101,3 +108,64 @@ def copy_archives(workspace, pattern=None): log.debug(e) os.makedirs(os.path.dirname(dest)) shutil.move(src, dest) + + +def deploy_archives(nexus_url, nexus_path, workspace, pattern=None): + """Archive files to a Nexus site repository named logs. + + Provides 2 ways to archive files: + 1) $WORKSPACE/archives directory provided by the user. + 2) globstar pattern provided by the user. + + Requirements: + + To use this API a Nexus server must have a site repository configured + with the name "logs" as this is a hardcoded path. + + Parameters: + + :nexus_url: URL of Nexus server. Eg: https://nexus.opendaylight.org + :nexus_path: Path on nexus logs repo to place the logs. Eg: + $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + :workspace: Directory in which to search, typically in Jenkins this is + $WORKSPACE + :pattern: Space-separated list of Globstar patterns of files to + archive. (optional) + """ + nexus_url = _format_url(nexus_url) + work_dir = tempfile.mkdtemp(prefix='lftools-da.') + os.chdir(work_dir) + log.debug('workspace: {}, work_dir: {}'.format(workspace, work_dir)) + + copy_archives(workspace, pattern) + + compress_types = [ + '**/*.log', + '**/*.txt', + ] + paths = [] + for _type in compress_types: + search = os.path.join(work_dir, _type) + paths.extend(glob2.glob(search, recursive=True)) + + for _file in paths: + with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest: + shutil.copyfileobj(src, dest) + os.remove(_file) + + archives_zip = shutil.make_archive( + '{}/archives'.format(workspace), 'zip') + log.debug('archives zip: {}'.format(archives_zip)) + + # TODO: Update to use I58ea1d7703b626f791dcd74e63251c4f3261ca7d once it's available. + upload_files = {'upload_file': open(archives_zip, 'rb')} + url = '{}/service/local/repositories/logs/content-compressed/{}'.format( + nexus_url, nexus_path) + r = requests.post(url, files=upload_files) + log.debug('{}: {}'.format(r.status_code, r.text)) + if r.status_code != 201: + raise requests.exceptions.HTTPError( + 'Failed to upload to Nexus with status code {}.\n{}'.format( + r.status_code, r.content)) + + shutil.rmtree(work_dir) diff --git a/releasenotes/notes/refactor-deploy-archives-5f86cfbe8415defc.yaml b/releasenotes/notes/refactor-deploy-archives-5f86cfbe8415defc.yaml new file mode 100644 index 00000000..5782fd0b --- /dev/null +++ b/releasenotes/notes/refactor-deploy-archives-5f86cfbe8415defc.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Refactored deploy_archives() function from shell/deploy to pure Python to + be more portable with Windows systems. +deprecations: + - | + shell/deploy script's deploy_archives() function is now deprecated and will + be removed in a future release. diff --git a/requirements-test.txt b/requirements-test.txt index b030c6c9..ce946138 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,3 +3,4 @@ pytest pytest-click pytest-datafiles pytest-mock +pytest-responses diff --git a/tests/test_deploy.py b/tests/test_deploy.py index dff98c6b..a3d5b4ed 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -88,6 +88,34 @@ def test_copy_archive_pattern(cli_runner, datafiles): stage_dir, 'aaa', 'aaa-cert', 'target', 'surefire-reports', 'org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt')) +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_archive(cli_runner, datafiles, responses): + """Test deploy_archives() command for expected upload cases.""" + os.chdir(str(datafiles)) + workspace_dir = os.path.join(str(datafiles), 'workspace') + + # Test successful upload + url = 'https://nexus.example.org/service/local/repositories/logs/content-compressed' + responses.add(responses.POST, '{}/test/path/abc'.format(url), + json=None, status=201) + result = cli_runner.invoke( + cli.cli, + ['--debug', 'deploy', 'archives', 'https://nexus.example.org', 'test/path/abc', workspace_dir], + obj={}) + assert result.exit_code == 0 + + # Test failed upload + url = 'https://nexus-fail.example.org/service/local/repositories/logs/content-compressed' + responses.add(responses.POST, '{}/test/fail/path'.format(url), + status=404) + result = cli_runner.invoke( + cli.cli, + ['--debug', 'deploy', 'archives', 'https://nexus-fail.example.org', 'test/fail/path', workspace_dir], + obj={}) + assert result.exit_code == 1 + def mocked_log_error(msg1=None, msg2=None): """Mock local_log_error_and_exit function. This function is modified to simply raise an Exception.