From dfab0ddcb3378c9fcaa21d2757babab4999ebf3e Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Thu, 8 Nov 2018 10:31:30 +0800 Subject: [PATCH] Refactor deploy-logs cmd to Python Issue: RELENG-1377 Change-Id: Ibf58a92fe8800ad77d8be32a123da603c6a36c56 Signed-off-by: Thanh Ha --- lftools/cli/deploy.py | 12 +- lftools/deploy.py | 127 ++++++++++++++++++--- .../refactor-deploy-logs-8631ffcf7eb7cad2.yaml | 9 ++ tests/test_deploy.py | 22 ++++ 4 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index 43129dc0..520ac3f1 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -135,11 +135,15 @@ def logs(ctx, nexus_url, nexus_path, build_url): for log archiving. To use this script the Nexus server must have a site repository configured - with the name "logs" as this is a hardcoded path. Also this script uses - ~/.netrc for it's authentication which must be provided. + with the name "logs" as this is a hardcoded path. """ - status = subprocess.call(['deploy', 'logs', nexus_url, nexus_path, build_url]) - sys.exit(status) + try: + deploy_sys.deploy_logs(nexus_url, nexus_path, build_url) + except HTTPError as e: + log.error(str(e)) + sys.exit(1) + + log.info('Logs upload complete.') @click.command(name='maven-file') diff --git a/lftools/deploy.py b/lftools/deploy.py index eceb398b..098d48fe 100644 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -14,6 +14,7 @@ import logging import os import re import shutil +import subprocess import sys import tempfile @@ -23,6 +24,28 @@ import requests log = logging.getLogger(__name__) +def _compress_text(dir): + """Compress all text files in directory.""" + save_dir = os.getcwd() + os.chdir(dir) + + compress_types = [ + '**/*.log', + '**/*.txt', + ] + paths = [] + for _type in compress_types: + search = os.path.join(dir, _type) + paths.extend(glob2.glob(search, recursive=True)) + + for _file in paths: + with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest: + shutil.copyfileobj(src, dest) + os.remove(_file) + + os.chdir(save_dir) + + def _format_url(url): """Ensure url starts with http and trim trailing '/'s.""" start_pattern = re.compile('^(http|https)://') @@ -138,20 +161,7 @@ def deploy_archives(nexus_url, nexus_path, workspace, pattern=None): log.debug('workspace: {}, work_dir: {}'.format(workspace, work_dir)) copy_archives(workspace, pattern) - - compress_types = [ - '**/*.log', - '**/*.txt', - ] - paths = [] - for _type in compress_types: - search = os.path.join(work_dir, _type) - paths.extend(glob2.glob(search, recursive=True)) - - for _file in paths: - with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest: - shutil.copyfileobj(src, dest) - os.remove(_file) + _compress_text(work_dir) archives_zip = shutil.make_archive( '{}/archives'.format(workspace), 'zip') @@ -169,3 +179,92 @@ def deploy_archives(nexus_url, nexus_path, workspace, pattern=None): r.status_code, r.content)) shutil.rmtree(work_dir) + + +def deploy_logs(nexus_url, nexus_path, build_url): + """Deploy logs to a Nexus site repository named logs. + + Fetches logs and system information and pushes them to Nexus + for log archiving. + Requirements: + + To use this API a Nexus server must have a site repository configured + with the name "logs" as this is a hardcoded path. + + Parameters: + + :nexus_url: URL of Nexus server. Eg: https://nexus.opendaylight.org + :nexus_path: Path on nexus logs repo to place the logs. Eg: + $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + :build_url: URL of the Jenkins build. Jenkins typically provides this + via the $BUILD_URL environment variable. + """ + nexus_url = _format_url(nexus_url) + work_dir = tempfile.mkdtemp(prefix='lftools-dl.') + os.chdir(work_dir) + log.debug('work_dir: {}'.format(work_dir)) + + build_details = open('_build-details.log', 'w+') + build_details.write('build-url: {}'.format(build_url)) + + with open('_sys-info.log', 'w+') as sysinfo_log: + sys_cmds = [] + + log.debug('Platform: {}'.format(sys.platform)) + if sys.platform == "linux" or sys.platform == "linux2": + sys_cmds = [ + ['uname', '-a'], + ['lscpu'], + ['nproc'], + ['df', '-h'], + ['free', '-m'], + ['ip', 'addr'], + ['sar', '-b', '-r', '-n', 'DEV'], + ['sar', '-P', 'ALL'], + ] + + for c in sys_cmds: + try: + output = subprocess.check_output(c).decode('utf-8') + except OSError: # TODO: Switch to FileNotFoundError when Python < 3.5 support is dropped. + log.debug('Command not found: {}'.format(c)) + continue + + output = '---> {}:\n{}\n'.format(' '.join(c), output) + sysinfo_log.write(output) + log.info(output) + + build_details.close() + + # Magic string used to trim console logs at the appropriate level during wget + MAGIC_STRING = "-----END_OF_BUILD-----" + log.info(MAGIC_STRING) + + resp = requests.get('{}/consoleText'.format(_format_url(build_url))) + with open('console.log', 'w+') as f: + f.write(resp.text.split(MAGIC_STRING)[0]) + + resp = requests.get('{}/timestamps?time=HH:mm:ss&appendLog'.format(_format_url(build_url))) + with open('console-timestamp.log', 'w+') as f: + f.write(resp.text.split(MAGIC_STRING)[0]) + + _compress_text(work_dir) + + console_zip = tempfile.NamedTemporaryFile(prefix='lftools-dl', delete=True) + log.debug('console-zip: {}'.format(console_zip.name)) + shutil.make_archive(console_zip.name, 'zip', work_dir) + log.debug('listdir: {}'.format(os.listdir(work_dir))) + + # TODO: Update to use I58ea1d7703b626f791dcd74e63251c4f3261ca7d once it's available. + upload_files = {'upload_file': open('{}.zip'.format(console_zip.name), 'rb')} + url = '{}/service/local/repositories/logs/content-compressed/{}'.format( + nexus_url, nexus_path) + r = requests.post(url, files=upload_files) + log.debug('{}: {}'.format(r.status_code, r.text)) + if r.status_code != 201: + raise requests.exceptions.HTTPError( + 'Failed to upload to Nexus with status code {}.\n{}'.format( + r.status_code, r.content)) + + console_zip.close() + shutil.rmtree(work_dir) diff --git a/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml b/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml new file mode 100644 index 00000000..cb29d13f --- /dev/null +++ b/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Refactored deploy_logs() function from shell/deploy to pure Python to + be more portable with Windows systems. +deprecations: + - | + shell/deploy script's deploy_logs() function is now deprecated and will + be removed in a future release. diff --git a/tests/test_deploy.py b/tests/test_deploy.py index a3d5b4ed..54585025 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -116,6 +116,28 @@ def test_deploy_archive(cli_runner, datafiles, responses): obj={}) assert result.exit_code == 1 +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_logs(cli_runner, datafiles, responses): + """Test deploy_logs() command for expected upload cases.""" + os.chdir(str(datafiles)) + workspace_dir = os.path.join(str(datafiles), 'workspace') + + # Test successful upload + build_url = 'https://jenkins.example.org/job/builder-check-poms/204' + nexus_url = 'https://nexus.example.org/service/local/repositories/logs/content-compressed' + responses.add(responses.GET, '{}/consoleText'.format(build_url), + status=201) + responses.add(responses.GET, '{}/timestamps?time=HH:mm:ss&appendLog'.format(build_url), + body='This is a console timestamped log.', status=201) + responses.add(responses.POST, '{}/test/log/upload'.format(nexus_url), status=201) + result = cli_runner.invoke( + cli.cli, + ['--debug', 'deploy', 'logs', 'https://nexus.example.org', 'test/log/upload', build_url], + obj={}) + assert result.exit_code == 0 + def mocked_log_error(msg1=None, msg2=None): """Mock local_log_error_and_exit function. This function is modified to simply raise an Exception. -- 2.16.6