__author__ = 'Thanh Ha'
+import logging
import subprocess
import sys
import click
+from requests.exceptions import HTTPError
import lftools.deploy as deploy_sys
+log = logging.getLogger(__name__)
+
@click.group()
@click.pass_context
def deploy(ctx):
- """Deploy files to a Nexus sites repository."""
+ """Deploy files to a Nexus sites repository.
+
+ Deploy commands use ~/.netrc for authentication. This file should be
+ pre-configured with an entry for the Nexus server. Eg.
+
+ \b
+ machine nexus.opendaylight.org login logs_user password logs_password
+ """
pass
@click.argument('nexus-url', envvar='NEXUS_URL')
@click.argument('nexus-path', envvar='NEXUS_PATH')
@click.argument('workspace', envvar='WORKSPACE')
-@click.option('-p', '--pattern', default='')
+@click.option('-p', '--pattern', multiple=True)
@click.pass_context
def archives(ctx, nexus_url, nexus_path, workspace, pattern):
"""Archive files to a Nexus site repository.
1) globstar pattern provided by the user.
2) $WORKSPACE/archives directory provided by the user.
- To use this script the Nexus server must have a site repository configured
- with the name "logs" as this is a hardcoded path. Also this script uses
- ~/.netrc for it's authentication which must be provided.
+ To use this command the Nexus server must have a site repository configured
+ with the name "logs" as this is a hardcoded log path.
"""
- status = subprocess.call(['deploy', 'archives', nexus_url, nexus_path, workspace, pattern])
- sys.exit(status)
+ if not pattern:
+ pattern = None
+
+ try:
+ deploy_sys.deploy_archives(nexus_url, nexus_path, workspace, pattern)
+ except HTTPError as e:
+ log.error(str(e))
+ sys.exit(1)
+
+ log.info('Archives upload complete.')
@click.command(name='copy-archives')
##############################################################################
"""Library of functions for deploying artifacts to Nexus."""
+import gzip
import logging
import os
import re
import shutil
import sys
+import tempfile
import glob2 # Switch to glob when Python < 3.5 support is dropped
import requests
The best way to use this function is to cd into the directory you wish to
store the files first before calling the function.
+ This function provides 2 ways to archive files:
+
+ 1) copy $WORKSPACE/archives directory
+ 2) copy globstar pattern
+
:params:
:arg str pattern: Space-separated list of Unix style glob patterns.
log.debug(e)
os.makedirs(os.path.dirname(dest))
shutil.move(src, dest)
+
+
+def deploy_archives(nexus_url, nexus_path, workspace, pattern=None):
+ """Archive files to a Nexus site repository named logs.
+
+ Provides 2 ways to archive files:
+ 1) $WORKSPACE/archives directory provided by the user.
+ 2) globstar pattern provided by the user.
+
+ Requirements:
+
+ To use this API a Nexus server must have a site repository configured
+ with the name "logs" as this is a hardcoded path.
+
+ Parameters:
+
+ :nexus_url: URL of Nexus server. Eg: https://nexus.opendaylight.org
+ :nexus_path: Path on nexus logs repo to place the logs. Eg:
+ $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
+ :workspace: Directory in which to search, typically in Jenkins this is
+ $WORKSPACE
+ :pattern: Space-separated list of Globstar patterns of files to
+ archive. (optional)
+ """
+ nexus_url = _format_url(nexus_url)
+ work_dir = tempfile.mkdtemp(prefix='lftools-da.')
+ os.chdir(work_dir)
+ log.debug('workspace: {}, work_dir: {}'.format(workspace, work_dir))
+
+ copy_archives(workspace, pattern)
+
+ compress_types = [
+ '**/*.log',
+ '**/*.txt',
+ ]
+ paths = []
+ for _type in compress_types:
+ search = os.path.join(work_dir, _type)
+ paths.extend(glob2.glob(search, recursive=True))
+
+ for _file in paths:
+ with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest:
+ shutil.copyfileobj(src, dest)
+ os.remove(_file)
+
+ archives_zip = shutil.make_archive(
+ '{}/archives'.format(workspace), 'zip')
+ log.debug('archives zip: {}'.format(archives_zip))
+
+ # TODO: Update to use I58ea1d7703b626f791dcd74e63251c4f3261ca7d once it's available.
+ upload_files = {'upload_file': open(archives_zip, 'rb')}
+ url = '{}/service/local/repositories/logs/content-compressed/{}'.format(
+ nexus_url, nexus_path)
+ r = requests.post(url, files=upload_files)
+ log.debug('{}: {}'.format(r.status_code, r.text))
+ if r.status_code != 201:
+ raise requests.exceptions.HTTPError(
+ 'Failed to upload to Nexus with status code {}.\n{}'.format(
+ r.status_code, r.content))
+
+ shutil.rmtree(work_dir)
--- /dev/null
+---
+features:
+ - |
+ Refactored deploy_archives() function from shell/deploy to pure Python to
+ be more portable with Windows systems.
+deprecations:
+ - |
+ shell/deploy script's deploy_archives() function is now deprecated and will
+ be removed in a future release.
pytest-click
pytest-datafiles
pytest-mock
+pytest-responses
stage_dir, 'aaa', 'aaa-cert', 'target', 'surefire-reports',
'org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt'))
+@pytest.mark.datafiles(
+ os.path.join(FIXTURE_DIR, 'deploy'),
+ )
+def test_deploy_archive(cli_runner, datafiles, responses):
+ """Test deploy_archives() command for expected upload cases."""
+ os.chdir(str(datafiles))
+ workspace_dir = os.path.join(str(datafiles), 'workspace')
+
+ # Test successful upload
+ url = 'https://nexus.example.org/service/local/repositories/logs/content-compressed'
+ responses.add(responses.POST, '{}/test/path/abc'.format(url),
+ json=None, status=201)
+ result = cli_runner.invoke(
+ cli.cli,
+ ['--debug', 'deploy', 'archives', 'https://nexus.example.org', 'test/path/abc', workspace_dir],
+ obj={})
+ assert result.exit_code == 0
+
+ # Test failed upload
+ url = 'https://nexus-fail.example.org/service/local/repositories/logs/content-compressed'
+ responses.add(responses.POST, '{}/test/fail/path'.format(url),
+ status=404)
+ result = cli_runner.invoke(
+ cli.cli,
+ ['--debug', 'deploy', 'archives', 'https://nexus-fail.example.org', 'test/fail/path', workspace_dir],
+ obj={})
+ assert result.exit_code == 1
+
def mocked_log_error(msg1=None, msg2=None):
"""Mock local_log_error_and_exit function.
This function is modified to simply raise an Exception.
Code Review / releng / lftools.git / commitdiff