From 5dfd489d3fe3e137f6845a046f3a69ed0fc24fbe Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Thu, 15 Nov 2018 00:00:24 -0800 Subject: [PATCH] Refactor deploy_maven_file from shell to python Refactoring the old shell script into pure python in order to improve compatibility with Windows. Issue: RELENG-1378 Change-Id: I4b9efbdd43fdc6605c5e152ca1a2a6b132770d1d Signed-off-by: Eric Ball --- lftools/cli/deploy.py | 64 ++++------------- lftools/nexus/__init__.py | 4 +- lftools/nexus/cmd.py | 83 ++++++++++++++++++++++ lftools/nexus/util.py | 43 +++++++++++ ...efactor-deploy-maven-file-766a7b05b4c31dbc.yaml | 26 +++++++ requirements.txt | 2 + tests/test_deploy.py | 38 ++++++++++ 7 files changed, 206 insertions(+), 54 deletions(-) create mode 100644 releasenotes/notes/refactor-deploy-maven-file-766a7b05b4c31dbc.yaml diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index 59625b1e..2c5780fb 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -13,13 +13,13 @@ __author__ = 'Thanh Ha' import logging -import subprocess import sys import click from requests.exceptions import HTTPError import lftools.deploy as deploy_sys +from lftools.nexus import cmd as nexuscmd log = logging.getLogger(__name__) @@ -148,15 +148,6 @@ def logs(ctx, nexus_url, nexus_path, build_url): @click.argument('nexus-url', envvar='NEXUS_URL') @click.argument('repo-id', envvar='REPO_ID') @click.argument('file-name', envvar='FILE_NAME') -# Maven Config -@click.option('-b', '--maven-bin', envvar='MAVEN_BIN', - help='Path of maven binary.') -@click.option('-gs', '--global-settings', envvar='GLOBAL_SETTINGS_FILE', - help='Global settings file.') -@click.option('-s', '--settings', envvar='SETTINGS_FILE', - help='Settings file.') -@click.option('-p', '--maven-params', - help='Pass Maven commandline options to the mvn command.') # Maven Artifact GAV @click.option('-a', '--artifact-id', help='Maven Artifact ID.') @@ -166,61 +157,30 @@ def logs(ctx, nexus_url, nexus_path, build_url): help='Pom file to extract GAV information from.') @click.option('-g', '--group-id', help='Maven Group ID') +@click.option('-p', '--packaging', + help='File packaging type.') @click.option('-v', '--version', help='Maven artifact version.') @click.pass_context def maven_file( - # Maven Config ctx, nexus_url, repo_id, file_name, - maven_bin, global_settings, settings, - maven_params, # Maven GAV - artifact_id, group_id, classifier, version, + artifact_id, group_id, classifier, packaging, version, pom_file): """Deploy a file to a Nexus maven2 repository. - As this script uses mvn to deploy. The server configuration should be - configured in your local settings.xml. By default the script uses the - mvn default "~/.m2/settings.xml" for the configuration but this can be - overrided in the following order: - - \b - 1. Passed through CLI option "-s" ("-gs" for global-settings) - 2. Environment variable "$SETTINGS_FILE" ("$GLOBAL_SETTINGS_FILE" for global-settings) - 3. Maven default "~/.m2/settings.xml". + Uses the local .netrc file for authorization. If pom-file is passed in via the "-f" option then the Maven GAV parameters are not necessary. pom-file setting overrides the Maven GAV parameters. """ - params = ['deploy', 'maven-file'] - - # Maven Configuration - if maven_bin: - params.extend(["-b", maven_bin]) - if global_settings: - params.extend(["-l", global_settings]) - if settings: - params.extend(["-s", settings]) - if maven_params: - params.extend(["-p", maven_params]) - - # Maven Artifact GAV - if artifact_id: - params.extend(["-a", artifact_id]) - if classifier: - params.extend(["-c", classifier]) - if group_id: - params.extend(["-g", group_id]) - if pom_file: - params.extend(["-f", pom_file]) - if version: - params.extend(["-v", version]) - - # Set required variables last as getopts get's processed first. - params.extend([nexus_url, repo_id, file_name]) - - status = subprocess.call(params) - sys.exit(status) + try: + nexuscmd.deploy_maven_file(nexus_url, repo_id, file_name, pom_file, + group_id, artifact_id, packaging, version, + classifier) + except HTTPError as e: + log.error(str(e)) + sys.exit(1) @click.command() diff --git a/lftools/nexus/__init__.py b/lftools/nexus/__init__.py index 6fa9db3a..c78a8277 100644 --- a/lftools/nexus/__init__.py +++ b/lftools/nexus/__init__.py @@ -36,9 +36,9 @@ class Nexus: self.baseurl = baseurl self.set_full_baseurl() if self.baseurl.find("local") < 0: - self.version = 2 - else: self.version = 3 + else: + self.version = 2 if username and password: self.add_credentials(username, password) diff --git a/lftools/nexus/cmd.py b/lftools/nexus/cmd.py index a1c83dc6..802470d2 100644 --- a/lftools/nexus/cmd.py +++ b/lftools/nexus/cmd.py @@ -281,6 +281,89 @@ def delete_images(settings_file, url, images): _nexus.delete_image(image) +def deploy_maven_file(nexus_url, nexus_repo_id, file_name, pom_file="", + group_id="", artifact_id="", packaging="", version="", + classifier=""): + """Deploy a file to a Nexus maven2 repository. + + :arg str nexus_url: URL of target Nexus server. + :arg str nexus_repo_id: Name of target Nexus repo. + :arg str file_name: Path to file that will be uploaded to Nexus. + :arg str pom_file: Optional path to POM file containing package info. + :arg str group_id: The artifact's groupId. Necessary if no pom is provided. + :arg str artifact_id: The artifact's artifactId. Necessary if no pom is + provided. + :arg str packaging: The artifact's packaging scheme. If not provided, this + will be guessed based on the file's extension. + :arg str version: The artifact's version. Necessary if no pom is provided. + :arg str classifier: Optional Nexus classifier for the artifact. + """ + _nexus = Nexus(nexus_url) + + if pom_file: + files = [ + ('file', (pom_file, open(pom_file, 'rb'))), + ('file', (file_name, open(file_name, 'rb'))) + ] + log.debug("Files: {}".format(files)) + + data = { + 'r': (None, nexus_repo_id), + 'hasPom': (None, 'true') + } + log.debug("Data: {}".format(data)) + + request_url = "{}/artifact/maven/content".format(_nexus.baseurl) + response = requests.post(request_url, data=data, files=files) + + else: + packaging = util.find_packaging_from_file_name(file_name) + log.debug("Packaging found: {}".format(packaging)) + + info_dict = {} + if packaging == "rpm": + info_dict = util.get_info_from_rpm(file_name) + elif packaging == "deb": + info_dict = util.get_info_from_deb(file_name) + if info_dict and not artifact_id: + artifact_id = info_dict["name"] + if info_dict and not version: + version = info_dict["version"] + + data = { + 'r': (None, nexus_repo_id), + 'p': (None, packaging), + 'hasPom': (None, 'false') + } + + files = { + 'file': (file_name, open(file_name, 'rb')) + } + log.debug("Files: {}".format(files)) + + if group_id: + data['g'] = (None, group_id) + if artifact_id: + data['a'] = (None, artifact_id) + if version: + data['v'] = (None, version) + if classifier: + data['c'] = (None, classifier) + log.debug("Data: {}".format(data)) + + request_url = "{}/artifact/maven/content".format(_nexus.baseurl) + log.debug("Request URL: {}".format(request_url)) + response = requests.post(request_url, data=data, files=files) + + if response.status_code != 201: + raise requests.HTTPError("Upload failed with the following " + "error:\n{}: {}".format(response.status_code, + response.text)) + else: + log.debug("Successfully uploaded {} to {}".format(file_name, + request_url)) + + def release_staging_repos(repos, nexus_url=""): """Release one or more staging repos. diff --git a/lftools/nexus/util.py b/lftools/nexus/util.py index 6938b6ec..d9ee259e 100644 --- a/lftools/nexus/util.py +++ b/lftools/nexus/util.py @@ -13,6 +13,11 @@ __author__ = 'Thanh Ha' import logging +import re +import sys + +from deb_pkg_tools.package import inspect_package_fields +import rpmfile log = logging.getLogger(__name__) @@ -20,3 +25,41 @@ log = logging.getLogger(__name__) def create_repo_target_regex(group_id): """Create a repo_target for Nexus use.""" return '^/{}/.*'.format(group_id.replace('.', '[/\.]')) + + +def find_packaging_from_file_name(file_name): + """Find packaging type based on file extension.""" + if file_name.endswith(".tar.gz"): + packaging = "tar.gz" + else: + ext_index = file_name.rfind(".") + if ext_index == -1: + log.error("ERROR: Could not determine packaging type. Please " + "provide an appropriate \"packaging\" parameter.") + sys.exit(1) + packaging = file_name[ext_index+1:] + return packaging + + +def get_info_from_rpm(path): + """Return data pulled from the headers of an RPM file.""" + info = {} + rpm = rpmfile.open(path) + info["name"] = rpm.headers.get("name") + ver = rpm.headers.get("version") + if re.search('\.s(rc\.)?rpm', path): + info["version"] = "{}.{}".format(ver, "src") + else: + info["version"] = "{}.{}".format(ver, rpm.headers.get("arch")) + log.debug("Info from rpm: {}".format(info)) + return info + + +def get_info_from_deb(path): + """Return data pulled from a deb archive.""" + info = {} + data = inspect_package_fields(path) + info["name"] = data["Package"] + info["version"] = data["Version"] + log.debug("Info from deb: {}".format(info)) + return info diff --git a/releasenotes/notes/refactor-deploy-maven-file-766a7b05b4c31dbc.yaml b/releasenotes/notes/refactor-deploy-maven-file-766a7b05b4c31dbc.yaml new file mode 100644 index 00000000..b98dd03d --- /dev/null +++ b/releasenotes/notes/refactor-deploy-maven-file-766a7b05b4c31dbc.yaml @@ -0,0 +1,26 @@ +--- +features: + - | + Refactored deploy_maven_file() function from shell/deploy to pure Python to + be more portable with Windows systems. +upgrade: + - | + The ``deploy maven-file`` command no longer calls maven (relying instead on + a direct REST call to Nexus). Due to this change, the following options + have been removed: + + * ``-b|--maven-bin`` + * ``-gs|--global-settings`` + * ``-s|--settings`` + * ``-p|--maven-params`` + + Additionally, the NEXUS_URL argument should now contain only the base URL + for the Nexus server being used, rather than the full path to the remote + repository. + + Any calls to this command should be updated to reflect the above changes. + Nexus credentials should be located in the local .netrc file. +deprecations: + - | + ``shell/deploy`` script's deploy_maven_file() function is now deprecated + and will be removed in a future release. diff --git a/requirements.txt b/requirements.txt index a1a78ce8..b0657f1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ click glob2 # Needed for Python < 3.5 recursive glob support +deb-pkg-tools~=5.2 defusedxml # Needed due to tox complains on parseString not safe pyyaml requests~=2.18.0 +rpmfile~=0.1.4 ruamel.yaml setuptools>=36.5.0 six~=1.11.0 diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 5bef101e..84d45223 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -634,3 +634,41 @@ def test_nexus_deploy_stage(datafiles, responses): #Execute test, should not return anything for successful run. deploy_sys.deploy_nexus_stage (url, staging_profile_id, deploy_dir) + + +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_maven_file(cli_runner, datafiles, responses): + """Test deploy_maven_file() command for expected upload cases.""" + os.chdir(str(datafiles)) + test_file = os.path.join(str(datafiles), 'm2repo', '4.0.3-SNAPSHOT', + 'odlparent-lite-4.0.3-20181120.113136-1.pom') + + # Prepare response for Nexus initialization + responses.add(responses.GET, + "https://nexus.example.org/service/local/repo_targets", + json=None, status=200) + # Test successful upload + url = 'https://nexus.example.org/service/local/artifact/maven/content' + responses.add(responses.POST, "{}".format(url), + json=None, status=201) + result = cli_runner.invoke( + cli.cli, + ["--debug", "deploy", "maven-file", "https://nexus.example.org", + "releases", test_file], + obj={}) + assert result.exit_code == 0 + + # Test failed upload + responses.add(responses.GET, + "https://nexus-fail.example.org/service/local/repo_targets", + json=None, status=200) + url = 'https://nexus-fail.example.org/service/local/artifact/maven/content' + responses.add(responses.POST, "{}".format(url), status=404) + result = cli_runner.invoke( + cli.cli, + ["--debug", "deploy", "maven-file", "https://nexus-fail.example.org", + "releases", test_file], + obj={}) + assert result.exit_code == 1 -- 2.16.6