From 837552cb3308a4cafaf8b283e6c78739f25410e8 Mon Sep 17 00:00:00 2001 From: Bengt Thuree Date: Tue, 13 Nov 2018 22:25:56 +1100 Subject: [PATCH] Refactor of Deploy_Nexus Refactoring deploy_nexus as part of the lftools-deploy-refactor from shell to python. Issue-ID: RELENG-1379 Issue-ID: RELENG-1374 Change-Id: Ibcf732643649600fa7049d92cdf1ac1072590371 Signed-off-by: Bengt Thuree --- lftools/cli/deploy.py | 16 +- lftools/deploy.py | 49 +- .../notes/deploy_nexus-4feb8fc7e24daaf0.yaml | 13 + .../m2repo/4.0.3-SNAPSHOT/maven-metadata.xml | 20 + .../m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.md5 | 1 + .../m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.sha1 | 1 + .../odlparent-lite-4.0.3-20181120.113136-1.pom | 542 +++++++++++++++++++++ .../odlparent-lite-4.0.3-20181120.113136-1.pom.md5 | 1 + ...odlparent-lite-4.0.3-20181120.113136-1.pom.sha1 | 1 + tests/fixtures/deploy/m2repo/maven-metadata.xml | 11 + .../fixtures/deploy/m2repo/maven-metadata.xml.md5 | 1 + .../fixtures/deploy/m2repo/maven-metadata.xml.sha1 | 1 + tests/test_deploy.py | 57 +++ 13 files changed, 704 insertions(+), 10 deletions(-) create mode 100644 releasenotes/notes/deploy_nexus-4feb8fc7e24daaf0.yaml create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.md5 create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.sha1 create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5 create mode 100644 tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1 create mode 100644 tests/fixtures/deploy/m2repo/maven-metadata.xml create mode 100644 tests/fixtures/deploy/m2repo/maven-metadata.xml.md5 create mode 100644 tests/fixtures/deploy/m2repo/maven-metadata.xml.sha1 diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index 4da3ac37..e9b93a45 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -239,15 +239,13 @@ def nexus(ctx, nexus_repo_url, deploy_dir, snapshot): https://nexus.example.org/content/repositories/release """ - params = ['deploy', 'nexus'] - - if snapshot: - params.extend(["-s"]) - - params.extend([nexus_repo_url, deploy_dir]) - - status = subprocess.call(params) - sys.exit(status) + log.debug("nexus_repo_url={}, deploy_dir={}, snapshot={}".format(nexus_repo_url, deploy_dir, snapshot)) + try: + deploy_sys.deploy_nexus(nexus_repo_url, deploy_dir, snapshot) + except IOError as e: + deploy_sys._log_error_and_exit(str(e)) + except HTTPError as e: + deploy_sys._log_error_and_exit(str(e)) @click.command(name='nexus-stage') diff --git a/lftools/deploy.py b/lftools/deploy.py index 5032ccd9..59404b93 100644 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -13,6 +13,8 @@ import errno import gzip import logging +import multiprocessing +from multiprocessing.dummy import Pool as ThreadPool import os import re import shutil @@ -352,7 +354,7 @@ def nexus_stage_repo_create(nexus_url, staging_profile_id): Returns: staging_repo_id Sample: - lftools deploy nexus-stage-repo-create 192.168.1.26:8081/nexsus/ 93fb68073c18 + lftools deploy nexus-stage-repo-create 192.168.1.26:8081/nexus/ 93fb68073c18 """ nexus_url = '{0}/service/local/staging/profiles/{1}/start'.format( _format_url(nexus_url), @@ -488,3 +490,48 @@ def upload_maven_file_to_nexus(nexus_url, nexus_repo_id, if re.search('nexus-error', resp.text): error_msg = _get_node_from_xml(resp.text, 'msg') raise requests.HTTPError("Nexus Error: {}".format(error_msg)) + + +def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False): + """Deploy Maven artifacts to Nexus. + + Parameters: + nexus_repo_url: URL to Nexus server. (Ex: https://nexus.example.org) + deploy_dir: The directory to deploy. (Ex: /tmp/m2repo) + + One purpose of this is so that we can get around the problematic + deploy-at-end configuration with upstream Maven. + https://issues.apache.org/jira/browse/MDEPLOY-193 + Sample: + lftools deploy nexus \ + http://192.168.1.26:8081/nexus/content/repositories/releases \ + tests/fixtures/deploy/zip-test-files + """ + def _deploy_nexus_upload(file): + """Fix file path, and call _request_post_file.""" + nexus_url_with_file = '{}/{}'.format(_format_url(nexus_repo_url), file) + log.debug('Uploading {} to nexus : {}'.format(file, nexus_url_with_file)) + _request_post_file(nexus_url_with_file, file) + + file_list = [] + + previous_dir = os.getcwd() + os.chdir(deploy_dir) + log.debug('Current dir is deploy_dir: {}'.format(deploy_dir)) + log.info('Uploading url : {}'.format(nexus_repo_url)) + files = glob2.glob('**/*') + for file in files: + if os.path.isfile(file): + if not file.endswith("_remote.repositories") and not file.endswith("resolver-status.properties"): + if snapshot: + file_list.append(file) + else: + if not "maven-metadata" in file: + file_list.append(file) + # Perform parallel upload + pool = ThreadPool(multiprocessing.cpu_count()) + pool.map(_deploy_nexus_upload, file_list) + pool.close() + pool.join() + + os.chdir(previous_dir) diff --git a/releasenotes/notes/deploy_nexus-4feb8fc7e24daaf0.yaml b/releasenotes/notes/deploy_nexus-4feb8fc7e24daaf0.yaml new file mode 100644 index 00000000..9ba34cee --- /dev/null +++ b/releasenotes/notes/deploy_nexus-4feb8fc7e24daaf0.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Refactored deploy_nexus function + from shell/deploy to pure Python to be more portable with Windows systems. + Also added a number of unit tests to cover all executable branches of the + code. + +deprecations: + - | + shell/deploy script's deploy_nexus + function is now deprecated and will be removed in a future release. + diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml new file mode 100644 index 00000000..a7d6af7f --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml @@ -0,0 +1,20 @@ + + + org.opendaylight.odlparent + odlparent-lite + 4.0.3-SNAPSHOT + + + 20181120.113136 + 1 + + 20181120113136 + + + pom + 4.0.3-20181120.113136-1 + 20181120113136 + + + + diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.md5 b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.md5 new file mode 100644 index 00000000..8952e38e --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.md5 @@ -0,0 +1 @@ +49343dad0e0b29452b785ff50ca8809b \ No newline at end of file diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.sha1 b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.sha1 new file mode 100644 index 00000000..07f42243 --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +90e26664185217e1fb1cd84b5a88626b01b452c4 \ No newline at end of file diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom new file mode 100644 index 00000000..ae2014b5 --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom @@ -0,0 +1,542 @@ + + + + + 4.0.0 + + + + org.opendaylight.odlparent + odlparent-lite + 4.0.3-SNAPSHOT + pom + ODL :: odlparent :: ${project.artifactId} + + + ${odl.site.url}/${project.groupId}/${stream}/${project.artifactId}/ + OpenDaylight is leading the transformation to Open Software Defined Networking (SDN). For more information, please see https://www.opendaylight.org + + + + Eclipse Public License v1.0 + https://www.eclipse.org/legal/epl-v10.html + + + + + OpenDaylight + https://www.opendaylight.org + + + + JIRA + https://jira.opendaylight.org/ + + + + Jenkins + https://jenkins.opendaylight.org/releng/ + + + + https://git.opendaylight.org/gerrit/ + + + + + * + Please consult the PROJECT_INFO.yaml, README* and/or CONTRIBUTORS which should be included with this JAR + https://www.opendaylight.org + helpdesk@opendaylight.org + + + + + https://nexus.opendaylight.org/content + + + file:${user.dir}/target/staged-site + https://nexus.opendaylight.org/content/sites/site/ + latest + + UTF-8 + UTF-8 + + + 3.0.1 + + + false + + + false + + + 0.8.2 + + + + + + + maven-archetype-plugin + ${maven.archetype.plugin.version} + + + maven-clean-plugin + 3.1.0 + + + maven-dependency-plugin + 3.1.1 + + + maven-deploy-plugin + 2.8.2 + + ${maven.deploy.skip} + + + + maven-help-plugin + 3.1.0 + + + maven-install-plugin + 2.5.2 + + ${maven.install.skip} + + + + maven-javadoc-plugin + 3.0.1 + + + true + + + + goal + t + Goal: + + + requiresProject + t + Requires project: + + + threadSafe + t + Threadsafe + + + phase + t + Phase: + + + + false + true + + + + attach-javadocs + + jar + + + + + + maven-project-info-reports-plugin + 3.0.0 + + + maven-release-plugin + 2.5.3 + + + maven-site-plugin + 3.7.1 + + + + ./images + ${project.build.directory}/site/images + font + coderay + style + + + asciidoctor-diagram + + + + + + org.asciidoctor + asciidoctor-maven-plugin + 1.5.7.1 + + + org.asciidoctor + asciidoctorj-diagram + 1.5.10 + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + org.codehaus.mojo + jdepend-maven-plugin + 2.0 + + + site + + generate-no-fork + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + pl.project13.maven + git-commit-id-plugin + [1,) + + revision + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + [2.10,) + + unpack + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + [1.0.0,) + + enforce + + + + + + + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + + + + maven-enforcer-plugin + 3.0.0-M2 + + + enforce-maven + + + + 1.8.0 + + + [3.5.0,) + + + + + enforce + + + + + + pl.project13.maven + git-commit-id-plugin + 2.2.5 + + + get-git-infos + + revision + + + + + false + true + ${project.build.outputDirectory}/META-INF/git.properties + + false + true + true + + + + + + + + + ide + + + m2e.version + + + + + target-ide + + + + + q + + true + + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + + + + + + + addInstallRepositoryPath + + + + + + maven-install-plugin + + + additional-install + install + + install-file + + + ${project.build.directory}/${project.build.finalName}.jar + ${addInstallRepositoryPath} + + + + + + + + + + maven-site + + + ${user.dir}/deploy-site.xml + + + + + + + maven-site-plugin + + + + generate-site + install + + site + attach-descriptor + + + + + + + + + + + maven-project-info-reports-plugin + + false + + + + + index + + + + + + maven-javadoc-plugin + + + + javadoc-no-fork + test-javadoc-no-fork + + + + + + + + + + javadoc-links + + + + maven-javadoc-plugin + + + http://www.atetric.com/atetric/javadoc/junit/junit/4.11/ + http://hamcrest.org/JavaHamcrest/javadoc/1.3/ + http://google.github.io/truth/api/0.42/ + https://www.slf4j.org/apidocs/ + https://xerces.apache.org/xerces2-j/javadocs/api/ + https://google.github.io/guava/releases/25.1-jre/api/docs/ + http://doc.akka.io/japi/akka/2.5.14/ + http://netty.io/4.1/api/ + https://commons.apache.org/proper/commons-lang/javadocs/api-2.6/ + https://commons.apache.org/proper/commons-lang/javadocs/api-3.8.1/ + https://commons.apache.org/proper/commons-codec/apidocs/ + + + + + + + + + + jdk9-javadoc + + [9,) + + + + + maven-javadoc-plugin + + + -html4 + + + + + + + + + + + opendaylight-release + ${nexusproxy}/repositories/opendaylight.release/ + + + opendaylight-snapshot + ${nexusproxy}/repositories/opendaylight.snapshot/ + + + opendaylight-site + ${nexus.site.url}/${project.artifactId}/ + + + diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5 b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5 new file mode 100644 index 00000000..6a8f929c --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5 @@ -0,0 +1 @@ +09be1f99c2ebd8952cdac8f9549c89ee \ No newline at end of file diff --git a/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1 b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1 new file mode 100644 index 00000000..06dd2fad --- /dev/null +++ b/tests/fixtures/deploy/m2repo/4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1 @@ -0,0 +1 @@ +68e4c8d328543f76dbc14a36c73d16e51f8ee55a \ No newline at end of file diff --git a/tests/fixtures/deploy/m2repo/maven-metadata.xml b/tests/fixtures/deploy/m2repo/maven-metadata.xml new file mode 100644 index 00000000..f133eaac --- /dev/null +++ b/tests/fixtures/deploy/m2repo/maven-metadata.xml @@ -0,0 +1,11 @@ + + + org.opendaylight.odlparent + odlparent-lite + + + 4.0.3-SNAPSHOT + + 20181120113136 + + diff --git a/tests/fixtures/deploy/m2repo/maven-metadata.xml.md5 b/tests/fixtures/deploy/m2repo/maven-metadata.xml.md5 new file mode 100644 index 00000000..b5368186 --- /dev/null +++ b/tests/fixtures/deploy/m2repo/maven-metadata.xml.md5 @@ -0,0 +1 @@ +b95fc576c7fdd297fcc015a9278cf287 \ No newline at end of file diff --git a/tests/fixtures/deploy/m2repo/maven-metadata.xml.sha1 b/tests/fixtures/deploy/m2repo/maven-metadata.xml.sha1 new file mode 100644 index 00000000..05cdd4d1 --- /dev/null +++ b/tests/fixtures/deploy/m2repo/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +aa3643f1cf459ad0ad17a3fa6f28c602e4511530 \ No newline at end of file diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 27b7d11a..a0f2e34a 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -542,3 +542,60 @@ def test__upload_maven_file_to_nexus(responses, mocker): with pytest.raises(requests.HTTPError) as excinfo: resp = deploy_sys.upload_maven_file_to_nexus(test_url, nexus_repo_id, group_id, artifact_id, version, packaging, zip_file) assert 'Something went wrong' in str(excinfo.value) + + +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_nexus_snapshot(datafiles, responses): + """Test deploy_nexus with snapshot. + + This test will send a directory of files to deploy_nexus, which should + call requests.post once for every valid (=3) file. + There are two files that should not be uploaded. + """ + os.chdir(str(datafiles)) + nexus_url = 'http://successfull.nexus.deploy/nexus/content/repositories/releases' + deploy_dir = 'm2repo' + + # Test success - Snapshot + snapshot = True + test_files = ['4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom', + '4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1', + '4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5', + '4.0.3-SNAPSHOT/maven-metadata.xml', + '4.0.3-SNAPSHOT/maven-metadata.xml.md5', + '4.0.3-SNAPSHOT/maven-metadata.xml.sha1', + 'maven-metadata.xml', + 'maven-metadata.xml.md5', + 'maven-metadata.xml.sha1'] + for file in test_files: + success_upload_url = '{}/{}'.format(nexus_url, file) + responses.add(responses.POST, success_upload_url, + status=201) + deploy_sys.deploy_nexus(nexus_url, deploy_dir, snapshot) + + +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_nexus_nosnapshot(datafiles, responses): + """Test deploy_nexus with no snapshot. + + This test will send a directory of files to deploy_nexus, which should + call requests.post once for every valid (=3) file. + There are six files that should not be uploaded, and three that should. + """ + os.chdir(str(datafiles)) + nexus_url = 'http://successfull.nexus.deploy/nexus/content/repositories/releases' + deploy_dir = 'm2repo' + + # Test success - No Snapshot + test_files = ['4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom', + '4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.sha1', + '4.0.3-SNAPSHOT/odlparent-lite-4.0.3-20181120.113136-1.pom.md5'] + for file in test_files: + success_upload_url = '{}/{}'.format(nexus_url, file) + responses.add(responses.POST, success_upload_url, + status=201) + deploy_sys.deploy_nexus(nexus_url, deploy_dir) -- 2.16.6