From: Bengt Thuree Date: Tue, 7 Apr 2020 01:39:46 +0000 (+1000) Subject: Add --repofile to releasedockerhub X-Git-Tag: v0.35.0~1 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F15%2F63615%2F12;p=releng%2Flftools.git Add --repofile to releasedockerhub Enables providing a file with the repo names. -f, --repofile Repo Name is a file name, which contains one repo per row ISSUE: RELENG-2553 Signed-off-by: Bengt Thuree Change-Id: I74a6eb5b95e1fef7d70d1e500c6df706b2eab8c9 --- diff --git a/lftools/cli/nexus.py b/lftools/cli/nexus.py index 7386228f..7f2b7dbb 100644 --- a/lftools/cli/nexus.py +++ b/lftools/cli/nexus.py @@ -204,11 +204,23 @@ def release(ctx, repos, verify, server): default=False, help="Display a progress bar for the time consuming jobs.", ) +@click.option( + "-f", + "--repofile", + is_flag=True, + default=False, + required=False, + help="Repo Name is a file name, which contains one repo per row." + " Syntax: " + " 'Nexus3 docker.release repo'; 'dockerhub dockername' " + " Sample: " + " onap/msb/msb_apigateway; onap/msb-msb_apigateway", +) @click.pass_context -def copy_from_nexus3_to_dockerhub(ctx, org, repo, exact, summary, verbose, copy, progbar): +def copy_from_nexus3_to_dockerhub(ctx, org, repo, exact, summary, verbose, copy, progbar, repofile): """Find missing repos in Docker Hub, Copy from Nexus3. Will by default list all missing repos in Docker Hub, compared to Nexus3. If -c (--copy) is provided, it will copy the repos from Nexus3 to Docker Hub. """ - rdh.start_point(org, repo, exact, summary, verbose, copy, progbar) + rdh.start_point(org, repo, exact, summary, verbose, copy, progbar, repofile) diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py index 3fb025b7..e0ee395e 100644 --- a/lftools/nexus/release_docker_hub.py +++ b/lftools/nexus/release_docker_hub.py @@ -126,15 +126,17 @@ class TagClass: Parameter: org_name : The organization part of the repository. (onap) repo_name : The Nexus3 repository name (aaf/aaf_service) + repo_from_file : Repository name was taken from input file. """ - def __init__(self, org_name, repo_name): + def __init__(self, org_name, repo_name, repo_from_file): """Initialize this class.""" self.valid = [] self.invalid = [] self.repository_exist = True self.org = org_name self.repo = repo_name + self.repofromfile = repo_from_file def _validate_tag(self, check_tag): """Local helper function to simplify validity check of version number. @@ -171,6 +173,8 @@ class NexusTagClass(TagClass): curl -s https://nexus3.onap.org:10002/v2/onap/aaf/aaf_service/tags/list which gives you the following output: {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5","2.1.6","2.1.7","2.1.8"]} + # https://nexus3.edgexfoundry.org/repository/docker.staging/v2/docker-device-rest-go/tags/list + # https://nexus3.edgexfoundry.org:10002/v2/docker-device-rest-go/tags/list When we fetch the tags from the Nexus3 repository url, they are returned like {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5"]} @@ -181,6 +185,7 @@ class NexusTagClass(TagClass): Parameter: org_name : The organization part of the repository. (onap) repo_name : The Nexus3 repository name (aaf/aaf_service) + repo_from_file : The reponame came from an input file. Result: Will fetch all tags from the Nexus3 repository URL, and store each tag @@ -188,14 +193,18 @@ class NexusTagClass(TagClass): If no repository is found, self.repository_exist will be set to False. """ - def __init__(self, org_name, repo_name): + def __init__(self, org_name, repo_name, repo_from_file): """Initialize this class.""" - TagClass.__init__(self, org_name, repo_name) - log.debug("Fetching nexus3 tags for {}/{}".format(org_name, repo_name)) + TagClass.__init__(self, org_name, repo_name, repo_from_file) retries = 0 + # Default to / + org_repo_name = "{}/{}".format(org_name, repo_name) + if repo_from_file: + org_repo_name = "{}".format(repo_name) + log.debug("Fetching nexus3 tags for {}".format(org_repo_name)) while retries < 20: try: - r = _request_get(NEXUS3_BASE + "/v2/" + org_name + "/" + repo_name + "/tags/list") + r = _request_get(NEXUS3_BASE + "/v2/" + org_repo_name + "/tags/list") break except requests.HTTPError as excinfo: log.debug("Fetching Nexus3 tags. {}".format(excinfo)) @@ -216,7 +225,7 @@ class NexusTagClass(TagClass): if len(TmpSplittedTags) > 0: for tag_2_add in TmpSplittedTags: self.add_tag(tag_2_add) - log.debug("Nexus {}/{} has tag {}".format(org_name, repo_name, tag_2_add)) + log.debug("Nexus {} has tag {}".format(org_repo_name, tag_2_add)) else: self.repository_exist = False @@ -242,6 +251,7 @@ class DockerTagClass(TagClass): Parameter: org_name : The organization part of the repository. (onap) repo_name : The Docker Hub repository name (aaf-aaf_service) + repo_from_file : The reponame came from an input file. Result: Will fetch all tags from the Docker Repository URL, and store each tag @@ -251,14 +261,18 @@ class DockerTagClass(TagClass): _docker_base = "https://registry.hub.docker.com/v1/repositories" - def __init__(self, org_name, repo_name): + def __init__(self, org_name, repo_name, repo_from_file): """Initialize this class.""" - TagClass.__init__(self, org_name, repo_name) - log.debug("Fetching docker tags for {}/{}".format(org_name, repo_name)) + TagClass.__init__(self, org_name, repo_name, repo_from_file) + if repo_from_file: + combined_repo_name = repo_name + else: + combined_repo_name = "{}/{}".format(org_name, repo_name) + log.debug("Fetching docker tags for {}".format(combined_repo_name)) retries = 0 while retries < 20: try: - r = _request_get(self._docker_base + "/" + org_name + "/" + repo_name + "/tags") + r = _request_get(self._docker_base + "/" + combined_repo_name + "/tags") break except requests.HTTPError as excinfo: log.debug("Fetching Docker Hub tags. {}".format(excinfo)) @@ -281,7 +295,7 @@ class DockerTagClass(TagClass): tmp_tuple = tuple.split(":") if len(tmp_tuple) > 1: self.add_tag(tmp_tuple[2].strip()) - log.debug("Docker {}/{} has tag {}".format(org_name, repo_name, tmp_tuple[2])) + log.debug("Docker {} has tag {}".format(combined_repo_name, tmp_tuple[2])) else: self.repository_exist = False @@ -293,8 +307,8 @@ class ProjectClass: Nexus3 to Docker Hub. Parameters: - nexus_proj : Tuple with 'org' and 'repo' - ('onap', 'aaf/aaf_service') + nexus_proj : list with ['org', 'repo', 'dockername'] + ['onap', 'aaf/aaf_service', 'aaf-aaf_service'] Upon class Initialize the following happens. * Set Nexus and Docker repository names. @@ -308,10 +322,14 @@ class ProjectClass: """Initialize this class.""" self.org_name = nexus_proj[0] self.nexus_repo_name = nexus_proj[1] - self._set_docker_repo_name(self.nexus_repo_name) - self.nexus_tags = NexusTagClass(self.org_name, self.nexus_repo_name) - self.docker_tags = DockerTagClass(self.org_name, self.docker_repo_name) - self.tags_2_copy = TagClass(self.org_name, self.nexus_repo_name) + repo_from_file = len(nexus_proj[2]) > 0 + if repo_from_file: + self.docker_repo_name = nexus_proj[2].strip() + else: + self._set_docker_repo_name(self.nexus_repo_name) + self.nexus_tags = NexusTagClass(self.org_name, self.nexus_repo_name, repo_from_file) + self.docker_tags = DockerTagClass(self.org_name, self.docker_repo_name, repo_from_file) + self.tags_2_copy = TagClass(self.org_name, self.nexus_repo_name, repo_from_file) self._populate_tags_to_copy() self.docker_client = docker.from_env() @@ -451,7 +469,52 @@ class ProjectClass: raise requests.HTTPError(retry_text) -def get_nexus3_catalog(org_name="", find_pattern="", exact_match=False): +def repo_is_in_file(check_repo="", repo_file_name=""): + """Function to verify of a repo name exists in a file name. + + The file contains rows of repo names to be included. + acumos-portal-fe + acumos/acumos-axure-client + + Function will return True if a match is found + + """ + with open("{}".format(repo_file_name)) as f: + for line in f.readlines(): + row = line.rstrip() + reponame = row.split(";")[0] + log.debug("Comparing {} with {} from file".format(check_repo, reponame)) + if check_repo == reponame: + log.debug("Found a match") + return True + log.debug("NO match found") + return False + + +def get_docker_name_from_file(check_repo="", repo_file_name=""): + """Function to verify of a repo name exists in a file name. + + The file contains rows of repo names to be included. + acumos-portal-fe + acumos/acumos-axure-client + + Function will return True if a match is found + + """ + with open("{}".format(repo_file_name)) as f: + for line in f.readlines(): + row = line.rstrip() + reponame = row.split(";")[0] + dockername = row.split(";")[1] + log.debug("Comparing {} with {} from file".format(check_repo, reponame)) + if check_repo == reponame: + log.debug("Found a match") + return dockername + log.debug("NO match found") + return "" + + +def get_nexus3_catalog(org_name="", find_pattern="", exact_match=False, repo_is_filename=False): """Main function to collect all Nexus3 repositories. This function will collect the Nexus catalog for all projects starting with @@ -469,14 +532,16 @@ def get_nexus3_catalog(org_name="", find_pattern="", exact_match=False): Nexus3 catalog starts with / Parameters: - org_name : Organizational name, for instance 'onap' - find_pattern : A pattern, that if specified, needs to be part of the - repository name. - for instance, - '' : this pattern finds all repositories. - 'eleo' : this pattern finds all repositories with 'eleo' - in its name. --> chameleon - exact_match : If specified, find_pattern is a unique repo name + org_name : Organizational name, for instance 'onap' + find_pattern : A pattern, that if specified, needs to be part of the + repository name. + for instance, + '' : this pattern finds all repositories. + 'eleo' : this pattern finds all repositories with 'eleo' + in its name. --> chameleon + exact_match : If specified, find_pattern is a unique repo name + repo_is_filename: If specified, find_pattern is a filename, which contains a repo name per row + org_name is irrelevant in this case """ global NexusCatalog @@ -488,7 +553,9 @@ def get_nexus3_catalog(org_name="", find_pattern="", exact_match=False): containing_str = ', and containing "{}"'.format(find_pattern) if exact_match: containing_str = ', and reponame = "{}"'.format(find_pattern) - info_str = "Collecting information from Nexus with projects with org = {}".format(org_name) + if repo_is_filename: + containing_str = ', and repos are found in "{}"'.format(find_pattern) + info_str = "Collecting information from Nexus from projects with org = {}".format(org_name) log.info("{}{}.".format(info_str, containing_str)) try: @@ -509,21 +576,25 @@ def get_nexus3_catalog(org_name="", find_pattern="", exact_match=False): TmpCatalog = raw_catalog[1].split(",") for word in TmpCatalog: # Remove all projects that do not start with org_name - if word.startswith(org_name): - use_this_repo = False - # Remove org_name/ from word, so we only get repository left - project = (org_name, word[len(org_name) + 1 :]) - # If a specific search string has been specified, search for it - # Empty string will match all words - if word.find(find_pattern) >= 0 and not exact_match: - use_this_repo = True - if exact_match and project[1] == find_pattern: - use_this_repo = True - if use_this_repo: - NexusCatalog.append(project) - log.debug("Added project {} to my list".format(project[1])) - if len(project[1]) > project_max_len_chars: - project_max_len_chars = len(project[1]) + use_this_repo = False + if repo_is_filename and repo_is_in_file(word, find_pattern): + use_this_repo = True + project = [org_name, word, get_docker_name_from_file(word, find_pattern)] + else: + if word.startswith(org_name): + # Remove org_name/ from word, so we only get repository left + project = [org_name, word[len(org_name) + 1 :], ""] + # If a specific search string has been specified, search for it + # Empty string will match all words + if word.find(find_pattern) >= 0 and not exact_match: + use_this_repo = True + if exact_match and project[1] == find_pattern: + use_this_repo = True + if use_this_repo: + NexusCatalog.append(project) + log.debug("Added project {} to my list".format(project[1])) + if len(project[1]) > project_max_len_chars: + project_max_len_chars = len(project[1]) log.debug( "# TmpCatalog {}, NexusCatalog {}, DIFF = {}".format( len(TmpCatalog), len(NexusCatalog), len(TmpCatalog) - len(NexusCatalog) @@ -742,14 +813,23 @@ def print_nbr_tags_to_copy(): log.info("Summary: {} tags that should be copied from Nexus3 to Docker Hub.".format(_tot_tags)) -def start_point(org_name, find_pattern="", exact_match=False, summary=False, verbose=False, copy=False, progbar=False): +def start_point( + org_name, + find_pattern="", + exact_match=False, + summary=False, + verbose=False, + copy=False, + progbar=False, + repofile=False, +): """Main function.""" # Verify find_pattern and specified_repo are not both used. if len(find_pattern) == 0 and exact_match: log.error("You need to provide a Pattern to go with the --exact flag") return initialize(org_name) - if not get_nexus3_catalog(org_name, find_pattern, exact_match): + if not get_nexus3_catalog(org_name, find_pattern, exact_match, repofile): log.info("Could not get any catalog from Nexus3 with org = {}".format(org_name)) return diff --git a/releasenotes/notes/ReleaseDockerHub-RepoInputFile-1027258b656a98e8.yaml b/releasenotes/notes/ReleaseDockerHub-RepoInputFile-1027258b656a98e8.yaml new file mode 100644 index 00000000..aedcad0d --- /dev/null +++ b/releasenotes/notes/ReleaseDockerHub-RepoInputFile-1027258b656a98e8.yaml @@ -0,0 +1,20 @@ +features: + - | + Add --repofile to releasedockerhub + + Enables providing a file with the repo names. + + -f, --repofile Repo Name is a file name, + which contains one repo per row + + Sample + lftools nexus docker releasedockerhub --org onap --repo /tmp/test_repos.txt --repofile + + Where the input file has the following syntax, one repo per row, + 'Nexus3 docker.release repo'; 'dockerhub dockername' + + Sample + onap/org.onap.dcaegen2.deployments.tls-init-container; onap/org.onap.dcaegen2.deployments.tls-init-container + onap/policy-api; onap/policy-api + onap/clamp-backend; onap/clamp-backend + onap/msb/msb_apigateway; onap/msb-msb_apigateway diff --git a/tests/fixtures/nexus/releasedockerhub_reponamelist1.txt b/tests/fixtures/nexus/releasedockerhub_reponamelist1.txt new file mode 100644 index 00000000..83694402 --- /dev/null +++ b/tests/fixtures/nexus/releasedockerhub_reponamelist1.txt @@ -0,0 +1,4 @@ +dcae_dmaapbc; onap/dcae_dmaapbc +onap/aaf/aaf_core; onap/aaf-aaf_core +onap/clamp; onap/clamp +onap/vfc/nfvo/svnfm/nokiav2; onap/vfc-nfvo-svnfm-nokiav2 diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py index cfe7a108..308ae413 100644 --- a/tests/test_release_docker_hub.py +++ b/tests/test_release_docker_hub.py @@ -19,6 +19,8 @@ import requests from lftools import cli import lftools.nexus.release_docker_hub as rdh +FIXTURE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures",) + def test_remove_http_from_url(): """Test _remove_http_from_url.""" @@ -58,9 +60,10 @@ def test_tag_class_valid_tags(): """Test TagClass""" org = "onap" repo = "base/sdc-sanity" + repo_from_file = False test_tags = ["1.2.3", "1.22.333", "111.22.3", "10.11.12", "1.0.3"] rdh.initialize(org) - tags = rdh.TagClass(org, repo) + tags = rdh.TagClass(org, repo, repo_from_file) for tag in test_tags: tags.add_tag(tag) assert len(tags.valid) == len(test_tags) @@ -71,6 +74,8 @@ def test_tag_class_invalid_tags(): """Test TagClass""" org = "onap" repo = "base/sdc-sanity" + + repo_from_file = False test_tags = [ "v1.2.3", "1.22", @@ -89,7 +94,7 @@ def test_tag_class_invalid_tags(): "1.1.2-STAGING-20181231T234559Z", ] rdh.initialize(org) - tags = rdh.TagClass(org, repo) + tags = rdh.TagClass(org, repo, repo_from_file) for tag in test_tags: tags.add_tag(tag) assert len(tags.invalid) == len(test_tags) @@ -100,8 +105,9 @@ def test_tag_class_repository_exist(): """Test TagClass""" org = "onap" repo = "base/sdc-sanity" + repo_from_file = False rdh.initialize(org) - tags = rdh.TagClass(org, repo) + tags = rdh.TagClass(org, repo, repo_from_file) assert tags.repository_exist == True @@ -109,13 +115,14 @@ def test_nexus_tag_class(responses): """Test NexusTagClass""" org = "onap" repo = "base/sdc-sanity" + repo_from_file = False url = "https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list" answer = '{"name":"onap/base_sdc-sanity","tags":["latest","1.3.0","1.3.1","1.4.0","1.4.1","v1.0.0"]}' answer_valid_tags = ["1.3.0", "1.3.1", "1.4.0", "1.4.1"] answer_invalid_tags = ["latest", "v1.0.0"] responses.add(responses.GET, url, body=answer, status=200) rdh.initialize(org) - test_tags = rdh.NexusTagClass(org, repo) + test_tags = rdh.NexusTagClass(org, repo, repo_from_file) for tag in answer_valid_tags: assert tag in test_tags.valid for tag in answer_invalid_tags: @@ -128,6 +135,7 @@ def test_docker_tag_class(responses): """Test DockerTagClass""" org = "onap" repo = "base-sdc-sanity" + repo_from_file = False url = "https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags" answer = """[{"layer": "", "name": "latest"}, {"layer": "", "name": "1.3.0"}, @@ -140,7 +148,7 @@ def test_docker_tag_class(responses): answer_invalid_tags = ["latest", "v1.0.0"] responses.add(responses.GET, url, body=answer, status=200) rdh.initialize(org) - test_tags = rdh.DockerTagClass(org, repo) + test_tags = rdh.DockerTagClass(org, repo, repo_from_file) for tag in answer_valid_tags: assert tag in test_tags.valid for tag in answer_invalid_tags: @@ -237,7 +245,7 @@ class TestProjectClass: "lftools.nexus.release_docker_hub.ProjectClass._docker_cleanup", side_effect=self.mocked_docker_cleanup ) - project = ("onap", "base/sdc-sanity") + project = ["onap", "base/sdc-sanity", ""] nexus_url = "https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list" nexus_answer = '{"name":"onap/base_sdc-sanity","tags":["1.3.0","1.3.1","1.4.0","1.4.1","v1.0.0"]}' @@ -290,7 +298,7 @@ class TestProjectClass: "lftools.nexus.release_docker_hub.ProjectClass._docker_cleanup", side_effect=self.mocked_docker_cleanup ) - project = ("onap", "base/sdc-sanity") + project = ["onap", "base/sdc-sanity", ""] nexus_url = "https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list" nexus_answer = '{"name":"onap/base_sdc-sanity","tags":["1.3.0","1.3.1","1.4.0","v1.0.0"]}' @@ -343,7 +351,7 @@ class TestProjectClass: "lftools.nexus.release_docker_hub.ProjectClass._docker_cleanup", side_effect=self.mocked_docker_cleanup ) - project = ("onap", "base/sdc-sanity") + project = ["onap", "base/sdc-sanity", ""] nexus_url = "https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list" nexus_answer = '{"name":"onap/base_sdc-sanity","tags":["1.3.0","1.3.1","1.4.0","v1.0.0"]}' docker_url = "https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags" @@ -413,6 +421,7 @@ class TestProjectClass: assert self.counter.cleanup == 90 +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "nexus"),) class TestFetchNexus3Catalog: url = "https://nexus3.onap.org:10002/v2/_catalog" @@ -459,6 +468,18 @@ class TestFetchNexus3Catalog: assert len(rdh.NexusCatalog) == 1 assert rdh.NexusCatalog[0][1] == "clamp-dashboard-logstash" + def test_get_all_onap_and_specify_repo_file(self, datafiles): + repo_names_file = os.path.join(str(datafiles), "releasedockerhub_reponamelist1.txt") + rdh.NexusCatalog = [] + rdh.initialize("onap") + responses.add(responses.GET, self.url, body=self.answer, status=200) + rdh.get_nexus3_catalog("onap", repo_names_file, False, True) + assert len(rdh.NexusCatalog) == 4 + assert rdh.NexusCatalog[0][1] == "dcae_dmaapbc" + assert rdh.NexusCatalog[1][1] == "onap/aaf/aaf_core" + assert rdh.NexusCatalog[2][1] == "onap/clamp" + assert rdh.NexusCatalog[3][1] == "onap/vfc/nfvo/svnfm/nokiav2" + class TestFetchAllTagsAndUpdate: _test_image_long_id = "sha256:3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c"