From eef14703efe2bc191b53108b2d60626cfeea98e2 Mon Sep 17 00:00:00 2001 From: Bengt Thuree Date: Fri, 28 Aug 2020 17:05:06 +1000 Subject: [PATCH] Add user regexp for version tag validation This patch adds the possibility to change the Version Schema. This is done by the --version_regexp parameter. If this parameter is not used, then the default #.#.# ("^\d+.\d+.\d+$") regexp will be used. The could be an regexp expression or a filename pointing to a file containing an regexp. Signed-off-by: Bengt Thuree Change-Id: I0099977f74c0da6bb87795111a48771fb3be4cd6 --- lftools/cli/nexus.py | 14 +- lftools/nexus/release_docker_hub.py | 48 +++++- ...kerhub_add_version_syntax-b1c18023ec130d81.yaml | 14 ++ tests/fixtures/nexus/releasedockerhub_bad_regexp | 1 + tests/fixtures/nexus/releasedockerhub_good_regexp | 1 + tests/test_release_docker_hub.py | 176 ++++++++++++++------- 6 files changed, 193 insertions(+), 61 deletions(-) create mode 100644 releasenotes/notes/releasedockerhub_add_version_syntax-b1c18023ec130d81.yaml create mode 100644 tests/fixtures/nexus/releasedockerhub_bad_regexp create mode 100644 tests/fixtures/nexus/releasedockerhub_good_regexp diff --git a/lftools/cli/nexus.py b/lftools/cli/nexus.py index 7f2b7dbb..e425e080 100644 --- a/lftools/cli/nexus.py +++ b/lftools/cli/nexus.py @@ -216,11 +216,21 @@ def release(ctx, repos, verify, server): " Sample: " " onap/msb/msb_apigateway; onap/msb-msb_apigateway", ) +@click.option( + "-x", + "--version_regexp", + type=str, + default="", + required=False, + help="Specify a file which contains a regexp expression to validate version number." + " File sample: " + " ^\d+.\d+.\d+$ ", +) @click.pass_context -def copy_from_nexus3_to_dockerhub(ctx, org, repo, exact, summary, verbose, copy, progbar, repofile): +def copy_from_nexus3_to_dockerhub(ctx, org, repo, exact, summary, verbose, copy, progbar, repofile, version_regexp): """Find missing repos in Docker Hub, Copy from Nexus3. Will by default list all missing repos in Docker Hub, compared to Nexus3. If -c (--copy) is provided, it will copy the repos from Nexus3 to Docker Hub. """ - rdh.start_point(org, repo, exact, summary, verbose, copy, progbar, repofile) + rdh.start_point(org, repo, exact, summary, verbose, copy, progbar, repofile, version_regexp) diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py index e0ee395e..37d75822 100644 --- a/lftools/nexus/release_docker_hub.py +++ b/lftools/nexus/release_docker_hub.py @@ -45,6 +45,7 @@ import multiprocessing from multiprocessing.dummy import Pool as ThreadPool import re import socket +import os import docker import requests @@ -61,6 +62,8 @@ NEXUS3_BASE = "" NEXUS3_CATALOG = "" NEXUS3_PROJ_NAME_HEADER = "" DOCKER_PROJ_NAME_HEADER = "" +VERSION_REGEXP = "" +DEFAULT_REGEXP = "^\d+.\d+.\d+$" def _remove_http_from_url(url): @@ -103,7 +106,35 @@ def _request_get(url): return resp -def initialize(org_name): +def which_version_regexp_to_use(input_regexp_or_filename): + """Set version regexp as per user request. + + regexp is either a regexp to be directly used, or its a file name, + and the file contains the regexp to use + """ + global VERSION_REGEXP + if len(input_regexp_or_filename) == 0: + VERSION_REGEXP = DEFAULT_REGEXP + else: + isFile = os.path.isfile(input_regexp_or_filename) + if isFile: + with open(input_regexp_or_filename, "r") as fp: + VERSION_REGEXP = fp.readline().strip() + else: + VERSION_REGEXP = input_regexp_or_filename + + +def validate_regexp(): + global VERSION_REGEXP + try: + re.compile(VERSION_REGEXP) + is_valid = True + except re.error: + is_valid = False + return is_valid + + +def initialize(org_name, input_regexp_or_filename=""): """Set constant strings.""" global NEXUS3_BASE global NEXUS3_CATALOG @@ -113,6 +144,7 @@ def initialize(org_name): NEXUS3_CATALOG = NEXUS3_BASE + "/v2/_catalog" NEXUS3_PROJ_NAME_HEADER = "Nexus3 Project Name" DOCKER_PROJ_NAME_HEADER = "Docker HUB Project Name" + which_version_regexp_to_use(input_regexp_or_filename) class TagClass: @@ -148,7 +180,7 @@ class TagClass: where keyword = STAGING or SNAPSHOT '^\d+.\d+.\d+-(STAGING|SNAPSHOT)-(20\d{2})(\d{2})(\d{2})T([01]\d|2[0-3])([0-5]\d)([0-5]\d)Z$' """ - pattern = re.compile(r"^\d+.\d+.\d+$") + pattern = re.compile(r"{}".format(VERSION_REGEXP)) log.debug("validate tag {} in {} --> {}".format(check_tag, self.repo, pattern.match(check_tag))) return pattern.match(check_tag) @@ -610,7 +642,11 @@ def fetch_all_tags(progbar=False): Nexus3 Catalog. """ NbrProjects = len(NexusCatalog) - log.info("Fetching tags from Nexus3 and Docker Hub for {} projects".format(NbrProjects)) + log.info( + "Fetching tags from Nexus3 and Docker Hub for {} projects with version regexp >>{}<<".format( + NbrProjects, VERSION_REGEXP + ) + ) if progbar: pbar = tqdm.tqdm(total=NbrProjects, bar_format="{l_bar}{bar}|{n_fmt}/{total_fmt} [{elapsed}]") @@ -822,13 +858,17 @@ def start_point( copy=False, progbar=False, repofile=False, + version_regexp="", ): """Main function.""" # Verify find_pattern and specified_repo are not both used. if len(find_pattern) == 0 and exact_match: log.error("You need to provide a Pattern to go with the --exact flag") return - initialize(org_name) + initialize(org_name, version_regexp) + if not validate_regexp(): + log.error("Found issues with the provided regexp >>{}<< ".format(VERSION_REGEXP)) + return if not get_nexus3_catalog(org_name, find_pattern, exact_match, repofile): log.info("Could not get any catalog from Nexus3 with org = {}".format(org_name)) return diff --git a/releasenotes/notes/releasedockerhub_add_version_syntax-b1c18023ec130d81.yaml b/releasenotes/notes/releasedockerhub_add_version_syntax-b1c18023ec130d81.yaml new file mode 100644 index 00000000..82df9da1 --- /dev/null +++ b/releasenotes/notes/releasedockerhub_add_version_syntax-b1c18023ec130d81.yaml @@ -0,0 +1,14 @@ +--- + +features: + - | + This patch adds the possibility for the user to change the Version Schema. + This is done by the new --version_regexp parameter. + If this parameter is not used, then the ONAP #.#.# ("^\d+.\d+.\d+$") regexp will be used. + + The parameter can be an regexp, like "^\d+.\d+.\d+$", or a file name, which contains a longer regexp. + + + Sample command + lftools nexus docker releasedockerhub -o onap -r aai -v --version_regexp "^\d+.\d+.\d+$" + diff --git a/tests/fixtures/nexus/releasedockerhub_bad_regexp b/tests/fixtures/nexus/releasedockerhub_bad_regexp new file mode 100644 index 00000000..558ed37d --- /dev/null +++ b/tests/fixtures/nexus/releasedockerhub_bad_regexp @@ -0,0 +1 @@ +[ diff --git a/tests/fixtures/nexus/releasedockerhub_good_regexp b/tests/fixtures/nexus/releasedockerhub_good_regexp new file mode 100644 index 00000000..e842e5a5 --- /dev/null +++ b/tests/fixtures/nexus/releasedockerhub_good_regexp @@ -0,0 +1 @@ +^\d+.\d+ diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py index 308ae413..80c20dc5 100644 --- a/tests/test_release_docker_hub.py +++ b/tests/test_release_docker_hub.py @@ -56,61 +56,6 @@ def test_format_image_id(): assert rdh._format_image_id(id[0]) == id[1] -def test_tag_class_valid_tags(): - """Test TagClass""" - org = "onap" - repo = "base/sdc-sanity" - repo_from_file = False - test_tags = ["1.2.3", "1.22.333", "111.22.3", "10.11.12", "1.0.3"] - rdh.initialize(org) - tags = rdh.TagClass(org, repo, repo_from_file) - for tag in test_tags: - tags.add_tag(tag) - assert len(tags.valid) == len(test_tags) - assert len(tags.invalid) == 0 - - -def test_tag_class_invalid_tags(): - """Test TagClass""" - org = "onap" - repo = "base/sdc-sanity" - - repo_from_file = False - test_tags = [ - "v1.2.3", - "1.22", - "111.22.3a", - "10.11.12.3", - "draft", - "1.2.jan14", - "1.2.3.4.5.6.7.8", - "1", - "latest", - "v0.1.0", - "1.1-20170906T011834", - "2.0-20180221T152423", - "1.3.0-20181121T1329", - "1.1.2-SNAPSHOT-20181231T234559Z", - "1.1.2-STAGING-20181231T234559Z", - ] - rdh.initialize(org) - tags = rdh.TagClass(org, repo, repo_from_file) - for tag in test_tags: - tags.add_tag(tag) - assert len(tags.invalid) == len(test_tags) - assert len(tags.valid) == 0 - - -def test_tag_class_repository_exist(): - """Test TagClass""" - org = "onap" - repo = "base/sdc-sanity" - repo_from_file = False - rdh.initialize(org) - tags = rdh.TagClass(org, repo, repo_from_file) - assert tags.repository_exist == True - - def test_nexus_tag_class(responses): """Test NexusTagClass""" org = "onap" @@ -157,6 +102,127 @@ def test_docker_tag_class(responses): assert len(test_tags.invalid) == len(answer_invalid_tags) +def test_tag_class_repository_exist(): + """Test TagClass""" + org = "onap" + repo = "base/sdc-sanity" + repo_from_file = False + rdh.initialize(org) + tags = rdh.TagClass(org, repo, repo_from_file) + assert tags.repository_exist == True + + +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "nexus"),) +class TestTagsRegExpClass: + """Test Tags and Regexp for Versions. + + This class contains all the test cases for Valid/Invalid tags and Version RegExp. + """ + + def test_tag_class_valid_tags(self): + """Test TagClass""" + org = "onap" + repo = "base/sdc-sanity" + repo_from_file = False + test_tags = ["1.2.3", "1.22.333", "111.22.3", "10.11.12", "1.0.3"] + rdh.initialize(org) + tags = rdh.TagClass(org, repo, repo_from_file) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.valid) == len(test_tags) + assert len(tags.invalid) == 0 + + def test_tag_class_invalid_tags(self): + """Test TagClass""" + org = "onap" + repo = "base/sdc-sanity" + + repo_from_file = False + test_tags = [ + "v1.2.3", + "1.22", + "111.22.3a", + "10.11.12.3", + "draft", + "1.2.jan14", + "1.2.3.4.5.6.7.8", + "1", + "latest", + "v0.1.0", + "1.1-20170906T011834", + "2.0-20180221T152423", + "1.3.0-20181121T1329", + "1.1.2-SNAPSHOT-20181231T234559Z", + "1.1.2-STAGING-20181231T234559Z", + ] + rdh.initialize(org) + tags = rdh.TagClass(org, repo, repo_from_file) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.invalid) == len(test_tags) + assert len(tags.valid) == 0 + + def test_tag_class_manual_version_regexp_str_valid_tags(self): + """Test TagClass""" + org = "onap" + repo = "base/sdc-sanity" + test_regexp = "^v\d+.\d+.\d+$" + repo_from_file = False + test_tags = ["v1.2.3", "v1.22.333", "v111.22.3", "v10.11.12", "v1.0.3"] + rdh.initialize(org, test_regexp) + tags = rdh.TagClass(org, repo, repo_from_file) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.valid) == len(test_tags) + assert len(tags.invalid) == 0 + + def test_tag_class_manual_version_regexp_str_invalid_tags(self): + """Test TagClass""" + org = "onap" + repo = "base/sdc-sanity" + test_regexp = "v^\d+.\d+.\d+$" + repo_from_file = False + test_tags = [ + "1.2.3", + "1.22", + "111.22.3a", + "10.11.12.3", + "draft", + "1.2.jan14", + "1.2.3.4.5.6.7.8", + "1", + "latest", + "0.1.0", + "1.1-20170906T011834", + "2.0-20180221T152423", + "1.3.0-20181121T1329", + "1.1.2-SNAPSHOT-20181231T234559Z", + "1.1.2-STAGING-20181231T234559Z", + ] + rdh.initialize(org, test_regexp) + tags = rdh.TagClass(org, repo, repo_from_file) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.invalid) == len(test_tags) + assert len(tags.valid) == 0 + + def test_tag_class_manual_version_regexp_str_from_file_valid(self, datafiles): + org = "onap" + test_regexp_from_file = os.path.join(str(datafiles), "releasedockerhub_good_regexp") + repo_from_file = False + rdh.initialize(org, test_regexp_from_file) + assert rdh.validate_regexp() == True + assert rdh.VERSION_REGEXP == "^\d+.\d+" + + def test_tag_class_manual_version_regexp_str_from_file_invalid(self, datafiles): + org = "onap" + test_regexp_from_file = os.path.join(str(datafiles), "releasedockerhub_bad_regexp") + repo_from_file = False + rdh.initialize(org, test_regexp_from_file) + assert rdh.validate_regexp() == False + assert rdh.VERSION_REGEXP == "[" + + class TestProjectClass: """Test ProjectClass. -- 2.16.6