From f3e77cc248e46c7135163aed74754516d99e2515 Mon Sep 17 00:00:00 2001 From: Bengt Thuree Date: Thu, 29 Nov 2018 21:44:27 +1100 Subject: [PATCH] Add tag classes to release_docker_hub 2(7). This is the second part, which contains the tag classes. Generic tag class, which contains two functions: Validate version number. Add new tag. The Nexus and Docker classes will fetch all tags upon init. Issue:RELENG-1549 Change-Id: I305c98977a7d5828875d3cfd9bc59871b87d08f1 Signed-off-by: Bengt Thuree --- lftools/nexus/release_docker_hub.py | 176 ++++++++++++++++++++++++++++++++++++ tests/test_release_docker_hub.py | 80 ++++++++++++++++ 2 files changed, 256 insertions(+) diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py index b39e269d..713dc979 100644 --- a/lftools/nexus/release_docker_hub.py +++ b/lftools/nexus/release_docker_hub.py @@ -41,6 +41,7 @@ lftools nexus docker releasedockerhub from __future__ import print_function import logging +import re import socket import requests @@ -108,3 +109,178 @@ def initialize(org_name): NEXUS3_CATALOG = NEXUS3_BASE + "/v2/_catalog" NEXUS3_PROJ_NAME_HEADER = "Nexus3 Project Name" DOCKER_PROJ_NAME_HEADER = "Docker HUB Project Name" + + +class TagClass: + """Base class for Nexus3 and Docker Hub tag class. + + This class contains the actual valid and invalid tags for a repository, + as well as an indication if the repository exist or not. + + A valid tag has the following format #.#.# (1.2.3, or 1.22.333) + + Parameter: + org_name : The organization part of the repository. (onap) + repo_name : The Nexus3 repository name (aaf/aaf_service) + """ + + def __init__(self, org_name, repo_name): + """Initialize this class.""" + self.valid = [] + self.invalid = [] + self.repository_exist = True + self.org = org_name + self.repo = repo_name + + def _validate_tag(self, check_tag): + """Local helper function to simplify validity check of version number. + + Returns true or false, depending if the version pattern is a valid one. + Valid pattern is #.#.#, or in computer term "^\d+.\d+.\d+$" + + Future pattern : x.y.z-KEYWORD-yyyymmddThhmmssZ + where keyword = STAGING or SNAPSHOT + '^\d+.\d+.\d+-(STAGING|SNAPSHOT)-(20\d{2})(\d{2})(\d{2})T([01]\d|2[0-3])([0-5]\d)([0-5]\d)Z$' + """ + pattern = re.compile(r'^\d+.\d+.\d+$') + log.debug("validate tag {} in {} --> {}".format(check_tag, self.repo, pattern.match(check_tag))) + return pattern.match(check_tag) + + def add_tag(self, new_tag): + """Add tag to a list. + + This function will take a tag, and add it to the correct list + (valid or invalid), depending on validate_tag result. + """ + if self._validate_tag(new_tag): + self.valid.append(new_tag) + else: + self.invalid.append(new_tag) + + +class NexusTagClass (TagClass): + """Nexus Tag class. + + This class fetches and stores all Nexus3 tags for a repository. + + Doing this manually from command line, you will give this command: + curl -s https://nexus3.onap.org:10002/v2/onap/aaf/aaf_service/tags/list + which gives you the following output: + {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5","2.1.6","2.1.7","2.1.8"]} + + When we fetch the tags from the Nexus3 repository url, they are returned like + {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5"]} + Hence, we need to extract all the tags, and add them to our list of valid or + invalid tags. + If we fail to collect the tags, we set the repository_exist flag to false. + + Parameter: + org_name : The organization part of the repository. (onap) + repo_name : The Nexus3 repository name (aaf/aaf_service) + + Result: + Will fetch all tags from the Nexus3 repository URL, and store each tag + in self.valid or self.invalid as a list. + If no repository is found, self.repository_exist will be set to False. + """ + + def __init__(self, org_name, repo_name): + """Initialize this class.""" + TagClass.__init__(self, org_name, repo_name) + log.debug("Fetching nexus3 tags for {}/{}".format(org_name, repo_name)) + retries = 0 + while retries < 20: + try: + r = _request_get(NEXUS3_BASE + "/v2/" + org_name + "/" + repo_name + "/tags/list") + break + except requests.HTTPError as excinfo: + log.debug("Fetching Nexus3 tags. {}".format(excinfo)) + retries = retries + 1 + if retries > 19: + self.repository_exist = False + return + + log.debug("r.status_code = {}, ok={}".format( + r.status_code, r.status_code == requests.codes.ok)) + if r.status_code == requests.codes.ok: + raw_tags = r.text + raw_tags = raw_tags.replace('"', '') + raw_tags = raw_tags.replace('}', '') + raw_tags = raw_tags.replace(']', '') + raw_tags = raw_tags.replace(' ', '') + raw_tags = raw_tags.split('[') + TmpSplittedTags = raw_tags[1].split(',') + if len(TmpSplittedTags) > 0: + for tag_2_add in TmpSplittedTags: + self.add_tag(tag_2_add) + log.debug("Nexus {}/{} has tag {}".format( + org_name, repo_name, tag_2_add)) + else: + self.repository_exist = False + + +class DockerTagClass (TagClass): + """Docker tag class. + + This class fetches and stores all docker tags for a repository. + + Doing this manually from command line, you will give this command: + curl -s https://registry.hub.docker.com/v1/repositories/onap/base_sdc-sanity/tags + which gives you the following output: + {"layer": "", "name": "latest"}, {"layer": "", "name": "1.3.0"}, + {"layer": "", "name": "1.3.1"}, {"layer": "", "name": "1.4.0"}, + {"layer": "", "name": "1.4.1"}, {"layer": "", "name": "v1.0.0"}] + + When we fetch the tags from the docker repository url, they are returned like + [{"layer": "", "name": "latest"}, {"layer": "", "name": "v1.0.0"}] + Hence, we need to extract all the tags, and add them to our list of valid or + invalid tags. + If we fail to collect the tags, we set the repository_exist flag to false. + + Parameter: + org_name : The organization part of the repository. (onap) + repo_name : The Docker Hub repository name (aaf-aaf_service) + + Result: + Will fetch all tags from the Docker Repository URL, and store each tag + in self.valid or self.invalid as a list. + If no repository is found, self.repository_exist will be set to False. + """ + + _docker_base = "https://registry.hub.docker.com/v1/repositories" + + def __init__(self, org_name, repo_name): + """Initialize this class.""" + TagClass.__init__(self, org_name, repo_name) + log.debug("Fetching docker tags for {}/{}".format(org_name, repo_name)) + retries = 0 + while retries < 20: + try: + r = _request_get(self._docker_base + "/" + org_name + "/" + repo_name + "/tags") + break + except requests.HTTPError as excinfo: + log.debug("Fetching Docker Hub tags. {}".format(excinfo)) + retries = retries + 1 + if retries > 19: + self.repository_exist = False + return + + log.debug("r.status_code = {}, ok={}".format( + r.status_code, r.status_code == requests.codes.ok)) + if r.status_code == requests.codes.ok: + raw_tags = r.text + raw_tags = raw_tags.replace('}]', '') + raw_tags = raw_tags.replace('[{', '') + raw_tags = raw_tags.replace('{', '') + raw_tags = raw_tags.replace('"', '') + raw_tags = raw_tags.replace(' ', '') + TmpSplittedTuple = raw_tags.split('}') + if len(TmpSplittedTuple) > 0: + for tuple in TmpSplittedTuple: + tmp_tuple = tuple.split(':') + if len(tmp_tuple) > 1: + self.add_tag(tmp_tuple[2].strip()) + log.debug("Docker {}/{} has tag {}".format( + org_name, repo_name, tmp_tuple[2])) + else: + self.repository_exist = False diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py index 463d925d..9880b241 100644 --- a/tests/test_release_docker_hub.py +++ b/tests/test_release_docker_hub.py @@ -44,3 +44,83 @@ def test_format_image_id(): for id in test_id: assert rdh._format_image_id(id[0]) == id[1] + + +def test_tag_class_valid_tags(): + """Test TagClass""" + org = 'onap' + repo = 'base/sdc-sanity' + test_tags =["1.2.3", "1.22.333", "111.22.3", "10.11.12", "1.0.3"] + rdh.initialize (org) + tags = rdh.TagClass (org, repo) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.valid) == len(test_tags) + assert len(tags.invalid) == 0 + +def test_tag_class_invalid_tags(): + """Test TagClass""" + org = 'onap' + repo = 'base/sdc-sanity' + test_tags =["v1.2.3", "1.22", "111.22.3a", "10.11.12.3", "draft", + "1.2.jan14", "1.2.3.4.5.6.7.8", "1", "latest", "v0.1.0", + "1.1-20170906T011834", "2.0-20180221T152423", + "1.3.0-20181121T1329", "1.1.2-SNAPSHOT-20181231T234559Z", + "1.1.2-STAGING-20181231T234559Z"] + rdh.initialize (org) + tags = rdh.TagClass (org, repo) + for tag in test_tags: + tags.add_tag(tag) + assert len(tags.invalid) == len(test_tags) + assert len(tags.valid) == 0 + +def test_tag_class_repository_exist(): + """Test TagClass""" + org = 'onap' + repo = 'base/sdc-sanity' + rdh.initialize (org) + tags = rdh.TagClass (org, repo) + assert tags.repository_exist == True + +def test_nexus_tag_class(responses): + """Test NexusTagClass""" + org = 'onap' + repo = 'base/sdc-sanity' + url = 'https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list' + answer = '{"name":"onap/base_sdc-sanity","tags":["latest","1.3.0","1.3.1","1.4.0","1.4.1","v1.0.0"]}' + answer_valid_tags = ["1.3.0","1.3.1","1.4.0","1.4.1"] + answer_invalid_tags = ["latest", "v1.0.0" ] + responses.add(responses.GET, url, body=answer, status=200) + rdh.initialize (org) + test_tags = rdh.NexusTagClass (org, repo) + for tag in answer_valid_tags: + assert tag in test_tags.valid + for tag in answer_invalid_tags: + assert tag in test_tags.invalid + assert len(test_tags.valid) == len(answer_valid_tags) + assert len(test_tags.invalid) == len(answer_invalid_tags) + + +def test_docker_tag_class(responses): + """Test DockerTagClass""" + org = 'onap' + repo = 'base-sdc-sanity' + url = 'https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags' + answer = """[{"layer": "", "name": "latest"}, + {"layer": "", "name": "1.3.0"}, + {"layer": "", "name": "1.3.1"}, + {"layer": "", "name": "1.4.0"}, + {"layer": "", "name": "1.4.1"}, + {"layer": "", "name": "v1.0.0"}] + """ + answer_valid_tags = ["1.3.0","1.3.1","1.4.0","1.4.1"] + answer_invalid_tags = ["latest", "v1.0.0"] + responses.add(responses.GET, url, body=answer, status=200) + rdh.initialize (org) + test_tags = rdh.DockerTagClass (org, repo) + for tag in answer_valid_tags: + assert tag in test_tags.valid + for tag in answer_invalid_tags: + assert tag in test_tags.invalid + assert len(test_tags.valid) == len(answer_valid_tags) + assert len(test_tags.invalid) == len(answer_invalid_tags) -- 2.16.6