Add tag classes to release_docker_hub 2(7).

This is the second part, which contains the tag classes.
Generic tag class, which contains two functions:
  Validate version number.
  Add new tag.
The Nexus and Docker classes will fetch all tags upon init.

Issue:RELENG-1549
Change-Id: I305c98977a7d5828875d3cfd9bc59871b87d08f1
Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>

diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py
index b39e269..713dc97 100644 (file)
--- a/lftools/nexus/release_docker_hub.py
+++ b/lftools/nexus/release_docker_hub.py
@@ -41,6 +41,7 @@ lftools nexus docker releasedockerhub
 from __future__ import print_function
 
 import logging
+import re
 import socket
 
 import requests
@@ -108,3 +109,178 @@ def initialize(org_name):
     NEXUS3_CATALOG = NEXUS3_BASE + "/v2/_catalog"
     NEXUS3_PROJ_NAME_HEADER = "Nexus3 Project Name"
     DOCKER_PROJ_NAME_HEADER = "Docker HUB Project Name"
+
+
+class TagClass:
+    """Base class for Nexus3 and Docker Hub tag class.
+
+    This class contains the actual valid and invalid tags for a repository,
+    as well as an indication if the repository exist or not.
+
+    A valid tag has the following format #.#.# (1.2.3, or 1.22.333)
+
+    Parameter:
+        org_name  : The organization part of the repository. (onap)
+        repo_name : The Nexus3 repository name (aaf/aaf_service)
+    """
+
+    def __init__(self, org_name, repo_name):
+        """Initialize this class."""
+        self.valid = []
+        self.invalid = []
+        self.repository_exist = True
+        self.org = org_name
+        self.repo = repo_name
+
+    def _validate_tag(self, check_tag):
+        """Local helper function to simplify validity check of version number.
+
+        Returns true or false, depending if the version pattern is a valid one.
+        Valid pattern is #.#.#, or in computer term "^\d+.\d+.\d+$"
+
+        Future pattern : x.y.z-KEYWORD-yyyymmddThhmmssZ
+          where keyword = STAGING or SNAPSHOT
+          '^\d+.\d+.\d+-(STAGING|SNAPSHOT)-(20\d{2})(\d{2})(\d{2})T([01]\d|2[0-3])([0-5]\d)([0-5]\d)Z$'
+        """
+        pattern = re.compile(r'^\d+.\d+.\d+$')
+        log.debug("validate tag {} in {} --> {}".format(check_tag, self.repo, pattern.match(check_tag)))
+        return pattern.match(check_tag)
+
+    def add_tag(self, new_tag):
+        """Add tag to a list.
+
+        This function will take a tag, and add it to the correct list
+        (valid or invalid), depending on validate_tag result.
+        """
+        if self._validate_tag(new_tag):
+            self.valid.append(new_tag)
+        else:
+            self.invalid.append(new_tag)
+
+
+class NexusTagClass (TagClass):
+    """Nexus Tag class.
+
+    This class fetches and stores all Nexus3 tags for a repository.
+
+    Doing this manually from command line, you will give this command:
+        curl -s https://nexus3.onap.org:10002/v2/onap/aaf/aaf_service/tags/list
+    which gives you the following output:
+        {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5","2.1.6","2.1.7","2.1.8"]}
+
+    When we fetch the tags from the Nexus3 repository url, they are returned like
+        {"name":"onap/aaf/aaf_service","tags":["2.1.1","2.1.3","2.1.4","2.1.5"]}
+    Hence, we need to extract all the tags, and add them to our list of valid or
+    invalid tags.
+    If we fail to collect the tags, we set the repository_exist flag to false.
+
+    Parameter:
+        org_name  : The organization part of the repository. (onap)
+        repo_name : The Nexus3 repository name (aaf/aaf_service)
+
+    Result:
+        Will fetch all tags from the Nexus3 repository URL, and store each tag
+        in self.valid or self.invalid as a list.
+        If no repository is found, self.repository_exist will be set to False.
+    """
+
+    def __init__(self, org_name, repo_name):
+        """Initialize this class."""
+        TagClass.__init__(self, org_name, repo_name)
+        log.debug("Fetching nexus3 tags for {}/{}".format(org_name, repo_name))
+        retries = 0
+        while retries < 20:
+            try:
+                r = _request_get(NEXUS3_BASE + "/v2/" + org_name + "/" + repo_name + "/tags/list")
+                break
+            except requests.HTTPError as excinfo:
+                log.debug("Fetching Nexus3 tags. {}".format(excinfo))
+                retries = retries + 1
+                if retries > 19:
+                    self.repository_exist = False
+                    return
+
+        log.debug("r.status_code = {}, ok={}".format(
+            r.status_code, r.status_code == requests.codes.ok))
+        if r.status_code == requests.codes.ok:
+            raw_tags = r.text
+            raw_tags = raw_tags.replace('"', '')
+            raw_tags = raw_tags.replace('}', '')
+            raw_tags = raw_tags.replace(']', '')
+            raw_tags = raw_tags.replace(' ', '')
+            raw_tags = raw_tags.split('[')
+            TmpSplittedTags = raw_tags[1].split(',')
+            if len(TmpSplittedTags) > 0:
+                for tag_2_add in TmpSplittedTags:
+                    self.add_tag(tag_2_add)
+                    log.debug("Nexus {}/{} has tag {}".format(
+                        org_name, repo_name, tag_2_add))
+        else:
+            self.repository_exist = False
+
+
+class DockerTagClass (TagClass):
+    """Docker tag class.
+
+    This class fetches and stores all docker tags for a repository.
+
+    Doing this manually from command line, you will give this command:
+        curl -s https://registry.hub.docker.com/v1/repositories/onap/base_sdc-sanity/tags
+    which gives you the following output:
+        {"layer": "", "name": "latest"}, {"layer": "", "name": "1.3.0"},
+        {"layer": "", "name": "1.3.1"}, {"layer": "", "name": "1.4.0"},
+        {"layer": "", "name": "1.4.1"}, {"layer": "", "name": "v1.0.0"}]
+
+    When we fetch the tags from the docker repository url, they are returned like
+        [{"layer": "", "name": "latest"}, {"layer": "", "name": "v1.0.0"}]
+    Hence, we need to extract all the tags, and add them to our list of valid or
+    invalid tags.
+    If we fail to collect the tags, we set the repository_exist flag to false.
+
+    Parameter:
+        org_name  : The organization part of the repository. (onap)
+        repo_name : The Docker Hub repository name (aaf-aaf_service)
+
+    Result:
+        Will fetch all tags from the Docker Repository URL, and store each tag
+        in self.valid or self.invalid as a list.
+        If no repository is found, self.repository_exist will be set to False.
+    """
+
+    _docker_base = "https://registry.hub.docker.com/v1/repositories"
+
+    def __init__(self, org_name, repo_name):
+        """Initialize this class."""
+        TagClass.__init__(self, org_name, repo_name)
+        log.debug("Fetching docker tags for {}/{}".format(org_name, repo_name))
+        retries = 0
+        while retries < 20:
+            try:
+                r = _request_get(self._docker_base + "/" + org_name + "/" + repo_name + "/tags")
+                break
+            except requests.HTTPError as excinfo:
+                log.debug("Fetching Docker Hub tags. {}".format(excinfo))
+                retries = retries + 1
+                if retries > 19:
+                    self.repository_exist = False
+                    return
+
+        log.debug("r.status_code = {}, ok={}".format(
+            r.status_code, r.status_code == requests.codes.ok))
+        if r.status_code == requests.codes.ok:
+            raw_tags = r.text
+            raw_tags = raw_tags.replace('}]', '')
+            raw_tags = raw_tags.replace('[{', '')
+            raw_tags = raw_tags.replace('{', '')
+            raw_tags = raw_tags.replace('"', '')
+            raw_tags = raw_tags.replace(' ', '')
+            TmpSplittedTuple = raw_tags.split('}')
+            if len(TmpSplittedTuple) > 0:
+                for tuple in TmpSplittedTuple:
+                    tmp_tuple = tuple.split(':')
+                    if len(tmp_tuple) > 1:
+                        self.add_tag(tmp_tuple[2].strip())
+                        log.debug("Docker {}/{} has tag {}".format(
+                            org_name, repo_name, tmp_tuple[2]))
+        else:
+            self.repository_exist = False

diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py
index 463d925..9880b24 100644 (file)
--- a/tests/test_release_docker_hub.py
+++ b/tests/test_release_docker_hub.py
@@ -44,3 +44,83 @@ def test_format_image_id():
 
     for id in test_id:
         assert rdh._format_image_id(id[0]) == id[1]
+
+
+def test_tag_class_valid_tags():
+    """Test TagClass"""
+    org = 'onap'
+    repo = 'base/sdc-sanity'
+    test_tags =["1.2.3", "1.22.333", "111.22.3", "10.11.12", "1.0.3"]
+    rdh.initialize (org)
+    tags = rdh.TagClass (org, repo)
+    for tag in test_tags:
+        tags.add_tag(tag)
+    assert len(tags.valid) == len(test_tags)
+    assert len(tags.invalid) == 0
+
+def test_tag_class_invalid_tags():
+    """Test TagClass"""
+    org = 'onap'
+    repo = 'base/sdc-sanity'
+    test_tags =["v1.2.3", "1.22", "111.22.3a", "10.11.12.3", "draft",
+        "1.2.jan14", "1.2.3.4.5.6.7.8", "1", "latest", "v0.1.0",
+        "1.1-20170906T011834", "2.0-20180221T152423",
+        "1.3.0-20181121T1329", "1.1.2-SNAPSHOT-20181231T234559Z",
+        "1.1.2-STAGING-20181231T234559Z"]
+    rdh.initialize (org)
+    tags = rdh.TagClass (org, repo)
+    for tag in test_tags:
+        tags.add_tag(tag)
+    assert len(tags.invalid) == len(test_tags)
+    assert len(tags.valid) == 0
+
+def test_tag_class_repository_exist():
+    """Test TagClass"""
+    org = 'onap'
+    repo = 'base/sdc-sanity'
+    rdh.initialize (org)
+    tags = rdh.TagClass (org, repo)
+    assert tags.repository_exist == True
+
+def test_nexus_tag_class(responses):
+    """Test NexusTagClass"""
+    org = 'onap'
+    repo = 'base/sdc-sanity'
+    url = 'https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list'
+    answer = '{"name":"onap/base_sdc-sanity","tags":["latest","1.3.0","1.3.1","1.4.0","1.4.1","v1.0.0"]}'
+    answer_valid_tags = ["1.3.0","1.3.1","1.4.0","1.4.1"]
+    answer_invalid_tags = ["latest", "v1.0.0" ]
+    responses.add(responses.GET, url, body=answer, status=200)
+    rdh.initialize (org)
+    test_tags = rdh.NexusTagClass (org, repo)
+    for tag in answer_valid_tags:
+        assert tag in test_tags.valid
+    for tag in answer_invalid_tags:
+        assert tag in test_tags.invalid
+    assert len(test_tags.valid) == len(answer_valid_tags)
+    assert len(test_tags.invalid) == len(answer_invalid_tags)
+
+
+def test_docker_tag_class(responses):
+    """Test DockerTagClass"""
+    org = 'onap'
+    repo = 'base-sdc-sanity'
+    url = 'https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags'
+    answer = """[{"layer": "", "name": "latest"},
+        {"layer": "", "name": "1.3.0"},
+        {"layer": "", "name": "1.3.1"},
+        {"layer": "", "name": "1.4.0"},
+        {"layer": "", "name": "1.4.1"},
+        {"layer": "", "name": "v1.0.0"}]
+    """
+    answer_valid_tags = ["1.3.0","1.3.1","1.4.0","1.4.1"]
+    answer_invalid_tags = ["latest", "v1.0.0"]
+    responses.add(responses.GET, url, body=answer, status=200)
+    rdh.initialize (org)
+    test_tags = rdh.DockerTagClass (org, repo)
+    for tag in answer_valid_tags:
+        assert tag in test_tags.valid
+    for tag in answer_invalid_tags:
+        assert tag in test_tags.invalid
+    assert len(test_tags.valid) == len(answer_valid_tags)
+    assert len(test_tags.invalid) == len(answer_invalid_tags)