Add multithread code to release_docker_hub 6(7).

This is the sixth part, with multithreading stuff,
fetching tags, copying to docker hub, and main script.

Issue:RELENG-1549
Change-Id: I03ec4adfe3225ec9ebbcc27092b578e4c0b36eae
Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>

diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py
index a4a27ea..e81cbe1 100644 (file)
--- a/lftools/nexus/release_docker_hub.py
+++ b/lftools/nexus/release_docker_hub.py
@@ -41,6 +41,8 @@ lftools nexus docker releasedockerhub
 from __future__ import print_function
 
 import logging
+import multiprocessing
+from multiprocessing.dummy import Pool as ThreadPool
 import re
 import socket
 
@@ -515,6 +517,76 @@ def get_nexus3_catalog(org_name='', find_pattern=''):
     return True
 
 
+def fetch_all_tags(progbar=False):
+    """Fetch all tags function.
+
+    This function will use multi-threading to fetch all tags for all projects in
+    Nexus3 Catalog.
+    """
+    NbrProjects = len(NexusCatalog)
+    log.info("Fetching tags from Nexus3 and Docker Hub for {} projects".format(NbrProjects))
+    if progbar:
+        pbar = tqdm.tqdm(total=NbrProjects, bar_format='{l_bar}{bar}|{n_fmt}/{total_fmt} [{elapsed}]')
+
+    def _fetch_all_tags(proj):
+        """Helper function for multi-threading.
+
+        This function, will create an instance of ProjectClass (which triggers
+        the project class fetching all Nexus3/Docker Hub tags)
+        Then adding this instance to the project list.
+
+            Parameters:
+                proj : Tuple with 'org' and 'repo'
+                    ('onap', 'aaf/aaf_service')
+        """
+        new_proj = ProjectClass(proj)
+        projects.append(new_proj)
+        if progbar:
+            pbar.update(1)
+
+    pool = ThreadPool(multiprocessing.cpu_count())
+    pool.map(_fetch_all_tags, NexusCatalog)
+    pool.close()
+    pool.join()
+
+    if progbar:
+        pbar.close()
+    projects.sort()
+
+
+def copy_from_nexus_to_docker(progbar=False):
+    """Copy all missing tags.
+
+    This function will use multi-threading to copy all missing tags in the project list.
+    """
+    _tot_tags = 0
+    for proj in projects:
+        _tot_tags = _tot_tags + len(proj.tags_2_copy.valid)
+    log.info("About to start copying from Nexus3 to Docker Hub for {} missing tags".format(_tot_tags))
+    if progbar:
+        pbar = tqdm.tqdm(total=_tot_tags, bar_format='{l_bar}{bar}|{n_fmt}/{total_fmt} [{elapsed}]')
+
+    def _docker_pull_tag_push(proj):
+        """Helper function for multi-threading.
+
+        This function, will call the ProjectClass proj's docker_pull_tag_push.
+
+            Parameters:
+                proj : Tuple with 'org' and 'repo'
+                    ('onap', 'aaf/aaf_service')
+        """
+        proj.docker_pull_tag_push(progbar)
+        if progbar:
+            pbar.update(len(proj.tags_2_copy.valid))
+
+    pool = ThreadPool(multiprocessing.cpu_count())
+    pool.map(_docker_pull_tag_push, projects)
+    pool.close()
+    pool.join()
+    if progbar:
+        pbar.close()
+
+
 def print_nexus_docker_proj_names():
     """Print Nexus3 - Docker Hub repositories."""
     fmt_str = '{:<'+str(project_max_len_chars)+'} : '
@@ -650,3 +722,28 @@ def print_nbr_tags_to_copy():
     for proj in projects:
         _tot_tags = _tot_tags + len(proj.tags_2_copy.valid)
     log.info("Summary: {} tags that should be copied from Nexus3 to Docker Hub.".format(_tot_tags))
+
+
+def start_point(org_name, find_pattern='', summary=False,
+                verbose=False, copy=False, progbar=False):
+    """Main function."""
+    initialize(org_name)
+    if not get_nexus3_catalog(org_name, find_pattern):
+        log.info("Could not get any catalog from Nexus3 with org = {}".format(org_name))
+        return
+
+    fetch_all_tags(progbar)
+    if verbose:
+        print_nexus_docker_proj_names()
+        print_nexus_valid_tags()
+        print_nexus_invalid_tags()
+        print_docker_valid_tags()
+        print_docker_invalid_tags()
+        print_stats()
+    if summary or verbose:
+        print_missing_docker_proj()
+        print_nexus_tags_to_copy()
+    if copy:
+        copy_from_nexus_to_docker(progbar)
+    else:
+        print_nbr_tags_to_copy()

diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py
index b8ea04c..d42aeb5 100644 (file)
--- a/tests/test_release_docker_hub.py
+++ b/tests/test_release_docker_hub.py
@@ -410,3 +410,197 @@ class TestFetchNexus3Catalog:
         responses.add(responses.GET, self.url, body=self.answer, status=200)
         rdh.get_nexus3_catalog ('onap', 'aaf')
         assert len(rdh.NexusCatalog) == 18
+
+
+class TestFetchAllTagsAndUpdate:
+    _test_image_long_id = 'sha256:3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c'
+    _test_image_short_id = 'sha256:3450464d68'
+    _expected_nexus_image_str = ['nexus3.onap.org:10002/onap/base/sdc-sanity:1.4.0',
+                                'nexus3.onap.org:10002/onap/gizmo2:1.3.1',
+                                'nexus3.onap.org:10002/onap/gizmo2:1.3.2'
+                                ]
+    class mock_image:
+        id = ''
+        short_id = ''
+        def __init__(self, id, short_id):
+            self.id = id
+            self.short_id = short_id
+
+    class count_mock_hits:
+        pull = 0
+        tag = 0
+        push = 0
+        cleanup = 0
+
+    counter = count_mock_hits
+
+    class nbr_exceptions:
+        pull = 0
+        tag = 0
+        push = 0
+        cleanup = 0
+
+    nbr_exc = nbr_exceptions
+
+    def mocked_docker_pull(self, nexus_image_str, count, tag, retry_text='', progbar=False):
+        """Mocking Pull an image from Nexus."""
+        if not nexus_image_str in self._expected_nexus_image_str:
+            print ("IMAGESTR {}".format(nexus_image_str))
+            raise ValueError('Wrong nexus project in pull')
+        image = self.mock_image (self._test_image_long_id, self._test_image_short_id)
+        self.counter.pull = self.counter.pull + 1
+        if self.counter.pull > self.nbr_exc.pull:
+            return image
+        else:
+            raise requests.exceptions.ConnectionError('Connection Error')
+
+    def mocked_docker_tag(self, count, image, tag, retry_text='', progbar=False):
+        """Mocking Tag the image with proper docker name and version."""
+        if not image.id == self._test_image_long_id:
+            raise ValueError('Wrong image id in remove')
+        if not tag in ["1.4.0","1.3.1","1.3.2"]:
+            raise ValueError('Wrong tag in docker_tag')
+        self.counter.tag = self.counter.tag + 1
+        if self.counter.tag <= self.nbr_exc.tag:
+            raise requests.exceptions.ConnectionError('Connection Error')
+
+    def mocked_docker_push(self, count, image, tag, retry_text, progbar=False):
+        """Mocking Tag the image with proper docker name and version."""
+        if not image.id == self._test_image_long_id:
+            raise ValueError('Wrong image id in remove')
+        if not tag in ["1.4.0","1.3.1","1.3.2"]:
+            raise ValueError('Wrong tag in push')
+        self.counter.push = self.counter.push + 1
+        if self.counter.push <= self.nbr_exc.push:
+            raise requests.exceptions.ConnectionError('Connection Error')
+
+    def mocked_docker_cleanup(self, count, image, tag, retry_text='', progbar=False):
+        """Mocking Tag the image with proper docker name and version."""
+        if not image.id == self._test_image_long_id:
+            raise ValueError('Wrong image id in remove')
+        self.counter.cleanup = self.counter.cleanup + 1
+        if self.counter.cleanup <= self.nbr_exc.cleanup:
+            raise requests.exceptions.ConnectionError('Connection Error')
+
+    def initiate_test_fetch(self, responses, mocker, repo=''):
+        mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_pull', side_effect=self.mocked_docker_pull)
+        mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_tag', side_effect=self.mocked_docker_tag)
+        mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_push', side_effect=self.mocked_docker_push)
+        mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_cleanup', side_effect=self.mocked_docker_cleanup)
+        url = 'https://nexus3.onap.org:10002/v2/_catalog'
+        answer = '{"repositories":["onap/base/sdc-sanity","onap/gizmo","onap/gizmo2"]}'
+
+        nexus_url1 = 'https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list'
+        nexus_answer1 = '{"name":"onap/base_sdc-sanity","tags":["1.3.0","1.3.1","1.4.0","v1.0.0"]}'
+        docker_url1 = 'https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags'
+        docker_answer1 = """[{"layer": "", "name": "1.3.0"},
+            {"layer": "", "name": "1.3.1"},
+            {"layer": "", "name": "v1.0.0"}]
+        """
+        nexus_url2 = 'https://nexus3.onap.org:10002/v2/onap/gizmo/tags/list'
+        nexus_answer2 = '{"name":"onap/gizmo","tags":["1.3.0"]}'
+        docker_url2 = 'https://registry.hub.docker.com/v1/repositories/onap/gizmo/tags'
+        docker_answer2 = """[{"layer": "", "name": "1.3.0"}]
+        """
+        nexus_url3 = 'https://nexus3.onap.org:10002/v2/onap/gizmo2/tags/list'
+        nexus_answer3 = '{"name":"onap/gizmo2","tags":["1.3.0", "1.3.1", "1.3.2"]}'
+        docker_url3 = 'https://registry.hub.docker.com/v1/repositories/onap/gizmo2/tags'
+        docker_answer3 = """[{"layer": "", "name": "1.3.0"}]
+        """
+        responses.add(responses.GET, url, body=answer, status=200)
+
+        rdh.NexusCatalog = []
+        rdh.projects = []
+
+        responses.add(responses.GET, nexus_url1, body=nexus_answer1, status=200)
+        responses.add(responses.GET, docker_url1, body=docker_answer1, status=200)
+        if len(repo) == 0:
+            responses.add(responses.GET, nexus_url2, body=nexus_answer2, status=200)
+            responses.add(responses.GET, docker_url2, body=docker_answer2, status=200)
+            responses.add(responses.GET, nexus_url3, body=nexus_answer3, status=200)
+            responses.add(responses.GET, docker_url3, body=docker_answer3, status=200)
+
+        self.counter.pull = self.counter.tag = self.counter.push = self.counter.cleanup = 0
+
+    def initiate_bogus_org_test_fetch(self, responses, org):
+        url = 'https://nexus3.{}.org:10002/v2/_catalog'.format(org)
+        exception = requests.HTTPError("Issues with URL: {} - <class 'requests.exceptions.ConnectionError'>".format(url))
+        responses.add(responses.GET, url, body=exception)
+        rdh.NexusCatalog = []
+        rdh.projects = []
+        self.counter.pull = self.counter.tag = self.counter.push = self.counter.cleanup = 0
+
+    def test_fetch_all_tags(self, responses, mocker):
+        self.initiate_test_fetch(responses, mocker)
+        rdh.initialize ('onap')
+        rdh.get_nexus3_catalog ('onap')
+        rdh.fetch_all_tags()
+        assert len(rdh.NexusCatalog) == 3
+        assert len(rdh.projects) == 3
+        assert len(rdh.projects[0].tags_2_copy.valid) == 1
+        assert len(rdh.projects[1].tags_2_copy.valid) == 0
+        assert len(rdh.projects[2].tags_2_copy.valid) == 2
+
+        assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0'
+        assert rdh.projects[2].tags_2_copy.valid[0] == '1.3.1'
+        assert rdh.projects[2].tags_2_copy.valid[1] == '1.3.2'
+
+    def test_fetch_from_bogus_orgs(self, responses, mocker):
+        self.initiate_bogus_org_test_fetch(responses, 'bogus_org321')
+        rdh.initialize ('bogus_org321')
+        rdh.get_nexus3_catalog ('bogus_org321')
+        assert len(rdh.NexusCatalog) == 0
+        assert len(rdh.projects) == 0
+
+    def test_copy(self, responses, mocker):
+        self.initiate_test_fetch(responses, mocker)
+        rdh.initialize ('onap')
+        rdh.get_nexus3_catalog ('onap')
+        rdh.fetch_all_tags()
+        rdh.copy_from_nexus_to_docker()
+        assert self.counter.pull == 3
+        assert self.counter.tag == 3
+        assert self.counter.push == 3
+        assert self.counter.cleanup == 3
+
+    def test_start_no_copy(self, responses, mocker):
+        self.initiate_test_fetch(responses, mocker)
+        rdh.start_point ('onap', '', False)
+        assert self.counter.pull == 0
+        assert self.counter.tag == 0
+        assert self.counter.push == 0
+        assert self.counter.cleanup == 0
+
+    def test_start_copy(self, responses, mocker):
+        self.initiate_test_fetch(responses, mocker)
+        rdh.start_point ('onap', '', False, False, True)
+        assert len(rdh.NexusCatalog) == 3
+        assert len(rdh.projects) == 3
+        assert len(rdh.projects[0].tags_2_copy.valid) == 1
+        assert len(rdh.projects[1].tags_2_copy.valid) == 0
+        assert len(rdh.projects[2].tags_2_copy.valid) == 2
+        assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0'
+        assert rdh.projects[2].tags_2_copy.valid[0] == '1.3.1'
+        assert rdh.projects[2].tags_2_copy.valid[1] == '1.3.2'
+        assert self.counter.pull == 3
+        assert self.counter.tag == 3
+        assert self.counter.push == 3
+        assert self.counter.cleanup == 3
+
+    def test_start_copy_repo(self, responses, mocker):
+        self.initiate_test_fetch(responses, mocker, 'sanity')
+        rdh.start_point ('onap', 'sanity', False, False, True)
+        assert len(rdh.NexusCatalog) == 1
+        assert len(rdh.projects) == 1
+        assert len(rdh.projects[0].tags_2_copy.valid) == 1
+        assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0'
+        assert self.counter.pull == 1
+        assert self.counter.tag == 1
+        assert self.counter.push == 1
+        assert self.counter.cleanup == 1
+
+    def test_start_bogus_orgs(self, responses):
+        self.initiate_bogus_org_test_fetch(responses, 'bogus_org321')
+        rdh.start_point ('bogus_org321')
+        assert len(rdh.NexusCatalog) == 0
+        assert len(rdh.projects) == 0