From 68a1eba6240d7e50c79723463dd4c40e84c8dc4a Mon Sep 17 00:00:00 2001 From: Bengt Thuree Date: Thu, 29 Nov 2018 22:18:44 +1100 Subject: [PATCH] Add multithread code to release_docker_hub 6(7). This is the sixth part, with multithreading stuff, fetching tags, copying to docker hub, and main script. Issue:RELENG-1549 Change-Id: I03ec4adfe3225ec9ebbcc27092b578e4c0b36eae Signed-off-by: Bengt Thuree --- lftools/nexus/release_docker_hub.py | 97 ++++++++++++++++++ tests/test_release_docker_hub.py | 194 ++++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+) diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py index a4a27ea8..e81cbe1b 100644 --- a/lftools/nexus/release_docker_hub.py +++ b/lftools/nexus/release_docker_hub.py @@ -41,6 +41,8 @@ lftools nexus docker releasedockerhub from __future__ import print_function import logging +import multiprocessing +from multiprocessing.dummy import Pool as ThreadPool import re import socket @@ -515,6 +517,76 @@ def get_nexus3_catalog(org_name='', find_pattern=''): return True +def fetch_all_tags(progbar=False): + """Fetch all tags function. + + This function will use multi-threading to fetch all tags for all projects in + Nexus3 Catalog. + """ + NbrProjects = len(NexusCatalog) + log.info("Fetching tags from Nexus3 and Docker Hub for {} projects".format(NbrProjects)) + if progbar: + pbar = tqdm.tqdm(total=NbrProjects, bar_format='{l_bar}{bar}|{n_fmt}/{total_fmt} [{elapsed}]') + + def _fetch_all_tags(proj): + """Helper function for multi-threading. + + This function, will create an instance of ProjectClass (which triggers + the project class fetching all Nexus3/Docker Hub tags) + Then adding this instance to the project list. + + Parameters: + proj : Tuple with 'org' and 'repo' + ('onap', 'aaf/aaf_service') + """ + new_proj = ProjectClass(proj) + projects.append(new_proj) + if progbar: + pbar.update(1) + + pool = ThreadPool(multiprocessing.cpu_count()) + pool.map(_fetch_all_tags, NexusCatalog) + pool.close() + pool.join() + + if progbar: + pbar.close() + projects.sort() + + +def copy_from_nexus_to_docker(progbar=False): + """Copy all missing tags. + + This function will use multi-threading to copy all missing tags in the project list. + """ + _tot_tags = 0 + for proj in projects: + _tot_tags = _tot_tags + len(proj.tags_2_copy.valid) + log.info("About to start copying from Nexus3 to Docker Hub for {} missing tags".format(_tot_tags)) + if progbar: + pbar = tqdm.tqdm(total=_tot_tags, bar_format='{l_bar}{bar}|{n_fmt}/{total_fmt} [{elapsed}]') + + def _docker_pull_tag_push(proj): + """Helper function for multi-threading. + + This function, will call the ProjectClass proj's docker_pull_tag_push. + + Parameters: + proj : Tuple with 'org' and 'repo' + ('onap', 'aaf/aaf_service') + """ + proj.docker_pull_tag_push(progbar) + if progbar: + pbar.update(len(proj.tags_2_copy.valid)) + + pool = ThreadPool(multiprocessing.cpu_count()) + pool.map(_docker_pull_tag_push, projects) + pool.close() + pool.join() + if progbar: + pbar.close() + + def print_nexus_docker_proj_names(): """Print Nexus3 - Docker Hub repositories.""" fmt_str = '{:<'+str(project_max_len_chars)+'} : ' @@ -650,3 +722,28 @@ def print_nbr_tags_to_copy(): for proj in projects: _tot_tags = _tot_tags + len(proj.tags_2_copy.valid) log.info("Summary: {} tags that should be copied from Nexus3 to Docker Hub.".format(_tot_tags)) + + +def start_point(org_name, find_pattern='', summary=False, + verbose=False, copy=False, progbar=False): + """Main function.""" + initialize(org_name) + if not get_nexus3_catalog(org_name, find_pattern): + log.info("Could not get any catalog from Nexus3 with org = {}".format(org_name)) + return + + fetch_all_tags(progbar) + if verbose: + print_nexus_docker_proj_names() + print_nexus_valid_tags() + print_nexus_invalid_tags() + print_docker_valid_tags() + print_docker_invalid_tags() + print_stats() + if summary or verbose: + print_missing_docker_proj() + print_nexus_tags_to_copy() + if copy: + copy_from_nexus_to_docker(progbar) + else: + print_nbr_tags_to_copy() diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py index b8ea04c3..d42aeb54 100644 --- a/tests/test_release_docker_hub.py +++ b/tests/test_release_docker_hub.py @@ -410,3 +410,197 @@ class TestFetchNexus3Catalog: responses.add(responses.GET, self.url, body=self.answer, status=200) rdh.get_nexus3_catalog ('onap', 'aaf') assert len(rdh.NexusCatalog) == 18 + + +class TestFetchAllTagsAndUpdate: + _test_image_long_id = 'sha256:3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c' + _test_image_short_id = 'sha256:3450464d68' + _expected_nexus_image_str = ['nexus3.onap.org:10002/onap/base/sdc-sanity:1.4.0', + 'nexus3.onap.org:10002/onap/gizmo2:1.3.1', + 'nexus3.onap.org:10002/onap/gizmo2:1.3.2' + ] + class mock_image: + id = '' + short_id = '' + def __init__(self, id, short_id): + self.id = id + self.short_id = short_id + + class count_mock_hits: + pull = 0 + tag = 0 + push = 0 + cleanup = 0 + + counter = count_mock_hits + + class nbr_exceptions: + pull = 0 + tag = 0 + push = 0 + cleanup = 0 + + nbr_exc = nbr_exceptions + + def mocked_docker_pull(self, nexus_image_str, count, tag, retry_text='', progbar=False): + """Mocking Pull an image from Nexus.""" + if not nexus_image_str in self._expected_nexus_image_str: + print ("IMAGESTR {}".format(nexus_image_str)) + raise ValueError('Wrong nexus project in pull') + image = self.mock_image (self._test_image_long_id, self._test_image_short_id) + self.counter.pull = self.counter.pull + 1 + if self.counter.pull > self.nbr_exc.pull: + return image + else: + raise requests.exceptions.ConnectionError('Connection Error') + + def mocked_docker_tag(self, count, image, tag, retry_text='', progbar=False): + """Mocking Tag the image with proper docker name and version.""" + if not image.id == self._test_image_long_id: + raise ValueError('Wrong image id in remove') + if not tag in ["1.4.0","1.3.1","1.3.2"]: + raise ValueError('Wrong tag in docker_tag') + self.counter.tag = self.counter.tag + 1 + if self.counter.tag <= self.nbr_exc.tag: + raise requests.exceptions.ConnectionError('Connection Error') + + def mocked_docker_push(self, count, image, tag, retry_text, progbar=False): + """Mocking Tag the image with proper docker name and version.""" + if not image.id == self._test_image_long_id: + raise ValueError('Wrong image id in remove') + if not tag in ["1.4.0","1.3.1","1.3.2"]: + raise ValueError('Wrong tag in push') + self.counter.push = self.counter.push + 1 + if self.counter.push <= self.nbr_exc.push: + raise requests.exceptions.ConnectionError('Connection Error') + + def mocked_docker_cleanup(self, count, image, tag, retry_text='', progbar=False): + """Mocking Tag the image with proper docker name and version.""" + if not image.id == self._test_image_long_id: + raise ValueError('Wrong image id in remove') + self.counter.cleanup = self.counter.cleanup + 1 + if self.counter.cleanup <= self.nbr_exc.cleanup: + raise requests.exceptions.ConnectionError('Connection Error') + + def initiate_test_fetch(self, responses, mocker, repo=''): + mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_pull', side_effect=self.mocked_docker_pull) + mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_tag', side_effect=self.mocked_docker_tag) + mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_push', side_effect=self.mocked_docker_push) + mocker.patch('lftools.nexus.release_docker_hub.ProjectClass._docker_cleanup', side_effect=self.mocked_docker_cleanup) + url = 'https://nexus3.onap.org:10002/v2/_catalog' + answer = '{"repositories":["onap/base/sdc-sanity","onap/gizmo","onap/gizmo2"]}' + + nexus_url1 = 'https://nexus3.onap.org:10002/v2/onap/base/sdc-sanity/tags/list' + nexus_answer1 = '{"name":"onap/base_sdc-sanity","tags":["1.3.0","1.3.1","1.4.0","v1.0.0"]}' + docker_url1 = 'https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags' + docker_answer1 = """[{"layer": "", "name": "1.3.0"}, + {"layer": "", "name": "1.3.1"}, + {"layer": "", "name": "v1.0.0"}] + """ + nexus_url2 = 'https://nexus3.onap.org:10002/v2/onap/gizmo/tags/list' + nexus_answer2 = '{"name":"onap/gizmo","tags":["1.3.0"]}' + docker_url2 = 'https://registry.hub.docker.com/v1/repositories/onap/gizmo/tags' + docker_answer2 = """[{"layer": "", "name": "1.3.0"}] + """ + nexus_url3 = 'https://nexus3.onap.org:10002/v2/onap/gizmo2/tags/list' + nexus_answer3 = '{"name":"onap/gizmo2","tags":["1.3.0", "1.3.1", "1.3.2"]}' + docker_url3 = 'https://registry.hub.docker.com/v1/repositories/onap/gizmo2/tags' + docker_answer3 = """[{"layer": "", "name": "1.3.0"}] + """ + responses.add(responses.GET, url, body=answer, status=200) + + rdh.NexusCatalog = [] + rdh.projects = [] + + responses.add(responses.GET, nexus_url1, body=nexus_answer1, status=200) + responses.add(responses.GET, docker_url1, body=docker_answer1, status=200) + if len(repo) == 0: + responses.add(responses.GET, nexus_url2, body=nexus_answer2, status=200) + responses.add(responses.GET, docker_url2, body=docker_answer2, status=200) + responses.add(responses.GET, nexus_url3, body=nexus_answer3, status=200) + responses.add(responses.GET, docker_url3, body=docker_answer3, status=200) + + self.counter.pull = self.counter.tag = self.counter.push = self.counter.cleanup = 0 + + def initiate_bogus_org_test_fetch(self, responses, org): + url = 'https://nexus3.{}.org:10002/v2/_catalog'.format(org) + exception = requests.HTTPError("Issues with URL: {} - ".format(url)) + responses.add(responses.GET, url, body=exception) + rdh.NexusCatalog = [] + rdh.projects = [] + self.counter.pull = self.counter.tag = self.counter.push = self.counter.cleanup = 0 + + def test_fetch_all_tags(self, responses, mocker): + self.initiate_test_fetch(responses, mocker) + rdh.initialize ('onap') + rdh.get_nexus3_catalog ('onap') + rdh.fetch_all_tags() + assert len(rdh.NexusCatalog) == 3 + assert len(rdh.projects) == 3 + assert len(rdh.projects[0].tags_2_copy.valid) == 1 + assert len(rdh.projects[1].tags_2_copy.valid) == 0 + assert len(rdh.projects[2].tags_2_copy.valid) == 2 + + assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0' + assert rdh.projects[2].tags_2_copy.valid[0] == '1.3.1' + assert rdh.projects[2].tags_2_copy.valid[1] == '1.3.2' + + def test_fetch_from_bogus_orgs(self, responses, mocker): + self.initiate_bogus_org_test_fetch(responses, 'bogus_org321') + rdh.initialize ('bogus_org321') + rdh.get_nexus3_catalog ('bogus_org321') + assert len(rdh.NexusCatalog) == 0 + assert len(rdh.projects) == 0 + + def test_copy(self, responses, mocker): + self.initiate_test_fetch(responses, mocker) + rdh.initialize ('onap') + rdh.get_nexus3_catalog ('onap') + rdh.fetch_all_tags() + rdh.copy_from_nexus_to_docker() + assert self.counter.pull == 3 + assert self.counter.tag == 3 + assert self.counter.push == 3 + assert self.counter.cleanup == 3 + + def test_start_no_copy(self, responses, mocker): + self.initiate_test_fetch(responses, mocker) + rdh.start_point ('onap', '', False) + assert self.counter.pull == 0 + assert self.counter.tag == 0 + assert self.counter.push == 0 + assert self.counter.cleanup == 0 + + def test_start_copy(self, responses, mocker): + self.initiate_test_fetch(responses, mocker) + rdh.start_point ('onap', '', False, False, True) + assert len(rdh.NexusCatalog) == 3 + assert len(rdh.projects) == 3 + assert len(rdh.projects[0].tags_2_copy.valid) == 1 + assert len(rdh.projects[1].tags_2_copy.valid) == 0 + assert len(rdh.projects[2].tags_2_copy.valid) == 2 + assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0' + assert rdh.projects[2].tags_2_copy.valid[0] == '1.3.1' + assert rdh.projects[2].tags_2_copy.valid[1] == '1.3.2' + assert self.counter.pull == 3 + assert self.counter.tag == 3 + assert self.counter.push == 3 + assert self.counter.cleanup == 3 + + def test_start_copy_repo(self, responses, mocker): + self.initiate_test_fetch(responses, mocker, 'sanity') + rdh.start_point ('onap', 'sanity', False, False, True) + assert len(rdh.NexusCatalog) == 1 + assert len(rdh.projects) == 1 + assert len(rdh.projects[0].tags_2_copy.valid) == 1 + assert rdh.projects[0].tags_2_copy.valid[0] == '1.4.0' + assert self.counter.pull == 1 + assert self.counter.tag == 1 + assert self.counter.push == 1 + assert self.counter.cleanup == 1 + + def test_start_bogus_orgs(self, responses): + self.initiate_bogus_org_test_fetch(responses, 'bogus_org321') + rdh.start_point ('bogus_org321') + assert len(rdh.NexusCatalog) == 0 + assert len(rdh.projects) == 0 -- 2.16.6