From: Bengt Thuree Date: Thu, 29 Nov 2018 10:32:51 +0000 (+1100) Subject: Add common functions to release_docker_hub 1(7). X-Git-Tag: v0.20.0~7^2 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F47%2F13047%2F17;p=releng%2Flftools.git Add common functions to release_docker_hub 1(7). This is the first part, which contains the common stuff. Issue:RELENG-1549 Change-Id: I6fd7658d85d696d39042f8210c7e2098d47f3bbc Signed-off-by: Bengt Thuree --- diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py new file mode 100644 index 00000000..b39e269d --- /dev/null +++ b/lftools/nexus/release_docker_hub.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Set of functions to facilitate copying nexus3 release images to docker hub. + +Workflow if you do it manually + + sudo docker login ---> DOCKER Credentials + sudo docker login nexus3.onap.org:10002 -u + + TAB1 https://nexus3.onap.org/#browse/search=repository_name%3Ddocker.release + TAB2 https://hub.docker.com/r/onap + + docker pull nexus3.onap.org:10002/onap/aaf/aaf_hello:2.1.3 + docker images --> imageid --> 991170554e6e + docker tag 991170554e6e onap/aaf-aaf_hello:2.1.3 + docker push onap/aaf-aaf_hello:2.1.3 + docker image rm --force 991170554e6e + +Filter +Find all projects that starts with org and contains repo (if specified). + +Set the repo to "" to find all projects that starts with org + +Set the repo to a str to find all projects that contains that string + and starts with org + repo = "aaf_co" # onap/aaf/aaf_config,onap/aaf/aaf_core + repo = "aaf_cm" # onap/aaf/aaf_cm + repo = "aa" + repo = "" # Find all projects + +lftools nexus docker releasedockerhub +""" +from __future__ import print_function + +import logging +import socket + +import requests +import urllib3 + +log = logging.getLogger(__name__) + +NexusCatalog = [] +projects = [] +TotTagsToBeCopied = 0 + +NEXUS3_BASE = "" +NEXUS3_CATALOG = "" +NEXUS3_PROJ_NAME_HEADER = "" +DOCKER_PROJ_NAME_HEADER = "" + + +def _remove_http_from_url(url): + """Remove http[s]:// from url.""" + if url.startswith('https://'): + return url[len('https://'):] + if url.startswith('http://'): + return url[len('http://'):] + return url + + +def _format_image_id(id): + """Remove sha256: from beginning of string.""" + if id.startswith("sha256:"): + return id[len('sha256:'):] + else: + return id + + +def _request_get(url): + """Execute a request get, return the resp.""" + resp = {} + try: + resp = requests.get(url) + except requests.exceptions.RequestException as excinfo: + log.debug("in _request_get RequestException. {}".format(type(excinfo))) + raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo))) + except socket.gaierror as excinfo: + log.debug("in _request_get gaierror. {}".format(type(excinfo))) + raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo))) + except requests.exceptions.ConnectionError as excinfo: + log.debug("in _request_get ConnectionError. {}".format(type(excinfo))) + raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo))) + except urllib3.exceptions.NewConnectionError as excinfo: + log.debug("in _request_get NewConnectionError. {}".format(type(excinfo))) + raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo))) + except urllib3.exceptions.MaxRetryError as excinfo: + log.debug("in _request_get MaxRetryError. {}".format(type(excinfo))) + raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo))) + return resp + + +def initialize(org_name): + """Set constant strings.""" + global NEXUS3_BASE + global NEXUS3_CATALOG + global NEXUS3_PROJ_NAME_HEADER + global DOCKER_PROJ_NAME_HEADER + NEXUS3_BASE = "https://nexus3.{}.org:10002".format(org_name) + NEXUS3_CATALOG = NEXUS3_BASE + "/v2/_catalog" + NEXUS3_PROJ_NAME_HEADER = "Nexus3 Project Name" + DOCKER_PROJ_NAME_HEADER = "Docker HUB Project Name" diff --git a/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml b/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml new file mode 100644 index 00000000..cfa9ae48 --- /dev/null +++ b/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml @@ -0,0 +1,35 @@ +--- +features: + - | + This command will collect all tags from both Nexus3 and Docker Hub, for + a particular org (for instance 'onap'), as well as a repo (default all repos). + With this information, it will calculate a list of valid tags that needs to + be copied to Docker Hub from Nexus3. + + Usage: + lftools nexus docker releasedockerhub + + Options: + -o, --org TEXT Specify repository organization. [required] + -r, --repo TEXT Only repos containing this string will be selected. + Default set to blank string, which is every repo. + -s, --summary Prints a summary of missing docker tags. + -v, --verbose Prints all collected repo/tag information. + -c, --copy Copy missing tags from Nexus3 repos to Docker Hub repos. + -p, --progbar Display a progress bar for the time consuming jobs. +issues: + - | + Currently, if the Docker Hub repo is missing, it is not created specifically, + but implicitly by docker itself when we push the docker image to an non- + existing Docker Hub repo. + + The command handles any org (onap or hyperledger for instance), "BUT" it + requires that the versioning pattern is #.#.# (1.2.3) for the project. + In regexp terms : ^\d+.\d+.\d+$ +critical: + - | + Before you give the "lftools nexus docker releasedockerhub" command please + ensure you have manually logged in to both Nexus as well as to Docker. + + sudo docker login ---> DOCKER Credentials + sudo docker login nexus3.onap.org:10002 -u diff --git a/requirements.txt b/requirements.txt index b0657f1f..de1bddd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click +docker glob2 # Needed for Python < 3.5 recursive glob support deb-pkg-tools~=5.2 defusedxml # Needed due to tox complains on parseString not safe @@ -9,6 +10,7 @@ ruamel.yaml setuptools>=36.5.0 six~=1.11.0 python-jenkins~=1.1.0 +tqdm xdg~=1.0.7;python_version<'3' xdg~=3.0.0;python_version>='3' diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py new file mode 100644 index 00000000..463d925d --- /dev/null +++ b/tests/test_release_docker_hub.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Test deploy command.""" + +import os +import sys + +import pytest +import responses +import requests + +from lftools import cli +import lftools.nexus.release_docker_hub as rdh + +def test_remove_http_from_url(): + """Test _remove_http_from_url.""" + test_url=[["192.168.1.1", "192.168.1.1"], + ["192.168.1.1:8081", "192.168.1.1:8081"], + ["http://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"], + ["192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"], + ["http://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"], + ["https://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"], + ["https://192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"], + ["http://www.goodnexussite.org:8081", "www.goodnexussite.org:8081"]] + + for url in test_url: + assert rdh._remove_http_from_url(url[0]) == url[1] + + +def test_format_image_id(): + """Test _remove_http_from_url.""" + test_id=[["b9e15a5d1e1a", "b9e15a5d1e1a"], + ["sha256:b9e15a5d1e1a", "b9e15a5d1e1a"], + ["sha256:3450464d68", "3450464d68"], + ["192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"], + ["sha256:3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c", "3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c"]] + + for id in test_id: + assert rdh._format_image_id(id[0]) == id[1]