Add common functions to release_docker_hub 1(7).

This is the first part, which contains the common stuff.

Issue:RELENG-1549
Change-Id: I6fd7658d85d696d39042f8210c7e2098d47f3bbc
Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>

diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py
new file mode 100644 (file)
index 0000000..b39e269
--- /dev/null
+++ b/lftools/nexus/release_docker_hub.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Set of functions to facilitate copying nexus3 release images to docker hub.
+
+Workflow if you do it manually
+
+    sudo docker login       ---> DOCKER Credentials
+    sudo docker login nexus3.onap.org:10002 -u <yourLFID>
+
+    TAB1 https://nexus3.onap.org/#browse/search=repository_name%3Ddocker.release
+    TAB2 https://hub.docker.com/r/onap
+
+    docker pull nexus3.onap.org:10002/onap/aaf/aaf_hello:2.1.3
+    docker images --> imageid --> 991170554e6e
+    docker tag 991170554e6e onap/aaf-aaf_hello:2.1.3
+    docker push onap/aaf-aaf_hello:2.1.3
+    docker image rm --force 991170554e6e
+
+Filter
+Find all projects that starts with org and contains repo (if specified).
+
+Set the repo to "" to find all projects that starts with org
+
+Set the repo to a str to find all projects that contains that string
+  and starts with org
+    repo = "aaf_co"   # onap/aaf/aaf_config,onap/aaf/aaf_core
+    repo = "aaf_cm"   # onap/aaf/aaf_cm
+    repo = "aa"
+    repo = ""         # Find all projects
+
+lftools nexus docker releasedockerhub
+"""
+from __future__ import print_function
+
+import logging
+import socket
+
+import requests
+import urllib3
+
+log = logging.getLogger(__name__)
+
+NexusCatalog = []
+projects = []
+TotTagsToBeCopied = 0
+
+NEXUS3_BASE = ""
+NEXUS3_CATALOG = ""
+NEXUS3_PROJ_NAME_HEADER = ""
+DOCKER_PROJ_NAME_HEADER = ""
+
+
+def _remove_http_from_url(url):
+    """Remove http[s]:// from url."""
+    if url.startswith('https://'):
+        return url[len('https://'):]
+    if url.startswith('http://'):
+        return url[len('http://'):]
+    return url
+
+
+def _format_image_id(id):
+    """Remove sha256: from beginning of string."""
+    if id.startswith("sha256:"):
+        return id[len('sha256:'):]
+    else:
+        return id
+
+
+def _request_get(url):
+    """Execute a request get, return the resp."""
+    resp = {}
+    try:
+        resp = requests.get(url)
+    except requests.exceptions.RequestException as excinfo:
+        log.debug("in _request_get RequestException. {}".format(type(excinfo)))
+        raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo)))
+    except socket.gaierror as excinfo:
+        log.debug("in _request_get gaierror. {}".format(type(excinfo)))
+        raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo)))
+    except requests.exceptions.ConnectionError as excinfo:
+        log.debug("in _request_get ConnectionError. {}".format(type(excinfo)))
+        raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo)))
+    except urllib3.exceptions.NewConnectionError as excinfo:
+        log.debug("in _request_get NewConnectionError. {}".format(type(excinfo)))
+        raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo)))
+    except urllib3.exceptions.MaxRetryError as excinfo:
+        log.debug("in _request_get MaxRetryError. {}".format(type(excinfo)))
+        raise requests.HTTPError("Issues with URL: {} - {}".format(url, type(excinfo)))
+    return resp
+
+
+def initialize(org_name):
+    """Set constant strings."""
+    global NEXUS3_BASE
+    global NEXUS3_CATALOG
+    global NEXUS3_PROJ_NAME_HEADER
+    global DOCKER_PROJ_NAME_HEADER
+    NEXUS3_BASE = "https://nexus3.{}.org:10002".format(org_name)
+    NEXUS3_CATALOG = NEXUS3_BASE + "/v2/_catalog"
+    NEXUS3_PROJ_NAME_HEADER = "Nexus3 Project Name"
+    DOCKER_PROJ_NAME_HEADER = "Docker HUB Project Name"

diff --git a/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml b/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml
new file mode 100644 (file)
index 0000000..cfa9ae4
--- /dev/null
+++ b/releasenotes/notes/release_docker_hub-5562e259be24b2c4.yaml
@@ -0,0 +1,35 @@
+---
+features:
+  - |
+    This command will collect all tags from both Nexus3 and Docker Hub, for
+    a particular org (for instance 'onap'), as well as a repo (default all repos).
+    With this information, it will calculate a list of valid tags that needs to
+    be copied to Docker Hub from Nexus3.
+
+    Usage:
+      lftools nexus docker releasedockerhub
+
+    Options:
+      -o, --org TEXT   Specify repository organization.  [required]
+      -r, --repo TEXT  Only repos containing this string will be selected.
+                       Default set to blank string, which is every repo.
+      -s, --summary    Prints a summary of missing docker tags.
+      -v, --verbose    Prints all collected repo/tag information.
+      -c, --copy       Copy missing tags from Nexus3 repos to Docker Hub repos.
+      -p, --progbar    Display a progress bar for the time consuming jobs.
+issues:
+  - |
+    Currently, if the Docker Hub repo is missing, it is not created specifically,
+    but implicitly by docker itself when we push the docker image to an non-
+    existing Docker Hub repo.
+
+    The command handles any org (onap or hyperledger for instance), "BUT" it
+    requires that the versioning pattern is #.#.# (1.2.3) for the project.
+    In regexp terms : ^\d+.\d+.\d+$
+critical:
+  - |
+    Before you give the "lftools nexus docker releasedockerhub" command please
+    ensure you have manually logged in to both Nexus as well as to Docker.
+
+    sudo docker login       ---> DOCKER Credentials
+    sudo docker login nexus3.onap.org:10002 -u <yourLFID>

diff --git a/requirements.txt b/requirements.txt
index b0657f1..de1bddd 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 click
+docker
 glob2  # Needed for Python < 3.5 recursive glob support
 deb-pkg-tools~=5.2
 defusedxml # Needed due to tox complains on parseString not safe
@@ -9,6 +10,7 @@ ruamel.yaml
 setuptools>=36.5.0
 six~=1.11.0
 python-jenkins~=1.1.0
+tqdm
 xdg~=1.0.7;python_version<'3'
 xdg~=3.0.0;python_version>='3'
 

diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py
new file mode 100644 (file)
index 0000000..463d925
--- /dev/null
+++ b/tests/test_release_docker_hub.py
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Test deploy command."""
+
+import os
+import sys
+
+import pytest
+import responses
+import requests
+
+from lftools import cli
+import lftools.nexus.release_docker_hub as rdh
+
+def test_remove_http_from_url():
+    """Test _remove_http_from_url."""
+    test_url=[["192.168.1.1", "192.168.1.1"],
+         ["192.168.1.1:8081", "192.168.1.1:8081"],
+         ["http://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"],
+         ["192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"],
+         ["http://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"],
+         ["https://192.168.1.1:8081/nexus", "192.168.1.1:8081/nexus"],
+         ["https://192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"],
+         ["http://www.goodnexussite.org:8081", "www.goodnexussite.org:8081"]]
+
+    for url in test_url:
+        assert rdh._remove_http_from_url(url[0]) == url[1]
+
+
+def test_format_image_id():
+    """Test _remove_http_from_url."""
+    test_id=[["b9e15a5d1e1a", "b9e15a5d1e1a"],
+         ["sha256:b9e15a5d1e1a", "b9e15a5d1e1a"],
+         ["sha256:3450464d68", "3450464d68"],
+         ["192.168.1.1:8081/nexus/", "192.168.1.1:8081/nexus/"],
+         ["sha256:3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c", "3450464d68c9443dedc8bfe3272a23e6441c37f707c42d32fee0ebdbcd319d2c"]]
+
+    for id in test_id:
+        assert rdh._format_image_id(id[0]) == id[1]