Fetch nexus3 catalog to release_docker_hub 4(7).

This is the fourth part, which contains fetching nexus3
catalog.

Issue:RELENG-1549
Change-Id: Ie627ad04cc0d84304b585d126ed1e468c95661b7
Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>

diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py
index d84e9b5..11053aa 100644 (file)
--- a/lftools/nexus/release_docker_hub.py
+++ b/lftools/nexus/release_docker_hub.py
@@ -444,3 +444,72 @@ class ProjectClass:
                     cnt_break_loop = cnt_break_loop + 1
                     if (cnt_break_loop > 90):
                         raise requests.HTTPError(retry_text)
+
+
+def get_nexus3_catalog(org_name='', find_pattern=''):
+    """Main function to collect all Nexus3 repositories.
+
+    This function will collect the Nexus catalog for all projects starting with
+    'org_name' as well as containing a pattern if specified.
+
+    If you do it manually, you give the following command.
+        curl -s https://nexus3.onap.org:10002/v2/_catalog
+
+    which gives you the following output.
+        {"repositories":["dcae_dmaapbc","onap/aaf/aaf-base-openssl_1.1.0",
+        "onap/aaf/aaf-base-xenial","onap/aaf/aaf_agent","onap/aaf/aaf_cass",
+        "onap/aaf/aaf_cm","onap/aaf/aaf_config","onap/aaf/aaf_core"]}
+
+    Nexus3 catalog starts with <org_name>/<repo name>
+
+    Parameters:
+        org_name     : Organizational name, for instance 'onap'
+        find_pattern : A pattern, that if specified, needs to be part of the
+                       repository name.
+                        for instance,
+                         ''     : this pattern finds all repositories.
+                         'eleo' : this pattern finds all repositories with 'eleo'
+                                 in its name. --> chameleon
+
+    """
+    global NexusCatalog
+    global project_max_len_chars
+
+    project_max_len_chars = 0
+    containing_str = ''
+    if len(find_pattern) > 0:
+        containing_str = ', and containing "{}"'.format(find_pattern)
+    info_str = "Collecting information from Nexus with projects with org = {}".format(org_name)
+    log.info("{}{}.".format(info_str, containing_str))
+
+    try:
+        r = _request_get(NEXUS3_CATALOG)
+    except requests.HTTPError as excinfo:
+        log.info("Fetching Nexus3 catalog. {}".format(excinfo))
+        return False
+
+    log.debug("r.status_code = {}, ok={}".format(r.status_code, r.status_code == requests.codes.ok))
+    if r.status_code == requests.codes.ok:
+        raw_catalog = r.text
+        raw_catalog = raw_catalog.replace('"', '')
+        raw_catalog = raw_catalog.replace(' ', '')
+        raw_catalog = raw_catalog.replace('}', '')
+        raw_catalog = raw_catalog.replace('[', '')
+        raw_catalog = raw_catalog.replace(']', '')
+        raw_catalog = raw_catalog.split(':')
+        TmpCatalog = raw_catalog[1].split(',')
+        for word in TmpCatalog:
+            # Remove all projects that do not start with org_name
+            if word.startswith(org_name):
+                # If  a specific search string has been specified, searc = h for it
+                # Empty string will match all words
+                if word.find(find_pattern) >= 0:
+                    # Remove onap/ from word, so we only get repository left
+                    project = (org_name, word[len(org_name)+1:])
+                    NexusCatalog.append(project)
+                    log.debug("Added project {} to my list".format(project[1]))
+                    if len(project[1]) > project_max_len_chars:
+                        project_max_len_chars = len(project[1])
+        log.debug("# TmpCatalog {}, NexusCatalog {}, DIFF = {}".format(
+            len(TmpCatalog), len(NexusCatalog), len(TmpCatalog)-len(NexusCatalog)))
+    return True

diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py
index 273b255..b8ea04c 100644 (file)
--- a/tests/test_release_docker_hub.py
+++ b/tests/test_release_docker_hub.py
@@ -379,3 +379,34 @@ class TestProjectClass:
         assert self.counter.tag == 90
         assert self.counter.push == 90
         assert self.counter.cleanup == 90
+
+
+class TestFetchNexus3Catalog:
+
+    url = 'https://nexus3.onap.org:10002/v2/_catalog'
+    answer = """
+         {"repositories":["dcae_dmaapbc","onap/aaf/aaf-base-openssl_1.1.0","onap/aaf/aaf-base-xenial","onap/aaf/aaf_agent","onap/aaf/aaf_cass","onap/aaf/aaf_cm","onap/aaf/aaf_config","onap/aaf/aaf_core","onap/aaf/aaf_fs","onap/aaf/aaf_gui","onap/aaf/aaf_hello","onap/aaf/aaf_locate","onap/aaf/aaf_oauth","onap/aaf/aaf_service","onap/aaf/abrmd","onap/aaf/distcenter","onap/aaf/sms","onap/aaf/smsquorumclient","onap/aaf/testcaservice","onap/aai-cacher","onap/aai-graphadmin","onap/aai-resources","onap/aai-traversal","onap/aai/esr-gui","onap/aai/esr-server","onap/admportal-sdnc-image","onap/appc-cdt-image","onap/appc-image","onap/babel","onap/base_sdc-cassandra","onap/base_sdc-cqlsh","onap/base_sdc-elasticsearch","onap/base_sdc-jetty","onap/base_sdc-kibana","onap/base_sdc-python","onap/base_sdc-sanity","onap/base_sdc-vnc","onap/ccsdk-ansible-server-image","onap/ccsdk-apps-ms-neng","onap/ccsdk-controllerblueprints","onap/ccsdk-dgbuilder-image","onap/ccsdk-odl-image","onap/ccsdk-odl-oxygen-image","onap/ccsdk-odlsli-image","onap/ccsdk-ubuntu-image","onap/chameleon","onap/champ","onap/clamp","onap/clamp-dashboard-kibana","onap/clamp-dashboard-logstash","onap/cli","onap/data-router","onap/dcae-be","onap/dcae-ci-tests","onap/dcae-dt","onap/dcae-fe","onap/dcae-tools","onap/dcae-tosca-app","onap/dmaap/buscontroller","onap/dmaap/datarouter-node","onap/dmaap/datarouter-prov","onap/dmaap/datarouter-subscriber","onap/dmaap/dmaap-mr","onap/dmaap/kafka01101","onap/externalapi/nbi","onap/gallifrey","onap/gizmo","onap/holmes/engine-management","onap/holmes/rule-management","onap/model-loader","onap/msb/msb_apigateway","onap/msb/msb_base","onap/msb/msb_discovery","onap/multicloud/azure","onap/multicloud/framework","onap/multicloud/openstack-newton","onap/multicloud/openstack-ocata","onap/multicloud/openstack-pike","onap/multicloud/openstack-windriver","onap/multicloud/openstack/openstack-ocata","onap/multicloud/vio","onap/multicloud/vio-vesagent","onap/music/cassandra_3_11","onap/music/cassandra_job","onap/music/cassandra_music","onap/music/music","onap/music/prom","onap/network-discovery","onap/oom/kube2msb","onap/optf-cmso-dbinit","onap/optf-cmso-service","onap/optf-has","onap/optf-osdf","onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server","onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main","onap/org.onap.dcaegen2.collectors.snmptrap","onap/org.onap.dcaegen2.collectors.ves.vescollector","onap/org.onap.dcaegen2.deployments.bootstrap","onap/org.onap.dcaegen2.deployments.cm-container","onap/org.onap.dcaegen2.deployments.healthcheck-container","onap/org.onap.dcaegen2.deployments.k8s-bootstrap-container","onap/org.onap.dcaegen2.deployments.redis-cluster-container","onap/org.onap.dcaegen2.deployments.tca-cdap-container","onap/org.onap.dcaegen2.deployments.tls-init-container","onap/org.onap.dcaegen2.platform.cdapbroker","onap/org.onap.dcaegen2.platform.configbinding","onap/org.onap.dcaegen2.platform.configbinding.app-app","onap/org.onap.dcaegen2.platform.deployment-handler","onap/org.onap.dcaegen2.platform.inventory-api","onap/org.onap.dcaegen2.platform.policy-handler","onap/org.onap.dcaegen2.platform.servicechange-handler","onap/org.onap.dcaegen2.services.prh.prh-app-server","onap/policy-apex-pdp","onap/policy-distribution","onap/policy-drools","onap/policy-pe","onap/policy/policy-db","onap/policy/policy-drools","onap/policy/policy-nexus","onap/policy/policy-pe","onap/pomba-aai-context-builder","onap/pomba-context-aggregator","onap/pomba-network-discovery-context-builder","onap/pomba-sdc-context-builder","onap/portal-app","onap/portal-apps","onap/portal-db","onap/portal-sdk","onap/portal-wms","onap/refrepo/postgres","onap/sdc-api-tests","onap/sdc-backend","onap/sdc-backend-init","onap/sdc-cassandra","onap/sdc-cassandra-init","onap/sdc-elasticsearch","onap/sdc-frontend","onap/sdc-init-elasticsearch","onap/sdc-kibana","onap/sdc-onboard-backend","onap/sdc-onboard-cassandra-init","onap/sdc-simulator","onap/sdc-ui-tests","onap/sdc/sdc-workflow-designer","onap/sdnc-ansible-server-image","onap/sdnc-dmaap-listener-image","onap/sdnc-image","onap/sdnc-ueb-listener-image","onap/search-data-service","onap/service-decomposition","onap/sniroemulator","onap/so/api-handler-infra","onap/so/asdc-controller","onap/so/base-image","onap/so/bpmn-infra","onap/so/catalog-db-adapter","onap/so/openstack-adapter","onap/so/request-db-adapter","onap/so/sdc-controller","onap/so/sdnc-adapter","onap/so/so-monitoring","onap/so/vfc-adapter","onap/sparky-be","onap/spike","onap/testsuite","onap/usecase-ui","onap/usecase-ui-server","onap/usecase-ui/usecase-ui-server","onap/validation","onap/vfc/catalog","onap/vfc/db","onap/vfc/emsdriver","onap/vfc/gvnfmdriver","onap/vfc/jujudriver","onap/vfc/multivimproxy","onap/vfc/nfvo/svnfm/huawei","onap/vfc/nfvo/svnfm/nokia","onap/vfc/nfvo/svnfm/nokiav2","onap/vfc/nslcm","onap/vfc/resmanagement","onap/vfc/vnflcm","onap/vfc/vnfmgr","onap/vfc/vnfres","onap/vfc/wfengine-activiti","onap/vfc/wfengine-mgrservice","onap/vfc/ztesdncdriver","onap/vfc/ztevmanagerdriver","onap/vfc/ztevnfmdriver","onap/vid","onap/vnfsdk/ice","onap/vnfsdk/refrepo","onap/vnfsdk/refrepo/postgres","onap/vnfsdk/vnftest","onap/vvp/cms","onap/vvp/engagementmgr","onap/vvp/gitlab","onap/vvp/image-scanner","onap/vvp/jenkins","onap/vvp/portal","onap/vvp/postgresql","onap/vvp/test-engine","onap/workflow-backend","onap/workflow-frontend","onap/workflow-init","openecomp/aai-cacher","openecomp/aai-resources","openecomp/aai-traversal","openecomp/appc-image","openecomp/base_sdc-backend","openecomp/base_sdc-cassandra","openecomp/base_sdc-elasticsearch","openecomp/base_sdc-frontend","openecomp/base_sdc-kibana","openecomp/base_sdc-sanity","openecomp/jacoco","openecomp/mso","openecomp/mso-arquillian","openecomp/portalapps","openecomp/portaldb","openecomp/sdc-backend","openecomp/sdc-cassandra","openecomp/sdc-elasticsearch","openecomp/sdc-frontend","openecomp/sdc-kibana","openecomp/sdc-sanity","openecomp/ubuntu-update","openecomp/vid","openecomp/wildfly"]}
+     """
+
+    def test_get_all_onap(self):
+        rdh.NexusCatalog = []
+        rdh.initialize ('onap')
+        responses.add(responses.GET, self.url, body=self.answer, status=200)
+        rdh.get_nexus3_catalog ('onap')
+        assert len(rdh.NexusCatalog) == 203
+
+    def test_get_all_onap_and_filter_1(self):
+        rdh.NexusCatalog = []
+        rdh.initialize ('onap')
+        responses.add(responses.GET, self.url, body=self.answer, status=200)
+        rdh.get_nexus3_catalog ('onap', 'spike')
+        assert len(rdh.NexusCatalog) == 1
+        assert rdh.NexusCatalog[0][0] == 'onap'
+        assert rdh.NexusCatalog[0][1] == 'spike'
+
+    def test_get_all_onap_and_filter_18(self):
+        rdh.NexusCatalog = []
+        rdh.initialize ('onap')
+        responses.add(responses.GET, self.url, body=self.answer, status=200)
+        rdh.get_nexus3_catalog ('onap', 'aaf')
+        assert len(rdh.NexusCatalog) == 18