From: Andrew Grimberg <agrimberg@linuxfoundation.org>
Date: Mon, 13 Feb 2017 14:17:03 +0000 (-0800)
Subject: Nexus2 and Gerrit project builder
X-Git-Tag: v0.0.8~9
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F57%2F4057%2F5;p=releng%2Flftools.git

Nexus2 and Gerrit project builder

Initial checkin of project build system for Nexus 2 and Gerrit

The script currently does the following:

* takes 2 yaml files (see examples) for configuration
  - settings.yaml == administrative / global settings
  - config.yaml == repository structure to build including nexus
    passwords
* Walks config.yaml for all repositories and creates targets,
  privileges, roles and users based upon our standard configuration

Presently does not create the Gerrit repositories

* Staging repo re-ordering hack script

NOTE: This does not work against Nexus 3 as the REST API has been
removed in Nexus 3. Presently scripting Nexus 3 requires groovy

Change-Id: Ia06444a85e167a1e5685f9e569322c9b0e0b8c97
Signed-off-by: Andrew Grimberg <agrimberg@linuxfoundation.org>
---

diff --git a/project_builder/.gitignore b/project_builder/.gitignore
new file mode 100644
index 00000000..2c6c3227
--- /dev/null
+++ b/project_builder/.gitignore
@@ -0,0 +1,3 @@
+# Do not store settings.yaml or config.yaml
+settings.yaml
+config.yaml
diff --git a/project_builder/config.example.yaml b/project_builder/config.example.yaml
new file mode 100644
index 00000000..7d21f3b6
--- /dev/null
+++ b/project_builder/config.example.yaml
@@ -0,0 +1,16 @@
+---
+# vim: sw=2 ts=2 sts=2 et :
+base_groupId: 'org.example'
+repositories:
+  foo:
+    password: 'foo user pass'
+    repositories:
+      bar:
+        password: 'foo-bar user pass'
+        repositories:
+          baz:
+            password: 'foo-bar-baz user pass'
+            extra_privs:
+              # extra roles that should be assigned to the user
+              # These need to be the full display name and not the roleId
+              - 'Staging: Deployer (autorelease)'
diff --git a/project_builder/nexus/__init__.py b/project_builder/nexus/__init__.py
new file mode 100644
index 00000000..13a93376
--- /dev/null
+++ b/project_builder/nexus/__init__.py
@@ -0,0 +1,241 @@
+# -*- code: utf-8 -*-
+# vim: sw=4 ts=4 sts=4 et :
+
+#
+
+"""
+Library for working with Sonatype Nexus REST API
+"""
+
+__title__ = 'nexus'
+__version__ = '0.1.0'
+__build__ = 0x000101
+__author__ = 'Andrew Grimberg'
+__license__ = 'Apache 2.0'
+__copyright__ = 'Copyright 2017 Andrew Grimberg'
+
+import requests
+import json
+from requests.auth import HTTPBasicAuth
+
+class Nexus:
+    def __init__(self, baseurl=None, username=None, password=None):
+        self.baseurl = baseurl
+
+        if username and password:
+            self.add_credentials(username, password)
+        else:
+            self.auth = None
+
+        self.headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+
+    def add_credentials(self, username, password):
+        """
+        Create an authentication object to be used.
+        """
+        self.auth = HTTPBasicAuth(username, password)
+
+    def add_baseurl(self, url):
+        """
+        Set the base URL for nexus
+        """
+        self.baseurl = url
+
+    def get_target(self, name):
+        """
+        Get the ID of a given target name
+        """
+        url = '/'.join([self.baseurl, 'service/local/repo_targets'])
+        targets = requests.get(url, auth=self.auth, headers=self.headers).json()
+
+        for priv in targets['data']:
+            if priv['name'] == name:
+                return priv['id']
+        raise LookupError("No target found named '%s'" % (name))
+
+    def create_target(self, name, patterns):
+        """
+        Create a target with the given patterns
+        """
+        url = '/'.join([self.baseurl, 'service/local/repo_targets'])
+
+        target = {
+            'data': {
+                'contentClass': 'any',
+                'patterns': patterns,
+                'name': name,
+            }
+        }
+
+        json_data = json.dumps(target, encoding='latin-1')
+
+        r = requests.post(url, auth=self.auth, headers=self.headers, data=json_data)
+
+        return r.json()['data']['id']
+
+    def get_priv(self, name, priv):
+        """
+        Get the ID for the privilege with the given name
+        """
+        url = '/'.join([self.baseurl, 'service/local/privileges'])
+
+        search_name = '%s - (%s)' % (name, priv)
+        privileges = requests.get(url, auth=self.auth, headers=self.headers).json()
+
+        for priv in privileges['data']:
+            if priv['name'] == search_name:
+                return priv['id']
+
+        raise LookupError("No privilege found named '%s'" % name)
+
+    def create_priv(self, name, target_id, priv):
+        """
+        Create a given privilege
+
+        Privilege must be one of the following:
+
+        create
+        read
+        delete
+        update
+        """
+        url = '/'.join([self.baseurl,'service/local/privileges_target'])
+
+        privileges = {
+            'data': {
+                'name': name,
+                'description': name,
+                'method': [
+                        priv,
+                    ],
+                'repositoryGroupId': '',
+                'repositoryId': '',
+                'repositoryTargetId': target_id,
+                'type': 'target',
+            }
+        }
+
+        json_data = json.dumps(privileges, encoding='latin-1')
+        privileges = requests.post(url, auth=self.auth, headers=self.headers, data=json_data).json()
+
+        return privileges['data'][0]['id']
+
+    def get_role(self, name):
+        """
+        Get the id of a role with a given name
+        """
+        url = '/'.join([self.baseurl, 'service/local/roles'])
+        roles = requests.get(url, auth=self.auth, headers=self.headers).json()
+
+        for role in roles['data']:
+            if role['name'] == name:
+                return role['id']
+
+        raise LookupError("No role with name '%s'" % (name))
+
+    def create_role(self, name, privs):
+        """
+        Create a role with the given privileges
+        """
+        url = '/'.join([self.baseurl, 'service/local/roles'])
+
+        role = {
+            'data': {
+                'id': name,
+                'name': name,
+                'description': name,
+                'privileges': privs,
+                'roles': [
+                        'repository-any-read',
+                    ],
+                'sessionTimeout': 60,
+            }
+        }
+
+        json_data = json.dumps(role, encoding='latin-1')
+
+        r = requests.post(url, auth=self.auth, headers=self.headers, data=json_data)
+
+        return r.json()['data']['id']
+
+    def get_user(self, user_id):
+        """
+        Determine if a user with a given userId exists
+        """
+        url = '/'.join([self.baseurl, 'service/local/users'])
+        users = requests.get(url, auth=self.auth, headers=self.headers).json()
+
+        for user in users['data']:
+            if user['userId'] == user_id:
+                return
+
+        raise LookupError("No user with id '%s'" % (user_id))
+
+    def create_user(self, name, domain, role_id, password, extra_roles=[]):
+        """
+        Create a Deployment user with a specific role_id and potentially extra roles
+
+        User is created with the nx-deployment role attached
+        """
+        url = '/'.join([self.baseurl, 'service/local/users'])
+
+        user = {
+            'data': {
+                'userId': name,
+                'email': '%s-deploy@%s' % (name, domain),
+                'firstName': name,
+                'lastName': 'Deployment',
+                'roles': [
+                        role_id,
+                        'nx-deployment',
+                    ],
+                'password': password,
+                'status': 'active',
+            }
+        }
+
+        for role in extra_roles:
+            user['data']['roles'].append(self.get_role(role))
+
+        json_data = json.dumps(user, encoding='latin-1')
+
+        r = requests.post(url, auth=self.auth, headers=self.headers, data=json_data)
+
+    def get_repo_group(self, name):
+        """
+        Get the repository ID for a repo group that has a specific name
+        """
+        url = '/'.join([self.baseurl, 'service/local/repo_groups'])
+
+        repos = requests.get(url, auth=self.auth, headers=self.headers).json()
+
+        for repo in repos['data']:
+            if repo['name'] == name:
+                return repo['id']
+
+        raise LookupError("No repository group named '%s'" % (name))
+
+    def get_repo_group_details(self, repoId):
+        """
+        Get the current configuration of a given repo group with a specific ID
+        """
+        url = '/'.join([self.baseurl, 'service/local/repo_groups', repoId])
+
+        return requests.get(url, auth=self.auth, headers=self.headers).json()['data']
+
+    def update_repo_group_details(self, repoId, data):
+        """
+        Update the given repo group with new configuration
+        """
+        url = '/'.join([self.baseurl, 'service/local/repo_groups', repoId])
+
+        repo = {
+            'data': data
+        }
+
+        json_data = json.dumps(repo, encoding='latin-1')
+
+        r = requests.put(url, auth=self.auth, headers=self.headers, data=json_data)
diff --git a/project_builder/reorder_staging_repo.py b/project_builder/reorder_staging_repo.py
new file mode 100755
index 00000000..866896ad
--- /dev/null
+++ b/project_builder/reorder_staging_repo.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- code: utf-8 -*-
+# vim: sw=4 ts=4 sts=4 et :
+
+#
+# NOTE: This is a hack for forcing the 'Staging Repositories' repo group
+# to be in the correct reverse sorted order. There is a problem with
+# Nexus where it is not doing this like it should be
+#
+
+import argparse
+import sys
+import nexus
+import yaml
+from operator import itemgetter
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-s', '--settings', type=str,
+    help='security and settings yaml file')
+args = parser.parse_args()
+
+if not args.settings:
+    sys.exit('Settings file is required')
+
+
+# open our settings file
+f = open(args.settings, 'r')
+settings = yaml.load(f)
+f.close()
+
+for setting in ['nexus', 'user', 'password']:
+    if not setting in settings:
+        sys.exit('{} needs to be defined'.format(setting))
+
+n = nexus.Nexus(settings['nexus'], settings['user'], settings['password'])
+
+try:
+    repo_id = n.get_repo_group('Staging Repositories')
+except LookupError as e:
+    sys.exit("Staging repository 'Staging Repositories' cannot be found")
+
+repo_details = n.get_repo_group_details(repo_id)
+
+sorted_repos = sorted(repo_details['repositories'], key=lambda k: k['id'], reverse=True)
+
+for repos in sorted_repos:
+    del repos['resourceURI']
+    del repos['name']
+
+repo_update = repo_details
+repo_update['repositories'] = sorted_repos
+del repo_update['contentResourceURI']
+del repo_update['repoType']
+
+n.update_repo_group_details(repo_id, repo_update)
diff --git a/project_builder/repo_build.py b/project_builder/repo_build.py
new file mode 100755
index 00000000..f2f1d93e
--- /dev/null
+++ b/project_builder/repo_build.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- code: utf-8 -*-
+# vim: sw=4 ts=4 sts=4 et :
+
+import argparse
+import sys
+import nexus
+import yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-s', '--settings', type=str,
+    help='security and settings yaml file')
+parser.add_argument('-c', '--config', type=str,
+    help='configuration to be created')
+args = parser.parse_args()
+
+if not args.settings:
+    sys.exit('Settings file is required')
+
+if not args.config:
+    sys.exit('Config file is required')
+
+
+# open our settings file
+f = open(args.settings, 'r')
+settings = yaml.load(f)
+f.close()
+
+for setting in ['nexus', 'user', 'password', 'email_domain']:
+    if not setting in settings:
+        sys.exit('{} needs to be defined'.format(setting))
+
+# open our config file
+f = open(args.config, 'r')
+config = yaml.load(f)
+f.close()
+
+n = nexus.Nexus(settings['nexus'], settings['user'], settings['password'])
+
+def create_nexus_perms(name, targets, email, password, extra_privs=[]):
+    # Create target
+    try:
+        target_id = n.get_target(name)
+    except LookupError as e:
+        target_id = n.create_target(name, targets)
+
+    # Create privileges
+    privs_set = [
+            'create',
+            'delete',
+            'read',
+            'update',
+        ]
+
+    privs = {}
+    for priv in privs_set:
+        try:
+            privs[priv] = n.get_priv(name, priv)
+        except LookupError as e:
+            privs[priv] = n.create_priv(name, target_id, priv)
+
+    # Create Role
+    try:
+        role_id = n.get_role(name)
+    except LookupError as e:
+        role_id = n.create_role(name, privs)
+
+    # Create user
+    try:
+        n.get_user(name)
+    except LookupError as e:
+        n.create_user(name, email, role_id, password, extra_privs)
+
+def do_build_repo(repo, repoId, config, base_groupId):
+    print('Building for %s.%s' % (base_groupId, repo))
+    groupId = '%s.%s' % (base_groupId, repo)
+    target = '^/%s/.*' % groupId.replace('.', '[/\.]')
+    if 'extra_privs' in config:
+        extra_privs = config['extra_privs']
+    else:
+        extra_privs = []
+    create_nexus_perms(repoId, [target], settings['email_domain'],
+        config['password'], extra_privs)
+    if 'repositories' in config:
+        for sub_repo in config['repositories']:
+            sub_repo_id = '%s-%s' % (repoId, sub_repo)
+            do_build_repo(sub_repo, sub_repo_id, config['repositories'][sub_repo],
+                groupId)
+
+for repo in config['repositories']:
+    do_build_repo(repo, repo, config['repositories'][repo], config['base_groupId'])
diff --git a/project_builder/requirements.txt b/project_builder/requirements.txt
new file mode 100644
index 00000000..a82933a3
--- /dev/null
+++ b/project_builder/requirements.txt
@@ -0,0 +1,12 @@
+appdirs==1.4.0
+ecdsa==0.11
+httplib2==0.10.3
+packaging==16.8
+paramiko==1.16.0
+pbr==1.10.0
+pycrypto==2.6.1
+pygerrit==1.0.0
+pyparsing==2.1.10
+PyYAML==3.12
+requests==2.9.1
+six==1.10.0
diff --git a/project_builder/settings.example.yaml b/project_builder/settings.example.yaml
new file mode 100644
index 00000000..5266cf9c
--- /dev/null
+++ b/project_builder/settings.example.yaml
@@ -0,0 +1,13 @@
+---
+# vim: sw=2 ts=2 sts=3 et :
+
+# Nexus 2 system to work against. This should be the full path to the base web
+# interface including any URL context.
+nexus: 'http://nexus2.example.com'
+
+# Administrative nexus user account credentials
+user: 'admin'
+password: 'admin123'
+
+# email domain that nexus user emails should be generated with
+email_domain: 'example.com'