From: Thanh Ha Date: Tue, 7 Mar 2017 04:59:03 +0000 (-0500) Subject: Convert project_builder code to lftools X-Git-Tag: v0.0.8~5 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F53%2F4053%2F4;p=releng%2Flftools.git Convert project_builder code to lftools Change-Id: Ie06569e69570065817a7fc2e96fa2e0930e4b2ed Signed-off-by: Thanh Ha --- diff --git a/.coafile b/.coafile index 74ead6f8..b1d56641 100644 --- a/.coafile +++ b/.coafile @@ -9,7 +9,7 @@ use_spaces = yeah [Python] bears = BanditBear,PyCommentedCodeBear,PyDocStyleBear,PyFlakesBear,PyImportSortBear files = lftools/**/*.py -pydocstyle_ignore = D213, D301 +pydocstyle_ignore = D203, D213, D301 [ShellCheck] bears = ShellCheckBear diff --git a/docs/commands/nexus.rst b/docs/commands/nexus.rst new file mode 100644 index 00000000..47ae8141 --- /dev/null +++ b/docs/commands/nexus.rst @@ -0,0 +1,26 @@ +***** +Nexus +***** + +.. program-output:: lftools nexus --help + +Commands +======== + +.. contents:: Nexus Commands + :local: + +create +------ + +.. program-output:: lftools nexus create --help + +repo +^^^^ + +.. program-output:: lftools nexus create repo --help + +reorder_staged_repos +-------------------- + +.. program-output:: lftools nexus reorder_staged_repos --help diff --git a/project_builder/.gitignore b/etc/nexus/.gitignore similarity index 100% rename from project_builder/.gitignore rename to etc/nexus/.gitignore diff --git a/project_builder/config.example.yaml b/etc/nexus/repo-config.example.yaml similarity index 100% rename from project_builder/config.example.yaml rename to etc/nexus/repo-config.example.yaml diff --git a/project_builder/settings.example.yaml b/etc/nexus/settings.example.yaml similarity index 100% rename from project_builder/settings.example.yaml rename to etc/nexus/settings.example.yaml diff --git a/lftools/cli/__init__.py b/lftools/cli/__init__.py index c6e93327..b36abded 100644 --- a/lftools/cli/__init__.py +++ b/lftools/cli/__init__.py @@ -13,6 +13,7 @@ __author__ = 'Thanh Ha' import click +from lftools.cli.nexus import nexus from lftools.cli.version import version @@ -24,6 +25,7 @@ def cli(ctx): pass +cli.add_command(nexus) cli.add_command(version) diff --git a/lftools/cli/nexus.py b/lftools/cli/nexus.py new file mode 100644 index 00000000..696e0855 --- /dev/null +++ b/lftools/cli/nexus.py @@ -0,0 +1,51 @@ +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""CLI entry point for nexus commands.""" +import click +from lftools.nexus import cmd as nexuscmd + + +@click.group() +@click.pass_context +def nexus(ctx): + """Provide an interface to Nexus.""" + pass + + +@click.command() +@click.option('-s', '--settings', type=str, required=True) +@click.pass_context +def reorder_staged_repos(ctx, settings): + """Reorder staging repositories in Nexus. + + Reorders staging repositories in Nexus such that the newest repository gets + priority in the aggregate repo. + """ + nexuscmd.reorder_staged_repos(settings) + +nexus.add_command(reorder_staged_repos) + + +@nexus.group() +@click.pass_context +def create(ctx): + """Create resources in Nexus.""" + pass + + +@create.command() +@click.option('-c', '--config', type=str, required=True, + help='Repo config file for how to the Nexus repository should be created.') +@click.option('-s', '--settings', type=str, required=True, + help='Config file containing administrative settings.') +@click.pass_context +def repo(ctx, config, settings): + """Create a Nexus repository as defined by a repo-config.yaml file.""" + nexuscmd.create_repos(config, settings) diff --git a/project_builder/nexus/__init__.py b/lftools/nexus/__init__.py similarity index 77% rename from project_builder/nexus/__init__.py rename to lftools/nexus/__init__.py index 13a93376..5df9c06e 100644 --- a/project_builder/nexus/__init__.py +++ b/lftools/nexus/__init__.py @@ -1,25 +1,33 @@ # -*- code: utf-8 -*- +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## # vim: sw=4 ts=4 sts=4 et : -# -""" -Library for working with Sonatype Nexus REST API -""" +"""Library for working with Sonatype Nexus REST API.""" -__title__ = 'nexus' -__version__ = '0.1.0' -__build__ = 0x000101 __author__ = 'Andrew Grimberg' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2017 Andrew Grimberg' -import requests import json + +import requests from requests.auth import HTTPBasicAuth + class Nexus: + """Nexus class to handle communicating with Nexus over a rest api.""" + def __init__(self, baseurl=None, username=None, password=None): + """Initialize Nexus instance.""" self.baseurl = baseurl if username and password: @@ -33,21 +41,15 @@ class Nexus: } def add_credentials(self, username, password): - """ - Create an authentication object to be used. - """ + """Create an authentication object to be used.""" self.auth = HTTPBasicAuth(username, password) def add_baseurl(self, url): - """ - Set the base URL for nexus - """ + """Set the base URL for nexus.""" self.baseurl = url def get_target(self, name): - """ - Get the ID of a given target name - """ + """Get the ID of a given target name.""" url = '/'.join([self.baseurl, 'service/local/repo_targets']) targets = requests.get(url, auth=self.auth, headers=self.headers).json() @@ -57,9 +59,7 @@ class Nexus: raise LookupError("No target found named '%s'" % (name)) def create_target(self, name, patterns): - """ - Create a target with the given patterns - """ + """Create a target with the given patterns.""" url = '/'.join([self.baseurl, 'service/local/repo_targets']) target = { @@ -77,9 +77,7 @@ class Nexus: return r.json()['data']['id'] def get_priv(self, name, priv): - """ - Get the ID for the privilege with the given name - """ + """Get the ID for the privilege with the given name.""" url = '/'.join([self.baseurl, 'service/local/privileges']) search_name = '%s - (%s)' % (name, priv) @@ -92,8 +90,7 @@ class Nexus: raise LookupError("No privilege found named '%s'" % name) def create_priv(self, name, target_id, priv): - """ - Create a given privilege + """Create a given privilege. Privilege must be one of the following: @@ -124,9 +121,7 @@ class Nexus: return privileges['data'][0]['id'] def get_role(self, name): - """ - Get the id of a role with a given name - """ + """Get the id of a role with a given name.""" url = '/'.join([self.baseurl, 'service/local/roles']) roles = requests.get(url, auth=self.auth, headers=self.headers).json() @@ -137,9 +132,7 @@ class Nexus: raise LookupError("No role with name '%s'" % (name)) def create_role(self, name, privs): - """ - Create a role with the given privileges - """ + """Create a role with the given privileges.""" url = '/'.join([self.baseurl, 'service/local/roles']) role = { @@ -162,9 +155,7 @@ class Nexus: return r.json()['data']['id'] def get_user(self, user_id): - """ - Determine if a user with a given userId exists - """ + """Determine if a user with a given userId exists.""" url = '/'.join([self.baseurl, 'service/local/users']) users = requests.get(url, auth=self.auth, headers=self.headers).json() @@ -175,8 +166,7 @@ class Nexus: raise LookupError("No user with id '%s'" % (user_id)) def create_user(self, name, domain, role_id, password, extra_roles=[]): - """ - Create a Deployment user with a specific role_id and potentially extra roles + """Create a Deployment user with a specific role_id and potentially extra roles. User is created with the nx-deployment role attached """ @@ -202,12 +192,10 @@ class Nexus: json_data = json.dumps(user, encoding='latin-1') - r = requests.post(url, auth=self.auth, headers=self.headers, data=json_data) + requests.post(url, auth=self.auth, headers=self.headers, data=json_data) def get_repo_group(self, name): - """ - Get the repository ID for a repo group that has a specific name - """ + """Get the repository ID for a repo group that has a specific name.""" url = '/'.join([self.baseurl, 'service/local/repo_groups']) repos = requests.get(url, auth=self.auth, headers=self.headers).json() @@ -219,17 +207,13 @@ class Nexus: raise LookupError("No repository group named '%s'" % (name)) def get_repo_group_details(self, repoId): - """ - Get the current configuration of a given repo group with a specific ID - """ + """Get the current configuration of a given repo group with a specific ID.""" url = '/'.join([self.baseurl, 'service/local/repo_groups', repoId]) return requests.get(url, auth=self.auth, headers=self.headers).json()['data'] def update_repo_group_details(self, repoId, data): - """ - Update the given repo group with new configuration - """ + """Update the given repo group with new configuration.""" url = '/'.join([self.baseurl, 'service/local/repo_groups', repoId]) repo = { @@ -238,4 +222,4 @@ class Nexus: json_data = json.dumps(repo, encoding='latin-1') - r = requests.put(url, auth=self.auth, headers=self.headers, data=json_data) + requests.put(url, auth=self.auth, headers=self.headers, data=json_data) diff --git a/lftools/nexus/cmd.py b/lftools/nexus/cmd.py new file mode 100644 index 00000000..ed1f5644 --- /dev/null +++ b/lftools/nexus/cmd.py @@ -0,0 +1,122 @@ +# -*- code: utf-8 -*- +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Contains functions for various Nexus tasks.""" +import sys + +from lftools.nexus import Nexus +import yaml + + +def reorder_staged_repos(settings_file): + """Reorder staging repositories in Nexus. + + NOTE: This is a hack for forcing the 'Staging Repositories' repo group + to be in the correct reverse sorted order. There is a problem with + Nexus where it is not doing this like it should be. + """ + with open(settings_file, 'r') as f: settings = yaml.safe_load(f) + + for setting in ['nexus', 'user', 'password']: + if not setting in settings: + sys.exit('{} needs to be defined'.format(setting)) + + _nexus = Nexus(settings['nexus'], settings['user'], settings['password']) + + try: + repo_id = _nexus.get_repo_group('Staging Repositories') + except LookupError as e: + sys.exit("Staging repository 'Staging Repositories' cannot be found") + + repo_details = _nexus.get_repo_group_details(repo_id) + + sorted_repos = sorted(repo_details['repositories'], key=lambda k: k['id'], reverse=True) + + for repos in sorted_repos: + del repos['resourceURI'] + del repos['name'] + + repo_update = repo_details + repo_update['repositories'] = sorted_repos + del repo_update['contentResourceURI'] + del repo_update['repoType'] + + _nexus.update_repo_group_details(repo_id, repo_update) + + +def create_repos(config_file, settings_file): + """Create repositories as defined by configuration file. + + :arg str config: Configuration file containing repository definitions that + will be used to create the new Nexus repositories. + :arg str settings: Settings file containing administrative credentials and + information. + """ + with open(config_file, 'r') as f: config = yaml.safe_load(f) + with open(settings_file, 'r') as f: settings = yaml.safe_load(f) + + for setting in ['nexus', 'user', 'password', 'email_domain']: + if not setting in settings: + sys.exit('{} needs to be defined'.format(setting)) + + _nexus = Nexus(settings['nexus'], settings['user'], settings['password']) + + def create_nexus_perms(name, targets, email, password, extra_privs=[]): + # Create target + try: + target_id = _nexus.get_target(name) + except LookupError as e: + target_id = _nexus.create_target(name, targets) + + # Create privileges + privs_set = [ + 'create', + 'delete', + 'read', + 'update', + ] + + privs = {} + for priv in privs_set: + try: + privs[priv] = _nexus.get_priv(name, priv) + except LookupError as e: + privs[priv] = _nexus.create_priv(name, target_id, priv) + + # Create Role + try: + role_id = _nexus.get_role(name) + except LookupError as e: + role_id = _nexus.create_role(name, privs) + + # Create user + try: + _nexus.get_user(name) + except LookupError as e: + _nexus.create_user(name, email, role_id, password, extra_privs) + + def build_repo(repo, repoId, config, base_groupId): + print('Building for %s.%s' % (base_groupId, repo)) + groupId = '%s.%s' % (base_groupId, repo) + target = '^/%s/.*' % groupId.replace('.', '[/\.]') + if 'extra_privs' in config: + extra_privs = config['extra_privs'] + else: + extra_privs = [] + create_nexus_perms(repoId, [target], settings['email_domain'], + config['password'], extra_privs) + if 'repositories' in config: + for sub_repo in config['repositories']: + sub_repo_id = '%s-%s' % (repoId, sub_repo) + build_repo(sub_repo, sub_repo_id, config['repositories'][sub_repo], + groupId) + + for repo in config['repositories']: + build_repo(repo, repo, config['repositories'][repo], config['base_groupId']) diff --git a/project_builder/reorder_staging_repo.py b/project_builder/reorder_staging_repo.py deleted file mode 100755 index 866896ad..00000000 --- a/project_builder/reorder_staging_repo.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -*- code: utf-8 -*- -# vim: sw=4 ts=4 sts=4 et : - -# -# NOTE: This is a hack for forcing the 'Staging Repositories' repo group -# to be in the correct reverse sorted order. There is a problem with -# Nexus where it is not doing this like it should be -# - -import argparse -import sys -import nexus -import yaml -from operator import itemgetter - -parser = argparse.ArgumentParser() -parser.add_argument('-s', '--settings', type=str, - help='security and settings yaml file') -args = parser.parse_args() - -if not args.settings: - sys.exit('Settings file is required') - - -# open our settings file -f = open(args.settings, 'r') -settings = yaml.load(f) -f.close() - -for setting in ['nexus', 'user', 'password']: - if not setting in settings: - sys.exit('{} needs to be defined'.format(setting)) - -n = nexus.Nexus(settings['nexus'], settings['user'], settings['password']) - -try: - repo_id = n.get_repo_group('Staging Repositories') -except LookupError as e: - sys.exit("Staging repository 'Staging Repositories' cannot be found") - -repo_details = n.get_repo_group_details(repo_id) - -sorted_repos = sorted(repo_details['repositories'], key=lambda k: k['id'], reverse=True) - -for repos in sorted_repos: - del repos['resourceURI'] - del repos['name'] - -repo_update = repo_details -repo_update['repositories'] = sorted_repos -del repo_update['contentResourceURI'] -del repo_update['repoType'] - -n.update_repo_group_details(repo_id, repo_update) diff --git a/project_builder/repo_build.py b/project_builder/repo_build.py deleted file mode 100755 index f2f1d93e..00000000 --- a/project_builder/repo_build.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python -# -*- code: utf-8 -*- -# vim: sw=4 ts=4 sts=4 et : - -import argparse -import sys -import nexus -import yaml - -parser = argparse.ArgumentParser() -parser.add_argument('-s', '--settings', type=str, - help='security and settings yaml file') -parser.add_argument('-c', '--config', type=str, - help='configuration to be created') -args = parser.parse_args() - -if not args.settings: - sys.exit('Settings file is required') - -if not args.config: - sys.exit('Config file is required') - - -# open our settings file -f = open(args.settings, 'r') -settings = yaml.load(f) -f.close() - -for setting in ['nexus', 'user', 'password', 'email_domain']: - if not setting in settings: - sys.exit('{} needs to be defined'.format(setting)) - -# open our config file -f = open(args.config, 'r') -config = yaml.load(f) -f.close() - -n = nexus.Nexus(settings['nexus'], settings['user'], settings['password']) - -def create_nexus_perms(name, targets, email, password, extra_privs=[]): - # Create target - try: - target_id = n.get_target(name) - except LookupError as e: - target_id = n.create_target(name, targets) - - # Create privileges - privs_set = [ - 'create', - 'delete', - 'read', - 'update', - ] - - privs = {} - for priv in privs_set: - try: - privs[priv] = n.get_priv(name, priv) - except LookupError as e: - privs[priv] = n.create_priv(name, target_id, priv) - - # Create Role - try: - role_id = n.get_role(name) - except LookupError as e: - role_id = n.create_role(name, privs) - - # Create user - try: - n.get_user(name) - except LookupError as e: - n.create_user(name, email, role_id, password, extra_privs) - -def do_build_repo(repo, repoId, config, base_groupId): - print('Building for %s.%s' % (base_groupId, repo)) - groupId = '%s.%s' % (base_groupId, repo) - target = '^/%s/.*' % groupId.replace('.', '[/\.]') - if 'extra_privs' in config: - extra_privs = config['extra_privs'] - else: - extra_privs = [] - create_nexus_perms(repoId, [target], settings['email_domain'], - config['password'], extra_privs) - if 'repositories' in config: - for sub_repo in config['repositories']: - sub_repo_id = '%s-%s' % (repoId, sub_repo) - do_build_repo(sub_repo, sub_repo_id, config['repositories'][sub_repo], - groupId) - -for repo in config['repositories']: - do_build_repo(repo, repo, config['repositories'][repo], config['base_groupId']) diff --git a/project_builder/requirements.txt b/project_builder/requirements.txt deleted file mode 100644 index a82933a3..00000000 --- a/project_builder/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -appdirs==1.4.0 -ecdsa==0.11 -httplib2==0.10.3 -packaging==16.8 -paramiko==1.16.0 -pbr==1.10.0 -pycrypto==2.6.1 -pygerrit==1.0.0 -pyparsing==2.1.10 -PyYAML==3.12 -requests==2.9.1 -six==1.10.0 diff --git a/requirements.txt b/requirements.txt index cfd78c8a..6bfbd0d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ click +pyyaml +requests sphinx>=1.4.9 sphinxcontrib-programoutput sphinx_bootstrap_theme>=0.4.14