From 4f06a413c6f2838eaf5ff8ae0154b9ba1e5bdd5e Mon Sep 17 00:00:00 2001 From: Aric Gardner Date: Wed, 30 Oct 2019 16:58:41 -0400 Subject: [PATCH] Create gerrit project via gerrit api internal jenkins implementation: gerrit.linuxfoundation.org/infra/c/ci-management/+/62158 Extend lfidapi to exit with good message and code if group does not exist. (check if group exists functionality) Remove pythonsix compatability for config parser Usage: lftools gerrit [OPTIONS] COMMAND [ARGS]... GERRIT TOOLS. Options: --help Show this message and exit. Commands: abandonchanges Abandon all OPEN changes for a gerrit project. addfile Add an file for review to a Project. addgithubrights Grant Github read for a project. addgitreview Add git review to a project. addinfojob Add an INFO job for a new Project. createproject Create a project via the gerrit API. list-project-inherits-from List who a project inherits from. list-project-permissions List Owners of a Project. ISSUE: RELENG-2370 Signed-off-by: Aric Gardner Change-Id: I3a24751eeb2e739dee183074bbef6742df58a793 --- docs/commands/gerrit.rst | 70 +++- docs/commands/github.rst | 4 +- lftools/api/client.py | 12 +- lftools/api/endpoints/gerrit.py | 399 +++++++++++++++++++++ lftools/cli/__init__.py | 2 +- lftools/cli/gerrit.py | 190 ++++++++-- lftools/cli/github_cli.py | 6 + lftools/cli/infofile.py | 71 ++-- lftools/config.py | 11 +- lftools/github_helper.py | 12 +- lftools/ldap_cli.py | 6 +- lftools/lfidapi.py | 32 +- ...-service-project-creation-28cc70ec9ea9ec3e.yaml | 34 ++ 13 files changed, 768 insertions(+), 81 deletions(-) create mode 100644 lftools/api/endpoints/gerrit.py create mode 100644 releasenotes/notes/self-service-project-creation-28cc70ec9ea9ec3e.yaml diff --git a/docs/commands/gerrit.rst b/docs/commands/gerrit.rst index f8fa32a8..025420bb 100644 --- a/docs/commands/gerrit.rst +++ b/docs/commands/gerrit.rst @@ -7,7 +7,71 @@ Gerrit Commands ======== -create --------- +list-project-permissions +------------------------ -.. program-output:: lftools gerrit create --help +.. program-output:: lftools gerrit list-project-permissions --help + + +list-project-inherits-from +-------------------------- + +.. program-output:: lftools gerrit list-project-inherits-from --help + + +abandonchanges +-------------- + +.. program-output:: lftools gerrit abandonchanges --help + +addgitreview +------------ + +.. program-output:: lftools gerrit addgitreview --help + + +addgithubrights +--------------- + +.. program-output:: lftools gerrit addgithubrights --help + + +addfile +------- + +.. program-output:: lftools gerrit addfile --help + + +createproject +------------- + +.. program-output:: lftools gerrit createproject --help + + +addinfojob +---------- +.. program-output:: lftools gerrit addinfojob --help + + +.. note:: + + Gerrit API methods require configuration in lftools.ini + in a global [gerrit] section. + support for [gerrit.umbrella.tld] exists as well + signed_off_by required to push changes. + Projects that do not allow self merge will require + as project.example.org.second section for submission + of their .gitreview on project creation. + + +.. code-block:: none + + [gerrit.example.org] + username = lfid + password = password + signed_off_by = Your Name + + [gerrit.example.org.second] + username = lfid2 + password = password2 + signed_off_by = Your Name diff --git a/docs/commands/github.rst b/docs/commands/github.rst index c64df6ad..ece4d517 100644 --- a/docs/commands/github.rst +++ b/docs/commands/github.rst @@ -49,10 +49,10 @@ votes -API requires a [github] section in ~/.config/lftools/lftools.ini: +API requires a [github] or [github.OrgName] section in ~/.config/lftools/lftools.ini: .. code-block:: bash - [github] + [github] or [github.org] token = REDACTED diff --git a/lftools/api/client.py b/lftools/api/client.py index d42bb52d..f9f3f769 100644 --- a/lftools/api/client.py +++ b/lftools/api/client.py @@ -34,6 +34,7 @@ class RestApi(object): self.password = self.creds['password'] self.r = requests.Session() self.r.auth = (self.username, self.password) + self.r.headers.update({'Content-Type': 'application/json; charset=UTF-8'}) if self.creds['authtype'] == 'token': self.token = self.creds['token'] @@ -42,15 +43,18 @@ class RestApi(object): .format(self.token)}) self.r.headers.update({'Content-Type': 'application/json'}) - def _request(self, url, method, data=None, timeout=10): + def _request(self, url, method, data=None, timeout=30): """Execute the request.""" resp = self.r.request(method, self.endpoint + url, data=data, timeout=timeout) + if resp.status_code == 409: + return resp + if resp.text: try: - print(resp.text) - if resp.headers['Content-Type'] == 'application/json': - body = json.loads(resp.text) + if 'application/json' in resp.headers['Content-Type']: + remove_xssi_magic = resp.text.replace(')]}\'', '') + body = json.loads(remove_xssi_magic) else: body = resp.text except ValueError: diff --git a/lftools/api/endpoints/gerrit.py b/lftools/api/endpoints/gerrit.py new file mode 100644 index 00000000..995e0086 --- /dev/null +++ b/lftools/api/endpoints/gerrit.py @@ -0,0 +1,399 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +"""Gerrit REST API interface.""" + +import json +import logging +import os +import time +import urllib + +from lftools import config +import lftools.api.client as client + +log = logging.getLogger(__name__) + + +class Gerrit(client.RestApi): + """API endpoint wrapper for Gerrit. + + Be sure to always include the trailing "/" when adding + new methods. + """ + + def __init__(self, **params): + """Initialize the class.""" + self.params = params + self.fqdn = self.params['fqdn'] + if 'creds' not in self.params: + creds = { + 'authtype': 'basic', + 'username': config.get_setting(self.fqdn, 'username'), + 'password': config.get_setting(self.fqdn, 'password'), + 'endpoint': config.get_setting(self.fqdn, 'endpoint') + } + params['creds'] = creds + + super(Gerrit, self).__init__(**params) + + def add_file(self, fqdn, gerrit_project, filename, issue_id, file_location, **kwargs): + """Add an file for review to a Project. + + File can be sourced from any location + but only lands in the root of the repo. + unless file_location is specified + Example: + + gerrit_url gerrit.o-ran-sc.org + gerrit_project test/test1 + filename /tmp/INFO.yaml + file_location="somedir/example-INFO.yaml" + """ + signed_off_by = config.get_setting(fqdn, 'sob') + basename = os.path.basename(filename) + payload = self.create_change(basename, gerrit_project, issue_id, signed_off_by) + + if file_location: + file_location = urllib.parse.quote(file_location, safe='', encoding=None, errors=None) + basename = file_location + log.info(payload) + + access_str = 'changes/' + result = self.post(access_str, data=payload)[1] + log.info(result['id']) + changeid = (result['id']) + + my_file = open(filename) + my_file_size = os.stat(filename) + headers = {'Content-Type': 'text/plain', + 'Content-length': '{}'.format(my_file_size)} + self.r.headers.update(headers) + access_str = 'changes/{}/edit/{}'.format(changeid, basename) + payload = my_file + result = self.put(access_str, data=payload) + log.info(result) + + access_str = 'changes/{}/edit:publish'.format(changeid) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + payload = json.dumps({ + "notify": "NONE", + }) + result = self.post(access_str, data=payload) + return result + ############################################################## + + def add_info_job(self, fqdn, gerrit_project, jjbrepo, reviewid, issue_id, **kwargs): + """Add an INFO job for a new Project. + + Adds info verify jenkins job for project. + result['id'] can be used to ammend a review + so that multiple projects can have info jobs added + in a single review + + Example: + + gerrit_fqdn gerrit.o-ran-sc.org + gerrit_project test/test1 + jjbrepo ci-mangement + """ + ############################################################### + # Setup + signed_off_by = config.get_setting(fqdn, 'sob') + gerrit_project_dashed = gerrit_project.replace("/", "-") + gerrit_project_encoded = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + filename = 'info-{}.yaml'.format(gerrit_project_dashed) + payload = self.create_change(filename, gerrit_project, issue_id, signed_off_by) + log.info(payload) + + access_str = 'changes/' + result = self.post(access_str, data=payload)[1] + log.info(result) + log.info(result['id']) + changeid = (result['id']) + + my_inline_file = """--- +- project: + name: {0}-info + project-name: {0} + jobs: + - gerrit-info-yaml-verify + project: {1} + branch: master\n""".format(gerrit_project_dashed, gerrit_project) + my_inline_file_size = len(my_inline_file.encode('utf-8')) + headers = {'Content-Type': 'text/plain', + 'Content-length': '{}'.format(my_inline_file_size)} + self.r.headers.update(headers) + access_str = 'changes/{0}/edit/jjb%2F{1}%2Finfo-{2}.yaml'.format( + changeid, gerrit_project_encoded, gerrit_project_dashed) + payload = my_inline_file + log.info(access_str) + result = self.put(access_str, data=payload) + log.info(result) + + if not reviewid: + access_str = 'changes/{}/edit:publish'.format(changeid) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + payload = json.dumps({ + "notify": "NONE", + }) + result = self.post(access_str, data=payload) + log.info(result) + return(result) + + def vote_on_change(self, fqdn, gerrit_project, changeid, **kwargs): + """Helper that votes on a change. + + POST /changes/{change-id}/revisions/{revision-id}/review + """ + log.info(fqdn, gerrit_project, changeid) + access_str = 'changes/{}/revisions/2/review'.format(changeid) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + payload = json.dumps({ + "tag": "automation", + "message": "Vote on file", + "labels": { + "Verified": +1, + "Code-Review": +2, + } + }) + + result = self.post(access_str, data=payload) + # Code for projects that don't allow self merge. + if config.get_setting(self.fqdn + '.second'): + second_username = config.get_setting(self.fqdn + '.second', 'username') + second_password = config.get_setting(self.fqdn + '.second', 'password') + self.r.auth = (second_username, second_password) + result = self.post(access_str, data=payload) + self.r.auth = (self.username, self.password) + return result + + def submit_change(self, fqdn, gerrit_project, changeid, payload, **kwargs): + """Method so submit a change.""" + # submit a change id + access_str = 'changes/{}/submit'.format(changeid) + log.info(access_str) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + result = self.post(access_str, data=payload) + return result + + def abandon_changes(self, fqdn, gerrit_project, **kwargs): + """.""" + gerrit_project_encoded = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + access_str = 'changes/?q=project:{}'.format(gerrit_project_encoded) + log.info(access_str) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + result = self.get(access_str)[1] + payload = {'message': 'Abandoned by automation'} + for id in result: + if (id['status']) == "NEW": + id = (id['id']) + access_str = 'changes/{}/abandon'.format(id) + log.info(access_str) + result = self.post(access_str, data=payload)[1] + return result + + def create_change(self, filename, gerrit_project, issue_id, signed_off_by, **kwargs): + """Method to create a gerrit change.""" + if issue_id: + subject = ( + 'Automation adds {0}\n\nIssue-ID: {1}\n\nSigned-off-by: {2}'.format(filename, issue_id, signed_off_by)) + else: + subject = ( + 'Automation adds {0}\n\nSigned-off-by: {1}'.format(filename, signed_off_by)) + payload = json.dumps({ + "project": '{}'.format(gerrit_project), + "subject": '{}'.format(subject), + "branch": 'master', + }) + return payload + + def sanity_check(self, fqdn, gerrit_project, **kwargs): + """Preform a sanity check.""" + # Sanity check + gerrit_project_encoded = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + mylist = ['projects/', 'projects/{}'.format(gerrit_project_encoded)] + for access_str in mylist: + log.info(access_str) + try: + result = self.get(access_str)[1] + except: + log.info("Not found {}".format(access_str)) + exit(1) + log.info("found {}".format(access_str)) + return result + + def add_git_review(self, fqdn, gerrit_project, issue_id, **kwargs): + """Add and Submit a .gitreview for a project. + + Example: + + gerrit_fqdn gerrit.o-ran-sc.org + gerrit_project test/test1 + issue_id: CIMAN-33 + """ + signed_off_by = config.get_setting(fqdn, 'sob') + self.sanity_check(fqdn, gerrit_project) + + ############################################################### + # Create A change set. + filename = ".gitreview" + payload = self.create_change(filename, gerrit_project, issue_id, signed_off_by) + log.info(payload) + + access_str = 'changes/' + result = self.post(access_str, data=payload)[1] + log.info(result) + changeid = (result['id']) + + ############################################################### + # Add a file to a change set. + my_inline_file = """ + [gerrit] + host={0} + port=29418 + project={1} + defaultbranch=master + asd=asdf + """.format(fqdn, gerrit_project) + my_inline_file_size = len(my_inline_file.encode('utf-8')) + headers = {'Content-Type': 'text/plain', + 'Content-length': '{}'.format(my_inline_file_size)} + self.r.headers.update(headers) + access_str = 'changes/{}/edit/{}'.format(changeid, filename) + payload = my_inline_file + result = self.put(access_str, data=payload) + + if result.status_code == 409: + log.info(result) + log.info("Conflict detected exiting") + exit(0) + + else: + access_str = 'changes/{}/edit:publish'.format(changeid) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + self.r.headers.update(headers) + payload = json.dumps({ + "notify": "NONE", + }) + result = self.post(access_str, data=payload) + log.info(result) + + result = self.vote_on_change(fqdn, gerrit_project, changeid) + log.info(result) + + time.sleep(5) + result = self.submit_change(fqdn, gerrit_project, changeid, payload) + log.info(result) + + def add_github_rights(self, fqdn, gerrit_project, **kwargs): + """Grant github read to a project.""" + ############################################################### + # Github Rights + + gerrit_project_encoded = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + # GET /groups/?m=test%2F HTTP/1.0 + access_str = 'groups/?m=GitHub%20Replication' + log.info(access_str) + result = self.get(access_str)[1] + time.sleep(5) + githubid = (result['GitHub Replication']['id']) + log.info(githubid) + + # POST /projects/MyProject/access HTTP/1.0 + if githubid: + payload = json.dumps({ + "add": { + "refs/*": { + "permissions": { + "read": { + "rules": { + "{}".format(githubid): { + "action": "{}".format("ALLOW") + }}}}}} + }) + access_str = 'projects/{}/access'.format(gerrit_project_encoded) + result = self.post(access_str, data=payload)[1] + pretty = json.dumps(result, indent=4, sort_keys=True) + log.info(pretty) + else: + log.info("Error no githubid found") + + def create_project(self, fqdn, gerrit_project, ldap_group, description, check): + """Create a project via the gerrit API. + + Creates a gerrit project. + Sets ldap group as owner. + + Example: + + gerrit_url gerrit.o-ran-sc.org/r + gerrit_project test/test1 + ldap_group oran-gerrit-test-test1-committers + --description="This is a demo project" + + """ + gerrit_project = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + + access_str = 'projects/{}'.format(gerrit_project) + + result = self.get(access_str)[0] + if result.status_code == 404: + log.info(result) + log.info("Project not found.") + projectexists = False + + else: + log.info("found {}".format(access_str)) + projectexists = True + + if projectexists: + log.info("Project already exists") + exit(1) + if check: + exit(0) + + ldapgroup = "ldap:cn={},ou=Groups,dc=freestandards,dc=org".format(ldap_group) + log.info(ldapgroup) + + access_str = 'projects/{}'.format(gerrit_project) + payload = json.dumps({ + "description": "{0}", + "submit_type": "INHERIT", + "create_empty_commit": "True", + "owners": [ + "{1}".format(description, ldapgroup) + ] + }) + + log.info(payload) + result = self.put(access_str, data=payload) + return result + + def list_project_permissions(self, project): + """List a projects owners.""" + result = self.get('access/?project={}'.format(project))[1][project]['local'] + group_list = [] + for k, v in result.items(): + for kk, vv in result[k]['permissions']['owner']['rules'].items(): + group_list.append(kk.replace('ldap:cn=', '').replace(',ou=Groups,dc=freestandards,dc=org', '')) + return group_list + + def list_project_inherits_from(self, gerrit_project): + """List who a project inherits from.""" + gerrit_project = urllib.parse.quote(gerrit_project, safe='', encoding=None, errors=None) + result = self.get('projects/{}/access'.format(gerrit_project))[1] + inherits = (result['inherits_from']['id']) + return inherits diff --git a/lftools/cli/__init__.py b/lftools/cli/__init__.py index 2b732ef2..534d229d 100644 --- a/lftools/cli/__init__.py +++ b/lftools/cli/__init__.py @@ -11,11 +11,11 @@ __author__ = 'Thanh Ha' +import configparser import getpass import logging import click -from six.moves import configparser from six.moves import input from lftools import config as conf diff --git a/lftools/cli/gerrit.py b/lftools/cli/gerrit.py index 9db2d089..8cf786ec 100644 --- a/lftools/cli/gerrit.py +++ b/lftools/cli/gerrit.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2018 The Linux Foundation and others. @@ -11,11 +12,15 @@ from __future__ import print_function -import subprocess -import sys +import logging +from pprint import pformat import click +from lftools.api.endpoints import gerrit + +log = logging.getLogger(__name__) + @click.group() @click.pass_context @@ -24,39 +29,162 @@ def gerrit_cli(ctx): pass -@click.command(name='create') -@click.argument('gerrit_url') +@click.command(name='addfile') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.argument('filename') +@click.option('--issue_id', type=str, required=False, + help='For projects that enforce an issue id for changesets') +@click.option('--file_location', type=str, required=False, + help='option allos you to specify full path and file name') +@click.pass_context +def addfile(ctx, gerrit_fqdn, gerrit_project, filename, issue_id, file_location): + """Add an file for review to a Project. + + Requires gerrit directory. + + Example: + + gerrit_url gerrit.o-ran-sc.org/r + gerrit_project test/test1 + """ + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.add_file(gerrit_fqdn, gerrit_project, filename, issue_id, file_location) + log.info(pformat(data)) + + +@click.command(name='addinfojob') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.argument('jjbrepo') +@click.option('--reviewid', type=str, required=False, + help='ammend a review rather than making a new one') +@click.option('--issue_id', type=str, required=False, + help='For projects that enforce an issue id for changesets') +@click.pass_context +def addinfojob(ctx, gerrit_fqdn, gerrit_project, jjbrepo, reviewid, issue_id): + """Add an INFO job for a new Project. + + Adds info verify jenkins job for project. + result['id'] can be used to ammend a review + so that multiple projects can have info jobs added + in a single review + + Example: + + gerrit_url gerrit.o-ran-sc.org/r + gerrit_project test/test1 + jjbrepo ci-mangement + """ + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.add_info_job(gerrit_fqdn, gerrit_project, jjbrepo, reviewid, issue_id) + log.info(pformat(data)) + + +@click.command(name='addgitreview') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.option('--issue_id', type=str, required=False, + help='For projects that enforce an issue id for changesets') +@click.pass_context +def addgitreview(ctx, gerrit_fqdn, gerrit_project, issue_id): + """Add git review to a project. + + Example: + gerrit_url gerrit.o-ran-sc.org + gerrit_project test/test1 + """ + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.add_git_review(gerrit_fqdn, gerrit_project, issue_id) + log.info(pformat(data)) + + +@click.command(name='addgithubrights') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.pass_context +def addgithubrights(ctx, gerrit_fqdn, gerrit_project): + """Grant Github read for a project. + + gerrit_url gerrit.o-ran-sc.org + gerrit_project test/test1 + """ + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.add_github_rights(gerrit_fqdn, gerrit_project) + log.info(pformat(data)) + + +@click.command(name='abandonchanges') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.pass_context +def abandonchanges(ctx, gerrit_fqdn, gerrit_project): + """Abandon all OPEN changes for a gerrit project. + + gerrit_url gerrit.o-ran-sc.org + gerrit_project test/test1 + """ + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.abandon_changes(gerrit_fqdn, gerrit_project) + log.info(pformat(data)) + +# Creates a gerrit project if project does not exist and adds ldap group as owner. +# Limits: does not support inherited permissions from other than All-Projects. +@click.command(name='createproject') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') @click.argument('ldap_group') -@click.argument('repo') -@click.argument('user') -@click.option('--enable', is_flag=True, - help='Enable replication to Github.') -@click.option('--parent', type=str, required=False, - help='Specify parent other than "All-Projects".') +@click.option('--description', type=str, required=True, + help='Project Description') +@click.option('--check', is_flag=True, + help='just check if the project exists') @click.pass_context -def create( - ctx, gerrit_url, ldap_group, repo, user, enable, parent): - """Create and configure permissions for a new gerrit repo. +def createproject(ctx, gerrit_fqdn, gerrit_project, ldap_group, description, check): + """Create a project via the gerrit API. - GERRIT_URL: server fqdn ex: gerrit.localhost + Creates a gerrit project. + Sets ldap group as owner. - LDAP_GROUP: owner ex: project-gerrit-group-committers + Example: - REPO: repo name ex: testrepo + gerrit_url gerrit.o-ran-sc.org/r + gerrit_project test/test1 + ldap_group oran-gerrit-test-test1-committers - USER: user that has permissions in gerrit """ - params = ['gerrit_create'] - params.extend(["-s", gerrit_url]) - params.extend(["-o", ldap_group]) - params.extend(["-r", repo]) - params.extend(["-u", user]) - if parent: - params.extend(["-p", parent]) - if enable: - params.extend(["-e"]) - status = subprocess.call(params) - sys.exit(status) - - -gerrit_cli.add_command(create) + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.create_project(gerrit_fqdn, gerrit_project, ldap_group, description, check) + log.info(pformat(data)) + + +@click.command(name='list-project-permissions') +@click.argument('gerrit_fqdn') +@click.argument('project') +@click.pass_context +def list_project_permissions(ctx, gerrit_fqdn, project): + """List Owners of a Project.""" + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.list_project_permissions(project) + for ldap_group in data: + log.info(pformat(ldap_group)) + + +@click.command(name='list-project-inherits-from') +@click.argument('gerrit_fqdn') +@click.argument('gerrit_project') +@click.pass_context +def list_project_inherits_from(ctx, gerrit_fqdn, gerrit_project): + """List who a project inherits from.""" + g = gerrit.Gerrit(fqdn=gerrit_fqdn) + data = g.list_project_inherits_from(gerrit_project) + log.info(data) + + +gerrit_cli.add_command(addinfojob) +gerrit_cli.add_command(addfile) +gerrit_cli.add_command(addgitreview) +gerrit_cli.add_command(addgithubrights) +gerrit_cli.add_command(createproject) +gerrit_cli.add_command(abandonchanges) +gerrit_cli.add_command(list_project_permissions) +gerrit_cli.add_command(list_project_inherits_from) diff --git a/lftools/cli/github_cli.py b/lftools/cli/github_cli.py index 357f24ac..70a00874 100644 --- a/lftools/cli/github_cli.py +++ b/lftools/cli/github_cli.py @@ -184,6 +184,12 @@ def updaterepo(ctx, organization, repository, has_issues, has_projects, has_wiki if repo.name == repository: repo_actual = (repo) + try: + repo_actual + except NameError: + print("repo not found") + exit(1) + for team in teams(): if team.name == add_team: print(team.id) diff --git a/lftools/cli/infofile.py b/lftools/cli/infofile.py index 64601fa3..b0a719ae 100644 --- a/lftools/cli/infofile.py +++ b/lftools/cli/infofile.py @@ -9,6 +9,7 @@ ############################################################################## """Script to insert missing values from ldap into a projects INFO.yaml.""" +import datetime import inspect import logging import re @@ -39,8 +40,12 @@ def infofile(ctx): @click.argument('gerrit_project', required=True) @click.option('--directory', type=str, required=False, default="r", help='custom gerrit directory, eg not /r/') +@click.option('--empty', is_flag=True, required=False, + help='Create info file for uncreated project.') +@click.option('--tsc_approval', type=str, required=False, default="missing", + help='optionally provde a tsc approval link') @click.pass_context -def create_info_file(ctx, gerrit_url, gerrit_project, directory): +def create_info_file(ctx, gerrit_url, gerrit_project, directory, empty, tsc_approval): """Create an initial INFO file. gerrit_project example: project/full-name @@ -55,30 +60,44 @@ def create_info_file(ctx, gerrit_url, gerrit_project, directory): headers = {'Content-Type': 'application/json; charset=UTF-8'} projectid_encoded = gerrit_project.replace("/", "%2F") access_str = 'projects/{}/access'.format(projectid_encoded) - result = rest.get(access_str, headers=headers) - project_dashed = gerrit_project.replace("/", "_") - project_dashed = project_dashed.replace("-", "_") + # project name with only underscores for info file anchors. + # project name with only dashes for ldap groups. + project_underscored = gerrit_project.replace("/", "_") + project_underscored = project_underscored.replace("-", "_") + project_dashed = project_underscored.replace("_", "-") + umbrella = gerrit_url.split(".")[1] match = re.search(r"(?<=\.).*", gerrit_url) umbrella_tld = match.group(0) - if 'inherits_from' in result: - inherits = (result['inherits_from']['id']) - if inherits != "All-Projects": - print(" Inherits from:", inherits) - print("Better Check this unconventional inherit") - try: - owner = (result['local']['refs/*']['permissions']['owner']['rules']) - except: - print("ERROR: Check project config, no owner set!") + if not empty: + result = rest.get(access_str, headers=headers) + + if 'inherits_from' in result: + inherits = (result['inherits_from']['id']) + if inherits != "All-Projects": + print(" Inherits from:", inherits) + print("Better Check this unconventional inherit") + + try: + owner = (result['local']['refs/*']['permissions']['owner']['rules']) + except: + print("ERROR: Check project config, no owner set!") + + for x in owner: + match = re.search(r"[^=]+(?=,)", x) + ldap_group = (match.group(0)) + + if umbrella == 'o-ran-sc': + umbrella = "oran" + + date = (datetime.datetime.now().strftime("%Y-%m-%d")) - for x in owner: - match = re.search(r"[^=]+(?=,)", x) - ldap_group = (match.group(0)) + ldap_group = "{}-gerrit-{}-committers".format(umbrella, project_dashed) long_string = """--- project: '{0}' -project_creation_date: '' +project_creation_date: '{3}' project_category: '' lifecycle_state: 'Incubation' project_lead: &{1}_{0}_ptl @@ -107,23 +126,33 @@ meetings: server: 'freenode.net' channel: '#{1}' repeats: '' - time: ''""".format(project_dashed, umbrella, umbrella_tld) + time: ''""".format(project_underscored, umbrella, umbrella_tld, date) tsc_string = """ tsc: - approval: '' + # yamllint disable rule:line-length + approval: '{}' changes: - type: '' name: '' link: '' +""".format(tsc_approval, end='') + empty_committer = """ - name: '' + email: '' + company: '' + id: '' """ tsc_string = inspect.cleandoc(tsc_string) print(long_string) print("repositories:") print(" - {}".format(gerrit_project)) print("committers:") - print(" - <<: *{1}_{0}_ptl".format(project_dashed, umbrella)) - helper_yaml4info(ldap_group) + print(" - <<: *{1}_{0}_ptl".format(project_underscored, umbrella, end='')) + if not empty: + this = helper_yaml4info(ldap_group) + print(this, end='') + else: + print(empty_committer, end='') print(tsc_string) diff --git a/lftools/config.py b/lftools/config.py index c30016c6..ff39e97e 100644 --- a/lftools/config.py +++ b/lftools/config.py @@ -12,10 +12,10 @@ __author__ = 'Thanh Ha' +import configparser import logging import os.path -from six.moves import configparser from xdg import XDG_CONFIG_HOME log = logging.getLogger(__name__) @@ -25,11 +25,18 @@ LFTOOLS_CONFIG_FILE = os.path.join(XDG_CONFIG_HOME, 'lftools', 'lftools.ini') def get_config(): """Get the config object.""" - config = configparser.ConfigParser() + #config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # noqa + config = configparser.ConfigParser() # noqa config.read(LFTOOLS_CONFIG_FILE) return config +def has_section(section): + """Get a configuration from a section.""" + config = get_config() + return config.has_section(section) + + def get_setting(section, option=None): """Get a configuration from a section.""" config = get_config() diff --git a/lftools/github_helper.py b/lftools/github_helper.py index 4b73775b..256ff233 100755 --- a/lftools/github_helper.py +++ b/lftools/github_helper.py @@ -24,7 +24,14 @@ log = logging.getLogger(__name__) def helper_list(ctx, organization, repos, audit, full, teams, team, repofeatures): """List options for github org repos.""" - token = config.get_setting("github", "token") + # Optionally pick token based on gitub org + + if config.has_section("github"): + token = config.get_setting("github", "token") + else: + section = "github.{}".format(organization) + token = config.get_setting(section, "token") + g = Github(token) orgName = organization @@ -33,8 +40,9 @@ def helper_list(ctx, organization, repos, audit, full, teams, team, repofeatures except GithubException as ghe: log.error(ghe) + # Extend this to check if a repo exists if repos: - log.info("All repos for organization: ", orgName) + print("All repos for organization: ", orgName) repos = org.get_repos() for repo in repos: log.info(repo.name) diff --git a/lftools/ldap_cli.py b/lftools/ldap_cli.py index 03ce4889..d6285b55 100644 --- a/lftools/ldap_cli.py +++ b/lftools/ldap_cli.py @@ -11,10 +11,12 @@ from __future__ import print_function -import subprocess +from subprocess import check_output +from subprocess import STDOUT def helper_yaml4info(group): """Build yaml of committers for your INFO.yaml.""" - status = subprocess.call(['yaml4info', group]) + command = ["yaml4info", group] + status = check_output(command, stderr=STDOUT).decode() return status diff --git a/lftools/lfidapi.py b/lftools/lfidapi.py index 755496fe..44556b60 100755 --- a/lftools/lfidapi.py +++ b/lftools/lfidapi.py @@ -11,6 +11,7 @@ import json import logging +import sys from email_validator import validate_email import requests @@ -46,19 +47,24 @@ def helper_check_group_exists(group): def helper_search_members(group): """List members of a group.""" - access_token, url = oauth_helper() - url = PARSE(url, group) - headers = {'Authorization': 'Bearer ' + access_token} - response = requests.get(url, headers=headers) - try: - check_response_code(response) - except requests.HTTPError as e: - log.error(e) - exit(1) - result = (response.json()) - members = result["members"] - log.debug(json.dumps(members, indent=4, sort_keys=True)) - return members + response_code = helper_check_group_exists(group) + if response_code != 200: + log.error("Code: {} Group {} does not exists exiting...".format(response_code, group)) + sys.exit(1) + else: + access_token, url = oauth_helper() + url = PARSE(url, group) + headers = {'Authorization': 'Bearer ' + access_token} + response = requests.get(url, headers=headers) + try: + check_response_code(response) + except requests.HTTPError as e: + log.error(e) + exit(1) + result = (response.json()) + members = result["members"] + log.debug(json.dumps(members, indent=4, sort_keys=True)) + return members def helper_user(user, group, delete): diff --git a/releasenotes/notes/self-service-project-creation-28cc70ec9ea9ec3e.yaml b/releasenotes/notes/self-service-project-creation-28cc70ec9ea9ec3e.yaml new file mode 100644 index 00000000..7224801d --- /dev/null +++ b/releasenotes/notes/self-service-project-creation-28cc70ec9ea9ec3e.yaml @@ -0,0 +1,34 @@ +--- +prelude: > + Changes to lftools needed for project creation to happen + via command line logic. +features: + - | + lftools gerrit [OPTIONS] COMMAND [ARGS] + abandonchanges Abandon all OPEN changes for a gerrit project. + addfile Add an file for review to a Project. + addgithubrights Grant Github read for a project. + addgitreview Add git review to a project. + addinfojob Add an INFO job for a new Project. + createproject Create a project via the gerrit API. + list-project-inherits-from List who a project inherits from. + list-project-permissions List Owners of a Project. +issues: + - | + Addinfofile trips up on extended characters in usernames. + Project lead must be added by hand to lftools infofile create. +upgrade: + - | + lftools.ini needs configuration on internal jenkins for auth. + Documenting and implementing this is an internal endevor and beyond + the scope of these release notes. +fixes: + - | + Use proper python3 config parser. + Add has_section check for configparser + lftools github update repo will properly return "repo not found" + lftools infofile create will now take tsc approval string and set date. + lftools infofile will allow INFO.yaml to be created before ldap group. + yaml4info now correctly outputs to STDOUT so that its output can be properly + captured and printed by python. + lfidapi now correctly exits if a group does not exist. -- 2.16.6