From: Aric Gardner Date: Thu, 24 Oct 2019 19:59:43 +0000 (-0400) Subject: lftools info file generator X-Git-Tag: v0.28.0~4 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=92f89d50c4c2d3e5155b2b4de10fb63f0045db36;p=releng%2Flftools.git lftools info file generator Creates a preliminary info file to streamline workflow. Missing fields are to be corrected in review. Includes needed fixes for lftools ldap for python3 Signed-off-by: Aric Gardner Change-Id: I3af2d768c356c8cd6950c527b67623e15032080e --- diff --git a/docs/commands/infofile.rst b/docs/commands/infofile.rst index 227254d4..8664bfbb 100644 --- a/docs/commands/infofile.rst +++ b/docs/commands/infofile.rst @@ -23,6 +23,25 @@ sync-committers .. program-output:: lftools infofile sync-committers --help +Creating an info file requires a connection to the VPN +and a working openldap configuration + + +.. code-block:: bash + + $ cat /etc/openldap/ldap.conf + TLS_REQCERT never + or + prereqs: For ldap lookups to work you must be on the VPN and have the cert to get the cert: log in to any collab system and grab /etc/ipa/ca.crt in /etc/openldap/ldap.conf, add 'TLS_CACERT /path/to/ipa.ca' + + +create-info-file +---------------- + + .. program-output:: lftools infofile create-info-file --help + + + API for check votes requires a [github] section in ~/.config/lftools/lftools.ini: .. code-block:: bash diff --git a/lftools/api/endpoints/readthedocs.py b/lftools/api/endpoints/readthedocs.py index bc501fc3..55ccaa3d 100644 --- a/lftools/api/endpoints/readthedocs.py +++ b/lftools/api/endpoints/readthedocs.py @@ -269,7 +269,6 @@ class ReadTheDocs(client.RestApi): :param alias: An alias (not required). (user-defined slug) :return: """ - data = { 'child': subproject, 'alias': alias @@ -280,7 +279,7 @@ class ReadTheDocs(client.RestApi): return result def subproject_delete(self, project, subproject): - """Deletes the project/sub relationship. + """Delete project/sub relationship. :param project: :param subproject: diff --git a/lftools/cli/infofile.py b/lftools/cli/infofile.py index 5d4dc918..64601fa3 100644 --- a/lftools/cli/infofile.py +++ b/lftools/cli/infofile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2019 The Linux Foundation and others. @@ -10,15 +9,20 @@ ############################################################################## """Script to insert missing values from ldap into a projects INFO.yaml.""" +import inspect import logging +import re import sys import click from pygerrit2 import GerritRestAPI +from pygerrit2 import HTTPBasicAuth import ruamel.yaml import yaml +from lftools import config from lftools.github_helper import prvotes +from lftools.ldap_cli import helper_yaml4info log = logging.getLogger(__name__) @@ -30,6 +34,99 @@ def infofile(ctx): pass +@click.command(name='create-info-file') +@click.argument('gerrit_url', required=True) +@click.argument('gerrit_project', required=True) +@click.option('--directory', type=str, required=False, default="r", + help='custom gerrit directory, eg not /r/') +@click.pass_context +def create_info_file(ctx, gerrit_url, gerrit_project, directory): + """Create an initial INFO file. + + gerrit_project example: project/full-name + gerrit_url example: gerrit.umbrella.com + directory example: /gerrit/ (rather than most projects /r/) + """ + user = config.get_setting("gerrit", "username") + pass1 = config.get_setting("gerrit", "password") + auth = HTTPBasicAuth(user, pass1) + url = ("https://{}/{}".format(gerrit_url, directory)) + rest = GerritRestAPI(url=url, auth=auth) + headers = {'Content-Type': 'application/json; charset=UTF-8'} + projectid_encoded = gerrit_project.replace("/", "%2F") + access_str = 'projects/{}/access'.format(projectid_encoded) + result = rest.get(access_str, headers=headers) + project_dashed = gerrit_project.replace("/", "_") + project_dashed = project_dashed.replace("-", "_") + umbrella = gerrit_url.split(".")[1] + match = re.search(r"(?<=\.).*", gerrit_url) + umbrella_tld = match.group(0) + if 'inherits_from' in result: + inherits = (result['inherits_from']['id']) + if inherits != "All-Projects": + print(" Inherits from:", inherits) + print("Better Check this unconventional inherit") + + try: + owner = (result['local']['refs/*']['permissions']['owner']['rules']) + except: + print("ERROR: Check project config, no owner set!") + + for x in owner: + match = re.search(r"[^=]+(?=,)", x) + ldap_group = (match.group(0)) + + long_string = """--- +project: '{0}' +project_creation_date: '' +project_category: '' +lifecycle_state: 'Incubation' +project_lead: &{1}_{0}_ptl + name: '' + email: '' + id: '' + company: '' + timezone: '' +primary_contact: *{1}_{0}_ptl +issue_tracking: + type: 'jira' + url: 'https://jira.{2}/projects/' + key: '{0}' +mailing_list: + type: 'groups.io' + url: 'technical-discuss@lists.{2}' + tag: '[]' +realtime_discussion: + type: 'irc' + server: 'freenode.net' + channel: '#{1}' +meetings: + - type: 'gotomeeting+irc' + agenda: 'https://wiki.{2}/display/' + url: '' + server: 'freenode.net' + channel: '#{1}' + repeats: '' + time: ''""".format(project_dashed, umbrella, umbrella_tld) + + tsc_string = """ +tsc: + approval: '' + changes: + - type: '' + name: '' + link: '' +""" + tsc_string = inspect.cleandoc(tsc_string) + print(long_string) + print("repositories:") + print(" - {}".format(gerrit_project)) + print("committers:") + print(" - <<: *{1}_{0}_ptl".format(project_dashed, umbrella)) + helper_yaml4info(ldap_group) + print(tsc_string) + + @click.command(name='get-committers') @click.argument('file', envvar='FILE_NAME', required=True) @click.option('--full', type=bool, required=False, @@ -43,6 +140,7 @@ def get_committers(ctx, file, full, id): project = yaml.safe_load(yaml_file) def print_committer_info(committer, full): + """Print committers.""" if full: print(" - name: {}".format(committer['name'])) print(" email: {}".format(committer['email'])) @@ -51,7 +149,6 @@ def get_committers(ctx, file, full, id): def list_committers(full, id, project): """List commiters from the INFO.yaml file.""" lookup = project.get('committers', []) - for item in lookup: if id: if item['id'] == id: @@ -60,7 +157,6 @@ def get_committers(ctx, file, full, id): else: continue print_committer_info(item, full) - list_committers(full, id, project) @@ -226,3 +322,4 @@ def check_votes(ctx, info_file, endpoint, change_number, tsc, github_repo): infofile.add_command(get_committers) infofile.add_command(sync_committers) infofile.add_command(check_votes) +infofile.add_command(create_info_file) diff --git a/lftools/cli/ldap_cli.py b/lftools/cli/ldap_cli.py index 8e271780..5fb74a53 100644 --- a/lftools/cli/ldap_cli.py +++ b/lftools/cli/ldap_cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2018 The Linux Foundation and others. @@ -76,7 +75,7 @@ def autocorrectinfofile(ctx, gerrit_clone_base, ldap_group, repo, purpose, revie @click.command() -@click.option('--ldap-server', default='ldaps://aws-us-west-2-lfit-openldap-2.linux-foundation.org', +@click.option('--ldap-server', default='ldaps://pdx-wl-lb-lfldap.web.codeaurora.org', envvar='LDAP_SERVER', type=str, required=True) @click.option('--ldap-user-base', default='ou=Users,dc=freestandards,dc=org', envvar='LDAP_USER_BASE_DN', type=str, required=True) @@ -146,8 +145,8 @@ def csv(ctx, ldap_server, ldap_group_base, ldap_user_base, groups): def user_to_csv(user): """Covert LDIF user info to CSV of uid,mail,cn.""" - attrs = user[0][0][1] - return ",".join([attrs['uid'][0], attrs['cn'][0], attrs['mail'][0]]) + attrs = (user[0][0][1]) + return ",".join([attrs['uid'][0].decode('utf-8'), attrs['cn'][0].decode('utf-8'), attrs['mail'][0].decode('utf-8')]) def main(groups): """Preform an LDAP query.""" @@ -160,6 +159,7 @@ def csv(ctx, ldap_server, ldap_group_base, ldap_user_base, groups): for group_bar in group_dict: group_name = group_bar['name'][3:-cut_length] for user in group_bar['members']: + user = (user.decode('utf-8')) user_info = ldap_query(l, ldap_user_base, user, ["uid", "cn", "mail"]) try: print("%s,%s" % (group_name, user_to_csv(user_info))) diff --git a/lftools/cli/lfidapi.py b/lftools/cli/lfidapi.py index 1f54cf59..c8109321 100755 --- a/lftools/cli/lfidapi.py +++ b/lftools/cli/lfidapi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2018 The Linux Foundation and others. diff --git a/lftools/cli/rtd.py b/lftools/cli/rtd.py index cd9579e8..c5d595ed 100644 --- a/lftools/cli/rtd.py +++ b/lftools/cli/rtd.py @@ -154,7 +154,7 @@ def subproject_details(ctx, project_slug, subproject_slug): @click.argument('subproject-slug') @click.pass_context def subproject_create(ctx, project_slug, subproject_slug): - """Creates a project-subproject relationship.""" + """Create a project-subproject relationship.""" r = readthedocs.ReadTheDocs() data = r.subproject_create(project_slug, subproject_slug) log.info(pformat(data)) @@ -165,7 +165,7 @@ def subproject_create(ctx, project_slug, subproject_slug): @click.argument('subproject-slug') @click.pass_context def subproject_delete(ctx, project_slug, subproject_slug): - """Deletes a project-subproject relationship.""" + """Delete a project-subproject relationship.""" r = readthedocs.ReadTheDocs() data = r.subproject_delete(project_slug, subproject_slug) if data: diff --git a/lftools/deploy.py b/lftools/deploy.py index fc3d349f..9e2fa2d8 100755 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -10,27 +10,25 @@ ############################################################################## """Library of functions for deploying artifacts to Nexus.""" -from datetime import timedelta -from defusedxml.minidom import parseString -from multiprocessing import cpu_count import concurrent.futures import errno -import glob2 # Switch to glob when Python < 3.5 support is dropped import gzip import io import logging import math +from multiprocessing import cpu_count import os import re -import requests import shutil -import six import subprocess import sys import tempfile -import time import zipfile +from defusedxml.minidom import parseString +import glob2 # Switch to glob when Python < 3.5 support is dropped +import requests +import six log = logging.getLogger(__name__) @@ -272,7 +270,7 @@ def copy_archives(workspace, pattern=None): try: shutil.move(src, dest) except IOError as e: # Switch to FileNotFoundError when Python 2 support is dropped. - log.debug("Missing path, will create it {}".format(os.path.dirname(dest))) + log.debug("Missing path, will create it {}.\n{}".format(os.path.dirname(dest), e)) os.makedirs(os.path.dirname(dest)) shutil.move(src, dest) else: @@ -592,9 +590,6 @@ def upload_maven_file_to_nexus(nexus_url, nexus_repo_id, raise requests.HTTPError("Nexus Error: {}".format(error_msg)) - - - def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False): """Deploy a local directory of files to a Nexus repository. @@ -671,6 +666,9 @@ def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False): filename = futures[future] try: data = future.result() + # remove pyflake warning + if data == data: + pass except Exception as e: log.error('Uploading {}: {}'.format(filename, e)) @@ -698,7 +696,7 @@ def deploy_nexus_stage(nexus_url, staging_profile_id, deploy_dir): staging repo. deploy_dir: The directory to deploy. (Ex: /tmp/m2repo) - # Sample: + # Sample: lftools deploy nexus-stage http://192.168.1.26:8081/nexus 4e6f95cd2344 /tmp/slask Deploying Maven artifacts to staging repo... Staging repository aaf-1005 created. diff --git a/lftools/ldap_cli.py b/lftools/ldap_cli.py new file mode 100644 index 00000000..03ce4889 --- /dev/null +++ b/lftools/ldap_cli.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Helper for INFO file generation.""" + +from __future__ import print_function + +import subprocess + + +def helper_yaml4info(group): + """Build yaml of committers for your INFO.yaml.""" + status = subprocess.call(['yaml4info', group]) + return status diff --git a/lftools/lfidapi.py b/lftools/lfidapi.py index 2bdeccc0..af4ae5a6 100755 --- a/lftools/lfidapi.py +++ b/lftools/lfidapi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2018 The Linux Foundation and others. diff --git a/releasenotes/notes/infofile-063db0be4acfe858.yaml b/releasenotes/notes/infofile-063db0be4acfe858.yaml new file mode 100644 index 00000000..f66c9c2c --- /dev/null +++ b/releasenotes/notes/infofile-063db0be4acfe858.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + New command lftools infofile create-info-file + Creates an initial info file for a project. + Must be on the VPN to use. diff --git a/shell/yaml4info b/shell/yaml4info index c282e665..6974f9dc 100755 --- a/shell/yaml4info +++ b/shell/yaml4info @@ -11,7 +11,6 @@ main() { - echo "committers:" while read -r line; do email="$(echo "$line" | awk -F"," '{print $NF}')" @@ -19,6 +18,10 @@ main() { fullname="$(echo "$line" | awk -F"," '{print $3}')" lfid="$(echo "$line" | awk -F"," '{print $2}')" + if [[ "$lfid" == "lfservices_releng" ]]; then + continue + else + cat << EOF - name: '$fullname' email: '$email' @@ -26,6 +29,7 @@ cat << EOF id: '$lfid' timezone: 'Unknown/Unknown' EOF + fi done < <(lftools ldap csv "$1") diff --git a/tox.ini b/tox.ini index c1c00b1e..d86a5ca8 100644 --- a/tox.ini +++ b/tox.ini @@ -21,13 +21,11 @@ basepython = python3 deps = coala coala-bears - detox nodeenv numpy commands = nodeenv -p npm install --global remark-cli remark-lint write-good - python3 -m nltk.downloader punkt maxent_treebank_pos_tagger averaged_perceptron_tagger coala --non-interactive [testenv:docs]