From: Aric Gardner Date: Tue, 26 Jun 2018 19:16:08 +0000 (-0400) Subject: Extend lftools with scripts for ldap lookups X-Git-Tag: v0.16.0~1^2 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F58%2F11558%2F46;p=releng%2Flftools.git Extend lftools with scripts for ldap lookups Add 2 new commands to lftools: - infofile - ldap prereqs: For ldap lookups to work you must be on the VPN and have the cert to get the cert: log in to any collab system and grab /etc/ipa/ca.crt in /etc/openldap/ldap.conf TLS_REQCERT always TLS_CACERTDIR /etc/openldap/certs TLS_CACERT /etc/openldap/certs/ca.crt To test: 1. Clone this patchset 2. start venv pip uninstall lftools && pip install -e . Usage: $ lftools ldap Usage: lftools ldap [OPTIONS] COMMAND [ARGS]... LDAP TOOLS. Commands: autocorrectinfofile Verify INFO.yaml against LDAP group. csv Query an Ldap server. inactivecommitters Check committer participation. yaml4info Build yaml of commiters for your INFO.yaml. $ lftools infofile Usage: lftools infofile [OPTIONS] COMMAND [ARGS]... INFO.yaml TOOLS. Commands: get-committers Extract Committer info from INFO.yaml or LDAP... sync-committers Script to insert missing values from ldap... Issue: RELENG-407 Change-Id: I4f4055441042d790008754bb085447f52e1c1a78 Signed-off-by: Aric Gardner --- diff --git a/lftools/cli/__init__.py b/lftools/cli/__init__.py index 25fc94e2..723f2a1b 100644 --- a/lftools/cli/__init__.py +++ b/lftools/cli/__init__.py @@ -16,7 +16,9 @@ import click from lftools.cli.config import config_sys from lftools.cli.dco import dco from lftools.cli.deploy import deploy +from lftools.cli.infofile import infofile from lftools.cli.jenkins import jenkins_cli +from lftools.cli.ldap_cli import ldap_cli from lftools.cli.license import license from lftools.cli.nexus import nexus from lftools.cli.sign import sign @@ -32,6 +34,7 @@ def cli(ctx): cli.add_command(config_sys) +cli.add_command(infofile) cli.add_command(deploy) cli.add_command(dco) cli.add_command(jenkins_cli, name='jenkins') @@ -39,6 +42,7 @@ cli.add_command(license) cli.add_command(nexus) cli.add_command(sign) cli.add_command(version) +cli.add_command(ldap_cli, name='ldap') try: from lftools.openstack.cmd import openstack diff --git a/lftools/cli/infofile.py b/lftools/cli/infofile.py new file mode 100644 index 00000000..7d5be7e1 --- /dev/null +++ b/lftools/cli/infofile.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Script to insert missing values from ldap into a projects INFO.yaml.""" + +import click +import ruamel.yaml +import yaml + + +@click.group() +@click.pass_context +def infofile(ctx): + """INFO.yaml TOOLS.""" + pass + + +@click.command(name='get-committers') +@click.argument('file', envvar='FILE_NAME', required=True) +@click.option('--full', type=bool, required=False, + help='Output name email and id for all committers in an infofile') +@click.option('--id', type=str, required=False, + help='Full output for a specific LFID') +@click.pass_context +def get_committers(ctx, file, full, id): + """Extract Committer info from INFO.yaml or LDAP dump.""" + with open(file, 'r') as yaml_file: + project = yaml.safe_load(yaml_file) + + def print_committer_info(committer, full): + if full: + print(" - name: {}".format(committer['name'])) + print(" email: {}".format(committer['email'])) + print(" id: {}".format(committer['id'])) + + def list_committers(full, id, project): + """List commiters from the INFO.yaml file.""" + lookup = project.get('committers', []) + + for item in lookup: + if id: + if item['id'] == id: + print_committer_info(item, full) + break + else: + continue + print_committer_info(item, full) + + list_committers(full, id, project) + + +@click.command(name='sync-committers') +@click.argument('info_file') +@click.argument('ldap_file') +@click.argument('id') +@click.option('--repo', type=str, required=False, + help='repo name') +@click.pass_context +def sync_committers(ctx, id, info_file, ldap_file, repo): + """Sync committer information from LDAP into INFO.yaml.""" + ryaml = ruamel.yaml.YAML() + ryaml.preserve_quotes = True + ryaml.indent(mapping=4, sequence=6, offset=4) + ryaml.explicit_start = True + with open(info_file, 'r') as stream: + try: + yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + with open(info_file) as f: + info_data = ryaml.load(f) + with open(ldap_file) as f: + ldap_data = ryaml.load(f) + + def readfile(data, ldap_data, id): + committer_info = info_data['committers'] + repo_info = info_data['repositories'] + committer_info_ldap = ldap_data['committers'] + readldap(id, ldap_file, committer_info, committer_info_ldap, repo, repo_info) + + def readldap(id, ldap_file, committer_info, committer_info_ldap, repo, repo_info): + for idx, val in enumerate(committer_info): + committer = info_data['committers'][idx]['id'] + if committer == id: + print('{} is alread in {}'.format(id, info_file)) + exit() + + for idx, val in enumerate(committer_info_ldap): + committer = ldap_data['committers'][idx]['id'] + if committer == id: + name = (ldap_data['committers'][idx]['name']) + email = (ldap_data['committers'][idx]['email']) + formatid = (ldap_data['committers'][idx]['id']) + company = (ldap_data['committers'][idx]['company']) + timezone = (ldap_data['committers'][idx]['timezone']) + try: + name + except NameError: + print('{} does not exist in {}'.format(id, ldap_file)) + exit() + + user = ruamel.yaml.comments.CommentedMap( + ( + ('name', name), ('company', company), ('email', email), ('id', formatid), ('timezone', timezone) + ) + ) + + info_data['repositories'][0] = repo + committer_info.append(user) + + with open(info_file, 'w') as f: + ryaml.dump(info_data, f) + + readfile(info_data, ldap_data, id) + + +infofile.add_command(get_committers) +infofile.add_command(sync_committers) diff --git a/lftools/cli/ldap_cli.py b/lftools/cli/ldap_cli.py new file mode 100644 index 00000000..b4bdf770 --- /dev/null +++ b/lftools/cli/ldap_cli.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Generate a CSV of a Projects Commiters. + +Prereqs: +- yum install python-devel openldap-devel +- pip install python-ldap +""" + +from __future__ import print_function + +import subprocess +import sys + +import click +import ldap + + +@click.group() +@click.pass_context +def ldap_cli(ctx): + """LDAP TOOLS.""" + pass + + +@click.command() +@click.argument('group') +@click.pass_context +def yaml4info(ctx, group): + """Build yaml of committers for your INFO.yaml.""" + status = subprocess.call(['yaml4info', group]) + sys.exit(status) + + +@click.command() +@click.argument('gerrit_url') +@click.argument('group') +@click.pass_context +def inactivecommitters(ctx, gerrit_url, group): + """Check committer participation.""" + status = subprocess.call(['inactivecommitters', gerrit_url, group]) + sys.exit(status) + + +@click.command() +@click.argument('gerrit_clone_base') +@click.argument('ldap_group') +@click.argument('repo') +@click.option('--purpose', envvar='purpose', type=str, + help='Must be one of READY_FOR_INFO LINT IN-REVIEW') +@click.option('--review', type=str, required=False, + help='review number in gerrit, required if purpose is IN-REVIEW') +@click.pass_context +def autocorrectinfofile(ctx, gerrit_clone_base, ldap_group, repo, purpose, review): + """Verify INFO.yaml against LDAP group.\n + PURPOSE must be one of: READY_FOR_INFO LINT IN-REVIEW\n + GERRITCLONEBASE must be a url: https://gerrit.opnfv.org/gerrit/\n + """ + params = ['autocorrectinfofile'] + params.extend([gerrit_clone_base, ldap_group, repo]) + if purpose: + params.extend([purpose]) + if review: + params.extend([review]) + status = subprocess.call(params) + sys.exit(status) + + +@click.command() +@click.option('--ldap-server', default='ldaps://pdx-wl-lb-lfldap.web.codeaurora.org', + envvar='LDAP_SERVER', type=str, required=True) +@click.option('--ldap-user-base', default='ou=Users,dc=freestandards,dc=org', + envvar='LDAP_USER_BASE_DN', type=str, required=True) +@click.option('--ldap-group-base', default='ou=Groups,dc=freestandards,dc=org', + envvar='LDAP_GROUP_BASE_DN', type=str, required=True) +@click.argument('groups') +@click.pass_context +def csv(ctx, ldap_server, ldap_group_base, ldap_user_base, groups): + """Query an Ldap server.""" + # groups needs to be a list + groups = groups.split(' ') + + def ldap_connect(ldap_object): + """Start the connection to LDAP.""" + try: + ldap_object.protocol_version = ldap.VERSION3 + ldap_object.simple_bind_s() + except ldap.LDAPError as e: + if type(e.message) == dict and e.message.has_key('desc'): + print(e.message['desc']) + else: + print(e) + sys.exit(0) + + def eprint(*args, **kwargs): + """Print to stderr.""" + print(*args, file=sys.stderr, **kwargs) + + def ldap_disconnect(ldap_object): + """Stop the connnection to LDAP.""" + ldap_object.unbind_s() + + def ldap_query(ldap_object, dn, search_filter, attrs): + """Perform an LDAP query and return the results.""" + try: + ldap_result_id = ldap_object.search(dn, ldap.SCOPE_SUBTREE, search_filter, attrs) + result_set = [] + while 1: + result_type, result_data = ldap_object.result(ldap_result_id, 0) + if (result_data == []): + break + else: + # if you are expecting multiple results you can append them + # otherwise you can just wait until the initial result and break out + if result_type == ldap.RES_SEARCH_ENTRY: + result_set.append(result_data) + return result_set + except ldap.LDAPError as e: + sys.exit(1) + + def package_groups(groups): + """Package a set of groups from LDIF into a Python dictionary. + + containing the groups member uids. + """ + group_list = [] + cut_length = len(ldap_user_base)+1 + for group in groups: + group_d = dict(name=group[0][0]) + members = [] + for group_attrs in group: + for member in group_attrs[1]['member']: + members.append(member[:-cut_length]) + group_d['members'] = members + group_list.append(group_d) + return group_list + + def user_to_csv(user): + """Covert LDIF user info to CSV of uid,mail,cn.""" + attrs = user[0][0][1] + return ",".join([attrs['uid'][0], attrs['cn'][0], attrs['mail'][0]]) + + def main(groups): + """Preform an LDAP query.""" + l = ldap.initialize(ldap_server) + ldap_connect(l) + for arg in groups: + groups = ldap_query(l, ldap_group_base, "cn=%s" % arg, ["member"]) + group_dict = package_groups(groups) + cut_length = len(ldap_group_base)+1 + for group_bar in group_dict: + group_name = group_bar['name'][3:-cut_length] + for user in group_bar['members']: + user_info = ldap_query(l, ldap_user_base, user, ["uid", "cn", "mail"]) + try: + print("%s,%s" % (group_name, user_to_csv(user_info))) + except: + eprint("Error parsing user: %s" % user) + continue + ldap_disconnect(l) + main(groups) + + +ldap_cli.add_command(csv) +ldap_cli.add_command(inactivecommitters) +ldap_cli.add_command(yaml4info) +ldap_cli.add_command(autocorrectinfofile) diff --git a/releasenotes/notes/ldap-info-017df79c3c8f9585.yaml b/releasenotes/notes/ldap-info-017df79c3c8f9585.yaml new file mode 100644 index 00000000..f8b2c9b3 --- /dev/null +++ b/releasenotes/notes/ldap-info-017df79c3c8f9585.yaml @@ -0,0 +1,23 @@ +--- +features: + - | + $ lftools ldap + + Usage: lftools ldap [OPTIONS] COMMAND [ARGS]... + + .. code-block:: none + + Commands: + autocorrectinfofile Verify INFO.yaml against LDAP group. + csv Query an Ldap server. + inactivecommitters Check committer participation. + yaml4info Build yaml of commiters for your INFO.yaml. + + - | + $ lftools infofile + + .. code-block:: none + + Commands: + get-committers Extract Committer info from INFO.yaml or LDAP... + sync-committers Sync committer information from LDAP into... diff --git a/requirements.txt b/requirements.txt index 3df89f7b..bba62593 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ click +python-ldap pyyaml requests~=2.18.0 +ruamel.yaml setuptools>=36.5.0 six~=1.11.0 python-jenkins~=1.1.0 diff --git a/setup.cfg b/setup.cfg index 4c3a91c4..9b70ca04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,8 +34,11 @@ data_files = scripts = shell/dco shell/deploy + shell/inactivecommitters shell/sign shell/version + shell/yaml4info + shell/autocorrectinfofile [entry_points] console_scripts = diff --git a/shell/autocorrectinfofile b/shell/autocorrectinfofile new file mode 100755 index 00000000..32f5e4f8 --- /dev/null +++ b/shell/autocorrectinfofile @@ -0,0 +1,169 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + + +#echo "Diff LDAP against INFO.yaml" + +tmpdir(){ + + DIR="/tmp/infofile/$purpose.$repo.$ldapgroup" + mkdir -p "$DIR" + +} + +parseoptions() { + + if [[ -z $purpose ]]; then + purpose=correct + fi + + echo " gerritclonebase $gerritclonebase" + echo " ldapgroup $ldapgroup" + echo " repo $repo" + echo " purpose $purpose" + + if [[ $purpose =~ "READY_FOR_INFO" ]]; then + + tmpdir "$@" + + if ! [[ -d "$DIR"/"$repo" ]]; then + echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1" + git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1 + fi + if [ ! -f "$DIR"/"$repo"/INFO.yaml ]; then + cp INFO.template.yaml "$DIR"/"$repo"/INFO.yaml || exit 1 + else + echo "INFO file already exists, refusing to overwrite" + exit 1 + fi + + fi + + if [[ $purpose =~ "LINT" ]]; then + + tmpdir "$@" + + if ! [[ -d "$DIR"/"$repo" ]]; then + echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1" + git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1 + fi + if ! yamllint "$DIR"/"$repo"/INFO.yaml; then + echo "ERROR LINT FAILED" + exit 1 + fi + + fi + + #I should only clone the review if their is a discrepancy in commiters + if [[ $purpose =~ "IN-REVIEW" ]]; then + + tmpdir "$@" + + if ! [[ -d "$DIR"/"$repo" ]]; then + echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1" + SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1 + cd "$DIR"/"$repo" || exit + git fetch origin "$review" && git checkout --quiet FETCH_HEAD + cd "$SCRIPTDIR" || exit + yamllint "$DIR"/"$repo"/INFO.yaml + if ! yamllint "$DIR"/"$repo"/INFO.yaml; then + echo " ERROR LINT FAILED CANNOT AUTO CORRECT FILE IN REVIEW" + exit 1 + fi + + fi + fi + + main "$@" +} + +main() { + + if [[ -z $DIR ]]; then + tmpdir "$@" + fi + echo " tmpdir = $DIR" + + if lftools ldap yaml4info "$ldapgroup" 2>&- > "$DIR"/LDAP.yaml + then + echo " LDAP lookup sucssesfull" + else + echo " LDAP lookup failed" + exit 1 + fi + + if ! [[ -d "$DIR"/"$repo" ]]; then + echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1" + git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1 + fi + + diff="$(diff <(lftools infofile get_committers "$DIR"/LDAP.yaml 2>&-| sort) <(lftools infofile get_committers "$DIR"/"$repo"/INFO.yaml 2>&- | sort))" + status="$?" + + diff_array=() + onlyinINFO=() + onlyinLDAP=() + + while IFS= read -r line; do + diff_array+=( "$line" ) + if [[ $(echo "$line" | grep ">") ]]; + then + onlyinINFO+=( "$(echo "$line" | awk -F"'" '{ print $2 }')" ) + fi + if [[ $(echo "$line" | grep "<") ]]; + then + onlyinLDAP+=( "$(echo "$line" | awk -F"'" '{ print $2 }')" ) + fi + done < <(echo "${diff[@]}" ) + + + if ! [ "${#onlyinINFO[@]}" -eq 0 ]; then + for missing in "${onlyinINFO[@]}"; do + if ! [ -z "$missing" ]; then + echo " DUMMY: sending invite to $missing" + lftools infofile get_committers --id "$missing" "$DIR"/"$repo"/INFO.yaml 2>&- + fi + done + fi + + if ! [ "${#onlyinLDAP[@]}" -eq 0 ]; then + echo " These users are listed as commiters in LDAP and not in the INFO.yaml" + for missing in "${onlyinLDAP[@]}"; do + echo " lftools infofile correct $DIR/$repo/INFO.yaml $DIR/LDAP.yaml $missing --repo $repo 2>&-" + lftools infofile correct "$DIR"/"$repo"/INFO.yaml "$DIR"/LDAP.yaml "$missing" --repo "$repo" 2>&- + done + + fi + + echo " Exit status = $status" + exit "$status" + +} + +usage() { +cat << EOF +Must be called from lftools +eg: lftools ldap autocorrectinfofile +EOF +exit 1 +} + +if [[ -z "$*" ]]; then usage +fi + +gerritclonebase="$1" +ldapgroup="$2" +repo="$3" +purpose="$4" +review="$5" + +parseoptions "$@" diff --git a/shell/inactivecommitters b/shell/inactivecommitters new file mode 100755 index 00000000..75e7e96d --- /dev/null +++ b/shell/inactivecommitters @@ -0,0 +1,108 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +datediff() { + d1=$(date -d "$1" +%s) + d2=$(date -d "$2" +%s) + echo $(( (d1 - d2) / 86400 )) days ago +} + +main() { + + echo "checking $gerriturl participation for group $group" + echo + + currentdate=$(date +%Y-%m-%d) + while read -r line; do + + currenetgroup="$group" + group="$(echo "$line" | awk -F"," '{print $1}')" + id="$(echo "$line" | awk -F"," '{print $2}')" + fullname="$(echo "$line" | awk -F"," '{print $3}')" + email="$(echo "$line" | awk -F"," '{print $NF}')" + + if [[ "$group" != "$currenetgroup" ]]; then + echo + echo "####### $group #####" + fi + + time_ago=$(date -d "-12 month" +%Y-%m-%d) + + updated=$(curl --silent -G \ + "https://$gerriturl/changes/?q=reviewedby:$id+after:$time_ago%20OR%20owner:$id+after:$time_ago&n=1" \ + | grep "updated" \ + | awk '{print $2}' \ + | sed s,\",,g + ) + + if [[ -z $updated ]]; then + + updated=$(curl --silent -G \ + "https://$gerriturl/changes/?q=reviewedby:$id%20OR%20owner:$id&n=1" \ + | grep "updated" \ + | awk '{print $2}' \ + | sed s,\",,g + ) + + if [[ -z $updated ]]; then + echo -n " ERROR $id, $fullname, $email has no activity " + + registered_on=$(ssh -n -p 29418 $gerritssh "gerrit gsql -c \"select registered_on from accounts where full_name = \\\"$fullname\\\" \" ") + whenreg=$(echo $registered_on | awk '{ print $3, $4 }' | awk '{print $1}'| sed s,\(,,g) + if [[ $whenreg == 0 ]]; then + echo "AND USER NEVER REGISTERED" + else + echo -n "REGISTERED ON $whenreg " + datediff "$currentdate" "$whenreg" + fi + else + echo -n " WARNING $id, $fullname, $email inactive since $updated " + datediff "$currentdate" "$updated" + fi + + else + echo " OK $id, $fullname, $email last submision or review $updated" + fi + + done < <(lftools ldap csv "$group") + +} + + +usage() { +cat << EOF +This program searches gerrit by email for user participation +Email list can be built with 'ldap-lookup' +Program outputs users who have not used gerrit at all, or in the last 6 months +users from this list can be slated to have their access revoked. + +If you encounter: "Cant contact LDAP server" + +you will need to add: +TLS_CACERT /path/to/ca.crt in /etc/openldap/ldap.conf +ca.crt can be found on any collab system in /etc/ipa/ca.crt + +Usage: + $1 target gerrit url + $2 target ldap group + +ex: $0 gerrit.opnfv.org/gerrit opnfv-gerrit-releng-submitters +EOF +} + +if [[ $# -eq 0 ]] ; then usage "$@" + exit 1 +fi + +gerriturl="$1" +gerritssh=${gerriturl%/*} +group="$2" +main diff --git a/shell/yaml4info b/shell/yaml4info new file mode 100755 index 00000000..c282e665 --- /dev/null +++ b/shell/yaml4info @@ -0,0 +1,60 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +main() { + + echo "committers:" + while read -r line; do + + email="$(echo "$line" | awk -F"," '{print $NF}')" + company=${email#*@}; company=${company%.*}; + fullname="$(echo "$line" | awk -F"," '{print $3}')" + lfid="$(echo "$line" | awk -F"," '{print $2}')" + +cat << EOF + - name: '$fullname' + email: '$email' + company: '$company' + id: '$lfid' + timezone: 'Unknown/Unknown' +EOF + + done < <(lftools ldap csv "$1") + +} + +usage() { +cat << EOF +Usage: + +$0 ldap-group-name + +calls lftools ldap --groups "\$1" +where \$1 is the ldap-group(s) name you would like to build yaml for your INFO.yaml file +Example: $0 'opnfv-gerrit-releng-*' (for multiple groups) + +EOF +} + +if [[ $# -eq 0 ]] ; then + usage + exit 1 +fi + +while getopts "h" OPTION +do + case $OPTION in + h ) usage; exit;; + \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;; + esac +done + +main "$@"