from lftools.cli.config import config_sys
from lftools.cli.dco import dco
from lftools.cli.deploy import deploy
+from lftools.cli.infofile import infofile
from lftools.cli.jenkins import jenkins_cli
+from lftools.cli.ldap_cli import ldap_cli
from lftools.cli.license import license
from lftools.cli.nexus import nexus
from lftools.cli.sign import sign
cli.add_command(config_sys)
+cli.add_command(infofile)
cli.add_command(deploy)
cli.add_command(dco)
cli.add_command(jenkins_cli, name='jenkins')
cli.add_command(nexus)
cli.add_command(sign)
cli.add_command(version)
+cli.add_command(ldap_cli, name='ldap')
try:
from lftools.openstack.cmd import openstack
--- /dev/null
+#!/usr/bin/env python2
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Script to insert missing values from ldap into a projects INFO.yaml."""
+
+import click
+import ruamel.yaml
+import yaml
+
+
+@click.group()
+@click.pass_context
+def infofile(ctx):
+ """INFO.yaml TOOLS."""
+ pass
+
+
+@click.command(name='get-committers')
+@click.argument('file', envvar='FILE_NAME', required=True)
+@click.option('--full', type=bool, required=False,
+ help='Output name email and id for all committers in an infofile')
+@click.option('--id', type=str, required=False,
+ help='Full output for a specific LFID')
+@click.pass_context
+def get_committers(ctx, file, full, id):
+ """Extract Committer info from INFO.yaml or LDAP dump."""
+ with open(file, 'r') as yaml_file:
+ project = yaml.safe_load(yaml_file)
+
+ def print_committer_info(committer, full):
+ if full:
+ print(" - name: {}".format(committer['name']))
+ print(" email: {}".format(committer['email']))
+ print(" id: {}".format(committer['id']))
+
+ def list_committers(full, id, project):
+ """List commiters from the INFO.yaml file."""
+ lookup = project.get('committers', [])
+
+ for item in lookup:
+ if id:
+ if item['id'] == id:
+ print_committer_info(item, full)
+ break
+ else:
+ continue
+ print_committer_info(item, full)
+
+ list_committers(full, id, project)
+
+
+@click.command(name='sync-committers')
+@click.argument('info_file')
+@click.argument('ldap_file')
+@click.argument('id')
+@click.option('--repo', type=str, required=False,
+ help='repo name')
+@click.pass_context
+def sync_committers(ctx, id, info_file, ldap_file, repo):
+ """Sync committer information from LDAP into INFO.yaml."""
+ ryaml = ruamel.yaml.YAML()
+ ryaml.preserve_quotes = True
+ ryaml.indent(mapping=4, sequence=6, offset=4)
+ ryaml.explicit_start = True
+ with open(info_file, 'r') as stream:
+ try:
+ yaml.safe_load(stream)
+ except yaml.YAMLError as exc:
+ print(exc)
+
+ with open(info_file) as f:
+ info_data = ryaml.load(f)
+ with open(ldap_file) as f:
+ ldap_data = ryaml.load(f)
+
+ def readfile(data, ldap_data, id):
+ committer_info = info_data['committers']
+ repo_info = info_data['repositories']
+ committer_info_ldap = ldap_data['committers']
+ readldap(id, ldap_file, committer_info, committer_info_ldap, repo, repo_info)
+
+ def readldap(id, ldap_file, committer_info, committer_info_ldap, repo, repo_info):
+ for idx, val in enumerate(committer_info):
+ committer = info_data['committers'][idx]['id']
+ if committer == id:
+ print('{} is alread in {}'.format(id, info_file))
+ exit()
+
+ for idx, val in enumerate(committer_info_ldap):
+ committer = ldap_data['committers'][idx]['id']
+ if committer == id:
+ name = (ldap_data['committers'][idx]['name'])
+ email = (ldap_data['committers'][idx]['email'])
+ formatid = (ldap_data['committers'][idx]['id'])
+ company = (ldap_data['committers'][idx]['company'])
+ timezone = (ldap_data['committers'][idx]['timezone'])
+ try:
+ name
+ except NameError:
+ print('{} does not exist in {}'.format(id, ldap_file))
+ exit()
+
+ user = ruamel.yaml.comments.CommentedMap(
+ (
+ ('name', name), ('company', company), ('email', email), ('id', formatid), ('timezone', timezone)
+ )
+ )
+
+ info_data['repositories'][0] = repo
+ committer_info.append(user)
+
+ with open(info_file, 'w') as f:
+ ryaml.dump(info_data, f)
+
+ readfile(info_data, ldap_data, id)
+
+
+infofile.add_command(get_committers)
+infofile.add_command(sync_committers)
--- /dev/null
+#!/usr/bin/env python2
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Generate a CSV of a Projects Commiters.
+
+Prereqs:
+- yum install python-devel openldap-devel
+- pip install python-ldap
+"""
+
+from __future__ import print_function
+
+import subprocess
+import sys
+
+import click
+import ldap
+
+
+@click.group()
+@click.pass_context
+def ldap_cli(ctx):
+ """LDAP TOOLS."""
+ pass
+
+
+@click.command()
+@click.argument('group')
+@click.pass_context
+def yaml4info(ctx, group):
+ """Build yaml of committers for your INFO.yaml."""
+ status = subprocess.call(['yaml4info', group])
+ sys.exit(status)
+
+
+@click.command()
+@click.argument('gerrit_url')
+@click.argument('group')
+@click.pass_context
+def inactivecommitters(ctx, gerrit_url, group):
+ """Check committer participation."""
+ status = subprocess.call(['inactivecommitters', gerrit_url, group])
+ sys.exit(status)
+
+
+@click.command()
+@click.argument('gerrit_clone_base')
+@click.argument('ldap_group')
+@click.argument('repo')
+@click.option('--purpose', envvar='purpose', type=str,
+ help='Must be one of READY_FOR_INFO LINT IN-REVIEW')
+@click.option('--review', type=str, required=False,
+ help='review number in gerrit, required if purpose is IN-REVIEW')
+@click.pass_context
+def autocorrectinfofile(ctx, gerrit_clone_base, ldap_group, repo, purpose, review):
+ """Verify INFO.yaml against LDAP group.\n
+ PURPOSE must be one of: READY_FOR_INFO LINT IN-REVIEW\n
+ GERRITCLONEBASE must be a url: https://gerrit.opnfv.org/gerrit/\n
+ """
+ params = ['autocorrectinfofile']
+ params.extend([gerrit_clone_base, ldap_group, repo])
+ if purpose:
+ params.extend([purpose])
+ if review:
+ params.extend([review])
+ status = subprocess.call(params)
+ sys.exit(status)
+
+
+@click.command()
+@click.option('--ldap-server', default='ldaps://pdx-wl-lb-lfldap.web.codeaurora.org',
+ envvar='LDAP_SERVER', type=str, required=True)
+@click.option('--ldap-user-base', default='ou=Users,dc=freestandards,dc=org',
+ envvar='LDAP_USER_BASE_DN', type=str, required=True)
+@click.option('--ldap-group-base', default='ou=Groups,dc=freestandards,dc=org',
+ envvar='LDAP_GROUP_BASE_DN', type=str, required=True)
+@click.argument('groups')
+@click.pass_context
+def csv(ctx, ldap_server, ldap_group_base, ldap_user_base, groups):
+ """Query an Ldap server."""
+ # groups needs to be a list
+ groups = groups.split(' ')
+
+ def ldap_connect(ldap_object):
+ """Start the connection to LDAP."""
+ try:
+ ldap_object.protocol_version = ldap.VERSION3
+ ldap_object.simple_bind_s()
+ except ldap.LDAPError as e:
+ if type(e.message) == dict and e.message.has_key('desc'):
+ print(e.message['desc'])
+ else:
+ print(e)
+ sys.exit(0)
+
+ def eprint(*args, **kwargs):
+ """Print to stderr."""
+ print(*args, file=sys.stderr, **kwargs)
+
+ def ldap_disconnect(ldap_object):
+ """Stop the connnection to LDAP."""
+ ldap_object.unbind_s()
+
+ def ldap_query(ldap_object, dn, search_filter, attrs):
+ """Perform an LDAP query and return the results."""
+ try:
+ ldap_result_id = ldap_object.search(dn, ldap.SCOPE_SUBTREE, search_filter, attrs)
+ result_set = []
+ while 1:
+ result_type, result_data = ldap_object.result(ldap_result_id, 0)
+ if (result_data == []):
+ break
+ else:
+ # if you are expecting multiple results you can append them
+ # otherwise you can just wait until the initial result and break out
+ if result_type == ldap.RES_SEARCH_ENTRY:
+ result_set.append(result_data)
+ return result_set
+ except ldap.LDAPError as e:
+ sys.exit(1)
+
+ def package_groups(groups):
+ """Package a set of groups from LDIF into a Python dictionary.
+
+ containing the groups member uids.
+ """
+ group_list = []
+ cut_length = len(ldap_user_base)+1
+ for group in groups:
+ group_d = dict(name=group[0][0])
+ members = []
+ for group_attrs in group:
+ for member in group_attrs[1]['member']:
+ members.append(member[:-cut_length])
+ group_d['members'] = members
+ group_list.append(group_d)
+ return group_list
+
+ def user_to_csv(user):
+ """Covert LDIF user info to CSV of uid,mail,cn."""
+ attrs = user[0][0][1]
+ return ",".join([attrs['uid'][0], attrs['cn'][0], attrs['mail'][0]])
+
+ def main(groups):
+ """Preform an LDAP query."""
+ l = ldap.initialize(ldap_server)
+ ldap_connect(l)
+ for arg in groups:
+ groups = ldap_query(l, ldap_group_base, "cn=%s" % arg, ["member"])
+ group_dict = package_groups(groups)
+ cut_length = len(ldap_group_base)+1
+ for group_bar in group_dict:
+ group_name = group_bar['name'][3:-cut_length]
+ for user in group_bar['members']:
+ user_info = ldap_query(l, ldap_user_base, user, ["uid", "cn", "mail"])
+ try:
+ print("%s,%s" % (group_name, user_to_csv(user_info)))
+ except:
+ eprint("Error parsing user: %s" % user)
+ continue
+ ldap_disconnect(l)
+ main(groups)
+
+
+ldap_cli.add_command(csv)
+ldap_cli.add_command(inactivecommitters)
+ldap_cli.add_command(yaml4info)
+ldap_cli.add_command(autocorrectinfofile)
--- /dev/null
+---
+features:
+ - |
+ $ lftools ldap
+
+ Usage: lftools ldap [OPTIONS] COMMAND [ARGS]...
+
+ .. code-block:: none
+
+ Commands:
+ autocorrectinfofile Verify INFO.yaml against LDAP group.
+ csv Query an Ldap server.
+ inactivecommitters Check committer participation.
+ yaml4info Build yaml of commiters for your INFO.yaml.
+
+ - |
+ $ lftools infofile
+
+ .. code-block:: none
+
+ Commands:
+ get-committers Extract Committer info from INFO.yaml or LDAP...
+ sync-committers Sync committer information from LDAP into...
click
+python-ldap
pyyaml
requests~=2.18.0
+ruamel.yaml
setuptools>=36.5.0
six~=1.11.0
python-jenkins~=1.1.0
scripts =
shell/dco
shell/deploy
+ shell/inactivecommitters
shell/sign
shell/version
+ shell/yaml4info
+ shell/autocorrectinfofile
[entry_points]
console_scripts =
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+
+#echo "Diff LDAP against INFO.yaml"
+
+tmpdir(){
+
+ DIR="/tmp/infofile/$purpose.$repo.$ldapgroup"
+ mkdir -p "$DIR"
+
+}
+
+parseoptions() {
+
+ if [[ -z $purpose ]]; then
+ purpose=correct
+ fi
+
+ echo " gerritclonebase $gerritclonebase"
+ echo " ldapgroup $ldapgroup"
+ echo " repo $repo"
+ echo " purpose $purpose"
+
+ if [[ $purpose =~ "READY_FOR_INFO" ]]; then
+
+ tmpdir "$@"
+
+ if ! [[ -d "$DIR"/"$repo" ]]; then
+ echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1"
+ git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1
+ fi
+ if [ ! -f "$DIR"/"$repo"/INFO.yaml ]; then
+ cp INFO.template.yaml "$DIR"/"$repo"/INFO.yaml || exit 1
+ else
+ echo "INFO file already exists, refusing to overwrite"
+ exit 1
+ fi
+
+ fi
+
+ if [[ $purpose =~ "LINT" ]]; then
+
+ tmpdir "$@"
+
+ if ! [[ -d "$DIR"/"$repo" ]]; then
+ echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1"
+ git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1
+ fi
+ if ! yamllint "$DIR"/"$repo"/INFO.yaml; then
+ echo "ERROR LINT FAILED"
+ exit 1
+ fi
+
+ fi
+
+ #I should only clone the review if their is a discrepancy in commiters
+ if [[ $purpose =~ "IN-REVIEW" ]]; then
+
+ tmpdir "$@"
+
+ if ! [[ -d "$DIR"/"$repo" ]]; then
+ echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1"
+ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1
+ cd "$DIR"/"$repo" || exit
+ git fetch origin "$review" && git checkout --quiet FETCH_HEAD
+ cd "$SCRIPTDIR" || exit
+ yamllint "$DIR"/"$repo"/INFO.yaml
+ if ! yamllint "$DIR"/"$repo"/INFO.yaml; then
+ echo " ERROR LINT FAILED CANNOT AUTO CORRECT FILE IN REVIEW"
+ exit 1
+ fi
+
+ fi
+ fi
+
+ main "$@"
+}
+
+main() {
+
+ if [[ -z $DIR ]]; then
+ tmpdir "$@"
+ fi
+ echo " tmpdir = $DIR"
+
+ if lftools ldap yaml4info "$ldapgroup" 2>&- > "$DIR"/LDAP.yaml
+ then
+ echo " LDAP lookup sucssesfull"
+ else
+ echo " LDAP lookup failed"
+ exit 1
+ fi
+
+ if ! [[ -d "$DIR"/"$repo" ]]; then
+ echo " git clone -q $gerritclonebase$repo $DIR/$repo || exit 1"
+ git clone -q "$gerritclonebase""$repo" "$DIR"/"$repo" || exit 1
+ fi
+
+ diff="$(diff <(lftools infofile get_committers "$DIR"/LDAP.yaml 2>&-| sort) <(lftools infofile get_committers "$DIR"/"$repo"/INFO.yaml 2>&- | sort))"
+ status="$?"
+
+ diff_array=()
+ onlyinINFO=()
+ onlyinLDAP=()
+
+ while IFS= read -r line; do
+ diff_array+=( "$line" )
+ if [[ $(echo "$line" | grep ">") ]];
+ then
+ onlyinINFO+=( "$(echo "$line" | awk -F"'" '{ print $2 }')" )
+ fi
+ if [[ $(echo "$line" | grep "<") ]];
+ then
+ onlyinLDAP+=( "$(echo "$line" | awk -F"'" '{ print $2 }')" )
+ fi
+ done < <(echo "${diff[@]}" )
+
+
+ if ! [ "${#onlyinINFO[@]}" -eq 0 ]; then
+ for missing in "${onlyinINFO[@]}"; do
+ if ! [ -z "$missing" ]; then
+ echo " DUMMY: sending invite to $missing"
+ lftools infofile get_committers --id "$missing" "$DIR"/"$repo"/INFO.yaml 2>&-
+ fi
+ done
+ fi
+
+ if ! [ "${#onlyinLDAP[@]}" -eq 0 ]; then
+ echo " These users are listed as commiters in LDAP and not in the INFO.yaml"
+ for missing in "${onlyinLDAP[@]}"; do
+ echo " lftools infofile correct $DIR/$repo/INFO.yaml $DIR/LDAP.yaml $missing --repo $repo 2>&-"
+ lftools infofile correct "$DIR"/"$repo"/INFO.yaml "$DIR"/LDAP.yaml "$missing" --repo "$repo" 2>&-
+ done
+
+ fi
+
+ echo " Exit status = $status"
+ exit "$status"
+
+}
+
+usage() {
+cat << EOF
+Must be called from lftools
+eg: lftools ldap autocorrectinfofile
+EOF
+exit 1
+}
+
+if [[ -z "$*" ]]; then usage
+fi
+
+gerritclonebase="$1"
+ldapgroup="$2"
+repo="$3"
+purpose="$4"
+review="$5"
+
+parseoptions "$@"
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+datediff() {
+ d1=$(date -d "$1" +%s)
+ d2=$(date -d "$2" +%s)
+ echo $(( (d1 - d2) / 86400 )) days ago
+}
+
+main() {
+
+ echo "checking $gerriturl participation for group $group"
+ echo
+
+ currentdate=$(date +%Y-%m-%d)
+ while read -r line; do
+
+ currenetgroup="$group"
+ group="$(echo "$line" | awk -F"," '{print $1}')"
+ id="$(echo "$line" | awk -F"," '{print $2}')"
+ fullname="$(echo "$line" | awk -F"," '{print $3}')"
+ email="$(echo "$line" | awk -F"," '{print $NF}')"
+
+ if [[ "$group" != "$currenetgroup" ]]; then
+ echo
+ echo "####### $group #####"
+ fi
+
+ time_ago=$(date -d "-12 month" +%Y-%m-%d)
+
+ updated=$(curl --silent -G \
+ "https://$gerriturl/changes/?q=reviewedby:$id+after:$time_ago%20OR%20owner:$id+after:$time_ago&n=1" \
+ | grep "updated" \
+ | awk '{print $2}' \
+ | sed s,\",,g
+ )
+
+ if [[ -z $updated ]]; then
+
+ updated=$(curl --silent -G \
+ "https://$gerriturl/changes/?q=reviewedby:$id%20OR%20owner:$id&n=1" \
+ | grep "updated" \
+ | awk '{print $2}' \
+ | sed s,\",,g
+ )
+
+ if [[ -z $updated ]]; then
+ echo -n " ERROR $id, $fullname, $email has no activity "
+
+ registered_on=$(ssh -n -p 29418 $gerritssh "gerrit gsql -c \"select registered_on from accounts where full_name = \\\"$fullname\\\" \" ")
+ whenreg=$(echo $registered_on | awk '{ print $3, $4 }' | awk '{print $1}'| sed s,\(,,g)
+ if [[ $whenreg == 0 ]]; then
+ echo "AND USER NEVER REGISTERED"
+ else
+ echo -n "REGISTERED ON $whenreg "
+ datediff "$currentdate" "$whenreg"
+ fi
+ else
+ echo -n " WARNING $id, $fullname, $email inactive since $updated "
+ datediff "$currentdate" "$updated"
+ fi
+
+ else
+ echo " OK $id, $fullname, $email last submision or review $updated"
+ fi
+
+ done < <(lftools ldap csv "$group")
+
+}
+
+
+usage() {
+cat << EOF
+This program searches gerrit by email for user participation
+Email list can be built with 'ldap-lookup'
+Program outputs users who have not used gerrit at all, or in the last 6 months
+users from this list can be slated to have their access revoked.
+
+If you encounter: "Cant contact LDAP server"
+
+you will need to add:
+TLS_CACERT /path/to/ca.crt in /etc/openldap/ldap.conf
+ca.crt can be found on any collab system in /etc/ipa/ca.crt
+
+Usage:
+ $1 target gerrit url
+ $2 target ldap group
+
+ex: $0 gerrit.opnfv.org/gerrit opnfv-gerrit-releng-submitters
+EOF
+}
+
+if [[ $# -eq 0 ]] ; then usage "$@"
+ exit 1
+fi
+
+gerriturl="$1"
+gerritssh=${gerriturl%/*}
+group="$2"
+main
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+main() {
+
+ echo "committers:"
+ while read -r line; do
+
+ email="$(echo "$line" | awk -F"," '{print $NF}')"
+ company=${email#*@}; company=${company%.*};
+ fullname="$(echo "$line" | awk -F"," '{print $3}')"
+ lfid="$(echo "$line" | awk -F"," '{print $2}')"
+
+cat << EOF
+ - name: '$fullname'
+ email: '$email'
+ company: '$company'
+ id: '$lfid'
+ timezone: 'Unknown/Unknown'
+EOF
+
+ done < <(lftools ldap csv "$1")
+
+}
+
+usage() {
+cat << EOF
+Usage:
+
+$0 ldap-group-name
+
+calls lftools ldap --groups "\$1"
+where \$1 is the ldap-group(s) name you would like to build yaml for your INFO.yaml file
+Example: $0 'opnfv-gerrit-releng-*' (for multiple groups)
+
+EOF
+}
+
+if [[ $# -eq 0 ]] ; then
+ usage
+ exit 1
+fi
+
+while getopts "h" OPTION
+do
+ case $OPTION in
+ h ) usage; exit;;
+ \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;;
+ esac
+done
+
+main "$@"
Code Review / releng / lftools.git / commitdiff