From: Aric Gardner <agardner@linuxfoundation.org>
Date: Thu, 21 Feb 2019 21:06:50 +0000 (-0500)
Subject: lfidapi for lftools
X-Git-Tag: v0.20.0~1
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F70%2F14670%2F23;p=releng%2Flftools.git

lfidapi for lftools

Note: as soon as by the end of the year
(whenever all clients have been migrated from LDAP to Auth0)
All groups will migrate to the "IAM"
product at which point this will be tooled to a different API.

All api methods implemented:

(search_members)
(user)
(invite)
(create_group)

uses oauth2_helper to retreive access_token from client_secret and
refresh_token

reads secrets from ~/.config/lftools/lftools.ini

[lfid]
clientid =
client_secret =
refresh_token =
token_uri = https://identitystg.linuxfoundation.org/oauth2/token
url = https://identitystg.linuxfoundation.org/rest/auth0/

Secrets are saved in LastPass under Release Engineering as
"Identity Groups REST API Client".

Change-Id: Ib7f575dda0c73a99ca65f0a7f87d2b62f85da03e
Signed-off-by: Aric Gardner <agardner@linuxfoundation.org>
---

diff --git a/.coafile b/.coafile
index a0e2259f..ab5bc5f0 100644
--- a/.coafile
+++ b/.coafile
@@ -36,13 +36,16 @@ ignore += docs/conf.py
 known_first_party_imports = lftools
 known_third_party_imports =
     defusedxml,
+    email_validator,
     glob2,
+    httplib2,
     jenkins,
     jsonschema,
+    oauth2client,
     pytest,
+    ruamel.yaml,
     six,
     shade,
-    ruamel.yaml,
     xdg,
     yaml
 pydocstyle_ignore = D203, D213, D301
diff --git a/docs/commands/index.rst b/docs/commands/index.rst
index 5187621a..5881a996 100644
--- a/docs/commands/index.rst
+++ b/docs/commands/index.rst
@@ -14,6 +14,7 @@ It supports the following commands:
     deploy
     dco
     gerrit
+    lfidapi
     license
     nexus
     openstack
diff --git a/docs/commands/lfidapi.rst b/docs/commands/lfidapi.rst
new file mode 100644
index 00000000..02776b9b
--- /dev/null
+++ b/docs/commands/lfidapi.rst
@@ -0,0 +1,29 @@
+******
+Schema
+******
+
+.. program-output:: lftools lfidapi --help
+
+Commands
+========
+
+create-group
+------------
+
+.. program-output:: lftools lfidapi create-group --help
+
+invite
+-------
+
+.. program-output:: lftools lfidapi invite --help
+
+search-members
+--------------
+
+.. program-output:: lftools lfidapi search-members --help
+
+
+user
+----
+
+.. program-output:: lftools lfidapi user --help
diff --git a/lftools/cli/__init__.py b/lftools/cli/__init__.py
index 3fd402e4..4bca34c4 100644
--- a/lftools/cli/__init__.py
+++ b/lftools/cli/__init__.py
@@ -25,6 +25,7 @@ from lftools.cli.deploy import deploy
 from lftools.cli.gerrit import gerrit_cli
 from lftools.cli.infofile import infofile
 from lftools.cli.jenkins import jenkins_cli
+from lftools.cli.lfidapi import lfidapi
 from lftools.cli.license import license
 from lftools.cli.nexus import nexus
 from lftools.cli.schema import schema
@@ -84,6 +85,7 @@ cli.add_command(jenkins_cli, name='jenkins')
 cli.add_command(license)
 cli.add_command(nexus)
 cli.add_command(schema)
+cli.add_command(lfidapi)
 cli.add_command(sign)
 cli.add_command(version)
 
diff --git a/lftools/cli/lfidapi.py b/lftools/cli/lfidapi.py
new file mode 100755
index 00000000..52891d56
--- /dev/null
+++ b/lftools/cli/lfidapi.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Use the LFIDAPI to add, remove and list members as well as create groups."""
+
+import subprocess
+import sys
+
+import click
+
+from lftools.lfidapi import helper_add_remove_committers
+from lftools.lfidapi import helper_create_group
+from lftools.lfidapi import helper_invite
+from lftools.lfidapi import helper_search_members
+from lftools.lfidapi import helper_user
+
+
+@click.group()
+@click.pass_context
+def lfidapi(ctx):
+    """LFID API TOOLS."""
+    pass
+
+
+@click.command()
+@click.argument('group')
+@click.pass_context
+def search_members(ctx, group):
+    """List members of a group."""
+    helper_search_members(group)
+
+
+@click.command()
+@click.argument('user')
+@click.option('--delete', is_flag=True, required=False,
+              help='remove user from group')
+@click.argument('group')
+@click.pass_context
+def user(ctx, user, group, delete):
+    """Add and remove users from groups."""
+    helper_user(user, group, delete)
+
+
+@click.command()
+@click.argument('email')
+@click.argument('group')
+@click.pass_context
+def invite(ctx, email, group):
+    """Email invitation to join group."""
+    helper_invite(email, group)
+
+
+@click.command()
+@click.argument('group')
+@click.pass_context
+def create_group(ctx, group):
+    """Create group."""
+    helper_create_group(group)
+
+
+@click.command()
+@click.argument('info_file')
+@click.argument('ldap_file')
+@click.argument('group')
+@click.argument('user')
+@click.pass_context
+def add_remove_committers(ctx, info_file, ldap_file, group, user):
+    """Used in automation."""
+    helper_add_remove_committers(info_file, ldap_file, group, user)
+
+
+@click.command()
+@click.argument('git_dir')
+@click.argument('gerrit_fqdn')
+@click.argument('gerrit_project')
+@click.pass_context
+def lfidapi_add_remove_users(ctx, git_dir, gerrit_fqdn, gerrit_project):
+    """Create a diff of the changes to the INFO.yaml.
+
+    Call the api to add and remove users as appropriate.
+    """
+    status = subprocess.call(['lfidapi_add_remove_users', git_dir, gerrit_fqdn, gerrit_project])
+
+    sys.exit(status)
+
+
+lfidapi.add_command(search_members)
+lfidapi.add_command(user)
+lfidapi.add_command(invite)
+lfidapi.add_command(create_group)
+lfidapi.add_command(add_remove_committers)
+lfidapi.add_command(lfidapi_add_remove_users)
diff --git a/lftools/lfidapi.py b/lftools/lfidapi.py
new file mode 100755
index 00000000..11f321a2
--- /dev/null
+++ b/lftools/lfidapi.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Use the LFIDAPI to add, remove and list members as well as create groups."""
+
+import json
+
+from email_validator import validate_email
+import requests
+from six.moves import urllib
+import yaml
+
+from lftools.oauth2_helper import oauth_helper
+
+PARSE = urllib.parse.urljoin
+
+
+def check_response_code(response):
+    """Response Code Helper function."""
+    if response.status_code != 200:
+        raise requests.HTTPError("Authorization failed with the following "
+                                 "error:\n{}: {}".format(response.status_code,
+                                                         response.text))
+
+
+def helper_search_members(group):
+    """List members of a group."""
+    access_token, url = oauth_helper()
+    url = PARSE(url, group)
+    headers = {'Authorization': 'Bearer ' + access_token}
+    response = requests.get(url, headers=headers)
+    check_response_code(response)
+    result = (response.json())
+    members = result["members"]
+    print(json.dumps(members, indent=4, sort_keys=True))
+
+
+def helper_user(user, group, delete):
+    """Add and remove users from groups."""
+    access_token, url = oauth_helper()
+    url = PARSE(url, group)
+    headers = {'Authorization': 'Bearer ' + access_token}
+    data = {"username": user}
+    if delete:
+        response = requests.delete(url, json=data, headers=headers)
+    else:
+        response = requests.put(url, json=data, headers=headers)
+    check_response_code(response)
+    result = (response.json())
+    print(json.dumps(result, indent=4, sort_keys=True))
+
+
+def helper_invite(email, group):
+    """Email invitation to join group."""
+    access_token, url = oauth_helper()
+    prejoin = group + '/invite'
+    url = PARSE(url, prejoin)
+    headers = {'Authorization': 'Bearer ' + access_token}
+    data = {"mail": email}
+    print('Validating email', email)
+    if validate_email(email):
+        response = requests.post(url, json=data, headers=headers)
+        check_response_code(response)
+        result = (response.json())
+        print(json.dumps(result, indent=4, sort_keys=True))
+    else:
+        print("Email is not valid")
+
+
+def helper_create_group(group):
+    """Create group."""
+    access_token, url = oauth_helper()
+    url = '{}/'.format(url)
+    headers = {'Authorization': 'Bearer ' + access_token}
+    data = {"title": group, "type": "group"}
+    print(data)
+    response = requests.post(url, json=data, headers=headers)
+    check_response_code(response)
+    result = (response.json())
+    print(json.dumps(result, indent=4, sort_keys=True))
+
+
+def helper_add_remove_committers(info_file, ldap_file, user, group):
+    """Helper only to be used in automation."""
+    with open(info_file) as file:
+        try:
+            info_data = yaml.safe_load(file)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+    with open(ldap_file, 'r') as file:
+        ldap_data = json.load(file)
+
+    committer_info = info_data['committers']
+
+    info_committers = []
+    for count, item in enumerate(committer_info):
+        committer = committer_info[count]['id']
+        info_committers.append(committer)
+
+    ldap_committers = []
+    for count, item in enumerate(ldap_data):
+        committer = ldap_data[count]['username']
+        ldap_committers.append(committer)
+
+    removed_by_patch = [item for item in ldap_committers if item not in info_committers]
+
+    if (user in removed_by_patch):
+        print(" {} found in group {} ".format(user, group))
+        print(" removing user {} from group {}".format(user, group))
+        helper_user(user, group, "--delete")
+
+    added_by_patch = [item for item in info_committers if item not in ldap_committers]
+
+    if (user in added_by_patch):
+        print(" {} not found in group {} ".format(user, group))
+        print(" adding user {} to group {}".format(user, group))
+        helper_user(user, group, "")
diff --git a/lftools/oauth2_helper.py b/lftools/oauth2_helper.py
new file mode 100644
index 00000000..7ba5b95a
--- /dev/null
+++ b/lftools/oauth2_helper.py
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Helper script to get access_token for lfid api."""
+
+import logging
+
+import httplib2
+from oauth2client import client
+
+from lftools import config
+
+
+def oauth_helper():
+    """Helper script to get access_token for lfid api."""
+    logging.getLogger("oauth2client").setLevel(logging.ERROR)
+    client_id = config.get_setting("lfid", "clientid")
+    client_secret = config.get_setting("lfid", "client_secret")
+    refresh_token = config.get_setting("lfid", "refresh_token")
+    token_uri = config.get_setting("lfid", "token_uri")
+    url = config.get_setting("lfid", "url")
+
+    credentials = client.OAuth2Credentials(
+        access_token=None,  # set access_token to None since we use a refresh token
+        client_id=client_id,
+        client_secret=client_secret,
+        refresh_token=refresh_token,
+        token_expiry=None,
+        token_uri=token_uri,
+        user_agent=None)
+    credentials.refresh(httplib2.Http())
+    access_token = credentials.access_token
+    return access_token, url
diff --git a/releasenotes/notes/lfidapi-74c7a5457203eec2.yaml b/releasenotes/notes/lfidapi-74c7a5457203eec2.yaml
new file mode 100644
index 00000000..912986bd
--- /dev/null
+++ b/releasenotes/notes/lfidapi-74c7a5457203eec2.yaml
@@ -0,0 +1,20 @@
+---
+features:
+  - |
+    LFID Api Tools.
+
+    Usage: lftools lfidapi [OPTIONS] COMMAND [ARGS]...
+
+
+    .. code-block:: none
+
+       Commands:
+         create-group    Create group.
+         invite          Email invitation to join group.
+         search-members  List members of a group.
+         user            Add and remove users from groups.
+
+    .. code-block:: none
+
+       Options:
+         --help    Show this message and exit
diff --git a/requirements.txt b/requirements.txt
index c4326eda..dd180e8e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,6 @@ glob2  # Needed for Python < 3.5 recursive glob support
 deb-pkg-tools~=5.2
 defusedxml # Needed due to tox complains on parseString not safe
 jsonschema~=2.6.0
-pyyaml
 requests~=2.18.0
 rpmfile~=0.1.4
 ruamel.yaml
@@ -14,6 +13,10 @@ python-jenkins~=1.1.0
 tqdm
 xdg~=1.0.7;python_version<'3'
 xdg~=3.0.0;python_version>='3'
+httplib2
+email_validator
+oauth2client
+pyyaml
 
 # workarounds to prevent upstream from breaking us
 netifaces==0.10.5
diff --git a/setup.cfg b/setup.cfg
index 8d020021..d31a57cf 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -38,6 +38,7 @@ scripts =
     shell/deploy
     shell/gerrit_create
     shell/inactivecommitters
+    shell/lfidapi_add_remove_users
     shell/sign
     shell/version
     shell/yaml4info
diff --git a/shell/lfidapi_add_remove_users b/shell/lfidapi_add_remove_users
new file mode 100755
index 00000000..21ca5eb5
--- /dev/null
+++ b/shell/lfidapi_add_remove_users
@@ -0,0 +1,74 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+git_dir="$1"
+gerrit_fqdn="$2"
+clonebase=https://$gerrit_fqdn/gerrit/
+gerrit_project="$3"
+
+cd "$git_dir" || exit
+pwd
+
+determine_ldap_group(){
+  get_group(){
+    ldap_group="$(curl -s "$clonebase"access/?project=$gerrit_project \
+      | tail -n +2 \
+      | jq  '.[].local[].permissions.owner.rules' \
+      | grep ldap \
+      | awk -F"=" '{print $2}' \
+      | awk -F"," '{print $1}')"
+  }
+
+  walkgroup(){
+    repo="$(curl -s "$clonebase"access/?project=$gerrit_project | tail -n +2 | jq -r '.[].inherits_from.id')"
+    get_group "$gerrit_project"
+  }
+
+  get_group "$gerrit_project"
+
+  #if ldap_group is null, check for a parent, there may be two levels of parent
+  #This looks stupid but it works.
+  if [ -z "$ldap_group" ]; then
+    walkgroup "$gerrit_project"
+    if [ -z "$ldap_group" ]; then
+      walkgroup "$gerrit_project"
+    fi
+  fi
+  if [ -z "$ldap_group" ]; then
+    echo "could not determine ldap group"
+    exit 1
+  fi
+}
+determine_ldap_group
+
+echo "LDAP GROUP IS $ldap_group for repo $repo"
+echo "Change as we see it"
+git --no-pager show INFO.yaml
+
+#define directions for diff
+added="'%>'"
+removed="'%<'"
+for direction in "$added" "$removed"; do
+unset diff
+
+  diff=$(diff --changed-group-format="$direction" --unchanged-group-format='' <(git show HEAD~1:INFO.yaml) <(git show HEAD:INFO.yaml))
+  if ! [ -z "$diff" ]; then
+    while IFS=$'\n' read -r id; do
+      user="$(echo "$id" | niet '.id')"
+      rm ldap_file.json
+      lftools lfidapi search-members "$ldap_group" > ldap_file.json
+      cat ldap_file.json
+      lftools lfidapi add-remove-committers INFO.yaml ldap_file.json "$user" "$ldap_group"
+
+    done < <(diff --changed-group-format="$direction" --unchanged-group-format='' <(git show HEAD~1:INFO.yaml) <(git show HEAD:INFO.yaml) |grep "id:")
+  fi
+
+done