From: Eric Ball Date: Wed, 2 Feb 2022 23:33:03 +0000 (-0800) Subject: Feat: Move git functions to use python git lib X-Git-Tag: v0.36.0~1 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F17%2F69717%2F9;p=releng%2Flftools.git Feat: Move git functions to use python git lib Issue: RELENG-4052 Change-Id: Iafaa078b24838135dd67ec0ac53b1d83de13ae8b Signed-off-by: Eric Ball --- diff --git a/lftools/api/endpoints/gerrit.py b/lftools/api/endpoints/gerrit.py index 423a5021..fffa2265 100644 --- a/lftools/api/endpoints/gerrit.py +++ b/lftools/api/endpoints/gerrit.py @@ -45,7 +45,7 @@ class Gerrit(client.RestApi): super(Gerrit, self).__init__(**params) def add_file(self, fqdn, gerrit_project, filename, issue_id, file_location, **kwargs): - """Add an file for review to a Project. + """Add a file for review to a Project. File can be sourced from any location but only lands in the root of the repo. @@ -110,7 +110,6 @@ class Gerrit(client.RestApi): # Setup signed_off_by = config.get_setting(fqdn, "sob") gerrit_project_dashed = gerrit_project.replace("/", "-") - urllib.parse.quote(gerrit_project, safe="", encoding=None, errors=None) filename = "{}.yaml".format(gerrit_project_dashed) if not reviewid: @@ -240,7 +239,7 @@ class Gerrit(client.RestApi): return payload def sanity_check(self, fqdn, gerrit_project, **kwargs): - """Preform a sanity check.""" + """Perform a sanity check.""" # Sanity check gerrit_project_encoded = urllib.parse.quote(gerrit_project, safe="", encoding=None, errors=None) mylist = ["projects/", "projects/{}".format(gerrit_project_encoded)] diff --git a/lftools/cli/gerrit.py b/lftools/cli/gerrit.py index 0ae99d1f..61bce6d2 100644 --- a/lftools/cli/gerrit.py +++ b/lftools/cli/gerrit.py @@ -18,6 +18,7 @@ from pprint import pformat import click from lftools.api.endpoints import gerrit +from lftools.git.gerrit import Gerrit as git_gerrit log = logging.getLogger(__name__) @@ -34,10 +35,10 @@ def gerrit_cli(ctx): @click.argument("gerrit_project") @click.argument("filename") @click.option("--issue_id", type=str, required=False, help="For projects that enforce an issue id for changesets") -@click.option("--file_location", type=str, required=False, help="option allos you to specify full path and file name") +@click.option("--file_location", type=str, required=False, help="File path within the repository") @click.pass_context def addfile(ctx, gerrit_fqdn, gerrit_project, filename, issue_id, file_location): - """Add an file for review to a Project. + """Add a file for review to a Project. Requires gerrit directory. @@ -55,10 +56,10 @@ def addfile(ctx, gerrit_fqdn, gerrit_project, filename, issue_id, file_location) @click.argument("gerrit_fqdn") @click.argument("gerrit_project") @click.argument("jjbrepo") -@click.option("--reviewid", type=str, required=False, help="ammend a review rather than making a new one") @click.option("--issue_id", type=str, required=False, help="For projects that enforce an issue id for changesets") +@click.option("--agent", type=str, required=False, help="Specify the Jenkins agent label to run the job on") @click.pass_context -def addinfojob(ctx, gerrit_fqdn, gerrit_project, jjbrepo, reviewid, issue_id): +def addinfojob(ctx, gerrit_fqdn, gerrit_project, jjbrepo, issue_id, agent): """Add an INFO job for a new Project. Adds info verify jenkins job for project. @@ -72,9 +73,8 @@ def addinfojob(ctx, gerrit_fqdn, gerrit_project, jjbrepo, reviewid, issue_id): gerrit_project test/test1 jjbrepo ci-mangement """ - g = gerrit.Gerrit(fqdn=gerrit_fqdn) - data = g.add_info_job(gerrit_fqdn, gerrit_project, jjbrepo, reviewid, issue_id) - log.info(pformat(data)) + git = git_gerrit(fqdn=gerrit_fqdn, project=jjbrepo) + git.add_info_job(gerrit_fqdn, gerrit_project, issue_id, agent) @click.command(name="addgitreview") @@ -89,9 +89,8 @@ def addgitreview(ctx, gerrit_fqdn, gerrit_project, issue_id): gerrit_url gerrit.o-ran-sc.org gerrit_project test/test1 """ - g = gerrit.Gerrit(fqdn=gerrit_fqdn) - data = g.add_git_review(gerrit_fqdn, gerrit_project, issue_id) - log.info(pformat(data)) + git = git_gerrit(fqdn=gerrit_fqdn, project=gerrit_project) + git.add_git_review(gerrit_fqdn, gerrit_project, issue_id) @click.command(name="addgithubrights") diff --git a/lftools/git/__init__.py b/lftools/git/__init__.py new file mode 100644 index 00000000..3012dc3b --- /dev/null +++ b/lftools/git/__init__.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Init for git module.""" diff --git a/lftools/git/gerrit.py b/lftools/git/gerrit.py new file mode 100644 index 00000000..6e1b243b --- /dev/null +++ b/lftools/git/gerrit.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2021 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +"""Gerrit git interface.""" + +import logging +import os +import shutil +import tempfile +import urllib + +import requests +from git import Repo +from jinja2 import Environment, PackageLoader, select_autoescape + +from lftools import config +from lftools.api.endpoints.gerrit import Gerrit as gerrit_api + +log = logging.getLogger(__name__) + + +class Gerrit: + """Wrapper for Gerrit-specific git methods.""" + + def __init__(self, **params): + """Initialize the class.""" + self.params = params + self.fqdn = self.params["fqdn"] + self.project = self.params["project"] + if "creds" not in self.params: + creds = { + "authtype": "basic", + "username": config.get_setting(self.fqdn, "username"), + "password": config.get_setting(self.fqdn, "password"), + "endpoint": config.get_setting(self.fqdn, "endpoint"), + "email": config.get_setting(self.fqdn, "email"), + } + params["creds"] = creds + + working_dir = tempfile.mkdtemp() + log.debug("Temporary working directory for git repo: {}".format(working_dir)) + os.chdir(working_dir) + + short_endpoint = self.params["creds"]["endpoint"].split("://")[-1] + project_endpoint = urllib.parse.urljoin(short_endpoint, self.project) + remote = "https://{}:{}@{}".format( + self.params["creds"]["username"], self.params["creds"]["password"], project_endpoint + ) + Repo.clone_from(remote, working_dir) + self.repo = Repo.init(working_dir) + self.get_commit_hook(self.params["creds"]["endpoint"], working_dir) + + with self.repo.config_writer() as git_config: + git_config.set_value("user", "name", self.params["creds"]["username"]) + git_config.set_value("user", "email", self.params["creds"]["email"]) + self.origin = self.repo.remote(name="origin") + + # Get the default branch from the repo + default_ref = self.repo.git.rev_parse("origin/HEAD", abbrev_ref=True) + self.default_branch = default_ref.split("/")[-1] + + def __del__(self): + try: + shutil.rmtree(self.repo.working_tree_dir) + except FileNotFoundError: + log.info("Could not remove working directory {}".format(self.repo.working_tree_dir)) + + def get_commit_hook(self, endpoint, working_dir): + """Pulls in the Gerrit server's commit hook to add a changeId.""" + hook_url = urllib.parse.urljoin(endpoint, "tools/hooks/commit-msg") + # The hook url does not include the /a that is typically part of a + # gerrit url for cloning. + hook_url = hook_url.replace("/a/", "/", 1) + local_hooks_path = os.path.join(working_dir, ".git/hooks") + commit_msg_hook_path = "{}/commit-msg".format(local_hooks_path) + + try: + os.mkdir(local_hooks_path) + except FileExistsError: + log.debug("Directory {} already exists".format(local_hooks_path)) + with requests.get(hook_url) as hook: + hook.raise_for_status() + with open(commit_msg_hook_path, "w") as file: + file.write(hook.text) + os.chmod(commit_msg_hook_path, 0o755) + + def add_file(self, filepath, content): + """Add a file to the current git repo. + + Example: + + local_path /tmp/INFO.yaml + file_path="somedir/example-INFO.yaml" + """ + if filepath.find("/") >= 0: + os.makedirs(os.path.split(filepath)[0]) + with open(filepath, "w") as newfile: + newfile.write(content) + self.repo.git.add(filepath) + + def commit(self, commit_msg, issue_id, push=False): + """Commit staged changes. + + This will commit all currently-staged changes, using the provided commit + message. This can be a single line, or a multi-line header-and-body + format. The footer is then added with optional Issue-ID, and signed- + off-by line. If push=True, the change will then be pushed to the + default branch's change creation link. + + Example: + + commit_msg "Chore: Add arbitrary files" + issue_id "EX-1234" + + Commit message will read: + + Chore: Add arbitrary files + + Issue-ID: EX-1234 + Signed-off-by: lf-automation + """ + sob = config.get_setting(self.fqdn, "sob") + # Add known \n\n gap to end of commit message by stripping, then adding + # exactly two newlines. + commit_msg = "{}\n\n".format(commit_msg.strip()) + if issue_id: + commit_msg += "Issue-ID: {}\n".format(issue_id) + commit_msg += "Signed-off-by: {}".format(sob) + self.repo.git.commit("-m{}".format(commit_msg)) + if push: + self.repo.git.push(self.origin, "HEAD:refs/for/{}".format(self.default_branch)) + + def add_info_job(self, fqdn, gerrit_project, issue_id, agent): + """Add info-verify jenkins job for the new project. + + Example: + + fqdn gerrit.o-ran-sc.org + gerrit_project test/test1 + jjbrepo ci-mangement + """ + gerrit_project_dashed = gerrit_project.replace("/", "-") + filename = "{}.yaml".format(gerrit_project_dashed) + + if not agent: + if fqdn == "gerrit.o-ran-sc.org": + buildnode = "centos7-builder-1c-1g" + else: + buildnode = "centos7-builder-2c-1g" + + jinja_env = Environment(loader=PackageLoader("lftools.git"), autoescape=select_autoescape()) + template = jinja_env.get_template("project.yaml") + content = template.render( + project_name_dashed=gerrit_project_dashed, + project_name=gerrit_project, + buildnode=buildnode, + default_branch=self.default_branch, + ) + log.debug("File contents:\n{}".format(content)) + + filepath = os.path.join(self.repo.working_tree_dir, "jjb/{0}/{0}.yaml".format(gerrit_project_dashed)) + self.add_file(filepath, content) + commit_msg = "Chore: Automation adds {}".format(filename) + self.commit(commit_msg, issue_id, push=True) + + def add_git_review(self, fqdn, gerrit_project, issue_id): + """Add and push a .gitreview for a project. + + Example: + + fqdn gerrit.o-ran-sc.org + gerrit_project test/test1 + issue_id: CIMAN-33 + """ + gerrit_api.sanity_check(self.fqdn, gerrit_project) + filename = ".gitreview" + + jinja_env = Environment(loader=PackageLoader("lftools.git"), autoescape=select_autoescape()) + template = jinja_env.get_template("gitreview") + content = template.render(fqdn=fqdn, project_name=gerrit_project, default_branch=self.default_branch) + log.debug(".gitreview contents:\n{}".format(content)) + + self.add_file(filename, content) + commit_msg = "Chore: Automation adds {}".format(filename) + self.commit(commit_msg, issue_id, push=True) diff --git a/lftools/git/templates/gitreview b/lftools/git/templates/gitreview new file mode 100644 index 00000000..5c83d343 --- /dev/null +++ b/lftools/git/templates/gitreview @@ -0,0 +1,5 @@ +[gerrit] +host={{ fqdn }} +port=29418 +project={{ project_name }} +defaultbranch={{ default_branch }} diff --git a/lftools/git/templates/project.yaml b/lftools/git/templates/project.yaml new file mode 100644 index 00000000..59d8ff13 --- /dev/null +++ b/lftools/git/templates/project.yaml @@ -0,0 +1,15 @@ +--- +project: + name: {{ project_name_dashed }}-project-view + project-name: {{ project_name_dashed }} + views: + - project-view + +project: + name: {{ project_name_dashed }}-info + project: {{ project_name }} + project-name: {{ project_name_dashed }} + build-node: {{ buildnode }} + branch: {{ default_branch }} + jobs: + - gerrit-info-yaml-verify diff --git a/releasenotes/notes/git-native-gerrit-747f772ddd1a9a2c.yaml b/releasenotes/notes/git-native-gerrit-747f772ddd1a9a2c.yaml new file mode 100644 index 00000000..15ad1589 --- /dev/null +++ b/releasenotes/notes/git-native-gerrit-747f772ddd1a9a2c.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add git-native functions for common Gerrit actions. diff --git a/requirements.txt b/requirements.txt index 2296e127..030df738 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,11 @@ dnspython docker==4.2.2 email-validator filelock +GitPython httplib2 identify idna +jinja2 jsonschema lxml multi-key-dict diff --git a/tests/fixtures/git/commit-msg b/tests/fixtures/git/commit-msg new file mode 100644 index 00000000..e3deb93c --- /dev/null +++ b/tests/fixtures/git/commit-msg @@ -0,0 +1,63 @@ +#!/bin/sh +# From Gerrit Code Review 3.2.3 +# +# Part of Gerrit Code Review (https://www.gerritcodereview.com/) +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# avoid [[ which is not POSIX sh. +if test "$#" != 1 ; then + echo "$0 requires an argument." + exit 1 +fi + +if test ! -f "$1" ; then + echo "file does not exist: $1" + exit 1 +fi + +# Do not create a change id if requested +if test "false" = "`git config --bool --get gerrit.createChangeId`" ; then + exit 0 +fi + +# $RANDOM will be undefined if not using bash, so don't use set -u +random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin) +dest="$1.tmp.${random}" + +trap 'rm -f "${dest}"' EXIT + +if ! git stripspace --strip-comments < "$1" > "${dest}" ; then + echo "cannot strip comments from $1" + exit 1 +fi + +if test ! -s "${dest}" ; then + echo "file is empty: $1" + exit 1 +fi + +# Avoid the --in-place option which only appeared in Git 2.8 +# Avoid the --if-exists option which only appeared in Git 2.15 +if ! git -c trailer.ifexists=doNothing interpret-trailers \ + --trailer "Change-Id: I${random}" < "$1" > "${dest}" ; then + echo "cannot insert change-id line in $1" + exit 1 +fi + +if ! mv "${dest}" "$1" ; then + echo "cannot mv ${dest} to $1" + exit 1 +fi diff --git a/tests/test_git.py b/tests/test_git.py new file mode 100644 index 00000000..70dc3923 --- /dev/null +++ b/tests/test_git.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2022 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Test git command.""" + +import os + +import pytest + +from lftools.git.gerrit import Gerrit, Repo, gerrit_api + +FIXTURE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures") + + +@pytest.fixture +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git")) +def mock_init(mocker, datafiles): + creds = { + "authtype": "basic", + "username": "myname", + "password": "mypass", + "endpoint": "http://gerrit.example.com/r/a/", + "email": "test@example.com", + } + + # Clone a sample ci-management repo for use with tests + remote = "https://gerrit.acumos.org/r/ci-management" + ciman_dir = os.path.join(str(datafiles), "ci-management") + os.makedirs(ciman_dir) + os.chdir(ciman_dir) + Repo.clone_from(remote, ciman_dir) + Repo.init(ciman_dir) + + mocker.patch("tempfile.mkdtemp", return_value=ciman_dir) + mocker.patch.object(Gerrit, "get_commit_hook") + Gerrit.get_commit_hook.start() # Needed for mocker.stopall() below + mocker.patch.object(Repo, "clone_from") + + git = Gerrit(creds=creds, fqdn="gerrit.example.com", project="test") + + Gerrit.get_commit_hook.assert_called_once() + Repo.clone_from.assert_called_once_with("https://myname:mypass@gerrit.example.com/r/a/test", ciman_dir) + mocker.stopall() + return git + + +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git")) +def test_get_commit_hook(mock_init, responses, datafiles): + os.chdir(str(datafiles)) + ciman_dir = os.path.join(str(datafiles), "ci-management") + hook_url = "http://gerrit.example.com/tools/hooks/commit-msg" + with open("commit-msg", "r") as hook: + hook_text = hook.read() + responses.add(responses.GET, hook_url, hook_text) + mock_init.get_commit_hook("http://gerrit.example.com", ciman_dir) + with open("ci-management/.git/hooks/commit-msg", "r") as new_hook: + assert hook_text == new_hook.read() + + +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git")) +def test_add_info_job(mock_init, datafiles, mocker): + fqdn = "gerrit.example.com" + gerrit_project = "project/subproject" + issue_id = "TEST-123" + agent = "" + commit_msg = "Chore: Automation adds project-subproject.yaml" + filepath = os.path.join(mock_init.repo.working_tree_dir, "jjb/project-subproject/project-subproject.yaml") + content = """--- +project: + name: project-subproject-project-view + project-name: project-subproject + views: + - project-view + +project: + name: project-subproject-info + project: project/subproject + project-name: project-subproject + build-node: centos7-builder-2c-1g + branch: master + jobs: + - gerrit-info-yaml-verify""" + + mocker.patch.object(Gerrit, "add_file") + mocker.patch.object(Gerrit, "commit") + + mock_init.add_info_job(fqdn, gerrit_project, issue_id, agent) + + Gerrit.add_file.assert_called_once_with(filepath, content) + Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True) + + +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git")) +def test_add_git_review(mock_init, datafiles, mocker): + fqdn = "gerrit.example.com" + gerrit_project = "project/subproject" + issue_id = "TEST-123" + commit_msg = "Chore: Automation adds .gitreview" + filepath = ".gitreview" + content = """[gerrit] +host=gerrit.example.com +port=29418 +project=project/subproject +defaultbranch=master""" + + mocker.patch.object(Gerrit, "add_file") + mocker.patch.object(Gerrit, "commit") + mocker.patch.object(gerrit_api, "sanity_check") + + mock_init.add_git_review(fqdn, gerrit_project, issue_id) + + Gerrit.add_file.assert_called_once_with(filepath, content) + Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True)