From 1d78aa9927898b7be4d3305aa4cea8291916a961 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Fri, 4 Mar 2022 12:21:19 -0800 Subject: [PATCH] Feat: New method add_maven_config for JCasC config This method allows for adding settings files for new projects via JCasC, rather than the old method of using a Groovy script to manually add the file to Jenkins. This also removes the __del__ function. The function was not entirely necessary, and was causing issues with unit testing. All work is done in /tmp, so these files will be automatically cleaned by the system. Issue: RELENG-3893 Change-Id: I03511b9c3e1c9f9c9f79054bd6f8991cd274e8f5 Signed-off-by: Eric Ball --- docs/commands/gerrit.rst | 20 ++++ lftools/cli/gerrit.py | 38 ++++++++ lftools/git/gerrit.py | 106 ++++++++++++++++++--- lftools/git/templates/config-params.yaml | 3 + .../git/templates/serverCredentialMappings.yaml | 18 ++++ .../notes/jcasc-add-settings-487d8fb321a8a4a4.yaml | 6 ++ tests/test_git.py | 59 ++++++++++++ 7 files changed, 235 insertions(+), 15 deletions(-) create mode 100644 lftools/git/templates/config-params.yaml create mode 100644 lftools/git/templates/serverCredentialMappings.yaml create mode 100644 releasenotes/notes/jcasc-add-settings-487d8fb321a8a4a4.yaml diff --git a/docs/commands/gerrit.rst b/docs/commands/gerrit.rst index 0b7f99ac..bbe0c9a0 100644 --- a/docs/commands/gerrit.rst +++ b/docs/commands/gerrit.rst @@ -81,3 +81,23 @@ addinfojob username = lfid2 password = password2 signed_off_by = Your Name + +addmavenconfig +-------------- +.. program-output:: lftools gerrit addmavenconfig --help + + +An example of the lftools.ini entry for a Gerrit server making use of a full +configuration: + +.. code-block:: none + + [gerrit.example.org] + username = lfid + password = password + signed_off_by = Your Name + endpoint = https://gerrit.example.org/ + default_servers = releases,snapshots,staging,site + nexus3 = nexus3.example.org + nexus3_ports = 10001,10002,10003,10004 + additional_credentials = {"docker.io": "dockerhub-cred", "nexus-iq": "nexus-iq-cred"} diff --git a/lftools/cli/gerrit.py b/lftools/cli/gerrit.py index 61bce6d2..44f9045a 100644 --- a/lftools/cli/gerrit.py +++ b/lftools/cli/gerrit.py @@ -184,6 +184,43 @@ def list_project_inherits_from(ctx, gerrit_fqdn, gerrit_project): log.info(data) +@click.command(name="addmavenconfig") +@click.argument("gerrit_fqdn") +@click.argument("gerrit_project") +@click.argument("jjbrepo") +@click.option("--issue_id", type=str, required=False, help="For projects that enforce an issue id for changesets") +@click.option("--nexus3", type=str, required=False, help="Specify a Nexus 3 server, e.g. nexus3.example.org") +@click.option( + "--nexus3_ports", + type=str, + required=False, + help="Comma-separated list of ports supported by the Nexus 3 server specified", +) +@click.pass_context +def addmavenconfig(ctx, gerrit_fqdn, gerrit_project, jjbrepo, issue_id, nexus3, nexus3_ports): + """Add maven config file for JCasC. + + \b + The following options can be set in the gerrit server's entry in lftools.ini: + * default_servers: Comma-separated list of servers using the + credential. Default: releases,snapshots,staging,site + * additional_credentials: JSON-formatted string containing + servername:credentialname pairings. This should be on a single line, + without quotes surrounding the string. + * nexus3: The nexus3 server url for a given project. + * nexus3_ports: Comma-separated list of ports used by Nexus3. + Default: 10001,10002,10003,10004 + + \f + The 'b' escape character above disables auto-formatting, so that the help + text will follow the exact formatting used here. The 'f' escape is to keep + this from appearing in the --help text. + https://click.palletsprojects.com/en/latest/documentation/ + """ + git = git_gerrit(fqdn=gerrit_fqdn, project=jjbrepo) + git.add_maven_config(gerrit_fqdn, gerrit_project, issue_id, nexus3, nexus3_ports) + + gerrit_cli.add_command(addinfojob) gerrit_cli.add_command(addfile) gerrit_cli.add_command(addgitreview) @@ -193,3 +230,4 @@ gerrit_cli.add_command(abandonchanges) gerrit_cli.add_command(create_saml_group) gerrit_cli.add_command(list_project_permissions) gerrit_cli.add_command(list_project_inherits_from) +gerrit_cli.add_command(addmavenconfig) diff --git a/lftools/git/gerrit.py b/lftools/git/gerrit.py index 6e1b243b..1d4a5e25 100644 --- a/lftools/git/gerrit.py +++ b/lftools/git/gerrit.py @@ -10,9 +10,10 @@ """Gerrit git interface.""" +import configparser +import json import logging import os -import shutil import tempfile import urllib @@ -66,12 +67,6 @@ class Gerrit: default_ref = self.repo.git.rev_parse("origin/HEAD", abbrev_ref=True) self.default_branch = default_ref.split("/")[-1] - def __del__(self): - try: - shutil.rmtree(self.repo.working_tree_dir) - except FileNotFoundError: - log.info("Could not remove working directory {}".format(self.repo.working_tree_dir)) - def get_commit_hook(self, endpoint, working_dir): """Pulls in the Gerrit server's commit hook to add a changeId.""" hook_url = urllib.parse.urljoin(endpoint, "tools/hooks/commit-msg") @@ -92,19 +87,26 @@ class Gerrit: os.chmod(commit_msg_hook_path, 0o755) def add_file(self, filepath, content): - """Add a file to the current git repo. - - Example: - - local_path /tmp/INFO.yaml - file_path="somedir/example-INFO.yaml" - """ + """Add a file to the current git repo.""" if filepath.find("/") >= 0: - os.makedirs(os.path.split(filepath)[0]) + try: + os.makedirs(os.path.split(filepath)[0]) + except FileExistsError: + pass with open(filepath, "w") as newfile: newfile.write(content) self.repo.git.add(filepath) + def add_symlink(self, filepath, target): + """Add a symlink to the current git repo.""" + try: + os.symlink(target, filepath) + except FileExistsError: + if not os.path.islink(filepath): + log.error("{} exists and is not a symlink".format(filepath)) + return + self.repo.git.add(filepath) + def commit(self, commit_msg, issue_id, push=False): """Commit staged changes. @@ -190,3 +192,77 @@ class Gerrit: self.add_file(filename, content) commit_msg = "Chore: Automation adds {}".format(filename) self.commit(commit_msg, issue_id, push=True) + + def add_maven_config(self, fqdn, gerrit_project, issue_id, nexus3_url="", nexus3_ports=""): + """Add the four required JCasC files to create settings for a new project.""" + project_dashed = gerrit_project.replace("/", "-") + params_path = "config-params.yaml" + creds_path = "serverCredentialMappings.yaml" + content_path = "content" + sb_creds_path = "serverCredentialMappings.sandbox.yaml" + nexus3_ports = nexus3_ports.split(",") + + try: + default_servers = config.get_setting(self.fqdn, "default_servers") + default_servers = default_servers.split(",") + except configparser.NoOptionError: + default_servers = ["releases", "snapshots", "staging", "site"] + + additional_credentials = "" + try: + credential_json = config.get_setting(self.fqdn, "additional_credentials") + additional_credentials = json.loads(credential_json) + except configparser.NoOptionError: + log.debug("No additional credentials found") + except json.decoder.JSONDecodeError: + log.error( + 'Improperly formatted JSON in "additional_credentials". ' + + "Please ensure that all credentials are on a single line, and are not quoted." + ) + + if not nexus3_url: + try: + nexus3_url = config.get_setting(self.fqdn, "nexus3") + except configparser.NoOptionError: + pass # If no url is passed in or present in lftools.ini, skip it. + + if nexus3_url and not nexus3_ports: + try: + nexus3_ports = config.get_setting(self.fqdn, "nexus3_ports") + nexus3_ports = nexus3_ports.split(",") + except configparser.NoOptionError: + nexus3_ports = ["10001", "10002", "10003", "10004"] + + jinja_env = Environment(loader=PackageLoader("lftools.git"), autoescape=select_autoescape()) + template = jinja_env.get_template(params_path) + config_params_content = template.render(project_dashed=project_dashed) + log.debug("config-params.yaml contents:\n{}".format(config_params_content)) + + template = jinja_env.get_template(creds_path) + server_creds_content = template.render( + project_dashed=project_dashed, + default_servers=default_servers, + nexus3_url=nexus3_url, + nexus3_ports=nexus3_ports, + additional_credentials=additional_credentials, + ) + log.debug("config-params.yaml contents:\n{}".format(server_creds_content)) + + config_path = "jenkins-config/managed-config-files/mavenSettings/{}".format(project_dashed) + try: + os.makedirs(config_path) + except FileExistsError: + pass + + self.add_symlink( + os.path.join(config_path, content_path), "../../../managed-config-templates/mavenSettings-content" + ) + self.add_symlink( + os.path.join(config_path, sb_creds_path), + "../../../managed-config-templates/mavenSettings-serverCredentialMappings.sandbox.yaml", + ) + self.add_file(os.path.join(config_path, params_path), config_params_content) + self.add_file(os.path.join(config_path, creds_path), server_creds_content) + + commit_msg = "Chore: Automation adds {} config files".format(gerrit_project) + self.commit(commit_msg, issue_id, push=True) diff --git a/lftools/git/templates/config-params.yaml b/lftools/git/templates/config-params.yaml new file mode 100644 index 00000000..f0747212 --- /dev/null +++ b/lftools/git/templates/config-params.yaml @@ -0,0 +1,3 @@ +--- +name: "{{ project_dashed }}" +comment: "{{ project_dashed }}" diff --git a/lftools/git/templates/serverCredentialMappings.yaml b/lftools/git/templates/serverCredentialMappings.yaml new file mode 100644 index 00000000..836021b5 --- /dev/null +++ b/lftools/git/templates/serverCredentialMappings.yaml @@ -0,0 +1,18 @@ +--- +serverCredentialMappings: +{%- for server in default_servers %} + - serverId: "{{ server }}" + credentialsId: "{{ project_dashed }}" +{%- endfor -%} +{%- if nexus3_url and nexus3_ports -%} + {% for port in nexus3_ports %} + - serverId: "{{ nexus3_url }}:{{port}}" + credentialsId: "{{ project_dashed }}" + {%- endfor -%} +{%- endif -%} +{%- if additional_credentials -%} + {%- for server, cred in additional_credentials.items() %} + - serverId: "{{ server }}" + credentialsId: "{{ cred }}" + {%- endfor -%} +{% endif %} diff --git a/releasenotes/notes/jcasc-add-settings-487d8fb321a8a4a4.yaml b/releasenotes/notes/jcasc-add-settings-487d8fb321a8a4a4.yaml new file mode 100644 index 00000000..f8e8e4c8 --- /dev/null +++ b/releasenotes/notes/jcasc-add-settings-487d8fb321a8a4a4.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + New method git.add_maven_config allows for adding settings files for new + projects via JCasC, rather than the old method of using a Groovy script to + manually add the file to Jenkins. diff --git a/tests/test_git.py b/tests/test_git.py index 70dc3923..341ad716 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -9,6 +9,7 @@ ############################################################################## """Test git command.""" +import configparser import os import pytest @@ -117,3 +118,61 @@ defaultbranch=master""" Gerrit.add_file.assert_called_once_with(filepath, content) Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True) + + +@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git")) +def test_add_maven_config(mock_init, datafiles, mocker): + fqdn = "gerrit.example.com" + gerrit_project = "project/subproject" + issue_id = "TEST-123" + commit_msg = "Chore: Automation adds project/subproject config files" + + creds_path = "jenkins-config/managed-config-files/mavenSettings/project-subproject/serverCredentialMappings.yaml" + server_creds_content = """--- +serverCredentialMappings: + - serverId: "releases" + credentialsId: "project-subproject" + - serverId: "snapshots" + credentialsId: "project-subproject" + - serverId: "staging" + credentialsId: "project-subproject" + - serverId: "site" + credentialsId: "project-subproject\"""" + + get_setting_mock = mocker.patch("lftools.config.get_setting") + get_setting_mock.side_effect = configparser.NoOptionError(fqdn, "default_servers") + mocker.patch.object(Gerrit, "add_symlink") + mocker.patch.object(Gerrit, "add_file") + mocker.patch.object(Gerrit, "commit") + + # Test 1 # + mock_init.add_maven_config(fqdn, gerrit_project, issue_id) + Gerrit.add_file.assert_called_with(creds_path, server_creds_content) + Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True) + + # Test 2 # + server_creds_content += """ + - serverId: "nexus3.example.com:10001" + credentialsId: "project-subproject" + - serverId: "nexus3.example.com:10002" + credentialsId: "project-subproject\"""" + + mock_init.add_maven_config(fqdn, gerrit_project, issue_id, "nexus3.example.com", "10001,10002") + Gerrit.add_file.assert_called_with(creds_path, server_creds_content) + + # Test 3 # + get_setting_mock = mocker.patch("lftools.config.get_setting") + + def setting_response(*args, **kwargs): + if "additional_credentials" in args: + return '{"docker.io": "dockerhub-cred"}' + raise configparser.NoOptionError(fqdn, "default_servers") + + get_setting_mock.side_effect = setting_response + + server_creds_content += """ + - serverId: "docker.io" + credentialsId: "dockerhub-cred\"""" + + mock_init.add_maven_config(fqdn, gerrit_project, issue_id, "nexus3.example.com", "10001,10002") + Gerrit.add_file.assert_called_with(creds_path, server_creds_content) -- 2.16.6