From a5438234d6ea32897b97a47044b0999fe501c532 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Tue, 3 Mar 2020 13:20:42 -0500 Subject: [PATCH] Add templates Tox + SonarQube Jenkins plugin New templates gerrit-tox-sonarqube and github-tox-sonarqube use the SonarQube Jenkins plug-in to analyze Python code and publish the results. Modeled after the generic gerrit/github-sonar templates. The new templates eliminate the need for mvn-settings in the job configuration and the need for a pom.xml file in the repo. Change-Id: I6d5bc5200937a0230c5d8eb0c522941d2affcee4 Issue-ID: RELENG-2767 Signed-off-by: Lott, Christopher (cl778h) --- .jjb-test/lf-python-jobs.yaml | 4 +- docs/jjb/lf-python-jobs.rst | 128 +++++++++++++++++++ jjb/lf-python-jobs.yaml | 141 +++++++++++++++++++++ .../notes/cmake-sonarqube-4cb43bea721775987.yaml | 8 ++ 4 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/cmake-sonarqube-4cb43bea721775987.yaml diff --git a/.jjb-test/lf-python-jobs.yaml b/.jjb-test/lf-python-jobs.yaml index bb7aadce..49f05ec0 100644 --- a/.jjb-test/lf-python-jobs.yaml +++ b/.jjb-test/lf-python-jobs.yaml @@ -6,8 +6,9 @@ - gerrit-pypi-merge - gerrit-pypi-release-merge - gerrit-pypi-release-verify - - gerrit-pypi-verify - gerrit-tox-sonar + - gerrit-tox-sonarqube + - gerrit-pypi-verify project-name: gerrit-python @@ -19,6 +20,7 @@ - github-pypi-release-merge - github-pypi-release-verify - github-tox-sonar + - github-tox-sonarqube - github-pypi-verify project-name: github-python diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index 117ab387..eb110095 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -111,6 +111,8 @@ Sonar scans for Python based repos. This job invokes tox to run tests and gather coverage statistics from the test results, then invokes Maven to publish the results to either a Sonar server or SonarCloud. +**Deprecated**, new projects should use Tox Sonarqube. + To get the Sonar coverage results, file tox.ini must exist and contain coverage commands to run. @@ -213,6 +215,132 @@ https://docs.sonarqube.org/display/PLUG/Python+Coverage+Results+Import .. comment Stop ignoring +Tox SonarQube +------------- + +The SonarQube job invokes tox to run tests and generate code-coverage +statistics, then runs the SonarQube Scanner Jenkins plug-in to analyze +code, gather coverage data, and upload the results to a SonarQube server +such as SonarCloud.io. Optionally runs a shell script before tox. + +Requires ``SonarQube Scanner for Jenkins`` + +This job runs on the master branch because the basic Sonar configuration +does not support multi-branch. + +Plug-in configurations + Manage Jenkins --> Configure System --> SonarQube servers + - Name: Sonar (fixed) + - Server URL: https://sonar.project.org/ or https://sonarcloud.io + - Server authentication token: none for local, API token (saved as + a "secret text" credential) for Sonarcloud + + Manage Jenkins --> Global Tool Configuration --> SonarQube Scanner + - Name: SonarQube Scanner (fixed) + - Install automatically + - Select latest version + +:Template Names: + + - {project-name}-tox-sonarqube + - gerrit-tox-sonarqube + - github-tox-sonarqube + +:Comment Trigger: ``run-sonar`` + +:Required parameters: + + :build-node: The node to run the build on. + (Commonly in defaults.yaml) + :jenkins-ssh-credential: Credential to use for SSH. + (Commonly in defaults.yaml) + :project: The git repository name. + :project-name: Prefix used to name jobs. + +.. comment Start ignoring WriteGoodLintBear + +:Optional Parameters: + + :archive-artifacts: Pattern for files to archive to the logs server + (default: '\*\*/\*.log') + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :cron: Cron schedule when to trigger the job. This parameter also + supports multiline input via YAML pipe | character in cases where + one may want to provide more than 1 cron timer. (default: @weekly) + :disable-job: Whether to disable the job (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :github-url: URL for Github. (default: https://github.com) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :pre-build-script: Shell script to run before tox. Useful for setting up + dependencies. (default: a string with a shell comment) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python3) + :sonar-additional-args: Command line arguments. (default: '') + :sonar-java-opts: JVM options. For example, use option -Xmx + to increase the memory size limit. (default: '') + :sonar-project-file: The file name with Sonar configuration properties + (default: sonar-project.properties) + :sonar-properties: Sonar configuration properties. (default: '') + :sonar-task: Sonar task to run. (default: '') + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + +.. comment Stop ignoring + +.. note:: A job definition must provide one of the optional parameters + ``sonar-project-file`` and ``sonar-properties``; they cannot both be + empty. Set Sonar properties directly in the job definition by setting + the ``sonar-project-file`` property to ``""`` and adding all properties + under ``sonar-properties``. + +:Required Sonar Properties: + + - sonar.login: The API token for authentication at SonarCloud. + Commonly defined as key "sonarcloud_api_token" in defaults.yaml. + - sonar.organization: The umbrella project name; e.g., "opendaylight". + Commonly defined as key "sonarcloud_project_organization" in defaults.yaml. + - sonar.projectName: The git repository name without slashes; e.g., "infrautils". + - sonar.projectKey: The globally unique key for the report in SonarCloud. Most + teams use the catenation of sonar.organization, an underscore, and + sonar.projectName; e.g., "opendaylight_infrautils". + +:Optional Sonar Properties: + + - sonar.cfamily.gcov.reportsPath: directory with GCOV output files + - Documentation of SonarQube properties is here: + https://docs.sonarqube.org/latest/analysis/overview/ + + +Example job definition +^^^^^^^^^^^^^^^^^^^^^^ + +The following example defines a job for a basic Python project. This definition +uses configuration parameters in the umbrella project's defaults.yaml file. + +.. code-block:: yaml + + - project: + name: my-package-sonar + project: my/package + project-name: my-package + sonar-project-file: "" + sonar-properties: | + sonar.login={sonarcloud_api_token} + sonar.projectKey={sonarcloud_project_organization}_{project-name} + sonar.projectName={project-name} + sonar.organization={sonarcloud_project_organization} + sonar.sourceEncoding=UTF-8 + sonar.sources=mypackage + sonar.exclusions=tests/*,setup.py + sonar.python.coverage.reportPaths=coverage.xml + jobs: + - gerrit-tox-sonarqube + Tox Verify ---------- diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index d215a673..59ae5494 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -418,6 +418,147 @@ - "{branch}" included-regions: "{obj:github_included_regions}" +################# +# Tox SonarQube # +################# + +- lf_tox_sonarqube: &lf_tox_sonarqube + name: lf-tox-sonarqube + + ###################### + # Default parameters # + ###################### + + archive-artifacts: > + **/*.log + branch: master # Sonar should always be run on master branch + build-days-to-keep: 7 + build-timeout: 15 + cron: "@weekly" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + parallel: false + pre-build-script: "# pre-build script goes here" + python-version: python3 + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + tox-dir: "." + tox-envs: "" + # Sonar properties + sonar-additional-args: "" + sonar-java-opts: "" + sonar-project-file: "sonar-project.properties" + sonar-properties: "" + sonar-task: "" + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + disabled: "{disable-job}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-tox-parameters: + tox-dir: "{tox-dir}" + tox-envs: "{tox-envs}" + - string: + name: ARCHIVE_ARTIFACTS + default: "{archive-artifacts}" + description: Artifacts to archive to the logs server. + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - lf-infra-sonar: + sonar-task: "{sonar-task}" + sonar-project-file: "{sonar-project-file}" + sonar-properties: "{sonar-properties}" + sonar-java-opts: "{sonar-java-opts}" + sonar-additional-args: "{sonar-additional-args}" + + publishers: + - lf-infra-publish + +- job-template: + name: "{project-name}-tox-sonarqube" + id: gerrit-tox-sonarqube + concurrent: false + <<: *lf_tox_sonarqube + + scm: + - lf-infra-gerrit-scm: + branch: "$GERRIT_BRANCH" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + + triggers: + - timed: "{obj:cron}" + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+run-sonar\s*$' + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**/{branch}" + +- job-template: + name: "{project-name}-tox-sonarqube" + id: github-tox-sonarqube + concurrent: false + <<: *lf_tox_sonarqube + + scm: + - lf-infra-github-scm: + branch: "$sha1" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + + triggers: + - github-pull-request: + trigger-phrase: "^run-sonar$" + only-trigger-phrase: false + status-context: "Tox Sonarqube" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + ############## # Tox Common # ############## diff --git a/releasenotes/notes/cmake-sonarqube-4cb43bea721775987.yaml b/releasenotes/notes/cmake-sonarqube-4cb43bea721775987.yaml new file mode 100644 index 00000000..8c4f740b --- /dev/null +++ b/releasenotes/notes/cmake-sonarqube-4cb43bea721775987.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + New templates gerrit-tox-sonarqube and github-tox-sonarqube + use the SonarQube Jenkins plug-in to analyze Python code and + publish the results. Modeled after the generic gerrit/github-sonar + templates. The new templates eliminate the need for mvn-settings in + the job configuration and the need for a pom.xml file in the repo. -- 2.16.6