From abb522a78975595e92197da01d72fbd2a1175085 Mon Sep 17 00:00:00 2001 From: Jessica Wagantall Date: Mon, 15 Aug 2022 12:30:10 -0700 Subject: [PATCH] Feat: Add CLI Sonar scanner job Add gerrit-cli-sonar and github-cli-sonar scanner job for non maven based repos. This job downloads a specific Sonar CLI version and runs sonnar-scanner on the code to produce a report which is pushed in SonarCloud. Issue: RELENG-4427 Co-authored-by: Jessica Wagantall Co-authored-by: Matthew Watkins Signed-off-by: Matthew Watkins Signed-off-by: Jessica Wagantall Change-Id: If02a2c78bddacdcc273fb5a0b9f60b99d2da221d --- .jjb-test/defaults.yaml | 4 + .jjb-test/lf-python-jobs.yaml | 2 + docs/jjb/lf-macros.rst | 5 + docs/jjb/lf-python-jobs.rst | 87 +++++++++++ jjb/lf-macros.yaml | 32 ++++ jjb/lf-python-jobs.yaml | 166 +++++++++++++++++++++ .../notes/sonar-cli-job-ec1ed9f61978a297.yaml | 7 + shell/sonar-cli.sh | 28 ++++ 8 files changed, 331 insertions(+) create mode 100644 releasenotes/notes/sonar-cli-job-ec1ed9f61978a297.yaml create mode 100644 shell/sonar-cli.sh diff --git a/.jjb-test/defaults.yaml b/.jjb-test/defaults.yaml index b8c798e1..a3c8e077 100644 --- a/.jjb-test/defaults.yaml +++ b/.jjb-test/defaults.yaml @@ -33,3 +33,7 @@ build-node: build-vm failure-notification: "fake@example.org" failure-notification-prefix: "[releng]" + + # Sonar CLI + sonarcloud-project-organization: org + sonarcloud-project-key: project diff --git a/.jjb-test/lf-python-jobs.yaml b/.jjb-test/lf-python-jobs.yaml index fcae63a7..1bd7dbc7 100644 --- a/.jjb-test/lf-python-jobs.yaml +++ b/.jjb-test/lf-python-jobs.yaml @@ -3,6 +3,7 @@ name: gerrit-python-jobs jobs: - "{project-name}-python-jobs" + - gerrit-cli-sonar - gerrit-pypi-merge - gerrit-pypi-release-merge - gerrit-pypi-release-verify @@ -18,6 +19,7 @@ name: github-python-jobs jobs: - "{project-name}-github-python-jobs" + - github-cli-sonar - github-pypi-merge - github-pypi-release-merge - github-pypi-release-verify diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 66a25a8e..0ad15060 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -435,6 +435,11 @@ lf-infra-node-parameters Provides parameters needed by NodeJS and NPM. Use in any jobs that need to run NodeJS or NPM. +lf-infra-sonar-cli-parameters +----------------------------- + +Provides parameters needed by Python jobs to run the SonarCloud CLI. + lf-infra-tox-parameters ----------------------- diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index fde103df..36f2779b 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -130,6 +130,93 @@ does not support multi-branch. :tox-envs: Tox environment with the appropriate pip freeze invocation. (default: 'clm') +Python Sonar with CLI +--------------------- + +Sonar scans for non Maven based repos. This job downloads the CLI +and runs a scan to publish the report to SonarCloud. + +As suggested in SonarCoud's UI instructions, the job downloads and unzips +the Sonar SLI and executes a sonar-scanner command to process the +report. + +For more details refer to sonar documentation: + +https://docs.sonarcloud.io/advanced-setup/ci-based-analysis/sonarscanner-cli/ + +:Template Names: + + - {project-name}-cli-sonar + - gerrit-cli-sonar + - github-cli-sonar + +:Comment Trigger: **run-sonar** post a comment with the trigger to launch + this job manually. Do not include any other text or vote in the + same comment. + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally should + get configured in defaults.yaml) + :mvn-settings: The name of the settings file with credentials for the project. + +.. comment Start ignoring WriteGoodLintBear + +:Optional parameters: + + :branch: Git branch, should be master (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 60) + :cron: Cron schedule when to trigger the job. This parameter also + supports multiline input via YAML pipe | character in cases where + one may want to provide more than 1 cron timer. (default: H 11 * * * + to run once a day) + :disable-job: Whether to disable the job (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :github-url: URL for Github. (default: https://github.com) + :java-version: Version of Java to use for the build. (default: openjdk11) + :mvn-global-settings: The name of the Maven global settings to use + :mvn-goals: The Maven goal to run first. (default: validate) + :mvn-version: Version of maven to use. (default: mvn35) + :parallel: If different from false, try pass this parameter to tox option + "--parallel" to parallelize jobs in the envlist (and then activate the + option "--parallel-live" to display output in logs). + Possible values are "auto" (equivalent to "true" for legacy), + "all" or any integer. Any other value is equivalent to "false". + (default: false, in series) + :pre-build-script: Shell script to execute before the Sonar builder. + For example, install prerequisites or move files to the repo root. + (default: a string with a shell comment) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python2) + :sonarcloud-project-key: SonarCloud project key. (default: '') + :sonarcloud-project-organization: SonarCloud project organization. + (default: '') + :sonarcloud-api-token: SonarCloud API Token. (default: '') + :sonar-scanner-home: Sonar scanner home directory. + (default: $WORKSPACE/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux) + :sonar-scanner-opts: Sonar scanner Java options. (default: '-server') + :sonar-scanner-version: Version of sonar scanner to use. (default: 4.7.0.2747) + :stream: Keyword used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_sonar_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://jenkins-job-builder.readthedocs.io/en/latest/triggers.html#triggers.gerrit + +.. comment Stop ignoring Python Sonar with Tox --------------------- diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 57edc3ab..667e75b7 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -578,6 +578,38 @@ default: "{node-version}" description: Version of NodeJS to install. +- parameter: + name: lf-infra-sonar-cli-parameters + parameters: + - string: + name: TOX_DIR + default: "{tox-dir}" + description: | + Path to directory containing tox.ini file. + - string: + name: TOX_ENVS + default: "{tox-envs}" + description: | + Tox environments to run build against. + - string: + name: SONAR_SCANNER_VERSION + default: "{sonar-scanner-version}" + description: | + Version of sonar scanner to use. + Example: 4.7.0.2747 + - string: + name: SONAR_SCANNER_HOME + default: "{sonar-scanner-home}" + description: | + Sonar scanner home directory. + Example: $WORKSPACE/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux + - string: + name: SONAR_SCANNER_OPTS + default: "{sonar-scanner-opts}" + description: | + Sonar scanner Java options. + Example: -server + - parameter: name: lf-infra-tox-parameters parameters: diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index 40fa9bf8..a0cf6bf1 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -222,6 +222,172 @@ white-list-target-branches: - "{branch}" +######################### +# Python Sonar with CLI # +######################### + +- lf_cli_sonar: &lf_cli_sonar + name: lf-cli-sonar + + ###################### + # Default parameters # + ###################### + + branch: master # Sonar should always be run on master branch + build-days-to-keep: 7 + build-timeout: 60 + cron: "H H * * *" # run daily + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + # SonarCloud scan using jdk8 will become deprecated by Oct, 2020 + # Projects not compatible with jdk11 can set java-version to something else + java-version: openjdk11 + parallel: true + pre-build-script: "# pre-build script goes here" + python-version: python3 + stream: master + sonar-scanner-version: "4.7.0.2747" + sonar-scanner-home: "$WORKSPACE/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux" + sonar-scanner-opts: "-server" + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + tox-dir: "." + tox-envs: "" + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: ".*" + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - ".*" + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-sonar-cli-parameters: + tox-dir: "{tox-dir}" + tox-envs: "{tox-envs}" + sonar-scanner-version: "{sonar-scanner-version}" + sonar-scanner-home: "{sonar-scanner-home}" + sonar-scanner-opts: "{sonar-scanner-opts}" + - string: + name: ARCHIVE_ARTIFACTS + default: "{archive-artifacts}" + description: Artifacts to archive to the logs server. + wrappers: + - credentials-binding: + - text: + credential-id: sonar-token-smo-ves + variable: SONAR_TOKEN + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + # With Sonar CLI + - inject: + properties-content: | + SONARCLOUD_PROJECT_ORGANIZATION={sonarcloud-project-organization} + SONARCLOUD_PROJECT_KEY={sonarcloud-project-key} + - shell: !include-raw-escape: ../shell/sonar-cli.sh + + publishers: + - lf-infra-publish + +- job-template: + name: "{project-name}-cli-sonar" + id: gerrit-cli-sonar + <<: *lf_python_common + <<: *lf_cli_sonar + + ###################### + # Default parameters # + ###################### + + gerrit_sonar_triggers: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+run-sonar\s*$' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: $GERRIT_REFSPEC + branch: $GERRIT_BRANCH + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + + triggers: + - timed: "{obj:cron}" + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_sonar_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + +- job-template: + name: "{project-name}-cli-sonar" + id: github-cli-sonar + <<: *lf_python_common + <<: *lf_cli_sonar + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^run-sonar$" + only-trigger-phrase: false + status-context: "Python Sonar" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + ######################### # Python Sonar with Tox # ######################### diff --git a/releasenotes/notes/sonar-cli-job-ec1ed9f61978a297.yaml b/releasenotes/notes/sonar-cli-job-ec1ed9f61978a297.yaml new file mode 100644 index 00000000..c192dd3a --- /dev/null +++ b/releasenotes/notes/sonar-cli-job-ec1ed9f61978a297.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add gerrit-cli-sonar and github-cli-sonar scanner job for + non maven based repos. This job downloads a specific Sonar + CLI version and runs sonnar-scanner on the code to produce + a report which is pushed in SonarCloud. diff --git a/shell/sonar-cli.sh b/shell/sonar-cli.sh new file mode 100644 index 00000000..d92334e6 --- /dev/null +++ b/shell/sonar-cli.sh @@ -0,0 +1,28 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2022 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Non-Maven Sonar CLI +echo "---> sonar-cli.sh" + +SCANNER_BIN="sonar-scanner-${SONAR_SCANNER_VERSION}-linux" +export SONAR_SCANNER_HOME="${WORKSPACE}/.sonar/${SCANNER_BIN}" +export PATH="${SONAR_SCANNER_HOME}/bin:${PATH}" + +curl --create-dirs -sSLo "${WORKSPACE}"/.sonar/sonar-scanner.zip \ +https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/\ +sonar-scanner-cli-"${SONAR_SCANNER_VERSION}"-linux.zip +unzip -o "${WORKSPACE}"/.sonar/sonar-scanner.zip -d "${WORKSPACE}"/.sonar/ + +echo "Running sonar-scanner" +sonar-scanner \ + -Dsonar.organization="${SONARCLOUD_PROJECT_ORGANIZATION}" \ + -Dsonar.projectKey="${SONARCLOUD_PROJECT_ORGANIZATION}_${SONARCLOUD_PROJECT_KEY}" \ + -Dsonar.sources=. \ + -Dsonar.host.url=https://sonarcloud.io -- 2.16.6