From 550f3c5a4b09929a629c508b43cf4bb803c810de Mon Sep 17 00:00:00 2001 From: Jessica Wagantall Date: Mon, 6 Mar 2023 19:23:03 -0800 Subject: [PATCH] Feat: Add Python Snyk CLI Scanner jobs Introduce Python Snyk CLI scanner jobs. These jobs can be triggered to download the latest version of Snyk's CLI scanner and trigger a scan for Python based repos. These jobs produce a report which is published into Snyk's dashboard. These reports are fetched and reflected back into the LFX Security tool. Issue: RELENG-4609 Signed-off-by: Jessica Wagantall Change-Id: I5414d04ccc7472a6b3cd2576da1cb6bc36d1ea25 --- .jjb-test/lf-python-jobs.yaml | 2 + docs/jjb/lf-python-jobs.rst | 55 ++++++++ jjb/lf-python-jobs.yaml | 146 +++++++++++++++++++++ ...d-python-snyk-cli-scanner-92cb49fe8ca39c51.yaml | 7 + shell/snyk-cli-scanner-run.sh | 9 ++ 5 files changed, 219 insertions(+) create mode 100644 releasenotes/notes/add-python-snyk-cli-scanner-92cb49fe8ca39c51.yaml diff --git a/.jjb-test/lf-python-jobs.yaml b/.jjb-test/lf-python-jobs.yaml index 1bd7dbc7..8d274290 100644 --- a/.jjb-test/lf-python-jobs.yaml +++ b/.jjb-test/lf-python-jobs.yaml @@ -9,6 +9,7 @@ - gerrit-pypi-release-verify - gerrit-pypi-stage - gerrit-pypi-verify + - gerrit-python-snyk-cli - gerrit-tox-nexus-iq-clm - gerrit-tox-sonar - gerrit-tox-sonarqube @@ -25,6 +26,7 @@ - github-pypi-release-verify - github-pypi-stage - github-pypi-verify + - gerrit-python-snyk-cli - github-tox-nexus-iq-clm - github-tox-sonar - github-tox-sonarqube diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index c5cac4e0..6a3e338c 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -132,6 +132,61 @@ does not support multi-branch. :tox-envs: Tox environment with the appropriate pip freeze invocation. (default: 'clm') +Python Snyk CLI +--------------- + +Builds the code, downloads and runs a Snyk CLI scan of the code into the Snyk dashboard. + +:Template Names: + + - {project-name}-python-snyk-cli-{stream} + - gerrit-python-snyk-cli + - github-python-snyk-cli + +:Comment Trigger: run-snyk + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally configured in defaults.yaml) + :snyk-token-credential-id: Snyk API token to communicate with Jenkins. + :snyk-org-credential-id: Snyk organization ID. + +:Optional parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 60) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :pre-build-script: Shell script to execute before the Tox builder. + For example, install prerequisites or move files to the repo root. + (default: a string with a shell comment) + :parallel: If different from false, try pass this parameter to tox option + "--parallel" to parallelize jobs in the envlist (and then activate the + option "--parallel-live" to display output in logs). + Possible values are "auto" (equivalent to "true" for legacy), + "all" or any integer. Any other value is equivalent to "false". + (default: false, in series) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python3) + :snyk-cli-options: Additional Snyk CLI options. (default: '') + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://jenkins-job-builder.readthedocs.io/en/latest/triggers.html#triggers.gerrit + Python Sonar with CLI --------------------- diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index 1c711c1b..4e1dcf92 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -227,6 +227,152 @@ white-list-target-branches: - "{branch}" +################### +# Python Snyk CLI # +################### + +- lf_python_snyk_cli: &lf_python_snyk_cli + name: lf-python-snyk_cli + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 30 # 30 days for troubleshooting purposes + build-timeout: 60 + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + java-version: openjdk11 + parallel: false + pre-build-script: "# pre-build script goes here" + python-version: python3 + snyk-cli-options: "" + snyk-token-credential-id: snyk-token + snyk-org-credential-id: snyk-org + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + tox-dir: "." + tox-envs: "" + + gerrit_snyk_triggers: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+run-snyk\s*$' + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - string: + name: SNYK_CLI_OPTIONS + default: "{snyk-cli-options}" + description: Additional Snyk CLI commands and options + - lf-infra-tox-parameters: + tox-dir: "{tox-dir}" + tox-envs: "{tox-envs}" + + wrappers: + - credentials-binding: + - text: + credential-id: "{snyk-token-credential-id}" + variable: SNYK_TOKEN + - text: + credential-id: "{snyk-org-credential-id}" + variable: SNYK_ORG + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - lf-infra-snyk-cli-scanner + +- job-template: + name: "{project-name}-python-snyk-cli-{stream}" + id: gerrit-python-snyk-cli + # yamllint disable-line rule:key-duplicates + <<: *lf_python_snyk_cli + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + + triggers: + # Build weekly on Saturdays + - timed: "H H * * 6" + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_snyk_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + +- job-template: + name: "{project-name}-python-snyk-cli-{stream}" + id: github-python-snyk-cli + # yamllint disable-line rule:key-duplicates + <<: *lf_python_snyk_cli + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "" + branch: "refs/heads/{branch}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + # Build weekly on Saturdays + - timed: "H H * * 6" + - github-pull-request: + trigger-phrase: "^run-snyk$" + only-trigger-phrase: true + status-context: "SNYK scan" + permit-all: true + github-hooks: true + org-list: + - "{github-org}" + white-list: "{obj:github_pr_allowlist}" + admin-list: "{obj:github_pr_admin_list}" + white-list-target-branches: + - "{branch}" + ######################### # Python Sonar with CLI # ######################### diff --git a/releasenotes/notes/add-python-snyk-cli-scanner-92cb49fe8ca39c51.yaml b/releasenotes/notes/add-python-snyk-cli-scanner-92cb49fe8ca39c51.yaml new file mode 100644 index 00000000..dd0d8e5e --- /dev/null +++ b/releasenotes/notes/add-python-snyk-cli-scanner-92cb49fe8ca39c51.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Introduce Python Snyk CLI scanner jobs. These jobs can be triggered to download the + latest version of Snyk's CLI scanner and trigger a scan for Python based repos. These + jobs produce a report which is published into Snyk's dashboard. These reports are + fetched and reflected back into the LFX Security tool. diff --git a/shell/snyk-cli-scanner-run.sh b/shell/snyk-cli-scanner-run.sh index 7e42784d..87681233 100644 --- a/shell/snyk-cli-scanner-run.sh +++ b/shell/snyk-cli-scanner-run.sh @@ -9,7 +9,16 @@ # http://www.eclipse.org/legal/epl-v10.html ############################################################################## echo "---> snyk-cli-scanner-run.sh" +# shellcheck disable=SC1090 +source ~/lf-env.sh +# Install Snyk CLI dependencies for Python +if [[ "$JOB_NAME" =~ "python" ]]; then + # Install Snyk CLI dependencies for Python based projects + lf-activate-venv flask flask-api flask-cors pg8000 pandas +else + lf-activate-venv +fi # Add mvn to PATH so that the Snyk CLI can use it export PATH=$PATH:"$M2_HOME"/bin # Download and install the latest Snyk scanner -- 2.16.6