name: "{project-name}-docker"
jobs:
- "{project-name}-gerrit-docker-jobs"
+ - gerrit-docker-snyk-cli
project: docker/project
project-name: docker-project
- gerrit-pypi-release-verify
- gerrit-pypi-stage
- gerrit-pypi-verify
+ - gerrit-python-snyk-cli
- gerrit-tox-nexus-iq-clm
- gerrit-tox-sonar
- gerrit-tox-sonarqube
- github-pypi-release-verify
- github-pypi-stage
- github-pypi-verify
+ - gerrit-python-snyk-cli
- github-tox-nexus-iq-clm
- github-tox-sonar
- github-tox-sonarqube
---
tag: 1.0.0
+
+Docker Snyk CLI
+---------------
+
+Builds the code, downloads and runs a Snyk CLI scan of the code into the Snyk dashboard.
+
+:Template Names:
+
+ - {project-name}-docker-snyk-cli-{stream}
+ - gerrit-docker-snyk-cli
+ - github-docker-snyk-cli
+
+:Comment Trigger: run-snyk
+
+:Required parameters:
+
+ :build-node: The node to run build on.
+ :container-public-registry: Docker registry source with base images.
+ :docker-name: Name of the Docker image.
+ :jenkins-ssh-credential: Credential to use for SSH. (Generally configured
+ in defaults.yaml)
+ :mvn-settings: Maven settings.xml file containing Docker credentials.
+ :snyk-token-credential-id: Snyk API token to communicate with Jenkins.
+ :snyk-org-credential-id: Snyk organization ID.
+
+:Optional parameters:
+
+ :branch: Git branch to fetch for the build. (default: master)
+ :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+ :build-timeout: Timeout in minutes before aborting build. (default: 60)
+ :container-tag-method: Specifies the docker tag-choosing method.
+ Options are "latest", "git-describe" or "yaml-file".
+ Option latest uses the "latest" tag.
+ Option git-describe uses the string returned by git-describe,
+ which requires a tag to exist in the repository.
+ Option yaml-file uses the string from file "container-tag.yaml"
+ in the repository. (default: latest)
+ :container-tag-yaml-dir: Directory with container-tag.yaml. (default: $DOCKER_ROOT)
+ :docker-build-args: Arguments for the docker build command.
+ :docker-get-container-tag-script: Path to script that chooses docker tag.
+ (default: ../shell/docker-get-container-tag.sh in global-jjb)
+ :docker-root: Build directory within the repo. (default: $WORKSPACE, the repo root)
+ :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+ :pre_docker_build_script: Build script to execute before the main verify
+ builder steps. (default: "")
+ :post_docker_build_script: Build script to execute after the main verify
+ builder steps. (default: "")
+ :snyk-cli-options: Additional Snyk CLI options. (default: '')
+ :stream: Keyword that represents a release code-name.
+ Often the same as the branch. (default: master)
+ :submodule-recursive: Whether to checkout submodules recursively.
+ (default: true)
+ :submodule-timeout: Timeout (in minutes) for checkout operation.
+ (default: 10)
+
+ :gerrit_snyk_triggers: Override Gerrit Triggers.
:tox-envs: Tox environment with the appropriate pip freeze invocation.
(default: 'clm')
+Python Snyk CLI
+---------------
+
+Builds the code, downloads and runs a Snyk CLI scan of the code into the Snyk dashboard.
+
+:Template Names:
+
+ - {project-name}-python-snyk-cli-{stream}
+ - gerrit-python-snyk-cli
+ - github-python-snyk-cli
+
+:Comment Trigger: run-snyk
+
+:Required parameters:
+
+ :build-node: The node to run build on.
+ :jenkins-ssh-credential: Credential to use for SSH. (Generally configured in defaults.yaml)
+ :snyk-token-credential-id: Snyk API token to communicate with Jenkins.
+ :snyk-org-credential-id: Snyk organization ID.
+
+:Optional parameters:
+
+ :branch: The branch to build against. (default: master)
+ :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+ :build-timeout: Timeout in minutes before aborting build. (default: 60)
+ :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+ :pre-build-script: Shell script to execute before the Tox builder.
+ For example, install prerequisites or move files to the repo root.
+ (default: a string with a shell comment)
+ :parallel: If different from false, try pass this parameter to tox option
+ "--parallel" to parallelize jobs in the envlist (and then activate the
+ option "--parallel-live" to display output in logs).
+ Possible values are "auto" (equivalent to "true" for legacy),
+ "all" or any integer. Any other value is equivalent to "false".
+ (default: false, in series)
+ :python-version: Python version to invoke pip install of tox-pyenv
+ (default: python3)
+ :snyk-cli-options: Additional Snyk CLI options. (default: '')
+ :stream: Keyword representing a release code-name.
+ Often the same as the branch. (default: master)
+ :submodule-recursive: Whether to checkout submodules recursively.
+ (default: true)
+ :submodule-timeout: Timeout (in minutes) for checkout operation.
+ (default: 10)
+ :submodule-disable: Disable submodule checkout operation.
+ (default: false)
+ :tox-dir: Directory containing the project's tox.ini relative to
+ the workspace. The default uses tox.ini at the project root.
+ (default: '.')
+ :tox-envs: Tox environments to run. If blank run everything described
+ in tox.ini. (default: '')
+ :gerrit_trigger_file_paths: Override file paths used to filter which file
+ modifications trigger a build. Refer to JJB documentation for "file-path" details.
+ https://jenkins-job-builder.readthedocs.io/en/latest/triggers.html#triggers.gerrit
+
Python Sonar with CLI
---------------------
white-list-target-branches:
- "{branch}"
included-regions: "{obj:github_included_regions}"
+
+##################
+# Docker Snyk CLI #
+##################
+
+- lf_docker_snyk_cli: &lf_docker_snyk_cli
+ name: lf-docker-snyk_cli
+
+ ######################
+ # Default parameters #
+ ######################
+
+ branch: master
+ build-days-to-keep: 30 # 30 days for troubleshooting purposes
+ build-timeout: 60
+ container-tag-method: "latest"
+ container-tag-yaml-dir: ""
+ disable-job: false
+ docker-get-container-tag-script: "../shell/docker-get-container-tag.sh"
+ docker-root: "$WORKSPACE"
+ docker-build-args: ""
+ git-url: "$GIT_URL/$PROJECT"
+ github-url: "https://github.com"
+ pre_docker_build_script: "# pre docker build script goes here"
+ post_docker_build_script: "# post docker build script goes here"
+ snyk-cli-options: ""
+ snyk-token-credential-id: snyk-token
+ snyk-org-credential-id: snyk-org
+ stream: master
+ submodule-recursive: true
+ submodule-timeout: 10
+ submodule-disable: false
+
+ gerrit_snyk_triggers:
+ - comment-added-contains-event:
+ comment-contains-value: '^Patch Set\s+\d+:\s+run-snyk\s*$'
+
+ parameters:
+ - lf-infra-parameters:
+ project: "{project}"
+ branch: "{branch}"
+ stream: "{stream}"
+ - string:
+ name: SNYK_CLI_OPTIONS
+ default: "{snyk-cli-options}"
+ description: Additional Snyk CLI commands and options
+
+ wrappers:
+ - credentials-binding:
+ - text:
+ credential-id: "{snyk-token-credential-id}"
+ variable: SNYK_TOKEN
+ - text:
+ credential-id: "{snyk-org-credential-id}"
+ variable: SNYK_ORG
+
+ #####################
+ # Job Configuration #
+ #####################
+
+ disabled: "{disable-job}"
+
+ builders:
+ - lf-infra-pre-build
+ - lf-infra-docker-login:
+ global-settings-file: "global-settings"
+ settings-file: "{mvn-settings}"
+ - shell: "{pre_docker_build_script}"
+ - lf-docker-get-container-tag:
+ container-tag-method: "{container-tag-method}"
+ container-tag-yaml-dir: "{container-tag-yaml-dir}"
+ docker-root: "{docker-root}"
+ docker-get-container-tag-script: "{docker-get-container-tag-script}"
+ - lf-docker-build:
+ docker-build-args: "{docker-build-args}"
+ docker-name: "{docker-name}"
+ docker-root: "{docker-root}"
+ container-public-registry: "{container-public-registry}"
+ container-push-registry: "{container-push-registry}"
+ - shell: "{post_docker_build_script}"
+ - lf-infra-snyk-cli-scanner
+ - lf-provide-maven-settings-cleanup
+ - shell: 'find . -regex ".*karaf/target" | xargs rm -rf'
+
+- job-template:
+ name: "{project-name}-docker-snyk-cli-{stream}"
+ id: gerrit-docker-snyk-cli
+ <<: *lf_docker_common
+ # yamllint disable-line rule:key-duplicates
+ <<: *lf_docker_snyk_cli
+
+ scm:
+ - lf-infra-gerrit-scm:
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+ git-url: "{git-url}"
+ refspec: "$GERRIT_REFSPEC"
+ branch: "$GERRIT_BRANCH"
+ submodule-recursive: "{submodule-recursive}"
+ submodule-timeout: "{submodule-timeout}"
+ submodule-disable: "{submodule-disable}"
+ choosing-strategy: default
+
+ triggers:
+ # Build weekly on Saturdays
+ - timed: "H H * * 6"
+ - gerrit:
+ server-name: "{gerrit-server-name}"
+ trigger-on: "{obj:gerrit_snyk_triggers}"
+ projects:
+ - project-compare-type: ANT
+ project-pattern: "{project}"
+ branches:
+ - branch-compare-type: ANT
+ branch-pattern: "**/{branch}"
+ skip-vote:
+ successful: true
+ failed: true
+ unstable: true
+ notbuilt: true
+
+- job-template:
+ name: "{project-name}-docker-snyk-cli-{stream}"
+ id: github-docker-snyk-cli
+ <<: *lf_docker_common
+ # yamllint disable-line rule:key-duplicates
+ <<: *lf_docker_snyk_cli
+
+ properties:
+ - lf-infra-properties:
+ build-days-to-keep: "{build-days-to-keep}"
+ - github:
+ url: "{github-url}/{github-org}/{project}"
+
+ scm:
+ - lf-infra-github-scm:
+ url: "{git-clone-url}{github-org}/{project}"
+ refspec: ""
+ branch: "refs/heads/{branch}"
+ submodule-recursive: "{submodule-recursive}"
+ submodule-timeout: "{submodule-timeout}"
+ submodule-disable: "{submodule-disable}"
+ choosing-strategy: default
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+ triggers:
+ # Build weekly on Saturdays
+ - timed: "H H * * 6"
+ - github-pull-request:
+ trigger-phrase: "^run-snyk$"
+ only-trigger-phrase: true
+ status-context: "SNYK scan"
+ permit-all: true
+ github-hooks: true
+ org-list:
+ - "{github-org}"
+ white-list: "{obj:github_pr_allowlist}"
+ admin-list: "{obj:github_pr_admin_list}"
+ white-list-target-branches:
+ - "{branch}"
white-list-target-branches:
- "{branch}"
+###################
+# Python Snyk CLI #
+###################
+
+- lf_python_snyk_cli: &lf_python_snyk_cli
+ name: lf-python-snyk_cli
+
+ ######################
+ # Default parameters #
+ ######################
+
+ branch: master
+ build-days-to-keep: 30 # 30 days for troubleshooting purposes
+ build-timeout: 60
+ disable-job: false
+ git-url: "$GIT_URL/$PROJECT"
+ github-url: "https://github.com"
+ java-version: openjdk11
+ parallel: false
+ pre-build-script: "# pre-build script goes here"
+ python-version: python3
+ snyk-cli-options: ""
+ snyk-token-credential-id: snyk-token
+ snyk-org-credential-id: snyk-org
+ stream: master
+ submodule-recursive: true
+ submodule-timeout: 10
+ submodule-disable: false
+ tox-dir: "."
+ tox-envs: ""
+
+ gerrit_snyk_triggers:
+ - comment-added-contains-event:
+ comment-contains-value: '^Patch Set\s+\d+:\s+run-snyk\s*$'
+
+ parameters:
+ - lf-infra-parameters:
+ project: "{project}"
+ branch: "{branch}"
+ stream: "{stream}"
+ - string:
+ name: SNYK_CLI_OPTIONS
+ default: "{snyk-cli-options}"
+ description: Additional Snyk CLI commands and options
+ - lf-infra-tox-parameters:
+ tox-dir: "{tox-dir}"
+ tox-envs: "{tox-envs}"
+
+ wrappers:
+ - credentials-binding:
+ - text:
+ credential-id: "{snyk-token-credential-id}"
+ variable: SNYK_TOKEN
+ - text:
+ credential-id: "{snyk-org-credential-id}"
+ variable: SNYK_ORG
+
+ #####################
+ # Job Configuration #
+ #####################
+
+ disabled: "{disable-job}"
+
+ builders:
+ - lf-infra-pre-build
+ - lf-infra-tox-install:
+ python-version: "{python-version}"
+ - shell: "{pre-build-script}"
+ - lf-infra-tox-run:
+ parallel: "{parallel}"
+ - lf-infra-snyk-cli-scanner
+
+- job-template:
+ name: "{project-name}-python-snyk-cli-{stream}"
+ id: gerrit-python-snyk-cli
+ # yamllint disable-line rule:key-duplicates
+ <<: *lf_python_snyk_cli
+
+ scm:
+ - lf-infra-gerrit-scm:
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+ git-url: "{git-url}"
+ refspec: "$GERRIT_REFSPEC"
+ branch: "$GERRIT_BRANCH"
+ submodule-recursive: "{submodule-recursive}"
+ submodule-timeout: "{submodule-timeout}"
+ submodule-disable: "{submodule-disable}"
+ choosing-strategy: default
+
+ triggers:
+ # Build weekly on Saturdays
+ - timed: "H H * * 6"
+ - gerrit:
+ server-name: "{gerrit-server-name}"
+ trigger-on: "{obj:gerrit_snyk_triggers}"
+ projects:
+ - project-compare-type: ANT
+ project-pattern: "{project}"
+ branches:
+ - branch-compare-type: ANT
+ branch-pattern: "**/{branch}"
+ skip-vote:
+ successful: true
+ failed: true
+ unstable: true
+ notbuilt: true
+
+- job-template:
+ name: "{project-name}-python-snyk-cli-{stream}"
+ id: github-python-snyk-cli
+ # yamllint disable-line rule:key-duplicates
+ <<: *lf_python_snyk_cli
+
+ properties:
+ - lf-infra-properties:
+ build-days-to-keep: "{build-days-to-keep}"
+ - github:
+ url: "{github-url}/{github-org}/{project}"
+
+ scm:
+ - lf-infra-github-scm:
+ url: "{git-clone-url}{github-org}/{project}"
+ refspec: ""
+ branch: "refs/heads/{branch}"
+ submodule-recursive: "{submodule-recursive}"
+ submodule-timeout: "{submodule-timeout}"
+ submodule-disable: "{submodule-disable}"
+ choosing-strategy: default
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+ triggers:
+ # Build weekly on Saturdays
+ - timed: "H H * * 6"
+ - github-pull-request:
+ trigger-phrase: "^run-snyk$"
+ only-trigger-phrase: true
+ status-context: "SNYK scan"
+ permit-all: true
+ github-hooks: true
+ org-list:
+ - "{github-org}"
+ white-list: "{obj:github_pr_allowlist}"
+ admin-list: "{obj:github_pr_admin_list}"
+ white-list-target-branches:
+ - "{branch}"
+
#########################
# Python Sonar with CLI #
#########################
--- /dev/null
+---
+features:
+ - |
+ Introduce Docker Snyk CLI scanner jobs. These jobs can be triggered to download the
+ latest version of Snyk's CLI scanner and trigger a scan for Docker based repos. These
+ jobs produce a report which is published into Snyk's dashboard. These reports are
+ fetched and reflected back into the LFX Security tool.
---
features:
+ - |
Introduce Go Snyk CLI scanner jobs. These jobs can be triggered to download the
latest version of Snyk's CLI scanner and trigger a scan for Go based repos. These
jobs produce a report which is published into Snyk's dashboard. These reports are
--- /dev/null
+---
+features:
+ - |
+ Introduce Python Snyk CLI scanner jobs. These jobs can be triggered to download the
+ latest version of Snyk's CLI scanner and trigger a scan for Python based repos. These
+ jobs produce a report which is published into Snyk's dashboard. These reports are
+ fetched and reflected back into the LFX Security tool.
--- /dev/null
+---
+fixes:
+ - |
+ The path and command for update-alternatives/alternatives was
+ not being set correctly between CentOS7/8 and was incorrect under all
+ tested ubuntu versions. It did not seem to cause jobs to break, so was
+ perhaps not being detected in all cases.
--- /dev/null
+---
+fixes:
+ - |
+ The latest (2.42.0.01) clm-maven-plugin introduced an error in our
+ environment.
+
+ Failed to execute goal com.sonatype.clm:clm-maven-plugin:2.42.0-01:index
+ (default-cli) on project babel: Failed to invoke Maven build.
+ Maven execution failed, exit code: 1 -> [Help 1]
+
+ This fix will pin the clm-maven-plugin to the previous version (2.41.0-02)
# http://www.eclipse.org/legal/epl-v10.html
##############################################################################
echo "---> snyk-cli-scanner-run.sh"
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+# Install Snyk CLI dependencies for Python
+if [[ "$JOB_NAME" =~ "python" ]]; then
+ # Install Snyk CLI dependencies for Python based projects
+ lf-activate-venv flask flask-api flask-cors pg8000 pandas
+else
+ lf-activate-venv
+fi
# Add mvn to PATH so that the Snyk CLI can use it
export PATH=$PATH:"$M2_HOME"/bin
# Download and install the latest Snyk scanner
echo "Authenticate with SNYK_TOKEN..."
snyk auth "$SNYK_CLI"
echo "Running Snyk CLI..."
-snyk test --json --severity-threshold=low "$SNYK_CLI_OPTIONS" --org="$SNYK_ORG"
-snyk monitor --severity-threshold=low "$SNYK_CLI_OPTIONS" --org="$SNYK_ORG"
+if [[ "$JOB_NAME" =~ "docker" ]]; then
+ snyk container test "$SNYK_CLI_OPTIONS" \
+ "$CONTAINER_PULL_REGISTRY/$DOCKER_NAME:$DOCKER_IMAGE_TAG" --org="$SNYK_ORG"
+ snyk container monitor "$SNYK_CLI_OPTIONS" \
+ "$CONTAINER_PULL_REGISTRY/$DOCKER_NAME:$DOCKER_IMAGE_TAG" --org="$SNYK_ORG"
+else
+ snyk test --json --severity-threshold=low "$SNYK_CLI_OPTIONS" --org="$SNYK_ORG"
+ snyk monitor --severity-threshold=low "$SNYK_CLI_OPTIONS" --org="$SNYK_ORG"
+fi
# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
# shellcheck disable=SC2086
-$MVN $MAVEN_GOALS dependency:tree com.sonatype.clm:clm-maven-plugin:index \
+$MVN $MAVEN_GOALS dependency:tree com.sonatype.clm:clm-maven-plugin:2.41.0-02:index \
--global-settings "$GLOBAL_SETTINGS_FILE" \
--settings "$SETTINGS_FILE" \
-DaltDeploymentRepository=staging::default::file:"$WORKSPACE"/m2repo \
JAVA_ENV_FILE="/tmp/java.env"
-JAVA_RELEASE=$(echo $SET_JDK_VERSION | sed 's/[a-zA-Z]//g')
-JAVA_RELEASE_NBR=$(echo $SET_JDK_VERSION | sed 's/[a-zA-Z:-]//g')
+JAVA_RELEASE=$(echo "$SET_JDK_VERSION" | sed 's/[a-zA-Z]//g')
+JAVA_RELEASE_NBR=$(echo "$SET_JDK_VERSION" | sed 's/[a-zA-Z:-]//g')
#TODO check whether is it worth keeping there 2 distinct variables
update_java_redhat() {
- if [ ${JAVA_RELEASE} -ge 9 ]; then
+ if [ "${JAVA_RELEASE}" -ge 9 ]; then
# Java 9 or newer: new version format
export JAVA_HOME="/usr/lib/jvm/java-${JAVA_RELEASE}-openjdk"
else
fedora|centos|redhat)
echo "---> RedHat type system detected"
update_java_redhat
- alternatives="/usr/sbin/alternatives"
+ alternatives="/usr/sbin/alternatives"
;;
ubuntu|debian)
echo "---> Ubuntu/Debian system detected"
update_java_ubuntu
- alternatives="/usr/sbin/update-alternatives"
+ alternatives=$(which update-alternatives)
;;
esac
if ! [ -d "$JAVA_HOME" ]; then
echo "$JAVA_HOME directory not found - trying to find an approaching one"
if ls -d "$JAVA_HOME"*; then
- export JAVA_HOME=$(ls -d "$JAVA_HOME"* | head -1)
+ JAVA_HOME=$(ls -d "$JAVA_HOME"* | head -1)
+ export JAVA_HOME
else
echo "no $JAVA_HOME directory nor candidate found -exiting " >&2
exit 17
fi
fi
-sudo $alternatives --install /usr/bin/java java "${JAVA_HOME}/bin/java" 1
-sudo $alternatives --install /usr/bin/javac javac "${JAVA_HOME}/bin/javac" 1
-sudo $alternatives --install /usr/lib/jvm/java-openjdk java_sdk_openjdk "${JAVA_HOME}" 1
-sudo $alternatives --set java "${JAVA_HOME}/bin/java"
-sudo $alternatives --set javac "${JAVA_HOME}/bin/javac"
-sudo $alternatives --set java_sdk_openjdk "${JAVA_HOME}"
+# If sudo is not found, the commands below will run anyway
+SUDO_CMD=$(which sudo)
+
+$SUDO_CMD "$alternatives" --install /usr/bin/java java "${JAVA_HOME}/bin/java" 1
+$SUDO_CMD "$alternatives" --install /usr/bin/javac javac "${JAVA_HOME}/bin/javac" 1
+$SUDO_CMD "$alternatives" --install /usr/lib/jvm/java-openjdk java_sdk_openjdk "${JAVA_HOME}" 1
+$SUDO_CMD "$alternatives" --set java "${JAVA_HOME}/bin/java"
+$SUDO_CMD "$alternatives" --set javac "${JAVA_HOME}/bin/javac"
+$SUDO_CMD "$alternatives" --set java_sdk_openjdk "${JAVA_HOME}"
echo JAVA_HOME="$JAVA_HOME" > "$JAVA_ENV_FILE"
java -version
Code Review / releng / global-jjb.git / commitdiff