From 232378ffaca9f7ef383b9371b4f4144259225d8c Mon Sep 17 00:00:00 2001 From: Jessica Wagantall Date: Thu, 21 Mar 2019 17:56:15 -0700 Subject: [PATCH] Add WhiteSource Scan jobs WhiteSource is an open source security and license compliance management platform. It is used to scan a great variety of languages includding Java, Python, C, Go, among others. {project-name}-whitesource-scan-{stream} uses WhiteSource's Unified Agent to perform the code scans. Issue: RELENG-1876 Change-Id: If511da2f51e08799352ee6841338f76dd63af6b6 Signed-off-by: Jessica Wagantall --- .jjb-test/lf-whitesource-jobs.yaml | 10 + docs/jjb/lf-whitesource-jobs.rst | 58 +++++ jjb/lf-whitesource-jobs.yaml | 209 ++++++++++++++++++ .../lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml | 15 ++ shell/whitesource-unified-agent-cli.sh | 27 +++ wss-unified-agent.config.example | 236 +++++++++++++++++++++ 6 files changed, 555 insertions(+) create mode 100644 .jjb-test/lf-whitesource-jobs.yaml create mode 100644 docs/jjb/lf-whitesource-jobs.rst create mode 100644 jjb/lf-whitesource-jobs.yaml create mode 100644 releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml create mode 100644 shell/whitesource-unified-agent-cli.sh create mode 100644 wss-unified-agent.config.example diff --git a/.jjb-test/lf-whitesource-jobs.yaml b/.jjb-test/lf-whitesource-jobs.yaml new file mode 100644 index 00000000..16186665 --- /dev/null +++ b/.jjb-test/lf-whitesource-jobs.yaml @@ -0,0 +1,10 @@ +--- +- project: + name: whitesource-scan-project-jobs + jobs: + - gerrit-whitesource-scan + branch: 'master' + project: 'whitesource/scan-project-jobs' + project-name: 'whitesource-scan-project' + wss-product-name: EXAMPLE-ORG + wss-unified-agent-config: '.config/wss-unified-agent.config' diff --git a/docs/jjb/lf-whitesource-jobs.rst b/docs/jjb/lf-whitesource-jobs.rst new file mode 100644 index 00000000..5750a0f4 --- /dev/null +++ b/docs/jjb/lf-whitesource-jobs.rst @@ -0,0 +1,58 @@ +################ +WhiteSource Jobs +################ + +Macros +====== + +lf-infra-wss-unified-agent-scan +------------------------------- + +Run WhiteSource Unified Agent for a project. + +Job Templates +============= + +WhiteSource Unified Agent scan +------------------------------ + +Trigger WhiteSource code scans using Unified Agent. For more details: +https://whitesource.atlassian.net/wiki/spaces/WD/pages/33718339/Unified+Agent + +The WhiteSource Unified Agent scanner runs using a configuration file: +https://s3.amazonaws.com/unified-agent/wss-unified-agent.config + +:Template Names: + + - {project-name}-whitesource-scan-{stream} + - gerrit-whitesource-scan + - github-whitesource-scan + +:Comment Trigger: run-whitesource + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Set in defaults.yaml) + :wss-product-name: Product to asociate the WhiteSource report in the dashboard. + :wss-unified-agent-config: Path to wss-unifed-agent.config. + +:Optional parameters: + + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 60) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :java-version: Version of Java to use for the build. (default: openjdk8) + :stream: Keyword used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :wss-unified-agent-version: WhiteSource Unified Agent version package to download + and use. + :gerrit_trigger_file_paths: Override file paths which used to filter which + file modifications will trigger a build. Refer to JJB documentation for + "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + :gerrit_wss_triggers: Override Gerrit Triggers. diff --git a/jjb/lf-whitesource-jobs.yaml b/jjb/lf-whitesource-jobs.yaml new file mode 100644 index 00000000..89673741 --- /dev/null +++ b/jjb/lf-whitesource-jobs.yaml @@ -0,0 +1,209 @@ +--- + +########## +# Macros # +########## + +- builder: + name: lf-infra-wss-unified-agent-scan + builders: + - shell: !include-raw: + - ../shell/whitesource-unified-agent-cli.sh + +#################### +# COMMON FUNCTIONS # +#################### + +- lf_whitesource_common: &lf_whitesource_common + name: lf-whitesource-common + + ###################### + # Default parameters # + ###################### + + archive-artifacts: > + **/*.log + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + - credentials-binding: + - text: + credential-id: wss-apiKey + variable: wss-apiKey + + publishers: + - lf-infra-publish + +################################## +# WHITESOURCE UNIFIED AGENT SCAN # +################################## + +- lf_wss_unified_agent_scan: &lf_wss_unified_agent_scan + name: lf-wss-unified-agent-scan + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 30 + build-timeout: 60 + cron: '@daily' + disable-job: false + git-url: '$GIT_URL/$PROJECT' + github-url: 'https://github.com' + java-version: openjdk8 + stream: master + submodule-recursive: true + submodule-timeout: 10 + wss-unified-agent-opts: '' + wss-unified-agent-version: 19.2.2.2 + + gerrit_trigger_file_paths: + - compare-type: ANT + pattern: '.*' + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - '.*' + + ##################### + # Job Configuration # + ##################### + + disabled: '{disable-job}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + - string: + name: WSS_PROJECT_NAME + default: '{project-name}' + description: Project to asociate the WhiteSource report in the dashboard. + - string: + name: WSS_PRODUCT_NAME + default: '{wss-product-name}' + description: Product to asociate the WhiteSource report in the dashboard. + - string: + name: WSS_UNIFIED_AGENT_CONFIG + default: '{wss-unified-agent-config}' + description: Path to wss-unifed-agent.config + - string: + name: WSS_UNIFIED_AGENT_OPTIONS + default: '{wss-unified-agent-opts}' + description: Options to pass to the WhiteSource Unified Agent CLI. + - string: + name: WSS_UNIFIED_AGENT_VERSION + default: '{wss-unified-agent-version}' + description: WhiteSource Unified Agent version package to download and use. + + builders: + - lf-infra-pre-build + - lf-update-java-alternatives: + java-version: '{java-version}' + - lf-infra-wss-unified-agent-scan + +- job-template: + name: '{project-name}-whitesource-scan-{stream}' + id: gerrit-whitesource-scan + <<: *lf_whitesource_common + # yamllint disable-line rule:key-duplicates + <<: *lf_wss_unified_agent_scan + + ###################### + # Default parameters # + ###################### + + gerrit_wss_triggers: + - comment-added-contains-event: + comment-contains-value: run-whitesource$ + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: default + + triggers: + # Build weekly on Saturdays + - timed: '{obj:cron}' + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: '{obj:gerrit_wss_triggers}' + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + file-paths: '{obj:gerrit_trigger_file_paths}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + +- job-template: + name: '{project-name}-whitesource-scan-{stream}' + id: github-whitesource-scan + <<: *lf_whitesource_common + # yamllint disable-line rule:key-duplicates + <<: *lf_wss_unified_agent_scan + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + - github: + url: '{github-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '' + branch: 'refs/heads/{branch}' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + # Build weekly on Saturdays + - timed: '{obj:cron}' + - github-pull-request: + trigger-phrase: '^run-whitesource$' + only-trigger-phrase: false + status-context: 'WhiteSource' + permit-all: true + github-hooks: true + white-list-target-branches: + - '{branch}' + included-regions: '{obj:github_included_regions}' diff --git a/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml b/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml new file mode 100644 index 00000000..dde27cd0 --- /dev/null +++ b/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml @@ -0,0 +1,15 @@ +--- +prelude: > + WhiteSource is a security and license compliance management platform. + It is used to perform scans on a great variety of coding and scripting + languages. +features: + - | + Job {project-name}-whitesource-scan-{stream} uses the WhiteSource Unified + Agent scanner CLI tool to perform the code scan and report the results into + the WhiteSource dashboard. +other: + - | + To run this job, a configuration file is needed (wss-unified-agent.config.example). + A new secret text credential will need to be created. (ID=wss-apiKey Secret=WhiteSource + organization API key) diff --git a/shell/whitesource-unified-agent-cli.sh b/shell/whitesource-unified-agent-cli.sh new file mode 100644 index 00000000..750a1523 --- /dev/null +++ b/shell/whitesource-unified-agent-cli.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script downloads wss-unified-agent-.jar and uses it to perform +# a scan on the code whithin a repo based on the wss-unified-agent.config provided. + +# DO NOT enable -u because $WSS_UNIFIED_AGENT_OPTIONS could be unbound. +# Ensure we fail the job if any steps fail. +set -xe -o pipefail +set -u +echo "---> whitesource-unified-agent-cli.sh" +jar_location="/tmp/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar" +wget -nv https://s3.amazonaws.com/unified-agent/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar \ + -O $jar_location +echo "---> Running WhiteSource Unified Agent CLI ..." +java -jar $jar_location -c $WSS_UNIFIED_AGENT_CONFIG \ + -product $WSS_PRODUCT_NAME -project $WSS_PROJECT_NAME \ + ${WSS_UNIFIED_AGENT_OPTIONS:-} +rm $jar_location diff --git a/wss-unified-agent.config.example b/wss-unified-agent.config.example new file mode 100644 index 00000000..4a9f091c --- /dev/null +++ b/wss-unified-agent.config.example @@ -0,0 +1,236 @@ +#################################################################### +# WhiteSource Unified-Agent configuration file +#################################################################### +########################################## +# GENERAL SCAN MODE: Files and Package Managers +########################################## + +checkPolicies=false +forceCheckAllDependencies=false +forceUpdate=false +forceUpdate.failBuildOnPolicyViolation=false +offline=false +#ignoreSourceFiles=true +#scanComment= +#updateInventory=false +#resolveAllDependencies=false +#failErrorLevel=ALL +#requireKnownSha1=false +#generateProjectDetailsJson=true +#generateScanReport=true +#scanReportTimeoutMinutes=10 +#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model + +#projectPerFolder=true +#projectPerFolderIncludes= +#projectPerFolderExcludes= + +#wss.connectionTimeoutMinutes=60 +# Change the below URL to your WhiteSource server. +# Use the 'WhiteSource Server URL' which can be retrieved +# from your 'Profile' page on the 'Server URLs' panel. +# Then, add the '/agent' path to it. +wss.url=https://saas.whitesourcesoftware.com/agent + +#npm.resolveDependencies=false +#npm.ignoreSourceFiles=false +#npm.includeDevDependencies=true +#npm.runPreStep=true +#npm.ignoreNpmLsErrors=true +#npm.ignoreScripts=true +#npm.yarnProject=true +#npm.accessToken= +#npm.identifyByNameAndVersion=true + +#bower.resolveDependencies=false +#bower.ignoreSourceFiles=true +#bower.runPreStep=true + +#nuget.resolvePackagesConfigFiles=false +#nuget.resolveCsProjFiles=false +#nuget.resolveDependencies=false +#nuget.restoreDependencies=true +#nuget.preferredEnvironment= +#nuget.packagesDirectory= +#nuget.ignoreSourceFiles=true +#nuget.runPreStep=true +#nuget.resolveNuspecFiles=false + +#python.resolveDependencies=false +#python.ignoreSourceFiles=false +#python.ignorePipInstallErrors=true +#python.installVirtualenv=true +#python.resolveHierarchyTree=false +#python.requirementsFileIncludes=requirements.txt +#python.resolveSetupPyFiles=true +#python.runPipenvPreStep=true +#python.pipenvDevDependencies=true +#python.IgnorePipenvInstallErrors=true + +#maven.ignoredScopes=test provided +#maven.resolveDependencies=false +#maven.ignoreSourceFiles=true +#maven.aggregateModules=true +#maven.ignorePomModules=false +#maven.runPreStep=true +#maven.ignoreMvnTreeErrors=true +#maven.environmentPath= +#maven.m2RepositoryPath= + +#gradle.ignoredScopes= +#gradle.resolveDependencies=false +#gradle.runAssembleCommand=false +#gradle.runPreStep=true +#gradle.ignoreSourceFiles=true +#gradle.aggregateModules=true +#gradle.preferredEnvironment=wrapper +#gradle.localRepositoryPath= + +#paket.resolveDependencies=false +#paket.ignoredGroups= +#paket.ignoreSourceFiles=false +#paket.runPreStep=true +#paket.exePath= + +#go.resolveDependencies=false +#go.collectDependenciesAtRuntime=true +#go.dependencyManager= +#go.ignoreSourceFiles=true +#go.glide.ignoreTestPackages=false +#go.gogradle.enableTaskAlias=true + +#ruby.resolveDependencies = false +#ruby.ignoreSourceFiles = false +#ruby.installMissingGems = true +#ruby.runBundleInstall = true +#ruby.overwriteGemFile = true + +#sbt.resolveDependencies=false +#sbt.ignoreSourceFiles=true +#sbt.aggregateModules=true +#sbt.runPreStep=true +#sbt.targetFolder= + +#php.resolveDependencies=false +#php.runPreStep=true +#php.includeDevDependencies=true + +#html.resolveDependencies=false + +#cocoapods.resolveDependencies=false +#cocoapods.runPreStep=true +#cocoapods.ignoreSourceFiles=false + +#hex.resolveDependencies=false +#hex.runPreStep=true +#hex.ignoreSourceFiles=false +#hex.aggregateModules=true + +################################## +# Organization tokens: +################################## +apiKey= + +#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option +#userKey= + +projectName= +projectVersion= +projectToken= + +productName= +productVersion= +productToken= +#updateType=APPEND +#requesterEmail=user@provider.com + +######################################################################################### +# Includes/Excludes Glob patterns - PLEASE USE ONLY ONE EXCLUDE LINE AND ONE INCLUDE LINE +######################################################################################### +includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx **/*.go **/*.py + +#includes=**/*.m **/*.mm **/*.js **/*.php +#includes=**/*.jar +#includes=**/*.gem **/*.rb +#includes=**/*.dll **/*.cs **/*.nupkg +#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2 +#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py + +## Exclude file extensions or specific directories by adding **/*. or **/** +excludes=**/*sources.jar **/*javadoc.jar + +case.sensitive.glob=false +followSymbolicLinks=true + +################################## +# Archive properties +################################## +#archiveExtractionDepth=2 +#archiveIncludes=**/*.war **/*.ear +#archiveExcludes=**/*sources.jar + +################################## +# Proxy settings +################################## +#proxy.host= +#proxy.port= +#proxy.user= +#proxy.pass= + +################################## +# SCM settings +################################## +#scm.type= +#scm.user= +#scm.pass= +#scm.ppk= +#scm.url= +#scm.branch= +#scm.tag= +#scm.npmInstall= +#scm.npmInstallTimeoutMinutes= +#scm.repositoriesFile= + +############################################## +# SCAN MODE: Linux package manager settings +############################################## +#scanPackageManager=true + +################################## +# SCAN MODE: Docker images +################################## +#docker.scanImages=true +#docker.includes=.*.* +#docker.excludes= +#docker.pull.enable=true +#docker.pull.images=.*.* +#docker.pull.maxImages=10 +#docker.pull.tags=.*.* +#docker.pull.digest= +#docker.delete.force=true +#docker.login.sudo=false + +#docker.aws.enable=true +#docker.aws.registryIds= + +#docker.azure.enable=true +#docker.azure.userName= +#docker.azure.userPassword= +#docker.azure.registryNames= + +################################## +# SCAN MODE: Docker containers +################################## +#docker.scanContainers=true +#docker.containerIncludes=.*.* +#docker.containerExcludes= + +################################ +# Serverless settings +################################ +#serverless.provider= +#serverless.scanFunctions=true +#serverless.includes= +#serverless.excludes= +#serverless.region= +#serverless.maxFunctions=10 -- 2.16.6