Add WhiteSource Scan jobs 97/14997/10
authorJessica Wagantall <jwagantall@linuxfoundation.org>
Fri, 22 Mar 2019 00:56:15 +0000 (17:56 -0700)
committerJessica Wagantall <jwagantall@linuxfoundation.org>
Thu, 4 Apr 2019 18:37:12 +0000 (11:37 -0700)
WhiteSource is an open source security and license compliance
management platform.
It is used to scan a great variety of languages includding Java,
Python, C, Go, among others.

{project-name}-whitesource-scan-{stream} uses WhiteSource's
Unified Agent to perform the code scans.

Issue: RELENG-1876
Change-Id: If511da2f51e08799352ee6841338f76dd63af6b6
Signed-off-by: Jessica Wagantall <jwagantall@linuxfoundation.org>
.jjb-test/lf-whitesource-jobs.yaml [new file with mode: 0644]
docs/jjb/lf-whitesource-jobs.rst [new file with mode: 0644]
jjb/lf-whitesource-jobs.yaml [new file with mode: 0644]
releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml [new file with mode: 0644]
shell/whitesource-unified-agent-cli.sh [new file with mode: 0644]
wss-unified-agent.config.example [new file with mode: 0644]

diff --git a/.jjb-test/lf-whitesource-jobs.yaml b/.jjb-test/lf-whitesource-jobs.yaml
new file mode 100644 (file)
index 0000000..1618666
--- /dev/null
@@ -0,0 +1,10 @@
+---
+- project:
+    name: whitesource-scan-project-jobs
+    jobs:
+      - gerrit-whitesource-scan
+    branch: 'master'
+    project: 'whitesource/scan-project-jobs'
+    project-name: 'whitesource-scan-project'
+    wss-product-name: EXAMPLE-ORG
+    wss-unified-agent-config: '.config/wss-unified-agent.config'
diff --git a/docs/jjb/lf-whitesource-jobs.rst b/docs/jjb/lf-whitesource-jobs.rst
new file mode 100644 (file)
index 0000000..5750a0f
--- /dev/null
@@ -0,0 +1,58 @@
+################
+WhiteSource Jobs
+################
+
+Macros
+======
+
+lf-infra-wss-unified-agent-scan
+-------------------------------
+
+Run WhiteSource Unified Agent for a project.
+
+Job Templates
+=============
+
+WhiteSource Unified Agent scan
+------------------------------
+
+Trigger WhiteSource code scans using Unified Agent. For more details:
+https://whitesource.atlassian.net/wiki/spaces/WD/pages/33718339/Unified+Agent
+
+The WhiteSource Unified Agent scanner runs using a configuration file:
+https://s3.amazonaws.com/unified-agent/wss-unified-agent.config
+
+:Template Names:
+
+    - {project-name}-whitesource-scan-{stream}
+    - gerrit-whitesource-scan
+    - github-whitesource-scan
+
+:Comment Trigger: run-whitesource
+
+:Required parameters:
+
+    :build-node: The node to run build on.
+    :jenkins-ssh-credential: Credential to use for SSH. (Set in defaults.yaml)
+    :wss-product-name: Product to asociate the WhiteSource report in the dashboard.
+    :wss-unified-agent-config: Path to wss-unifed-agent.config.
+
+:Optional parameters:
+
+    :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+    :build-timeout: Timeout in minutes before aborting build. (default: 60)
+    :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+    :java-version: Version of Java to use for the build. (default: openjdk8)
+    :stream: Keyword used to represent a release code-name.
+        Often the same as the branch. (default: master)
+    :submodule-recursive: Whether to checkout submodules recursively.
+        (default: true)
+    :submodule-timeout: Timeout (in minutes) for checkout operation.
+        (default: 10)
+    :wss-unified-agent-version: WhiteSource Unified Agent version package to download
+        and use.
+    :gerrit_trigger_file_paths: Override file paths which used to filter which
+        file modifications will trigger a build. Refer to JJB documentation for
+        "file-path" details.
+        https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit
+    :gerrit_wss_triggers: Override Gerrit Triggers.
diff --git a/jjb/lf-whitesource-jobs.yaml b/jjb/lf-whitesource-jobs.yaml
new file mode 100644 (file)
index 0000000..8967374
--- /dev/null
@@ -0,0 +1,209 @@
+---
+
+##########
+# Macros #
+##########
+
+- builder:
+    name: lf-infra-wss-unified-agent-scan
+    builders:
+      - shell: !include-raw:
+          - ../shell/whitesource-unified-agent-cli.sh
+
+####################
+# COMMON FUNCTIONS #
+####################
+
+- lf_whitesource_common: &lf_whitesource_common
+    name: lf-whitesource-common
+
+    ######################
+    # Default parameters #
+    ######################
+
+    archive-artifacts: >
+      **/*.log
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+      - credentials-binding:
+          - text:
+              credential-id: wss-apiKey
+              variable: wss-apiKey
+
+    publishers:
+      - lf-infra-publish
+
+##################################
+# WHITESOURCE UNIFIED AGENT SCAN #
+##################################
+
+- lf_wss_unified_agent_scan: &lf_wss_unified_agent_scan
+    name: lf-wss-unified-agent-scan
+
+    ######################
+    # Default parameters #
+    ######################
+
+    branch: master
+    build-days-to-keep: 30
+    build-timeout: 60
+    cron: '@daily'
+    disable-job: false
+    git-url: '$GIT_URL/$PROJECT'
+    github-url: 'https://github.com'
+    java-version: openjdk8
+    stream: master
+    submodule-recursive: true
+    submodule-timeout: 10
+    wss-unified-agent-opts: ''
+    wss-unified-agent-version: 19.2.2.2
+
+    gerrit_trigger_file_paths:
+      - compare-type: ANT
+        pattern: '.*'
+
+    # github_included_regions MUST match gerrit_trigger_file_paths
+    github_included_regions:
+      - '.*'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    disabled: '{disable-job}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+      - string:
+          name: WSS_PROJECT_NAME
+          default: '{project-name}'
+          description: Project to asociate the WhiteSource report in the dashboard.
+      - string:
+          name: WSS_PRODUCT_NAME
+          default: '{wss-product-name}'
+          description: Product to asociate the WhiteSource report in the dashboard.
+      - string:
+          name: WSS_UNIFIED_AGENT_CONFIG
+          default: '{wss-unified-agent-config}'
+          description: Path to wss-unifed-agent.config
+      - string:
+          name: WSS_UNIFIED_AGENT_OPTIONS
+          default: '{wss-unified-agent-opts}'
+          description: Options to pass to the WhiteSource Unified Agent CLI.
+      - string:
+          name: WSS_UNIFIED_AGENT_VERSION
+          default: '{wss-unified-agent-version}'
+          description: WhiteSource Unified Agent version package to download and use.
+
+    builders:
+      - lf-infra-pre-build
+      - lf-update-java-alternatives:
+          java-version: '{java-version}'
+      - lf-infra-wss-unified-agent-scan
+
+- job-template:
+    name: '{project-name}-whitesource-scan-{stream}'
+    id: gerrit-whitesource-scan
+    <<: *lf_whitesource_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_wss_unified_agent_scan
+
+    ######################
+    # Default parameters #
+    ######################
+
+    gerrit_wss_triggers:
+      - comment-added-contains-event:
+          comment-contains-value: run-whitesource$
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          submodule-timeout: '{submodule-timeout}'
+          choosing-strategy: default
+
+    triggers:
+      # Build weekly on Saturdays
+      - timed: '{obj:cron}'
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_wss_triggers}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+          skip-vote:
+            successful: true
+            failed: true
+            unstable: true
+            notbuilt: true
+
+- job-template:
+    name: '{project-name}-whitesource-scan-{stream}'
+    id: github-whitesource-scan
+    <<: *lf_whitesource_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_wss_unified_agent_scan
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+      - github:
+          url: '{github-url}/{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '{git-clone-url}{github-org}/{project}'
+          refspec: ''
+          branch: 'refs/heads/{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          submodule-timeout: '{submodule-timeout}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      # Build weekly on Saturdays
+      - timed: '{obj:cron}'
+      - github-pull-request:
+          trigger-phrase: '^run-whitesource$'
+          only-trigger-phrase: false
+          status-context: 'WhiteSource'
+          permit-all: true
+          github-hooks: true
+          white-list-target-branches:
+            - '{branch}'
+          included-regions: '{obj:github_included_regions}'
diff --git a/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml b/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml
new file mode 100644 (file)
index 0000000..dde27cd
--- /dev/null
@@ -0,0 +1,15 @@
+---
+prelude: >
+    WhiteSource is a security and license compliance management platform.
+    It is used to perform scans on a great variety of coding and scripting
+    languages.
+features:
+  - |
+    Job {project-name}-whitesource-scan-{stream} uses the WhiteSource Unified
+    Agent scanner CLI tool to perform the code scan and report the results into
+    the WhiteSource dashboard.
+other:
+  - |
+    To run this job, a configuration file is needed (wss-unified-agent.config.example).
+    A new secret text credential will need to be created. (ID=wss-apiKey Secret=WhiteSource
+    organization API key)
diff --git a/shell/whitesource-unified-agent-cli.sh b/shell/whitesource-unified-agent-cli.sh
new file mode 100644 (file)
index 0000000..750a152
--- /dev/null
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script downloads wss-unified-agent-<version>.jar and uses it to perform
+# a scan on the code whithin a repo based on the wss-unified-agent.config provided.
+
+# DO NOT enable -u because $WSS_UNIFIED_AGENT_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -xe -o pipefail
+set -u
+echo "---> whitesource-unified-agent-cli.sh"
+jar_location="/tmp/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar"
+wget -nv https://s3.amazonaws.com/unified-agent/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar \
+    -O $jar_location
+echo "---> Running WhiteSource Unified Agent CLI ..."
+java -jar $jar_location -c $WSS_UNIFIED_AGENT_CONFIG \
+    -product $WSS_PRODUCT_NAME -project $WSS_PROJECT_NAME \
+    ${WSS_UNIFIED_AGENT_OPTIONS:-}
+rm $jar_location
diff --git a/wss-unified-agent.config.example b/wss-unified-agent.config.example
new file mode 100644 (file)
index 0000000..4a9f091
--- /dev/null
@@ -0,0 +1,236 @@
+####################################################################\r
+# WhiteSource Unified-Agent configuration file\r
+####################################################################\r
+##########################################\r
+# GENERAL SCAN MODE: Files and Package Managers\r
+##########################################\r
+\r
+checkPolicies=false\r
+forceCheckAllDependencies=false\r
+forceUpdate=false\r
+forceUpdate.failBuildOnPolicyViolation=false\r
+offline=false\r
+#ignoreSourceFiles=true\r
+#scanComment=\r
+#updateInventory=false\r
+#resolveAllDependencies=false\r
+#failErrorLevel=ALL\r
+#requireKnownSha1=false\r
+#generateProjectDetailsJson=true\r
+#generateScanReport=true\r
+#scanReportTimeoutMinutes=10\r
+#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model\r
+\r
+#projectPerFolder=true\r
+#projectPerFolderIncludes=\r
+#projectPerFolderExcludes=\r
+\r
+#wss.connectionTimeoutMinutes=60\r
+# Change the below URL to your WhiteSource server.\r
+# Use the 'WhiteSource Server URL' which can be retrieved\r
+# from your 'Profile' page on the 'Server URLs' panel.\r
+# Then, add the '/agent' path to it.\r
+wss.url=https://saas.whitesourcesoftware.com/agent\r
+\r
+#npm.resolveDependencies=false\r
+#npm.ignoreSourceFiles=false\r
+#npm.includeDevDependencies=true\r
+#npm.runPreStep=true\r
+#npm.ignoreNpmLsErrors=true\r
+#npm.ignoreScripts=true\r
+#npm.yarnProject=true\r
+#npm.accessToken=\r
+#npm.identifyByNameAndVersion=true\r
+\r
+#bower.resolveDependencies=false\r
+#bower.ignoreSourceFiles=true\r
+#bower.runPreStep=true\r
+\r
+#nuget.resolvePackagesConfigFiles=false\r
+#nuget.resolveCsProjFiles=false\r
+#nuget.resolveDependencies=false\r
+#nuget.restoreDependencies=true\r
+#nuget.preferredEnvironment=\r
+#nuget.packagesDirectory=\r
+#nuget.ignoreSourceFiles=true\r
+#nuget.runPreStep=true\r
+#nuget.resolveNuspecFiles=false\r
+\r
+#python.resolveDependencies=false\r
+#python.ignoreSourceFiles=false\r
+#python.ignorePipInstallErrors=true\r
+#python.installVirtualenv=true\r
+#python.resolveHierarchyTree=false\r
+#python.requirementsFileIncludes=requirements.txt\r
+#python.resolveSetupPyFiles=true\r
+#python.runPipenvPreStep=true\r
+#python.pipenvDevDependencies=true\r
+#python.IgnorePipenvInstallErrors=true\r
+\r
+#maven.ignoredScopes=test provided\r
+#maven.resolveDependencies=false\r
+#maven.ignoreSourceFiles=true\r
+#maven.aggregateModules=true\r
+#maven.ignorePomModules=false\r
+#maven.runPreStep=true\r
+#maven.ignoreMvnTreeErrors=true\r
+#maven.environmentPath=\r
+#maven.m2RepositoryPath=\r
+\r
+#gradle.ignoredScopes=\r
+#gradle.resolveDependencies=false\r
+#gradle.runAssembleCommand=false\r
+#gradle.runPreStep=true\r
+#gradle.ignoreSourceFiles=true\r
+#gradle.aggregateModules=true\r
+#gradle.preferredEnvironment=wrapper\r
+#gradle.localRepositoryPath=\r
+\r
+#paket.resolveDependencies=false\r
+#paket.ignoredGroups=\r
+#paket.ignoreSourceFiles=false\r
+#paket.runPreStep=true\r
+#paket.exePath=\r
+\r
+#go.resolveDependencies=false\r
+#go.collectDependenciesAtRuntime=true\r
+#go.dependencyManager=\r
+#go.ignoreSourceFiles=true\r
+#go.glide.ignoreTestPackages=false\r
+#go.gogradle.enableTaskAlias=true\r
+\r
+#ruby.resolveDependencies = false\r
+#ruby.ignoreSourceFiles = false\r
+#ruby.installMissingGems = true\r
+#ruby.runBundleInstall = true\r
+#ruby.overwriteGemFile = true\r
+\r
+#sbt.resolveDependencies=false\r
+#sbt.ignoreSourceFiles=true\r
+#sbt.aggregateModules=true\r
+#sbt.runPreStep=true\r
+#sbt.targetFolder=\r
+\r
+#php.resolveDependencies=false\r
+#php.runPreStep=true\r
+#php.includeDevDependencies=true\r
+\r
+#html.resolveDependencies=false\r
+\r
+#cocoapods.resolveDependencies=false\r
+#cocoapods.runPreStep=true\r
+#cocoapods.ignoreSourceFiles=false\r
+\r
+#hex.resolveDependencies=false\r
+#hex.runPreStep=true\r
+#hex.ignoreSourceFiles=false\r
+#hex.aggregateModules=true\r
+\r
+##################################\r
+# Organization tokens:\r
+##################################\r
+apiKey=\r
+\r
+#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option\r
+#userKey=\r
+\r
+projectName=\r
+projectVersion=\r
+projectToken=\r
+\r
+productName=\r
+productVersion=\r
+productToken=\r
+#updateType=APPEND\r
+#requesterEmail=user@provider.com\r
+\r
+#########################################################################################\r
+# Includes/Excludes Glob patterns - PLEASE USE ONLY ONE EXCLUDE LINE AND ONE INCLUDE LINE\r
+#########################################################################################\r
+includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx **/*.go **/*.py\r
+\r
+#includes=**/*.m **/*.mm  **/*.js **/*.php\r
+#includes=**/*.jar\r
+#includes=**/*.gem **/*.rb\r
+#includes=**/*.dll **/*.cs **/*.nupkg\r
+#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2\r
+#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py\r
+\r
+## Exclude file extensions or specific directories by adding **/*.<extension> or **<excluded_dir>/**\r
+excludes=**/*sources.jar **/*javadoc.jar\r
+\r
+case.sensitive.glob=false\r
+followSymbolicLinks=true\r
+\r
+##################################\r
+# Archive properties\r
+##################################\r
+#archiveExtractionDepth=2\r
+#archiveIncludes=**/*.war **/*.ear\r
+#archiveExcludes=**/*sources.jar\r
+\r
+##################################\r
+# Proxy settings\r
+##################################\r
+#proxy.host=\r
+#proxy.port=\r
+#proxy.user=\r
+#proxy.pass=\r
+\r
+##################################\r
+# SCM settings\r
+##################################\r
+#scm.type=\r
+#scm.user=\r
+#scm.pass=\r
+#scm.ppk=\r
+#scm.url=\r
+#scm.branch=\r
+#scm.tag=\r
+#scm.npmInstall=\r
+#scm.npmInstallTimeoutMinutes=\r
+#scm.repositoriesFile=\r
+\r
+##############################################\r
+# SCAN MODE: Linux package manager settings\r
+##############################################\r
+#scanPackageManager=true\r
+\r
+##################################\r
+# SCAN MODE: Docker images\r
+##################################\r
+#docker.scanImages=true\r
+#docker.includes=.*.*\r
+#docker.excludes=\r
+#docker.pull.enable=true\r
+#docker.pull.images=.*.*\r
+#docker.pull.maxImages=10\r
+#docker.pull.tags=.*.*\r
+#docker.pull.digest=\r
+#docker.delete.force=true\r
+#docker.login.sudo=false\r
+\r
+#docker.aws.enable=true\r
+#docker.aws.registryIds=\r
+\r
+#docker.azure.enable=true\r
+#docker.azure.userName=\r
+#docker.azure.userPassword=\r
+#docker.azure.registryNames=\r
+\r
+##################################\r
+# SCAN MODE: Docker containers\r
+##################################\r
+#docker.scanContainers=true\r
+#docker.containerIncludes=.*.*\r
+#docker.containerExcludes=\r
+\r
+################################\r
+# Serverless settings\r
+################################\r
+#serverless.provider=\r
+#serverless.scanFunctions=true\r
+#serverless.includes=\r
+#serverless.excludes=\r
+#serverless.region=\r
+#serverless.maxFunctions=10\r