From: Jessica Wagantall <jwagantall@linuxfoundation.org>
Date: Fri, 22 Mar 2019 00:56:15 +0000 (-0700)
Subject: Add WhiteSource Scan jobs
X-Git-Tag: v0.36.0~11^2
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=232378ffaca9f7ef383b9371b4f4144259225d8c;p=releng%2Fglobal-jjb.git

Add WhiteSource Scan jobs

WhiteSource is an open source security and license compliance
management platform.
It is used to scan a great variety of languages includding Java,
Python, C, Go, among others.

{project-name}-whitesource-scan-{stream} uses WhiteSource's
Unified Agent to perform the code scans.

Issue: RELENG-1876
Change-Id: If511da2f51e08799352ee6841338f76dd63af6b6
Signed-off-by: Jessica Wagantall <jwagantall@linuxfoundation.org>
---

diff --git a/.jjb-test/lf-whitesource-jobs.yaml b/.jjb-test/lf-whitesource-jobs.yaml
new file mode 100644
index 00000000..16186665
--- /dev/null
+++ b/.jjb-test/lf-whitesource-jobs.yaml
@@ -0,0 +1,10 @@
+---
+- project:
+    name: whitesource-scan-project-jobs
+    jobs:
+      - gerrit-whitesource-scan
+    branch: 'master'
+    project: 'whitesource/scan-project-jobs'
+    project-name: 'whitesource-scan-project'
+    wss-product-name: EXAMPLE-ORG
+    wss-unified-agent-config: '.config/wss-unified-agent.config'
diff --git a/docs/jjb/lf-whitesource-jobs.rst b/docs/jjb/lf-whitesource-jobs.rst
new file mode 100644
index 00000000..5750a0f4
--- /dev/null
+++ b/docs/jjb/lf-whitesource-jobs.rst
@@ -0,0 +1,58 @@
+################
+WhiteSource Jobs
+################
+
+Macros
+======
+
+lf-infra-wss-unified-agent-scan
+-------------------------------
+
+Run WhiteSource Unified Agent for a project.
+
+Job Templates
+=============
+
+WhiteSource Unified Agent scan
+------------------------------
+
+Trigger WhiteSource code scans using Unified Agent. For more details:
+https://whitesource.atlassian.net/wiki/spaces/WD/pages/33718339/Unified+Agent
+
+The WhiteSource Unified Agent scanner runs using a configuration file:
+https://s3.amazonaws.com/unified-agent/wss-unified-agent.config
+
+:Template Names:
+
+    - {project-name}-whitesource-scan-{stream}
+    - gerrit-whitesource-scan
+    - github-whitesource-scan
+
+:Comment Trigger: run-whitesource
+
+:Required parameters:
+
+    :build-node: The node to run build on.
+    :jenkins-ssh-credential: Credential to use for SSH. (Set in defaults.yaml)
+    :wss-product-name: Product to asociate the WhiteSource report in the dashboard.
+    :wss-unified-agent-config: Path to wss-unifed-agent.config.
+
+:Optional parameters:
+
+    :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+    :build-timeout: Timeout in minutes before aborting build. (default: 60)
+    :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+    :java-version: Version of Java to use for the build. (default: openjdk8)
+    :stream: Keyword used to represent a release code-name.
+        Often the same as the branch. (default: master)
+    :submodule-recursive: Whether to checkout submodules recursively.
+        (default: true)
+    :submodule-timeout: Timeout (in minutes) for checkout operation.
+        (default: 10)
+    :wss-unified-agent-version: WhiteSource Unified Agent version package to download
+        and use.
+    :gerrit_trigger_file_paths: Override file paths which used to filter which
+        file modifications will trigger a build. Refer to JJB documentation for
+        "file-path" details.
+        https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit
+    :gerrit_wss_triggers: Override Gerrit Triggers.
diff --git a/jjb/lf-whitesource-jobs.yaml b/jjb/lf-whitesource-jobs.yaml
new file mode 100644
index 00000000..89673741
--- /dev/null
+++ b/jjb/lf-whitesource-jobs.yaml
@@ -0,0 +1,209 @@
+---
+
+##########
+# Macros #
+##########
+
+- builder:
+    name: lf-infra-wss-unified-agent-scan
+    builders:
+      - shell: !include-raw:
+          - ../shell/whitesource-unified-agent-cli.sh
+
+####################
+# COMMON FUNCTIONS #
+####################
+
+- lf_whitesource_common: &lf_whitesource_common
+    name: lf-whitesource-common
+
+    ######################
+    # Default parameters #
+    ######################
+
+    archive-artifacts: >
+      **/*.log
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+      - credentials-binding:
+          - text:
+              credential-id: wss-apiKey
+              variable: wss-apiKey
+
+    publishers:
+      - lf-infra-publish
+
+##################################
+# WHITESOURCE UNIFIED AGENT SCAN #
+##################################
+
+- lf_wss_unified_agent_scan: &lf_wss_unified_agent_scan
+    name: lf-wss-unified-agent-scan
+
+    ######################
+    # Default parameters #
+    ######################
+
+    branch: master
+    build-days-to-keep: 30
+    build-timeout: 60
+    cron: '@daily'
+    disable-job: false
+    git-url: '$GIT_URL/$PROJECT'
+    github-url: 'https://github.com'
+    java-version: openjdk8
+    stream: master
+    submodule-recursive: true
+    submodule-timeout: 10
+    wss-unified-agent-opts: ''
+    wss-unified-agent-version: 19.2.2.2
+
+    gerrit_trigger_file_paths:
+      - compare-type: ANT
+        pattern: '.*'
+
+    # github_included_regions MUST match gerrit_trigger_file_paths
+    github_included_regions:
+      - '.*'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    disabled: '{disable-job}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+      - string:
+          name: WSS_PROJECT_NAME
+          default: '{project-name}'
+          description: Project to asociate the WhiteSource report in the dashboard.
+      - string:
+          name: WSS_PRODUCT_NAME
+          default: '{wss-product-name}'
+          description: Product to asociate the WhiteSource report in the dashboard.
+      - string:
+          name: WSS_UNIFIED_AGENT_CONFIG
+          default: '{wss-unified-agent-config}'
+          description: Path to wss-unifed-agent.config
+      - string:
+          name: WSS_UNIFIED_AGENT_OPTIONS
+          default: '{wss-unified-agent-opts}'
+          description: Options to pass to the WhiteSource Unified Agent CLI.
+      - string:
+          name: WSS_UNIFIED_AGENT_VERSION
+          default: '{wss-unified-agent-version}'
+          description: WhiteSource Unified Agent version package to download and use.
+
+    builders:
+      - lf-infra-pre-build
+      - lf-update-java-alternatives:
+          java-version: '{java-version}'
+      - lf-infra-wss-unified-agent-scan
+
+- job-template:
+    name: '{project-name}-whitesource-scan-{stream}'
+    id: gerrit-whitesource-scan
+    <<: *lf_whitesource_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_wss_unified_agent_scan
+
+    ######################
+    # Default parameters #
+    ######################
+
+    gerrit_wss_triggers:
+      - comment-added-contains-event:
+          comment-contains-value: run-whitesource$
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          submodule-timeout: '{submodule-timeout}'
+          choosing-strategy: default
+
+    triggers:
+      # Build weekly on Saturdays
+      - timed: '{obj:cron}'
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_wss_triggers}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+          skip-vote:
+            successful: true
+            failed: true
+            unstable: true
+            notbuilt: true
+
+- job-template:
+    name: '{project-name}-whitesource-scan-{stream}'
+    id: github-whitesource-scan
+    <<: *lf_whitesource_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_wss_unified_agent_scan
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+      - github:
+          url: '{github-url}/{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '{git-clone-url}{github-org}/{project}'
+          refspec: ''
+          branch: 'refs/heads/{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          submodule-timeout: '{submodule-timeout}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      # Build weekly on Saturdays
+      - timed: '{obj:cron}'
+      - github-pull-request:
+          trigger-phrase: '^run-whitesource$'
+          only-trigger-phrase: false
+          status-context: 'WhiteSource'
+          permit-all: true
+          github-hooks: true
+          white-list-target-branches:
+            - '{branch}'
+          included-regions: '{obj:github_included_regions}'
diff --git a/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml b/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml
new file mode 100644
index 00000000..dde27cd0
--- /dev/null
+++ b/releasenotes/notes/lf-whitesource-jobs-bf3c81f9c2ea4b73.yaml
@@ -0,0 +1,15 @@
+---
+prelude: >
+    WhiteSource is a security and license compliance management platform.
+    It is used to perform scans on a great variety of coding and scripting
+    languages.
+features:
+  - |
+    Job {project-name}-whitesource-scan-{stream} uses the WhiteSource Unified
+    Agent scanner CLI tool to perform the code scan and report the results into
+    the WhiteSource dashboard.
+other:
+  - |
+    To run this job, a configuration file is needed (wss-unified-agent.config.example).
+    A new secret text credential will need to be created. (ID=wss-apiKey Secret=WhiteSource
+    organization API key)
diff --git a/shell/whitesource-unified-agent-cli.sh b/shell/whitesource-unified-agent-cli.sh
new file mode 100644
index 00000000..750a1523
--- /dev/null
+++ b/shell/whitesource-unified-agent-cli.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script downloads wss-unified-agent-<version>.jar and uses it to perform
+# a scan on the code whithin a repo based on the wss-unified-agent.config provided.
+
+# DO NOT enable -u because $WSS_UNIFIED_AGENT_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -xe -o pipefail
+set -u
+echo "---> whitesource-unified-agent-cli.sh"
+jar_location="/tmp/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar"
+wget -nv https://s3.amazonaws.com/unified-agent/wss-unified-agent-$WSS_UNIFIED_AGENT_VERSION.jar \
+    -O $jar_location
+echo "---> Running WhiteSource Unified Agent CLI ..."
+java -jar $jar_location -c $WSS_UNIFIED_AGENT_CONFIG \
+    -product $WSS_PRODUCT_NAME -project $WSS_PROJECT_NAME \
+    ${WSS_UNIFIED_AGENT_OPTIONS:-}
+rm $jar_location
diff --git a/wss-unified-agent.config.example b/wss-unified-agent.config.example
new file mode 100644
index 00000000..4a9f091c
--- /dev/null
+++ b/wss-unified-agent.config.example
@@ -0,0 +1,236 @@
+####################################################################
+# WhiteSource Unified-Agent configuration file
+####################################################################
+##########################################
+# GENERAL SCAN MODE: Files and Package Managers
+##########################################
+
+checkPolicies=false
+forceCheckAllDependencies=false
+forceUpdate=false
+forceUpdate.failBuildOnPolicyViolation=false
+offline=false
+#ignoreSourceFiles=true
+#scanComment=
+#updateInventory=false
+#resolveAllDependencies=false
+#failErrorLevel=ALL
+#requireKnownSha1=false
+#generateProjectDetailsJson=true
+#generateScanReport=true
+#scanReportTimeoutMinutes=10
+#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model
+
+#projectPerFolder=true
+#projectPerFolderIncludes=
+#projectPerFolderExcludes=
+
+#wss.connectionTimeoutMinutes=60
+# Change the below URL to your WhiteSource server.
+# Use the 'WhiteSource Server URL' which can be retrieved
+# from your 'Profile' page on the 'Server URLs' panel.
+# Then, add the '/agent' path to it.
+wss.url=https://saas.whitesourcesoftware.com/agent
+
+#npm.resolveDependencies=false
+#npm.ignoreSourceFiles=false
+#npm.includeDevDependencies=true
+#npm.runPreStep=true
+#npm.ignoreNpmLsErrors=true
+#npm.ignoreScripts=true
+#npm.yarnProject=true
+#npm.accessToken=
+#npm.identifyByNameAndVersion=true
+
+#bower.resolveDependencies=false
+#bower.ignoreSourceFiles=true
+#bower.runPreStep=true
+
+#nuget.resolvePackagesConfigFiles=false
+#nuget.resolveCsProjFiles=false
+#nuget.resolveDependencies=false
+#nuget.restoreDependencies=true
+#nuget.preferredEnvironment=
+#nuget.packagesDirectory=
+#nuget.ignoreSourceFiles=true
+#nuget.runPreStep=true
+#nuget.resolveNuspecFiles=false
+
+#python.resolveDependencies=false
+#python.ignoreSourceFiles=false
+#python.ignorePipInstallErrors=true
+#python.installVirtualenv=true
+#python.resolveHierarchyTree=false
+#python.requirementsFileIncludes=requirements.txt
+#python.resolveSetupPyFiles=true
+#python.runPipenvPreStep=true
+#python.pipenvDevDependencies=true
+#python.IgnorePipenvInstallErrors=true
+
+#maven.ignoredScopes=test provided
+#maven.resolveDependencies=false
+#maven.ignoreSourceFiles=true
+#maven.aggregateModules=true
+#maven.ignorePomModules=false
+#maven.runPreStep=true
+#maven.ignoreMvnTreeErrors=true
+#maven.environmentPath=
+#maven.m2RepositoryPath=
+
+#gradle.ignoredScopes=
+#gradle.resolveDependencies=false
+#gradle.runAssembleCommand=false
+#gradle.runPreStep=true
+#gradle.ignoreSourceFiles=true
+#gradle.aggregateModules=true
+#gradle.preferredEnvironment=wrapper
+#gradle.localRepositoryPath=
+
+#paket.resolveDependencies=false
+#paket.ignoredGroups=
+#paket.ignoreSourceFiles=false
+#paket.runPreStep=true
+#paket.exePath=
+
+#go.resolveDependencies=false
+#go.collectDependenciesAtRuntime=true
+#go.dependencyManager=
+#go.ignoreSourceFiles=true
+#go.glide.ignoreTestPackages=false
+#go.gogradle.enableTaskAlias=true
+
+#ruby.resolveDependencies = false
+#ruby.ignoreSourceFiles = false
+#ruby.installMissingGems = true
+#ruby.runBundleInstall = true
+#ruby.overwriteGemFile = true
+
+#sbt.resolveDependencies=false
+#sbt.ignoreSourceFiles=true
+#sbt.aggregateModules=true
+#sbt.runPreStep=true
+#sbt.targetFolder=
+
+#php.resolveDependencies=false
+#php.runPreStep=true
+#php.includeDevDependencies=true
+
+#html.resolveDependencies=false
+
+#cocoapods.resolveDependencies=false
+#cocoapods.runPreStep=true
+#cocoapods.ignoreSourceFiles=false
+
+#hex.resolveDependencies=false
+#hex.runPreStep=true
+#hex.ignoreSourceFiles=false
+#hex.aggregateModules=true
+
+##################################
+# Organization tokens:
+##################################
+apiKey=
+
+#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option
+#userKey=
+
+projectName=
+projectVersion=
+projectToken=
+
+productName=
+productVersion=
+productToken=
+#updateType=APPEND
+#requesterEmail=user@provider.com
+
+#########################################################################################
+# Includes/Excludes Glob patterns - PLEASE USE ONLY ONE EXCLUDE LINE AND ONE INCLUDE LINE
+#########################################################################################
+includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx **/*.go **/*.py
+
+#includes=**/*.m **/*.mm  **/*.js **/*.php
+#includes=**/*.jar
+#includes=**/*.gem **/*.rb
+#includes=**/*.dll **/*.cs **/*.nupkg
+#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2
+#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py
+
+## Exclude file extensions or specific directories by adding **/*.<extension> or **<excluded_dir>/**
+excludes=**/*sources.jar **/*javadoc.jar
+
+case.sensitive.glob=false
+followSymbolicLinks=true
+
+##################################
+# Archive properties
+##################################
+#archiveExtractionDepth=2
+#archiveIncludes=**/*.war **/*.ear
+#archiveExcludes=**/*sources.jar
+
+##################################
+# Proxy settings
+##################################
+#proxy.host=
+#proxy.port=
+#proxy.user=
+#proxy.pass=
+
+##################################
+# SCM settings
+##################################
+#scm.type=
+#scm.user=
+#scm.pass=
+#scm.ppk=
+#scm.url=
+#scm.branch=
+#scm.tag=
+#scm.npmInstall=
+#scm.npmInstallTimeoutMinutes=
+#scm.repositoriesFile=
+
+##############################################
+# SCAN MODE: Linux package manager settings
+##############################################
+#scanPackageManager=true
+
+##################################
+# SCAN MODE: Docker images
+##################################
+#docker.scanImages=true
+#docker.includes=.*.*
+#docker.excludes=
+#docker.pull.enable=true
+#docker.pull.images=.*.*
+#docker.pull.maxImages=10
+#docker.pull.tags=.*.*
+#docker.pull.digest=
+#docker.delete.force=true
+#docker.login.sudo=false
+
+#docker.aws.enable=true
+#docker.aws.registryIds=
+
+#docker.azure.enable=true
+#docker.azure.userName=
+#docker.azure.userPassword=
+#docker.azure.registryNames=
+
+##################################
+# SCAN MODE: Docker containers
+##################################
+#docker.scanContainers=true
+#docker.containerIncludes=.*.*
+#docker.containerExcludes=
+
+################################
+# Serverless settings
+################################
+#serverless.provider=
+#serverless.scanFunctions=true
+#serverless.includes=
+#serverless.excludes=
+#serverless.region=
+#serverless.maxFunctions=10