From bbaf6b55269a0cb6fb129af73d82c2efd6d83bcf Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Thu, 28 Sep 2017 11:26:37 -0400 Subject: [PATCH] Add license check job using new lhc Create a job using https://gerrit.linuxfoundation.org/infra/#/admin/projects/releng/license-checker as the license checker tool for projects. This job is meant to be managed by LFRE to ensure projects are providing the appropriate license headers. Change-Id: I5ca6da61dc6ee250a2c6d451b09a0075fb1ff0b0 Signed-off-by: Thanh Ha --- docs/jjb/lf-ci-jobs.rst | 28 +++++++++++ docs/jjb/lf-macros.rst | 20 ++++++++ jjb/lf-ci-jobs.yaml | 126 +++++++++++++++++++++++++++++++++++++++++++++++- jjb/lf-macros.yaml | 13 +++++ shell/license-check.sh | 62 ++++++++++++++++++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 shell/license-check.sh diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 70a6fa21..d752039d 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -426,6 +426,34 @@ other file changes. Verifies INFO.yaml files follow the schema defined in (default: true) :gerrit_verify_triggers: Override Gerrit Triggers. +.. _license-checker: + +License Checker +--------------- + +Job to scan projects for files missing license headers. + +:Template Names: + - {project-name}-license-check + - gerrit-license-check + - github-license-check + +:Optional parameters: + + :file-patterns: Space-separated list of file patterns to scan. + (default: \*.go \*.groovy \*.java \*.py \*.sh) + :spdx-disable: Disable the SPDX-Identifier checker. (default: false) + :lhc-version: Version of LHC to use. (default: 0.2.0) + :license-exclude-paths: Comma-separated list of paths to exclude from the + license checker. The paths used here will be matched using a contains + rule so it is best to be as precise with the path as possible. + For example a path of '/src/generated/' will be searched as + '**/src/generated/**'. + Example: org/opendaylight/yang/gen,protobuff/messages + (default: '') + :licenses-allowed: Comma-separated list of allowed licenses. + (default: Apache-2.0,EPL-1.0,MIT) + .. _gjjb-packer-merge: Packer Merge diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 08722cf7..1ab522cb 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -16,6 +16,26 @@ called DEPENDENCY_BUILD_ORDER which can be used if necessary to build projects in the specified order. The order is determined by first patch instance for a project in the patch list. +lf-license-check +---------------- + +Checks files for + +:Required parameters: + + :file-patterns: Space-separated list of file patterns to scan. + For example: \*.go \*.groovy \*.java \*.py \*.sh + :spdx-disable: Disable the SPDX-Identifier checker. + :lhc-version: Version of LHC to use. + :license-exclude-paths: Comma-separated list of paths to exclude from the + license checker. The paths used here will be matched using a contains + rule so it is best to be as precise with the path as possible. + For example a path of '/src/generated/' will be searched as + '**/src/generated/**'. + Example: org/opendaylight/yang/gen,protobuff/messages + :licenses-allowed: Comma-separated list of allowed licenses. + For example: Apache-2.0,EPL-1.0,MIT + lf-infra-create-netrc --------------------- diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 788dc46c..86459411 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -36,7 +36,6 @@ - gerrit-packer-merge - gerrit-packer-verify - - job-group: name: '{project-name}-github-packer-jobs' @@ -714,6 +713,131 @@ github-hooks: true included-regions: '{obj:github_included_regions}' +################# +# License Check # +################# + +- lf_license_check: &lf_license_check + name: lf-license-check + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 15 + file-patterns: > + *.go + *.groovy + *.java + *.py + *.sh + git-url: '$GIT_URL/$PROJECT' + lhc-version: 0.2.0 + license-exclude-paths: '' + licenses-allowed: Apache-2.0,EPL-1.0,MIT + spdx-disable: false + stream: master + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + concurrent: true + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + lftools-version: '{lftools-version}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + builders: + - lf-license-check: + file-patterns: '{file-patterns}' + spdx-disable: '{spdx-disable}' + lhc-version: '{lhc-version}' + license-exclude-paths: '{license-exclude-paths}' + licenses-allowed: '{licenses-allowed}' + + publishers: + - lf-infra-publish + +- job-template: + name: '{project-name}-license-check' + id: gerrit-license-check + <<: *lf_license_check + + git-url: '$GIT_URL/$GERRIT_PROJECT' + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + # Submodules are out of the project's control + submodule-recursive: false + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + # Trigger should not be overridable as we want to always run + trigger-on: + - patchset-created-event: + exclude-drafts: false + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: recheck$ + projects: + - project-compare-type: ANT + project-pattern: '**' + branches: + - branch-compare-type: ANT + branch-pattern: '**' + +- job-template: + name: '{project-name}-license-check' + id: github-license-check + <<: *lf_license_check + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '$sha1' + # Submodules are out of the project's control + submodule-recursive: false + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - github-pull-request: + trigger-phrase: '^recheck$' + only-trigger-phrase: false + status-context: 'License Check' + permit-all: true + github-hooks: true + #################### # Info YAML Verify # #################### diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 6a0cab9d..2fbb8256 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -10,6 +10,19 @@ - inject: properties-file: $WORKSPACE/.dependency.properties +- builder: + name: lf-license-check + builders: + - inject: + properties-content: | + FILE_PATTERNS={file-patterns} + SPDX_DISABLE={spdx-disable} + LHC_VERSION={lhc-version} + LICENSE_EXCLUDE_PATHS={license-exclude-paths} + LICENSES_ALLOWED={licenses-allowed} + - shell: !include-raw-escape: + - ../shell/license-check.sh + - builder: name: lf-infra-create-netrc builders: diff --git a/shell/license-check.sh b/shell/license-check.sh new file mode 100644 index 00000000..8466cf99 --- /dev/null +++ b/shell/license-check.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Scans files for license header violations +# +# This script is meant to be called by a Jenkins job and was not designed to +# run standalone refer to the below Inputs for details on optional ENVVAR +# inputs. + +echo "---> license-check.sh" + +# --- Inputs + +# Space separated list of file patterns to scan for license headers. +file_patterns=("${FILE_PATTERNS:-*.go *.groovy *.java *.py *.sh}") +# Version of the License Header Checker to install +lhc_version="${LHC_VERSION:-0.2.0}" +# Comma-separated list of paths to exclude from license checking +license_exclude_paths="${LICENSE_EXCLUDE_PATHS:-}" +# Comma-separated list of allowed licenses +licenses_allowed="${LICENSES_ALLOWED:-Apache-2.0,EPL-1.0,MIT}" + +if [[ "${SPDX_DISABLE}" == "true" ]]; then + disable_spdx="--disable-spdx" +else + disable_spdx="" +fi + +# --- Script start + +# DO NOT enable -u because LICENSE_EXCLUDE_PATHS is unbound. +# Ensure we fail the job if any steps fail. +set -eux -o pipefail + +if hash lhc 2>/dev/null; then + echo "License Header Checker is installed." + lhc --version +else + echo "License Header Checker is not installed. Installing..." + mkdir "$WORKSPACE/bin" + wget -nv -O "/tmp/lhc.tar.gz" "https://nexus.opendaylight.org/content/repositories/hosted_installers/org/linuxfoundation/lhc/${lhc_version}/lhc-${lhc_version}.tar.gz" + tar -zxvf /tmp/lhc.tar.gz -C "$WORKSPACE/bin" + chmod +x "$WORKSPACE/bin/lhc" + export PATH="$WORKSPACE/bin:$PATH" + lhc --version +fi + + +set -f # Disable globbing for $file_patterns to pass '*' +# Purposely disable SC2068 for $file_patterns +# shellcheck disable=SC2068 +lhc --license "$licenses_allowed" ${disable_spdx} \ + --exclude "$license_exclude_paths" \ + ${file_patterns[@]} +set +f -- 2.16.6