From b9bf2930c67afd6ba69c1be0f672c0a21de34303 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Wed, 9 Dec 2020 08:19:51 -0800 Subject: [PATCH] Add pipeline-verify jobs to lint and check agents This adds a simple pipeline verification job that will lint any pipelines found, and check to make sure that they are not set up to run on master. Issue: RELENG-2185 Change-Id: I76d400d145857887a6d28d7c2a9869ec4acf414d Signed-off-by: Eric Ball --- docs/jjb/lf-macros.rst | 6 + docs/jjb/lf-pipeline-jobs.rst | 62 +++++++++ jjb/lf-macros.yaml | 5 + jjb/lf-pipeline-jobs.yaml | 153 +++++++++++++++++++++ .../notes/pipeline-verify-3e6b2c896faee653.yaml | 6 + shell/pipeline-linter.sh | 41 ++++++ 6 files changed, 273 insertions(+) create mode 100644 docs/jjb/lf-pipeline-jobs.rst create mode 100644 jjb/lf-pipeline-jobs.yaml create mode 100644 releasenotes/notes/pipeline-verify-3e6b2c896faee653.yaml create mode 100644 shell/pipeline-linter.sh diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 4280cb80..1ea2cfaf 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -374,6 +374,12 @@ Requires ``SonarQube Scanner for Jenkins`` :sonar-java-opts: JVM options. (default: "") :sonar-additional-args: Additional command line arguments. (default: "") +lf-infra-pipeline-verify +------------------------ + +Verify a Jenkins pipeline by linting it and ensuring that it cannot run on the +Jenkins master. + Parameters ========== diff --git a/docs/jjb/lf-pipeline-jobs.rst b/docs/jjb/lf-pipeline-jobs.rst new file mode 100644 index 00000000..18375f81 --- /dev/null +++ b/docs/jjb/lf-pipeline-jobs.rst @@ -0,0 +1,62 @@ +.. _lf-global-jjb-pipeline-jobs: + +############# +Pipeline Jobs +############# + + +Macros +====== + +lf-pipeline-common +------------------ + +Common definitions for use within all pipeline jobs. + + +Job Templates +============= + +Pipeline Verify +--------------- + +Verify job that checks a Jenkins pipeline by linting it and ensuring that it +cannot run on the master. + +:Template Names: + - {project-name}-pipeline-verify-{stream} + - gerrit-pipeline-verify + - github-pipeline-verify + +:Comment Trigger: recheck|reverify + +:Required Parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-node: The node to run build on. + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :project-pattern: Project to trigger build against. (default: \*\*) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + + :gerrit_verify_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: REG_EXP + pattern: "Jenkinsfile.*" diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index f583181b..f5021d26 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -397,6 +397,11 @@ java-opts: "{sonar-java-opts}" additional-arguments: "{sonar-additional-args}" +- builder: + name: lf-infra-pipeline-verify + builders: + - shell: !include-raw-escape: ../shell/pipeline-linter.sh + ############## # PARAMETERS # ############## diff --git a/jjb/lf-pipeline-jobs.yaml b/jjb/lf-pipeline-jobs.yaml new file mode 100644 index 00000000..6f11408d --- /dev/null +++ b/jjb/lf-pipeline-jobs.yaml @@ -0,0 +1,153 @@ +--- +#################### +# COMMON FUNCTIONS # +#################### + +- lf_pipeline_common: &lf_pipeline_common + name: lf-pipeline-common + + ###################### + # Default parameters # + ###################### + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: "Jenkinsfile.*" + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - "Jenkinsfile.*" + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + + properties: + - lf-infra-properties: + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + refspec: "refs/heads/{branch}" + stream: "{stream}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + publishers: + - lf-infra-publish + +################## +# PIPELINE VERIFY# +################## + +- lf_pipeline_verify: &lf_pipeline_verify + name: lf-pipeline-verify + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + project-pattern: "**" + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + builders: + - lf-infra-pipeline-verify + +- job-template: + name: "{project-name}-pipeline-verify-{stream}" + id: gerrit-pipeline-verify + concurrent: true + <<: *lf_pipeline_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pipeline_verify + + scm: + - lf-infra-gerrit-scm: + branch: "$GERRIT_BRANCH" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_verify_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project-pattern}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-pipeline-verify-{stream}" + id: github-pipeline-verify + concurrent: true + <<: *lf_pipeline_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pipeline_verify + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^(recheck|reverify)$" + only-trigger-phrase: false + status-context: "Pipeline Verify" + permit-all: true + github-hooks: true + included-regions: "{obj:github_included_regions}" + white-list-target-branches: + - "{branch}" diff --git a/releasenotes/notes/pipeline-verify-3e6b2c896faee653.yaml b/releasenotes/notes/pipeline-verify-3e6b2c896faee653.yaml new file mode 100644 index 00000000..93cc85e3 --- /dev/null +++ b/releasenotes/notes/pipeline-verify-3e6b2c896faee653.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add pipeline-verify jobs. This adds a simple pipeline verification job that + will lint any pipelines found, and check to make sure that they are not set + up to run on master. diff --git a/shell/pipeline-linter.sh b/shell/pipeline-linter.sh new file mode 100644 index 00000000..8ed681d5 --- /dev/null +++ b/shell/pipeline-linter.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2020 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> pipeline-linter.sh" + +set -eu -o pipefail + +# shellcheck disable=SC2153 +JENKINS_CRUMB=$(curl --silent "$JENKINS_URL/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)") +JENKINS_VAL="$JENKINS_URL/pipeline-model-converter/validate" +mapfile -t JENKINS_FILE_LIST < <(grep -lr "^pipeline\s*{" vars src Jenkinsfile*) +exit_code=0 + +for JENKINS_FILE in "${JENKINS_FILE_LIST[@]}" +do + ret=$(curl --silent -X POST -H "$JENKINS_CRUMB" -F "jenkinsfile=<$JENKINS_FILE" "$JENKINS_VAL") + if [[ $ret == *"Errors"* ]];then + echo "ERROR: Linting error for $JENKINS_FILE" + exit_code=1 + # Check for the line "agent any". This includes master as an agent. + elif grep "^\s*agent any" "$JENKINS_FILE"; then + echo "ERROR: $JENKINS_FILE is set to use any agent. Please specify a label instead." + exit_code=1 + # If "any" is not used, check for master as a specified label or node. + elif grep -Pz "agent\s*\{\s*(label|node)\s+['\"]*master" "$JENKINS_FILE"; then + echo "ERROR: $JENKINS_FILE is set to use master as an agent. Please specify a different label." + exit_code=1 + else + echo "$JENKINS_FILE successfully validated" + fi +done + +# Set non-zero exit if linter reports any errors +exit "$exit_code" -- 2.16.6