From 851b0b6d0cfe84e986548fd596a9b2f18413fefd Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Fri, 7 Apr 2017 10:00:12 -0700 Subject: [PATCH] Add lf ci-management jjb jobs This patch ports the jjb ci-management jobs over from OpenDaylight and makes necessary adjustments to make it generically re-useable as possible. This patch requires some environment variables defined. For example: JENKINS_HOSTNAME=jenkins092 LOGS_SERVER=https://logs.opendaylight.org NEXUS_URL=https://nexus.opendaylight.org SILO=releng Issue: RELENG-32 Change-Id: Ic19c5fa652885270d8e16992e82e642af95b0f4d Signed-off-by: Thanh Ha --- README.md | 66 ++++++++++++++ jjb/lf-ci-jobs.yaml | 183 ++++++++++++++++++++++++++++++++++++++ jjb/lf-macros.yaml | 142 +++++++++++++++++++++++++++++ shell/create-netrc.sh | 22 +++++ shell/gpg-verify-git-signature.sh | 20 +++++ shell/jjb-check-unicode.sh | 18 ++++ shell/jjb-install.sh | 18 ++++ shell/jjb-merge-job.sh | 13 +++ shell/jjb-verify-job.sh | 28 ++++++ shell/lftools-install.sh | 18 ++++ shell/logs-clear-credentials.sh | 14 +++ shell/logs-deploy.sh | 24 +++++ 12 files changed, 566 insertions(+) create mode 100644 README.md create mode 100644 jjb/lf-ci-jobs.yaml create mode 100644 jjb/lf-macros.yaml create mode 100644 shell/create-netrc.sh create mode 100644 shell/gpg-verify-git-signature.sh create mode 100644 shell/jjb-check-unicode.sh create mode 100644 shell/jjb-install.sh create mode 100644 shell/jjb-merge-job.sh create mode 100644 shell/jjb-verify-job.sh create mode 100644 shell/lftools-install.sh create mode 100644 shell/logs-clear-credentials.sh create mode 100644 shell/logs-deploy.sh diff --git a/README.md b/README.md new file mode 100644 index 00000000..ce7d420f --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# Global JJB + +The purpose of this repository is store generically define reusable JJB +templates that can be deployed across LF projects. + +The following variables are necessary to be defined in the Jenkins server as +global environment variables as scripts in this repo expect these variables to +be available. + +For example: + +``` +GIT_URL=ssh://jenkins-$SILO@git.opendaylight.org:29418 +JENKINS_HOSTNAME=jenkins092 +LOGS_SERVER=https://logs.opendaylight.org +NEXUS_URL=https://nexus.opendaylight.org +SILO=releng +``` + +## Jenkins Plugin Requirements + +**Required** + +- Config File Provider +- Description Setter +- Gerrit Trigger +- Post Build Script +- SSH Agent +- Workspace Cleanup + +**Optional** + +- Mask Passwords +- MsgInject +- OpenStack Cloud +- Timestamps + +## Deploying ci-jobs + +The CI job group contains multiple jobs that should be deployed in all LF +Jenkins infra. The minimal configuration needed to deploy the ci-management +jobs is as follows which deploys the **{project-name}-ci-jobs** job group as +defined in **lf-ci-jobs.yaml**. + +``` +- project: + name: ci-jobs + + jobs: + - '{project-name}-ci-jobs' + + project: ci-management + project-name: ci-management + build-node: centos7-basebuild-2c-1g +``` + +Required parameters: + +**project**: is the project repo as defined in Gerrit. +**project-name**: is a custom name to call the job in Jenkins. +**build-node**: is the name of the builder to use when building (Jenkins label). + +Optional parameters: + +**branch**: is the git branch to build from. +**jjb-version**: is the version of JJB to install in the build minion. diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml new file mode 100644 index 00000000..6a3b6b12 --- /dev/null +++ b/jjb/lf-ci-jobs.yaml @@ -0,0 +1,183 @@ +--- +- job-group: + name: '{project-name}-ci-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci. + + jjb-version: 1.6.2 + + jobs: + - '{project-name}-jjb-merge' + - '{project-name}-jjb-verify' + +#################### +# Anchors & Macros # +#################### + +- builder: + name: lf-infra-jjbini + builders: + - config-file-provider: + files: + - file-id: jjbini + target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini' + +- lf_jjb_file_paths: &lf_jjb_file_paths + name: lf-jjb-file-paths + file-paths: + - compare-type: ANT + pattern: '**/*.sh' + - compare-type: ANT + pattern: '**/*.yaml' + +- parameter: + name: lf-infra-jjb-parameters + parameters: + - string: + name: JJB_VERSION + default: '{jjb-version}' + description: Jenkins Job Builder version to download and install. + +################# +# Job Templates # +################# + +- job-template: + name: '{project-name}-jjb-merge' + project-type: freestyle + + node: '{build-node}' + + ###################### + # Default parameters # + ###################### + + branch: master + submodule-recursive: true + + ##################### + # Job Configuration # + ##################### + + properties: + - lf-infra-properties: + project: '{project}' + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + - lf-infra-jjb-parameters: + jjb-version: '{jjb-version}' + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + + wrappers: + - lf-infra-wrappers: + build-timeout: 10 + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: ^remerge$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_jjb_file_paths + + builders: + - lf-infra-jjbini + - shell: !include-raw-escape: + - ../shell/jjb-install.sh + - ../shell/jjb-merge-job.sh + + publishers: + - lf-infra-publish + +- job-template: + name: '{project-name}-jjb-verify' + project-type: freestyle + + node: '{build-node}' + concurrent: true + + ###################### + # Default parameters # + ###################### + + branch: master + submodule-recursive: true + + ##################### + # Job Configuration # + ##################### + + properties: + - lf-infra-properties: + project: '{project}' + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + - lf-infra-jjb-parameters: + jjb-version: '{jjb-version}' + + scm: + - lf-infra-gerrit-scm: + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + wrappers: + - lf-infra-wrappers: + build-timeout: 10 + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - patchset-created-event: + exclude-drafts: false + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: ^recheck$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_jjb_file_paths + + builders: + - lf-infra-jjbini + - shell: !include-raw-escape: + - ../shell/jjb-install.sh + - ../shell/jjb-verify-job.sh + - ../shell/jjb-check-unicode.sh + - gpg-verify-git-signature + + publishers: + - lf-infra-publish diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml new file mode 100644 index 00000000..0c0e69b3 --- /dev/null +++ b/jjb/lf-macros.yaml @@ -0,0 +1,142 @@ +--- +- builder: + name: create-netrc + # Macro to create a ~/.netrc file from a Maven settings.xml + # Parameters: + # {server-id} The id of a server as defined in settings.xml + builders: + - inject: + properties-content: 'SERVER_ID={server-id}' + - shell: !include-raw-escape: ../shell/create-netrc.sh + +- builder: + name: gpg-verify-git-signature + # Verify gpg signature of the latest commit message in $WORKSPACE + # + # This command assumes that $WORKSPACE is a git repo. + # + # TODO: Verify signature after downloading users public key from a locally + # created repository instead of the public keymesh. This requires a process + # in place to get ODL developers public keys into a local repository without + # increasing the job thoughput. + builders: + - shell: !include-raw: ../shell/gpg-verify-git-signature.sh + +- builder: + name: lf-infra-ship-logs + builders: + - config-file-provider: + files: + - file-id: 'jenkins-log-archives-settings' + variable: 'SETTINGS_FILE' + - create-netrc: + server-id: logs + - shell: !include-raw: + - ../shell/lftools-install.sh + - ../shell/logs-deploy.sh + - shell: !include-raw: + - ../shell/logs-clear-credentials.sh + - description-setter: + regexp: '^Build logs: .*' + +- parameter: + name: lf-infra-parameters + parameters: + - string: + name: PROJECT + default: '{project}' + description: | + Parameter to identify a Gerrit project. This is typically the + project repo path as exists in Gerrit. + For example: ofextensions/circuitsw + + (Deprecated) Please use GERRIT_PROJECT instead. A future version + of global-jjb will remove this variable. + - string: + name: GERRIT_PROJECT + default: '{project}' + description: | + Parameter to identify Gerrit project. This is typically the + project repo path as exists in Gerrit. + For example: ofextensions/circuitsw + + Note that Gerrit will override this parameter automatically if a + job is triggered by Gerrit. + - string: + name: GERRIT_BRANCH + default: '{branch}' + description: | + Parameter to identify a Gerrit branch. + + Note that Gerrit will override this parameter automatically if a + job is triggered by Gerrit. + - string: + name: GERRIT_REFSPEC + default: '' + description: | + Parameter to identify a refspec when pulling from Gerrit. + + Note that Gerrit will override this parameter automatically if a + job is triggered by Gerrit. + +- property: + name: lf-infra-properties + properties: + - build-discarder: + # Allow build data to be stored at a length configured by the + # downstream project. + days-to-keep: '{build-days-to-keep}' + # Do not allow artifacts to be stored in Jenkins. + artifact-num-to-keep: 0 + +- publisher: + name: lf-infra-publish + # lf-infra macro to finish up a build. + # + # Handles the following: + # - Shipping logs to Nexus logs site repository + # - Cleanup workspace + publishers: + - postbuildscript: + builders: + - lf-infra-ship-logs + script-only-if-succeeded: false + script-only-if-failed: false + mark-unstable-if-failed: false + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - '**/*.jenkins-trigger' + fail-build: false + +- scm: + name: lf-infra-gerrit-scm + scm: + - git: + credentials-id: '{jenkins-ssh-credential}' + url: '$GIT_URL/$GERRIT_PROJECT' + refspec: '{refspec}' + branches: + - 'refs/heads/{branch}' + skip-tag: true + wipe-workspace: true + submodule: + recursive: '{submodule-recursive}' + choosing-strategy: '{choosing-strategy}' + +- wrapper: + name: lf-infra-wrappers + wrappers: + - mask-passwords + - timeout: + type: absolute + timeout: '{build-timeout}' + timeout-var: 'BUILD_TIMEOUT' + fail: true + - timestamps + - ssh-agent-credentials: + users: + - '{jenkins-ssh-credential}' + - openstack: + single-use: true diff --git a/shell/create-netrc.sh b/shell/create-netrc.sh new file mode 100644 index 00000000..4a15e3b6 --- /dev/null +++ b/shell/create-netrc.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc diff --git a/shell/gpg-verify-git-signature.sh b/shell/gpg-verify-git-signature.sh new file mode 100644 index 00000000..b26359e6 --- /dev/null +++ b/shell/gpg-verify-git-signature.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> gpg-verify-git-signature.sh" + +if git log --show-signature -1 | egrep -q 'gpg: Signature made.*key ID'; then + echo "Git commit is GPG signed." +else + echo "WARNING: GPG signature missing for the commit." +fi + +# Do NOT fail the job for unsigned commits +exit 0 diff --git a/shell/jjb-check-unicode.sh b/shell/jjb-check-unicode.sh new file mode 100644 index 00000000..5d65dad4 --- /dev/null +++ b/shell/jjb-check-unicode.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2015 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-check-unicode.sh" + +if LC_ALL=C grep -r '[^[:print:][:space:]]' jjb/; then + echo "Found files containing non-ascii characters." + exit 1 +fi + +echo "All files are ASCII only" diff --git a/shell/jjb-install.sh b/shell/jjb-install.sh new file mode 100644 index 00000000..63d16947 --- /dev/null +++ b/shell/jjb-install.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-install.sh" + +virtualenv "$WORKSPACE/.virtualenvs/jjb" +# shellcheck source=./.virtualenvs/jjb/bin/activate disable=SC1091 +source "$WORKSPACE/.virtualenvs/jjb/bin/activate" +pip install --upgrade pip +pip install --upgrade "jenkins-job-builder==$JJB_VERSION" +pip freeze diff --git a/shell/jjb-merge-job.sh b/shell/jjb-merge-job.sh new file mode 100644 index 00000000..d3db1910 --- /dev/null +++ b/shell/jjb-merge-job.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-merge-job.sh" + +jenkins-jobs update --recursive --delete-old --workers 4 jjb/ diff --git a/shell/jjb-verify-job.sh b/shell/jjb-verify-job.sh new file mode 100644 index 00000000..bfa2622a --- /dev/null +++ b/shell/jjb-verify-job.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-verify-job.sh" + +jenkins-jobs -l DEBUG test --recursive -o archives/job-configs jjb/ + +# Sort job output into sub-directories. On large Jenkins systems that have +# many jobs archiving so many files into the same directory makes NGINX return +# the directory list slow. +pushd archives/job-configs +for letter in {a..z} +do + ls "$letter"* > /dev/null 2>&1 + if [[ "$?" -eq 0 ]] + then + mkdir "$letter" + find . -type f -maxdepth 1 -name "$letter*" -exec mv {} "$letter" \; + fi +done +popd diff --git a/shell/lftools-install.sh b/shell/lftools-install.sh new file mode 100644 index 00000000..527a7345 --- /dev/null +++ b/shell/lftools-install.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +virtualenv "$WORKSPACE/.virtualenvs/lftools" +# shellcheck source=./.virtualenvs/lftools/bin/activate disable=SC1091 +source "$WORKSPACE/.virtualenvs/lftools/bin/activate" +pip install --upgrade pip +pip install --upgrade "lftools<1.0.0" +pip freeze diff --git a/shell/logs-clear-credentials.sh b/shell/logs-clear-credentials.sh new file mode 100644 index 00000000..8ebb5a6f --- /dev/null +++ b/shell/logs-clear-credentials.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm $SETTINGS_FILE +rm ~/.netrc diff --git a/shell/logs-deploy.sh b/shell/logs-deploy.sh new file mode 100644 index 00000000..222bb7f1 --- /dev/null +++ b/shell/logs-deploy.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +set -x # Trace commands for this script to make debugging easier. + +LOGS_SERVER="${LOGS_SERVER:-WARNING: Logging Server Not Set.}" +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" +BUILD_URL="${BUILD_URL}" + +lftools deploy archives "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" +lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + +set +x # Disable trace since we no longer need it. + +echo "Build logs: $LOGS_SERVER/$NEXUS_PATH" -- 2.16.6