From: Thanh Ha Date: Fri, 8 Sep 2017 01:40:56 +0000 (-0400) Subject: Add job to deploy to sandbox X-Git-Tag: v0.9.0~9 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F19%2F6419%2F8;p=releng%2Fglobal-jjb.git Add job to deploy to sandbox This job allows a contributor to leave a comment for example 'jjb-deploy builder-jjb-*' to deploy a job (or several jobs if wildcards are used) to the Jenkins Sandbox. This allows jjb contributors to no longer have to install jjb on their local systems in order to push the job to the Jenkins Sandbox. ATTN: GitHub version of the job is untested and likely does NOT work as currently designed. Issue: RELENG-373 Change-Id: I4f26c640804a0b7db1a0087cbd805b9263821326 Signed-off-by: Thanh Ha --- diff --git a/.jjb-test/expected-xml/gerrit-ciman-jjb-deploy-job b/.jjb-test/expected-xml/gerrit-ciman-jjb-deploy-job new file mode 100644 index 00000000..fa7cca57 --- /dev/null +++ b/.jjb-test/expected-xml/gerrit-ciman-jjb-deploy-job @@ -0,0 +1,539 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 1 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + refs/heads/master + + + LFTOOLS_VERSION + Version of lftools to install. Can be a specific version like +'0.6.0' or a PEP-440 definition. +https://www.python.org/dev/peps/pep-0440/ +For example '<1.0.0' or '>=1.0.0,<2.0.0'. + + <1.0.0 + + + + + + 2 + + + origin + refs/heads/master + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/master + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + ANT + releng/ciman + + + ANT + **/master + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + jjb-deploy (?!\*+$).+$ + + + + + + + + + + test-server + + + + + + PIP_PACKAGES=jenkins-job-builder + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# pip install packages into a virtualenv using the first listed package as venv name +# +# PIP_PACKAGES is a space separated list of pypi packages to install. The first +# listed package is used as the virtualenv directory name. +echo "---> pip-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +# Install git-review using virtualenv to the latest version that supports +# --reviewers option, available through pip install. Existing minion image has a +# version that does not have it. +virtualenv "/tmp/v/${PIP_PACKAGES%% *}" +# shellcheck source=/tmp/v/venv/bin/activate disable=SC1091 +source "/tmp/v/${PIP_PACKAGES%% *}/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree + +# PIP_PACKAGES needs to be passed through as a space separated list of packages +# shellcheck disable=SC2086 +pip install --upgrade $PIP_PACKAGES + +echo "----> Pip Dependency Tree" +pipdeptree + + + + + + jjbini-sandbox + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Push a job to jenkins-sandbox via Gerrit / GitHub comment +# Comment Pattern: jjb-deploy JOB_NAME +# JOB_NAME: Can also include * wildcards too. Additional parameters are ignored. +echo "---> jjb-deploy-job.sh" + +# Ensure we fail the job if any steps fail. +set -e -o pipefail + +# shellcheck source=/tmp/v/jenkins-job-builder/bin/activate disable=SC1091 +source "/tmp/v/jenkins-job-builder/bin/activate" + +echo "-----> Fetching project" +git fetch origin "$GERRIT_REFSPEC" && git checkout FETCH_HEAD + +# If not Gerrit Trigger than assume GitHub +COMMENT="${GERRIT_EVENT_COMMENT_TEXT:-$ghprbCommentBody}" +JOB_NAME=$(echo "$COMMENT" | grep jjb-deploy | awk '{print $2}') + +# Strip all * characters to prevent pushing all jobs to Jenkins +if [ -z "${JOB_NAME//*/}" ]; then + echo "ERROR: JOB_NAME cannot be empty or '*'." + exit 1 +fi + +jenkins-jobs update --recursive --workers 4 jjb/ "$JOB_NAME" + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Script to install lftools via a version passed in via lf-infra-parameters +# +# Required parameters: +# +# LFTOOLS_VERSION: Passed in via lf-infra-parameters configuration. Can be +# set to a strict version number like '1.2.3' or using +# PEP-440 definitions. +# +# Examples: +# <1.0.0 +# >=1.0.0,<2.0.0 +# +# By default a released version of lftools should always be used. +# The purpose of the 2 variables below is so that lftools devs can test +# unreleased versions of lftools. There are 2 methods to install a dev version +# of lftools: +# +# 1) gerrit patch: Used to test a patch that has not yet been merged. +# To do this set something like this: +# LFTOOLS_MODE=gerrit +# LFTOOLS_REFSPEC=refs/changes/96/5296/7 +# +# 2) git branch: Used to install an lftools version from a specific branch. +# To use this set the variables as follows: +# LFTOOLS_MODE=git +# LFTOOLS_REFSPEC=master +# +# 3) release : The intended use case and default setting. +# Set LFTOOLS_MODE=release, in this case LFTOOLS_REFSPEC is unused. + +LFTOOLS_MODE=release # release | git | gerrit +LFTOOLS_REFSPEC=master + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip + +case $LFTOOLS_MODE in + gerrit) + git clone https://gerrit.linuxfoundation.org/infra/releng/lftools.git /tmp/lftools + pushd /tmp/lftools + git fetch origin "$LFTOOLS_REFSPEC" + git checkout FETCH_HEAD + pip install --quiet --upgrade -r requirements.txt + pip install --quiet --upgrade -e . + popd + ;; + + git) + pip install --quiet --upgrade git+https://gerrit.linuxfoundation.org/infra/releng/lftools.git@"$BRANCH" + ;; + + release) + if [[ $LFTOOLS_VERSION =~ ^[0-9] ]]; then + LFTOOLS_VERSION="==$LFTOOLS_VERSION" + fi + + pip install --quiet --upgrade "lftools${LFTOOLS_VERSION}" + ;; +esac + +lftools --version + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 5 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/.jjb-test/expected-xml/github-ciman-jjb-deploy-job b/.jjb-test/expected-xml/github-ciman-jjb-deploy-job new file mode 100644 index 00000000..a214b2df --- /dev/null +++ b/.jjb-test/expected-xml/github-ciman-jjb-deploy-job @@ -0,0 +1,507 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + $GIT_URL/$GERRIT_PROJECT/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + refs/heads/master + + + LFTOOLS_VERSION + Version of lftools to install. Can be a specific version like +'0.6.0' or a PEP-440 definition. +https://www.python.org/dev/peps/pep-0440/ +For example '<1.0.0' or '>=1.0.0,<2.0.0'. + + <1.0.0 + + + + + + 2 + + + origin + +refs/pull/*:refs/remotes/origin/pr/* + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + false + + + + ^jjb-deploy (?!\*+$).+$ + true + true + true + false + + + JJB Deploy Job + + + + + + + + PIP_PACKAGES=jenkins-job-builder + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# pip install packages into a virtualenv using the first listed package as venv name +# +# PIP_PACKAGES is a space separated list of pypi packages to install. The first +# listed package is used as the virtualenv directory name. +echo "---> pip-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +# Install git-review using virtualenv to the latest version that supports +# --reviewers option, available through pip install. Existing minion image has a +# version that does not have it. +virtualenv "/tmp/v/${PIP_PACKAGES%% *}" +# shellcheck source=/tmp/v/venv/bin/activate disable=SC1091 +source "/tmp/v/${PIP_PACKAGES%% *}/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree + +# PIP_PACKAGES needs to be passed through as a space separated list of packages +# shellcheck disable=SC2086 +pip install --upgrade $PIP_PACKAGES + +echo "----> Pip Dependency Tree" +pipdeptree + + + + + + jjbini-sandbox + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Push a job to jenkins-sandbox via Gerrit / GitHub comment +# Comment Pattern: jjb-deploy JOB_NAME +# JOB_NAME: Can also include * wildcards too. Additional parameters are ignored. +echo "---> jjb-deploy-job.sh" + +# Ensure we fail the job if any steps fail. +set -e -o pipefail + +# shellcheck source=/tmp/v/jenkins-job-builder/bin/activate disable=SC1091 +source "/tmp/v/jenkins-job-builder/bin/activate" + +echo "-----> Fetching project" +git fetch origin "$GERRIT_REFSPEC" && git checkout FETCH_HEAD + +# If not Gerrit Trigger than assume GitHub +COMMENT="${GERRIT_EVENT_COMMENT_TEXT:-$ghprbCommentBody}" +JOB_NAME=$(echo "$COMMENT" | grep jjb-deploy | awk '{print $2}') + +# Strip all * characters to prevent pushing all jobs to Jenkins +if [ -z "${JOB_NAME//*/}" ]; then + echo "ERROR: JOB_NAME cannot be empty or '*'." + exit 1 +fi + +jenkins-jobs update --recursive --workers 4 jjb/ "$JOB_NAME" + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Script to install lftools via a version passed in via lf-infra-parameters +# +# Required parameters: +# +# LFTOOLS_VERSION: Passed in via lf-infra-parameters configuration. Can be +# set to a strict version number like '1.2.3' or using +# PEP-440 definitions. +# +# Examples: +# <1.0.0 +# >=1.0.0,<2.0.0 +# +# By default a released version of lftools should always be used. +# The purpose of the 2 variables below is so that lftools devs can test +# unreleased versions of lftools. There are 2 methods to install a dev version +# of lftools: +# +# 1) gerrit patch: Used to test a patch that has not yet been merged. +# To do this set something like this: +# LFTOOLS_MODE=gerrit +# LFTOOLS_REFSPEC=refs/changes/96/5296/7 +# +# 2) git branch: Used to install an lftools version from a specific branch. +# To use this set the variables as follows: +# LFTOOLS_MODE=git +# LFTOOLS_REFSPEC=master +# +# 3) release : The intended use case and default setting. +# Set LFTOOLS_MODE=release, in this case LFTOOLS_REFSPEC is unused. + +LFTOOLS_MODE=release # release | git | gerrit +LFTOOLS_REFSPEC=master + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip + +case $LFTOOLS_MODE in + gerrit) + git clone https://gerrit.linuxfoundation.org/infra/releng/lftools.git /tmp/lftools + pushd /tmp/lftools + git fetch origin "$LFTOOLS_REFSPEC" + git checkout FETCH_HEAD + pip install --quiet --upgrade -r requirements.txt + pip install --quiet --upgrade -e . + popd + ;; + + git) + pip install --quiet --upgrade git+https://gerrit.linuxfoundation.org/infra/releng/lftools.git@"$BRANCH" + ;; + + release) + if [[ $LFTOOLS_VERSION =~ ^[0-9] ]]; then + LFTOOLS_VERSION="==$LFTOOLS_VERSION" + fi + + pip install --quiet --upgrade "lftools${LFTOOLS_VERSION}" + ;; +esac + +lftools --version + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 5 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 09dc191a..46089ae7 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -8,6 +8,7 @@ jjb-version: 1.6.2 jobs: + - gerrit-jjb-deploy-job - gerrit-jjb-merge - gerrit-jjb-verify @@ -20,6 +21,7 @@ jjb-version: 1.6.2 jobs: + - github-jjb-deploy-job - github-jjb-merge - github-jjb-verify @@ -61,6 +63,15 @@ - file-id: jjbini target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini' +- builder: + name: lf-infra-jjbini-sandbox + # TODO: Consolidate this into jjbini when JJB2.0 is available. + builders: + - config-file-provider: + files: + - file-id: jjbini-sandbox + target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini' + - lf_packer_file_paths: &lf_packer_file_paths name: lf-packer-file-paths file-paths: @@ -252,6 +263,136 @@ publishers: - lf-infra-publish +################## +# JJB DEPLOY JOB # +################## + +- lf_jjb_deploy_job: &lf_jjb_deploy_job + name: lf-jjb-deploy-job + + # Deploy jobs to jenkins-sandbox system via code review comment + # + # This job checks out the current code review patch and then runs a + # `jenkins-jobs update` to push a patch defined by the comment. + # + # Comment Trigger: jjb-deploy JOB_NAME + # + # JOB_NAME can include the * wildcard character to push multiple jobs + # matching the pattern. For example `jjb-deploy builder-jjb-*`` will push + # all builder-jjb-* jobs to the sandbox system. + # + # Required parameters: + # + # :build-node: The node to run build on. + # :jenkins-ssh-credential: Credential to use for SSH. (Generally should + # be configured in defaults.yaml) + # + # Optional parameters: + # + # :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + # :gerrit_jjb_deploy_job_triggers: Override Gerrit Triggers. + + ###################### + # Default parameters # + ###################### + + branch: master + git-url: '$GIT_URL/$GERRIT_PROJECT' + stream: master + gerrit_jjb_deploy_job_triggers: + - comment-added-contains-event: + comment-contains-value: jjb-deploy (?!\*+$).+$ + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + concurrent: true + + properties: + - lf-infra-properties: + project: '{project}' + build-days-to-keep: 1 + + parameters: + - lf-infra-parameters: + project: '{project}' + stream: '{stream}' + branch: '{branch}' + lftools-version: '{lftools-version}' + + wrappers: + - lf-infra-wrappers: + build-timeout: 5 + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + builders: + - lf-pip-install: + pip-packages: jenkins-job-builder + - lf-infra-jjbini-sandbox + - shell: !include-raw-escape: ../shell/jjb-deploy-job.sh + + publishers: + - lf-infra-publish + +- job-template: + name: '{project-name}-jjb-deploy-job' + id: gerrit-jjb-deploy-job + <<: *lf_jjb_deploy_job + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: 'refs/heads/{branch}' + branch: '{branch}' + submodule-recursive: true + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: '{obj:gerrit_jjb_deploy_job_triggers}' + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + +- job-template: + name: '{project-name}-jjb-deploy-job' + id: github-jjb-deploy-job + <<: *lf_jjb_deploy_job + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '{branch}' + submodule-recursive: true + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - lf-infra-github-pr-trigger: + trigger-phrase: '^jjb-deploy (?!\*+$).+$' + only-trigger-phrase: true + status-context: 'JJB Deploy Job' + permit-all: true + github-hooks: true + github-org: '' + github_pr_whitelist: + - '' + github_pr_admin_list: + - '' + ############# # JJB Merge # ############# diff --git a/shell/jjb-deploy-job.sh b/shell/jjb-deploy-job.sh new file mode 100644 index 00000000..c7b8d8d7 --- /dev/null +++ b/shell/jjb-deploy-job.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Push a job to jenkins-sandbox via Gerrit / GitHub comment +# Comment Pattern: jjb-deploy JOB_NAME +# JOB_NAME: Can also include * wildcards too. Additional parameters are ignored. +echo "---> jjb-deploy-job.sh" + +# Ensure we fail the job if any steps fail. +set -e -o pipefail + +# shellcheck source=/tmp/v/jenkins-job-builder/bin/activate disable=SC1091 +source "/tmp/v/jenkins-job-builder/bin/activate" + +echo "-----> Fetching project" +git fetch origin "$GERRIT_REFSPEC" && git checkout FETCH_HEAD + +# If not Gerrit Trigger than assume GitHub +COMMENT="${GERRIT_EVENT_COMMENT_TEXT:-$ghprbCommentBody}" +JOB_NAME=$(echo "$COMMENT" | grep jjb-deploy | awk '{print $2}') + +# Strip all * characters to prevent pushing all jobs to Jenkins +if [ -z "${JOB_NAME//*/}" ]; then + echo "ERROR: JOB_NAME cannot be empty or '*'." + exit 1 +fi + +jenkins-jobs update --recursive --workers 4 jjb/ "$JOB_NAME"