From cee820144e0741dee01b9eb755827f2c263a0cf8 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 9 Aug 2017 19:01:52 -0400 Subject: [PATCH] Refactor tests into jjb-tests directory Issue: RELENG-250 Change-Id: Idc2ddc9c28a8ab835b3558f4076305053ee43235 Signed-off-by: Thanh Ha --- jjb-compare-xml.sh | 39 ++ jjb-test/defaults.yaml | 25 + jjb-test/expected-xml/gerrit-ciman-jjb-merge | 452 +++++++++++++++ jjb-test/expected-xml/gerrit-ciman-jjb-verify | 545 ++++++++++++++++++ .../gerrit-ciman-packer-merge-centos-java-builder | 633 +++++++++++++++++++++ .../gerrit-ciman-packer-merge-centos-mininet | 633 +++++++++++++++++++++ ...rrit-ciman-packer-merge-ubuntu1604-java-builder | 633 +++++++++++++++++++++ .../gerrit-ciman-packer-merge-ubuntu1604-mininet | 633 +++++++++++++++++++++ jjb-test/expected-xml/gerrit-ciman-packer-verify | 521 +++++++++++++++++ .../expected-xml/gerrit-maven-maven-release-master | 580 +++++++++++++++++++ .../expected-xml/gerrit-python-tox-verify-{stream} | 484 ++++++++++++++++ jjb-test/expected-xml/github-ciman-jjb-merge | 418 ++++++++++++++ jjb-test/expected-xml/github-ciman-jjb-verify | 497 ++++++++++++++++ .../github-ciman-packer-merge-centos-java-builder | 591 +++++++++++++++++++ .../github-ciman-packer-merge-centos-mininet | 591 +++++++++++++++++++ ...thub-ciman-packer-merge-ubuntu1604-java-builder | 591 +++++++++++++++++++ .../github-ciman-packer-merge-ubuntu1604-mininet | 591 +++++++++++++++++++ jjb-test/expected-xml/github-ciman-packer-verify | 465 +++++++++++++++ .../expected-xml/github-maven-maven-release-master | 575 +++++++++++++++++++ .../expected-xml/github-python-tox-verify-{stream} | 446 +++++++++++++++ jjb-test/lf-ci-jobs.yaml | 45 ++ jjb-test/lf-maven-jobs.yaml | 18 + jjb-test/lf-python-jobs.yaml | 14 + test.template | 108 ---- tox.ini | 15 +- 25 files changed, 10028 insertions(+), 115 deletions(-) create mode 100755 jjb-compare-xml.sh create mode 100644 jjb-test/defaults.yaml create mode 100644 jjb-test/expected-xml/gerrit-ciman-jjb-merge create mode 100644 jjb-test/expected-xml/gerrit-ciman-jjb-verify create mode 100644 jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-java-builder create mode 100644 jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-mininet create mode 100644 jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-java-builder create mode 100644 jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-mininet create mode 100644 jjb-test/expected-xml/gerrit-ciman-packer-verify create mode 100644 jjb-test/expected-xml/gerrit-maven-maven-release-master create mode 100644 jjb-test/expected-xml/gerrit-python-tox-verify-{stream} create mode 100644 jjb-test/expected-xml/github-ciman-jjb-merge create mode 100644 jjb-test/expected-xml/github-ciman-jjb-verify create mode 100644 jjb-test/expected-xml/github-ciman-packer-merge-centos-java-builder create mode 100644 jjb-test/expected-xml/github-ciman-packer-merge-centos-mininet create mode 100644 jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-java-builder create mode 100644 jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-mininet create mode 100644 jjb-test/expected-xml/github-ciman-packer-verify create mode 100644 jjb-test/expected-xml/github-maven-maven-release-master create mode 100644 jjb-test/expected-xml/github-python-tox-verify-{stream} create mode 100644 jjb-test/lf-ci-jobs.yaml create mode 100644 jjb-test/lf-maven-jobs.yaml create mode 100644 jjb-test/lf-python-jobs.yaml delete mode 100644 test.template diff --git a/jjb-compare-xml.sh b/jjb-compare-xml.sh new file mode 100755 index 00000000..5017674b --- /dev/null +++ b/jjb-compare-xml.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script tests jjb templates by comparing the result with expected output. + +test_dir=$(mktemp -d) +script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +expected_xml_dir="$script_dir/jjb-test/expected-xml" + +echo "Script Directory: $script_dir" +echo "Test Directory: $test_dir" + +jenkins-jobs test --recursive -o "$test_dir" "$script_dir" + +fail=false +for xml in "$test_dir"/*; do + job=$(basename "$xml") + if ! cmp "$expected_xml_dir/$job" "$xml"; then + echo "Differences detected in $job" + diff "$expected_xml_dir/$job" "$xml" + fail=true + fi +done + +# Cleanup +rm -rf "$test_dir" + +if $fail; then + echo "Differences detected. Check above for jobs that have been changed." + exit 1 +fi diff --git a/jjb-test/defaults.yaml b/jjb-test/defaults.yaml new file mode 100644 index 00000000..95a951ce --- /dev/null +++ b/jjb-test/defaults.yaml @@ -0,0 +1,25 @@ +--- +- defaults: + name: global + + # General + jenkins-ssh-credential: test-credential + + # Gerrit Infra + gerrit-server-name: test-server + + # GitHub Infra + git-url: https://github.com + git-clone-url: 'git@github.com:' + github-org: example-org + github_pr_whitelist: + - jpwku + - tykeal + - zxiiro + github_pr_admin_list: + - tykeal + + # Common test config + project: releng/ciman + stream: latest + build-node: build-vm diff --git a/jjb-test/expected-xml/gerrit-ciman-jjb-merge b/jjb-test/expected-xml/gerrit-ciman-jjb-merge new file mode 100644 index 00000000..0cc8742c --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-jjb-merge @@ -0,0 +1,452 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + JJB_VERSION + Jenkins Job Builder version to download and install. + 1.6.2 + + + + + + 2 + + + origin + + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/master + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + **/*.sh + + + ANT + **/*.yaml + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + + remerge$ + + + + + + + + + + test-server + + + + + + + jjbini + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv "/tmp/v/jjb" +# shellcheck source=/tmp/v/jjb/bin/activate disable=SC1091 +source "/tmp/v/jjb/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade "jenkins-job-builder==$JJB_VERSION" + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-merge-job.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +jenkins-jobs update --recursive --delete-old --workers 4 --exclude jjb-test jjb + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-jjb-verify b/jjb-test/expected-xml/gerrit-ciman-jjb-verify new file mode 100644 index 00000000..4c982804 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-jjb-verify @@ -0,0 +1,545 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + JJB_VERSION + Jenkins Job Builder version to download and install. + 1.6.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + **/*.sh + + + ANT + **/*.yaml + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + false + false + false + + + + recheck$ + + + + + + + + + + test-server + + + + + + + jjbini + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script will make sure that there are no JIRA URLs in the commit +# message. JIRA URLs will break the its-jira plugin + +# Ensure we fail the job if any steps fail. +# Do not treat undefined variables as errors as in this case we are allowed +# to have JIRA_URL undefined +set -e -o pipefail +set +u + +if [ -n "${JIRA_URL}" ]; +then + BASE_URL=$(echo $JIRA_URL | awk -F'/' '{print $3}') + JIRA_LINK=$(git rev-list --format=%B --max-count=1 HEAD | grep -io "http[s]*://$BASE_URL/" || true) + if [[ ! -z "$JIRA_LINK" ]] + then + echo 'Remove JIRA URLs from commit message' + echo 'Add jira references as: Issue: <JIRAKEY>-<ISSUE#>, instead of URLs' + exit 1 + fi +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv "/tmp/v/jjb" +# shellcheck source=/tmp/v/jjb/bin/activate disable=SC1091 +source "/tmp/v/jjb/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade "jenkins-job-builder==$JJB_VERSION" + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-verify-job.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +jenkins-jobs -l DEBUG test --recursive -o archives/job-configs --exclude jjb-test jjb + +# Sort job output into sub-directories. On large Jenkins systems that have +# many jobs archiving so many files into the same directory makes NGINX return +# the directory list slow. +pushd archives/job-configs +for letter in {a..z} +do + if [[ $(ls "$letter"* > /dev/null 2>&1) -eq 0 ]] + then + mkdir "$letter" + find . -maxdepth 1 -type f -name "$letter*" -exec mv {} "$letter" \; + fi +done +popd + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2015 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-check-unicode.sh" + +if LC_ALL=C grep -r '[^[:print:][:space:]]' jjb/; then + echo "Found files containing non-ascii characters." + exit 1 +fi + +echo "All files are ASCII only" + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> gpg-verify-git-signature.sh" + +if git log --show-signature -1 | grep -E -q 'gpg: Signature made.*key ID'; then + echo "Git commit is GPG signed." +else + echo "WARNING: GPG signature missing for the commit." +fi + +# Do NOT fail the job for unsigned commits +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-java-builder b/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-java-builder new file mode 100644 index 00000000..869ba6d8 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-java-builder @@ -0,0 +1,633 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + packer/vars/centos.json + + + ANT + packer/templates/java-builder.json + + + ANT + packer/provision/java-builder.sh + + + ANT + packer/provision/lib/** + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + + remerge$ + + + + + + + + + + test-server + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=centos +PACKER_TEMPLATE=java-builder +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-mininet b/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-mininet new file mode 100644 index 00000000..bcfd8910 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-packer-merge-centos-mininet @@ -0,0 +1,633 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + packer/vars/centos.json + + + ANT + packer/templates/mininet.json + + + ANT + packer/provision/mininet.sh + + + ANT + packer/provision/lib/** + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + + remerge$ + + + + + + + + + + test-server + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=centos +PACKER_TEMPLATE=mininet +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-java-builder b/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-java-builder new file mode 100644 index 00000000..4e2eaebc --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-java-builder @@ -0,0 +1,633 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + packer/vars/ubuntu1604.json + + + ANT + packer/templates/java-builder.json + + + ANT + packer/provision/java-builder.sh + + + ANT + packer/provision/lib/** + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + + remerge$ + + + + + + + + + + test-server + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=ubuntu1604 +PACKER_TEMPLATE=java-builder +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-mininet b/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-mininet new file mode 100644 index 00000000..97ccd566 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-packer-merge-ubuntu1604-mininet @@ -0,0 +1,633 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + packer/vars/ubuntu1604.json + + + ANT + packer/templates/mininet.json + + + ANT + packer/provision/mininet.sh + + + ANT + packer/provision/lib/** + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + + remerge$ + + + + + + + + + + test-server + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=ubuntu1604 +PACKER_TEMPLATE=mininet +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-ciman-packer-verify b/jjb-test/expected-xml/gerrit-ciman-packer-verify new file mode 100644 index 00000000..8681cd84 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-ciman-packer-verify @@ -0,0 +1,521 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + ANT + releng/ciman + + + ANT + **/master + + + + + ANT + packer/vars/['centos', 'ubuntu1604'].json + + + ANT + packer/templates/['java-builder', 'mininet'].json + + + ANT + packer/provision/['java-builder', 'mininet'].sh + + + ANT + packer/provision/lib/** + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + false + false + false + + + + recheck$ + + + + + + + + + + test-server + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-maven-maven-release-master b/jjb-test/expected-xml/gerrit-maven-maven-release-master new file mode 100644 index 00000000..5e0bba8f --- /dev/null +++ b/jjb-test/expected-xml/gerrit-maven-maven-release-master @@ -0,0 +1,580 @@ + + + openjdk8 + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + build-vm + false + + + + 30 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + MAVEN_OPTS + Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m + + + + + MAVEN_PARAMS + Maven parameters to pass to the mvn command. + + + + + MVN + Maven selector to be used by shell scripts + /w/tools/hudson.tasks.Maven_MavenInstallation/mvn33/bin/mvn + + + STAGING_PROFILE_ID + Nexus staging profile ID. + + uuddlrlrba + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes + + + echo "quiet=on" > "$HOME/.wgetrc" + + + --version + + mvn33 + false + + + + + rm "$HOME/.wgetrc" + + + + + global-settings + + GLOBAL_SETTINGS_FILE + + + gerrit-maven-project-settings + + SETTINGS_FILE + + + + + + SERVER_ID=opendaylight-staging + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This file contains a list of variables that are generally useful in many +# scripts. It is meant to be sourced in other scripts so that the variables can +# be called. + +MAVEN_OPTIONS="$(echo --show-version \ + --batch-mode \ + -Djenkins \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dmaven.repo.local=/tmp/r \ + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)" +echo "$MAVEN_OPTIONS" + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script removes the -SNAPSHOT from a project to prepare it for release. + +PATCH_DIR="$WORKSPACE/archives/patches" +mkdir -p "$PATCH_DIR" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +echo "$PROJECT" "$(git rev-parse --verify HEAD)" | tee -a "$PATCH_DIR/taglist.log" + +# Strip -SNAPSHOT from version to prepare release. +find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g' + +git commit -am "Release $PROJECT" +git format-patch --stdout "origin/$GERRIT_BRANCH" > "$PATCH_DIR/${PROJECT//\//-}.patch" +git bundle create "$PATCH_DIR/${PROJECT//\//-}.bundle" "origin/${GERRIT_BRANCH}..HEAD" + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script builds a Maven project and deploys it into a staging repo which +# can be used to deploy elsewhere later eg. Nexus staging / snapshot repos. + +# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound. +# Ensure we fail the job if any steps fail. +set -e -o pipefail +set +u + +export MAVEN_OPTS + +# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters. +# shellcheck disable=SC2086 +$MVN clean deploy \ + --global-settings "$GLOBAL_SETTINGS_FILE" \ + --settings "$SETTINGS_FILE" \ + -DaltDeploymentRepository=staging::default::file:"$WORKSPACE"/m2repo \ + $MAVEN_PARAMS $MAVEN_OPTIONS + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script publishes artifacts to a staging repo in Nexus. +# +# $WORKSPACE/m2repo : Exists and used to deploy the staging repository. +# $NEXUS_URL : Jenkins global variable should be defined. +# $STAGING_PROFILE_ID : Provided by a job parameter. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +TMP_FILE="$(mktemp)" +lftools deploy nexus-stage "$NEXUS_URL" "$STAGING_PROFILE_ID" "$WORKSPACE/m2repo" | tee "$TMP_FILE" +staging_repo=$(sed -n -e 's/Staging repository \(.*\) created\./\1/p' "$TMP_FILE") + +# Store repo info to a file in archives +mkdir -p "$WORKSPACE/archives" +echo "$staging_repo" > "$WORKSPACE/archives/staging-repo.txt" + +# Cleanup +rm "$TMP_FILE" + + + + #!/bin/bash +set +e # DO NOT cause build failure if any of the rm calls fail. + +rm "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE" + +# In some cases we use the lf-provide-maven-settings macro to produce +# a "$HOME/.netrc" file containing credentials. Remove that file here +# too if it exists. +rm "$HOME/.netrc" + +# DO NOT fail build if any of the above lines fail. +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 30 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/gerrit-python-tox-verify-{stream} b/jjb-test/expected-xml/gerrit-python-tox-verify-{stream} new file mode 100644 index 00000000..2284a3c3 --- /dev/null +++ b/jjb-test/expected-xml/gerrit-python-tox-verify-{stream} @@ -0,0 +1,484 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + + 7 + -1 + -1 + 0 + + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + {stream} + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + TOX_DIR + Path to directory containing tox.ini file. + + + + + TOX_ENVS + Tox environments to run build against. +Example: docs,py2,py3 + + + + + + + + 2 + + + origin + $GERRIT_REFSPEC + $GIT_URL/$GERRIT_PROJECT + test-credential + + + + + refs/heads/$GERRIT_BRANCH + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + ANT + releng/ciman + + + ANT + **/master + + + false + + + + false + false + false + false + + false + false + true + false + false + + + False + + false + + + false + false + false + + + + recheck$ + + + + + + + + + + test-server + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/tox" +# shellcheck source=/tmp/v/tox/bin/activate disable=SC1091 +source "/tmp/v/tox/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade tox argparse + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +cd "$WORKSPACE/$TOX_DIR" + +if [ -n "$TOX_ENVS" ]; +then + tox -e "$TOX_ENVS" +else + tox +fi + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-jjb-merge b/jjb-test/expected-xml/github-ciman-jjb-merge new file mode 100644 index 00000000..ce75fbc3 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-jjb-merge @@ -0,0 +1,418 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + JJB_VERSION + Jenkins Job Builder version to download and install. + 1.6.2 + + + + + + 2 + + + origin + + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + + + false + + + + tykeal + false + jpwku +tykeal +zxiiro + example-org + + ^remerge$ + true + true + false + false + + + JJB Merge + + + + + + + + + jjbini + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv "/tmp/v/jjb" +# shellcheck source=/tmp/v/jjb/bin/activate disable=SC1091 +source "/tmp/v/jjb/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade "jenkins-job-builder==$JJB_VERSION" + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-merge-job.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +jenkins-jobs update --recursive --delete-old --workers 4 --exclude jjb-test jjb + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-jjb-verify b/jjb-test/expected-xml/github-ciman-jjb-verify new file mode 100644 index 00000000..dc8b94a7 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-jjb-verify @@ -0,0 +1,497 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + JJB_VERSION + Jenkins Job Builder version to download and install. + 1.6.2 + + + + + + 2 + + + origin + +refs/pull/*:refs/remotes/origin/pr/* + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + false + + + + ^recheck$ + false + true + true + false + + + JJB Verify + + + + + + + + + jjbini + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script will make sure that there are no JIRA URLs in the commit +# message. JIRA URLs will break the its-jira plugin + +# Ensure we fail the job if any steps fail. +# Do not treat undefined variables as errors as in this case we are allowed +# to have JIRA_URL undefined +set -e -o pipefail +set +u + +if [ -n "${JIRA_URL}" ]; +then + BASE_URL=$(echo $JIRA_URL | awk -F'/' '{print $3}') + JIRA_LINK=$(git rev-list --format=%B --max-count=1 HEAD | grep -io "http[s]*://$BASE_URL/" || true) + if [[ ! -z "$JIRA_LINK" ]] + then + echo 'Remove JIRA URLs from commit message' + echo 'Add jira references as: Issue: <JIRAKEY>-<ISSUE#>, instead of URLs' + exit 1 + fi +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv "/tmp/v/jjb" +# shellcheck source=/tmp/v/jjb/bin/activate disable=SC1091 +source "/tmp/v/jjb/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade "jenkins-job-builder==$JJB_VERSION" + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-verify-job.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +jenkins-jobs -l DEBUG test --recursive -o archives/job-configs --exclude jjb-test jjb + +# Sort job output into sub-directories. On large Jenkins systems that have +# many jobs archiving so many files into the same directory makes NGINX return +# the directory list slow. +pushd archives/job-configs +for letter in {a..z} +do + if [[ $(ls "$letter"* > /dev/null 2>&1) -eq 0 ]] + then + mkdir "$letter" + find . -maxdepth 1 -type f -name "$letter*" -exec mv {} "$letter" \; + fi +done +popd + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2015 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> jjb-check-unicode.sh" + +if LC_ALL=C grep -r '[^[:print:][:space:]]' jjb/; then + echo "Found files containing non-ascii characters." + exit 1 +fi + +echo "All files are ASCII only" + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> gpg-verify-git-signature.sh" + +if git log --show-signature -1 | grep -E -q 'gpg: Signature made.*key ID'; then + echo "Git commit is GPG signed." +else + echo "WARNING: GPG signature missing for the commit." +fi + +# Do NOT fail the job for unsigned commits +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-packer-merge-centos-java-builder b/jjb-test/expected-xml/github-ciman-packer-merge-centos-java-builder new file mode 100644 index 00000000..2f044d9c --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-packer-merge-centos-java-builder @@ -0,0 +1,591 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + + false + + + + tykeal + false + jpwku +tykeal +zxiiro + example-org + + ^remerge$ + true + true + false + false + + + Packer centos-$java-builder Merge + + + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=centos +PACKER_TEMPLATE=java-builder +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-packer-merge-centos-mininet b/jjb-test/expected-xml/github-ciman-packer-merge-centos-mininet new file mode 100644 index 00000000..a2b2c665 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-packer-merge-centos-mininet @@ -0,0 +1,591 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + + false + + + + tykeal + false + jpwku +tykeal +zxiiro + example-org + + ^remerge$ + true + true + false + false + + + Packer centos-$mininet Merge + + + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=centos +PACKER_TEMPLATE=mininet +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-java-builder b/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-java-builder new file mode 100644 index 00000000..e734f840 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-java-builder @@ -0,0 +1,591 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + + false + + + + tykeal + false + jpwku +tykeal +zxiiro + example-org + + ^remerge$ + true + true + false + false + + + Packer ubuntu1604-$java-builder Merge + + + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=ubuntu1604 +PACKER_TEMPLATE=java-builder +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-mininet b/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-mininet new file mode 100644 index 00000000..ce8a79e7 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-packer-merge-ubuntu1604-mininet @@ -0,0 +1,591 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + 00 10 1 * * + + + + + + + false + + + + tykeal + false + jpwku +tykeal +zxiiro + example-org + + ^remerge$ + true + true + false + false + + + Packer ubuntu1604-$mininet Merge + + + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_PLATFORM=ubuntu1604 +PACKER_TEMPLATE=mininet +PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 45 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-ciman-packer-verify b/jjb-test/expected-xml/github-ciman-packer-verify new file mode 100644 index 00000000..b46c68c9 --- /dev/null +++ b/jjb-test/expected-xml/github-ciman-packer-verify @@ -0,0 +1,465 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + https://github.com/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + PACKER_VERSION + Packer version to download and install. + 1.0.2 + + + + + + 2 + + + origin + +refs/pull/*:refs/remotes/origin/pr/* + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + false + + + + ^recheck$ + false + true + true + false + + + Packer Verify + + + + + + + + + ci-packer-cloud-env + + CLOUDENV + + + + + + PACKER_VERSION=1.0.2 + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*.json) +templates=(../packer/templates/*.json) + +for varfile in "${varfiles[@]}"; do + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-maven-maven-release-master b/jjb-test/expected-xml/github-maven-maven-release-master new file mode 100644 index 00000000..e669719a --- /dev/null +++ b/jjb-test/expected-xml/github-maven-maven-release-master @@ -0,0 +1,575 @@ + + + openjdk8 + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + build-vm + false + + + $GIT_URL/$PROJECT/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + master + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + MAVEN_OPTS + Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m + + + + + MAVEN_PARAMS + Maven parameters to pass to the mvn command. + + + + + MVN + Maven selector to be used by shell scripts + /w/tools/hudson.tasks.Maven_MavenInstallation/mvn33/bin/mvn + + + STAGING_PROFILE_ID + Nexus staging profile ID. + + uuddlrlrba + + + + + + 2 + + + origin + +refs/pull/*:refs/remotes/origin/pr/* + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes + + + echo "quiet=on" > "$HOME/.wgetrc" + + + --version + + mvn33 + false + + + + + rm "$HOME/.wgetrc" + + + + + global-settings + + GLOBAL_SETTINGS_FILE + + + aproject-settings + + SETTINGS_FILE + + + + + + SERVER_ID=opendaylight-staging + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This file contains a list of variables that are generally useful in many +# scripts. It is meant to be sourced in other scripts so that the variables can +# be called. + +MAVEN_OPTIONS="$(echo --show-version \ + --batch-mode \ + -Djenkins \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dmaven.repo.local=/tmp/r \ + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)" +echo "$MAVEN_OPTIONS" + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script removes the -SNAPSHOT from a project to prepare it for release. + +PATCH_DIR="$WORKSPACE/archives/patches" +mkdir -p "$PATCH_DIR" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +echo "$PROJECT" "$(git rev-parse --verify HEAD)" | tee -a "$PATCH_DIR/taglist.log" + +# Strip -SNAPSHOT from version to prepare release. +find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g' + +git commit -am "Release $PROJECT" +git format-patch --stdout "origin/$GERRIT_BRANCH" > "$PATCH_DIR/${PROJECT//\//-}.patch" +git bundle create "$PATCH_DIR/${PROJECT//\//-}.bundle" "origin/${GERRIT_BRANCH}..HEAD" + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script builds a Maven project and deploys it into a staging repo which +# can be used to deploy elsewhere later eg. Nexus staging / snapshot repos. + +# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound. +# Ensure we fail the job if any steps fail. +set -e -o pipefail +set +u + +export MAVEN_OPTS + +# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters. +# shellcheck disable=SC2086 +$MVN clean deploy \ + --global-settings "$GLOBAL_SETTINGS_FILE" \ + --settings "$SETTINGS_FILE" \ + -DaltDeploymentRepository=staging::default::file:"$WORKSPACE"/m2repo \ + $MAVEN_PARAMS $MAVEN_OPTIONS + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script publishes artifacts to a staging repo in Nexus. +# +# $WORKSPACE/m2repo : Exists and used to deploy the staging repository. +# $NEXUS_URL : Jenkins global variable should be defined. +# $STAGING_PROFILE_ID : Provided by a job parameter. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +TMP_FILE="$(mktemp)" +lftools deploy nexus-stage "$NEXUS_URL" "$STAGING_PROFILE_ID" "$WORKSPACE/m2repo" | tee "$TMP_FILE" +staging_repo=$(sed -n -e 's/Staging repository \(.*\) created\./\1/p' "$TMP_FILE") + +# Store repo info to a file in archives +mkdir -p "$WORKSPACE/archives" +echo "$staging_repo" > "$WORKSPACE/archives/staging-repo.txt" + +# Cleanup +rm "$TMP_FILE" + + + + #!/bin/bash +set +e # DO NOT cause build failure if any of the rm calls fail. + +rm "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE" + +# In some cases we use the lf-provide-maven-settings macro to produce +# a "$HOME/.netrc" file containing credentials. Remove that file here +# too if it exists. +rm "$HOME/.netrc" + +# DO NOT fail build if any of the above lines fail. +exit 0 + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 30 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/expected-xml/github-python-tox-verify-{stream} b/jjb-test/expected-xml/github-python-tox-verify-{stream} new file mode 100644 index 00000000..af531186 --- /dev/null +++ b/jjb-test/expected-xml/github-python-tox-verify-{stream} @@ -0,0 +1,446 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + true + build-vm + false + + + $GIT_URL/$GERRIT_PROJECT/example-org/releng/ciman + + + + + PROJECT + Parameter to identify a Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + + releng/ciman + + + STREAM + Stream is often set to the same name as 'branch' but can +sometimes be used as a name representing a project's release code +name. + + {stream} + + + GERRIT_PROJECT + Parameter to identify Gerrit project. This is typically the +project repo path as exists in Gerrit. +For example: ofextensions/circuitsw + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + releng/ciman + + + GERRIT_BRANCH + Parameter to identify a Gerrit branch. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + master + + + GERRIT_REFSPEC + Parameter to identify a refspec when pulling from Gerrit. + +Note that Gerrit will override this parameter automatically if a +job is triggered by Gerrit. + + + + + TOX_DIR + Path to directory containing tox.ini file. + + + + + TOX_ENVS + Tox environments to run build against. +Example: docs,py2,py3 + + + + + + + + 2 + + + origin + +refs/pull/*:refs/remotes/origin/pr/* + git@github.com:example-org/releng/ciman + test-credential + + + + + ** + + + + + false + false + true + false + false + Default + + + + + + true + + false + false + + + false + true + false + + 10 + + + + + + + + + false + + + + ^recheck$ + false + true + true + false + + + JJB Verify + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/tox" +# shellcheck source=/tmp/v/tox/bin/activate disable=SC1091 +source "/tmp/v/tox/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade pipdeptree +pip install --quiet --upgrade tox argparse + +echo "----> Pip Dependency Tree" +pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +cd "$WORKSPACE/$TOX_DIR" + +if [ -n "$TOX_ENVS" ]; +then + tox -e "$TOX_ENVS" +else + tox +fi + + + + + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sysstat.sh" +set +e # DON'T fail build if script fails. + +OS=$(facter operatingsystem) +case "$OS" in + Ubuntu) + SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi + ;; + CentOS|RedHat) + SYSSTAT_PATH="/var/log/sa" + ;; + *) + # nothing to do + exit 0 + ;; +esac + +SAR_DIR="$WORKSPACE/archives/sar-reports" +mkdir -p "$SAR_DIR" +cp "$SYSSTAT_PATH/"* "$_" +# convert sar data to ascii format +while IFS="" read -r s +do + [ -f "$s" ] && LC_TIME=POSIX sar -A -f "$s" > "$SAR_DIR/sar${s//[!0-9]/}" +done < <(find "$SYSSTAT_PATH" -name "sa[0-9]*" || true) + +# DON'T fail build if script fails. +exit 0 + + + + + + jenkins-log-archives-settings + + SETTINGS_FILE + + + + + + SERVER_ID=logs + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> create-netrc.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" +CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[x:id='${SERVER_ID}']" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + +machine=$(echo "$NEXUS_URL" | awk -F/ '{print $3}') +user=$(echo "$CREDENTIAL" | cut -f1 -d:) +pass=$(echo "$CREDENTIAL" | cut -f2 -d:) + +echo "machine $machine login $user password $pass" > ~/.netrc + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> lftools-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "/tmp/v/lftools" +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +pip install --quiet --upgrade pip +pip install --quiet --upgrade "lftools<1.0.0" + +# pipdeptree prints out a lot of information because lftools pulls in many +# dependencies. Let's only print it if we want to debug. +# echo "----> Pip Dependency Tree" +# pip install --quiet --upgrade pipdeptree +# pipdeptree + +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> logs-deploy.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +set -x # Trace commands for this script to make debugging easier. + +ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}" +LOGS_SERVER="${LOGS_SERVER:-None}" + +if [ "${LOGS_SERVER}" == 'None' ] +then + set +x # Disable trace since we no longer need it + + echo "WARNING: Logging server not set" +else + NEXUS_URL="${NEXUS_URL:-$NEXUSPROXY}" + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}" + BUILD_URL="${BUILD_URL}" + + lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE" + lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL" + + set +x # Disable trace since we no longer need it. + + echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>" +fi + + + + #!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Clear log credential files +rm "$SETTINGS_FILE" +rm ~/.netrc + + + + ^Build logs: .* + + + false + false + false + + + + + **/*.jenkins-trigger + EXCLUDE + + + false + false + + true + true + true + true + true + true + + + + + + 10 + BUILD_TIMEOUT + true + false + 0 + 3 + absolute + + + + test-credential + + + + diff --git a/jjb-test/lf-ci-jobs.yaml b/jjb-test/lf-ci-jobs.yaml new file mode 100644 index 00000000..bb2dd42a --- /dev/null +++ b/jjb-test/lf-ci-jobs.yaml @@ -0,0 +1,45 @@ +--- +- project: + name: gerrit-ci-jobs + jobs: + - "{project-name}-ci-jobs" + + project-name: gerrit-ciman + +- project: + name: github-ci-jobs + jobs: + - "{project-name}-github-ci-jobs" + + project-name: github-ciman + + +- project: + name: gerrit-packer-jobs + jobs: + - "{project-name}-packer-jobs" + + packer-cloud-settings: ci-packer-cloud-env + project-name: gerrit-ciman + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet + +- project: + name: github-packer-jobs + jobs: + - "{project-name}-github-packer-jobs" + + packer-cloud-settings: ci-packer-cloud-env + project-name: github-ciman + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet diff --git a/jjb-test/lf-maven-jobs.yaml b/jjb-test/lf-maven-jobs.yaml new file mode 100644 index 00000000..58367054 --- /dev/null +++ b/jjb-test/lf-maven-jobs.yaml @@ -0,0 +1,18 @@ +--- +- project: + name: gerrit-maven-jobs + jobs: + - gerrit-maven-release + + project-name: gerrit-maven + staging-profile-id: uuddlrlrba + mvn-settings: gerrit-maven-project-settings + +- project: + name: github-maven-jobs + jobs: + - github-maven-release + + project-name: github-maven + staging-profile-id: uuddlrlrba + mvn-settings: aproject-settings diff --git a/jjb-test/lf-python-jobs.yaml b/jjb-test/lf-python-jobs.yaml new file mode 100644 index 00000000..49214fb2 --- /dev/null +++ b/jjb-test/lf-python-jobs.yaml @@ -0,0 +1,14 @@ +--- +- project: + name: gerrit-python-jobs + jobs: + - "{project-name}-python-jobs" + + project-name: gerrit-python + +- project: + name: github-python-jobs + jobs: + - "{project-name}-github-python-jobs" + + project-name: github-python diff --git a/test.template b/test.template deleted file mode 100644 index 81bd9073..00000000 --- a/test.template +++ /dev/null @@ -1,108 +0,0 @@ -- defaults: - name: global - - # General - jenkins-ssh-credential: test-credential - - # Gerrit Infra - gerrit-server-name: test-server - - # GitHub Infra - git-url: https://github.com - git-clone-url: 'git@github.com:' - github-org: example-org - github_pr_whitelist: - - jpwku - - tykeal - - zxiiro - github_pr_admin_list: - - tykeal - - # Common test config - project: releng/ciman - stream: latest - build-node: build-vm - -################ -# GERRIT TESTS # -################ - -- project: - name: gerrit-ci-jobs - jobs: - - "{project-name}-ci-jobs" - - project-name: gerrit-ciman - -- project: - name: gerrit-maven-jobs - jobs: - - gerrit-maven-release - - project-name: gerrit-maven - staging-profile-id: uuddlrlrba - mvn-settings: gerrit-maven-project-settings - -- project: - name: gerrit-packer-jobs - jobs: - - "{project-name}-packer-jobs" - - packer-cloud-settings: ci-packer-cloud-env - project-name: gerrit-ciman - platforms: - - centos - - ubuntu1604 - - templates: - - java-builder - - mininet - -- project: - name: gerrit-python-jobs - jobs: - - "{project-name}-python-jobs" - - project-name: gerrit-python - -################ -# GITHUB TESTS # -################ - -- project: - name: github-ci-jobs - jobs: - - "{project-name}-github-ci-jobs" - - project-name: github-ciman - -- project: - name: github-maven-jobs - jobs: - - github-maven-release - - project-name: github-maven - staging-profile-id: uuddlrlrba - mvn-settings: aproject-settings - -- project: - name: github-packer-jobs - jobs: - - "{project-name}-github-packer-jobs" - - packer-cloud-settings: ci-packer-cloud-env - project-name: github-ciman - platforms: - - centos - - ubuntu1604 - - templates: - - java-builder - - mininet - -- project: - name: github-python-jobs - jobs: - - "{project-name}-github-python-jobs" - - project-name: github-python diff --git a/tox.ini b/tox.ini index c27b3635..06424c0d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = coala,jjb,license +envlist = coala,jjb,jjb-compare-xml,license skipsdist = true [testenv:coala] @@ -20,12 +20,13 @@ commands = deps = jenkins-job-builder commands = - cp test.template {toxinidir}/jjb/test.yaml - jenkins-jobs -l DEBUG test --recursive -o {toxinidir}/archives/job-configs {toxinidir}/jjb/ - rm {toxinidir}/jjb/test.yaml -whitelist_externals = - cp - rm + jenkins-jobs -l DEBUG test --recursive -o {toxinidir}/archives/job-configs {toxinidir} + +[testenv:jjb-compare-xml] +deps = + jenkins-job-builder +commands = + ./jjb-compare-xml.sh [testenv:license] deps = lftools -- 2.16.6