Add Maven job to build dependencies with patches 41/6841/5
authorThanh Ha <thanh.ha@linuxfoundation.org>
Fri, 13 Oct 2017 03:34:15 +0000 (23:34 -0400)
committerThanh Ha <thanh.ha@linuxfoundation.org>
Sat, 14 Oct 2017 18:58:07 +0000 (14:58 -0400)
Provide a Maven job that can verify a patch including firstly building
projects it depends on with their patches first. This allows projects to
validate a patch that has upstream project dependencies which need to be
merged first.

Requires a new global variable to be defined in Jenkins GERRIT_URL
containing the base path to the Gerrit Web UI.

Issue: RELENG-526
Change-Id: Ie9ccb88c21ab5674d0156566da1daee4aa43d641
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
.jjb-test/expected-xml/gerrit-maven-maven-verify-deps-master-mvn33-openjdk8 [new file with mode: 0644]
.jjb-test/lf-maven-jobs.yaml
README.md
jjb/lf-macros.yaml
jjb/lf-maven-jobs.yaml
shell/gerrit-fetch-dependencies.sh [new file with mode: 0644]
shell/maven-build-deps.sh [new file with mode: 0644]

diff --git a/.jjb-test/expected-xml/gerrit-maven-maven-verify-deps-master-mvn33-openjdk8 b/.jjb-test/expected-xml/gerrit-maven-maven-verify-deps-master-mvn33-openjdk8
new file mode 100644 (file)
index 0000000..8903c2b
--- /dev/null
@@ -0,0 +1,740 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <jdk>openjdk8</jdk>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>true</concurrentBuild>
+  <assignedNode>build-vm</assignedNode>
+  <canRoam>false</canRoam>
+  <properties>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>7</daysToKeep>
+        <numToKeep>-1</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>0</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+    <hudson.model.ParametersDefinitionProperty>
+      <parameterDefinitions>
+        <hudson.model.StringParameterDefinition>
+          <name>PROJECT</name>
+          <description>Parameter to identify a Gerrit project. This is typically the
+project repo path as exists in Gerrit.
+For example: ofextensions/circuitsw
+</description>
+          <defaultValue>releng/ciman</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>STREAM</name>
+          <description>Stream is often set to the same name as 'branch' but can
+sometimes be used as a name representing a project's release code
+name.
+</description>
+          <defaultValue>master</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>GERRIT_PROJECT</name>
+          <description>Parameter to identify Gerrit project. This is typically the
+project repo path as exists in Gerrit.
+For example: ofextensions/circuitsw
+
+Note that Gerrit will override this parameter automatically if a
+job is triggered by Gerrit.
+</description>
+          <defaultValue>releng/ciman</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>GERRIT_BRANCH</name>
+          <description>Parameter to identify a Gerrit branch.
+
+Note that Gerrit will override this parameter automatically if a
+job is triggered by Gerrit.
+</description>
+          <defaultValue>master</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>GERRIT_REFSPEC</name>
+          <description>Parameter to identify a refspec when pulling from Gerrit.
+
+Note that Gerrit will override this parameter automatically if a
+job is triggered by Gerrit.
+</description>
+          <defaultValue>refs/heads/master</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>LFTOOLS_VERSION</name>
+          <description>Version of lftools to install. Can be a specific version like
+'0.6.0' or a PEP-440 definition.
+https://www.python.org/dev/peps/pep-0440/
+For example '&lt;1.0.0' or '&gt;=1.0.0,&lt;2.0.0'.
+</description>
+          <defaultValue>&lt;1.0.0</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>MAVEN_OPTS</name>
+          <description>Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
+</description>
+          <defaultValue/>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>MAVEN_PARAMS</name>
+          <description>Maven parameters to pass to the mvn command.
+</description>
+          <defaultValue>-Dstream=$STREAM</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>MVN</name>
+          <description>Maven selector to be used by shell scripts</description>
+          <defaultValue>/w/tools/hudson.tasks.Maven_MavenInstallation/mvn33/bin/mvn</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>STAGING_PROFILE_ID</name>
+          <description>Nexus staging profile ID.
+</description>
+          <defaultValue>uuddlrlrba</defaultValue>
+        </hudson.model.StringParameterDefinition>
+      </parameterDefinitions>
+    </hudson.model.ParametersDefinitionProperty>
+  </properties>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>$GERRIT_REFSPEC</refspec>
+        <url>$GIT_URL/$PROJECT</url>
+        <credentialsId>test-credential</credentialsId>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>refs/heads/$GERRIT_BRANCH</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <excludedUsers/>
+    <buildChooser class="com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser"/>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <wipeOutWorkspace>true</wipeOutWorkspace>
+    <pruneBranches>false</pruneBranches>
+    <remotePoll>false</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <skipTag>true</skipTag>
+    <scmName/>
+    <useShallowClone>false</useShallowClone>
+    <ignoreNotifyCommit>false</ignoreNotifyCommit>
+    <extensions>
+      <hudson.plugins.git.extensions.impl.SubmoduleOption>
+        <disableSubmodules>false</disableSubmodules>
+        <recursiveSubmodules>true</recursiveSubmodules>
+        <trackingSubmodules>false</trackingSubmodules>
+        <reference/>
+        <timeout>10</timeout>
+      </hudson.plugins.git.extensions.impl.SubmoduleOption>
+      <hudson.plugins.git.extensions.impl.WipeWorkspace/>
+    </extensions>
+  </scm>
+  <triggers class="vector">
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec/>
+      <gerritProjects>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>ANT</compareType>
+          <pattern>releng/ciman</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>ANT</compareType>
+              <pattern>**/master</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+          <filePaths>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.FilePath>
+              <compareType>ANT</compareType>
+              <pattern>**</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.FilePath>
+          </filePaths>
+          <disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+      </gerritProjects>
+      <skipVote>
+        <onSuccessful>false</onSuccessful>
+        <onFailed>false</onFailed>
+        <onUnstable>false</onUnstable>
+        <onNotBuilt>false</onNotBuilt>
+      </skipVote>
+      <silentMode>false</silentMode>
+      <silentStartMode>false</silentStartMode>
+      <escapeQuotes>true</escapeQuotes>
+      <noNameAndEmailParameters>false</noNameAndEmailParameters>
+      <readableMessage>false</readableMessage>
+      <dependencyJobsNames/>
+      <notificationLevel/>
+      <dynamicTriggerConfiguration>False</dynamicTriggerConfiguration>
+      <triggerConfigURL/>
+      <allowTriggeringUnreviewedPatches>false</allowTriggeringUnreviewedPatches>
+      <triggerOnEvents>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
+          <commentAddedCommentContains>recheck: [0-9 ]+</commentAddedCommentContains>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
+      </triggerOnEvents>
+      <buildStartMessage/>
+      <buildFailureMessage/>
+      <buildSuccessfulMessage/>
+      <buildUnstableMessage/>
+      <buildNotBuiltMessage/>
+      <buildUnsuccessfulFilepath/>
+      <customUrl/>
+      <serverName>test-server</serverName>
+    </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+  </triggers>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;quiet=on&quot; &gt; &quot;$HOME/.wgetrc&quot;</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Maven>
+      <targets>--version</targets>
+      <properties/>
+      <mavenName>mvn33</mavenName>
+      <usePrivateRepository>false</usePrivateRepository>
+      <settings class="jenkins.mvn.DefaultSettingsProvider"/>
+      <globalSettings class="jenkins.mvn.DefaultGlobalSettingsProvider"/>
+    </hudson.tasks.Maven>
+    <hudson.tasks.Shell>
+      <command>rm &quot;$HOME/.wgetrc&quot;</command>
+    </hudson.tasks.Shell>
+    <org.jenkinsci.plugins.configfiles.builder.ConfigFileBuildStep plugin="config-file-provider">
+      <managedFiles>
+        <org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+          <fileId>global-settings</fileId>
+          <targetLocation/>
+          <variable>GLOBAL_SETTINGS_FILE</variable>
+        </org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+        <org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+          <fileId>gerrit-maven-project-settings</fileId>
+          <targetLocation/>
+          <variable>SETTINGS_FILE</variable>
+        </org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+      </managedFiles>
+    </org.jenkinsci.plugins.configfiles.builder.ConfigFileBuildStep>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# Fetches patches all projects provided by comment trigger
+#
+# Takes a list of Gerrit patches and fetches all projects and cherry-pick
+# patches for projects. The trigger is
+#     'recheck: SPACE_SEPERATED_LIST_OF_PATCHES'
+#
+# NOTE: This script assumes the user will provide the correct dependency order
+#       via the PATCHES list.
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+REPOS_DIR=&quot;$WORKSPACE/.repos&quot;
+
+PATCHES=($(echo &quot;$GERRIT_EVENT_COMMENT_TEXT&quot; | grep 'recheck:' | awk -F: '{print $2}'))
+
+projects=()
+for patch in $(echo &quot;${PATCHES[@]}&quot;); do
+    json=$(curl -s &quot;$GERRIT_URL/changes/$patch&quot; | sed -e &quot;s/)]}'//&quot;)
+    project=$(echo &quot;$json&quot; | jq -r '.project')
+    branch=$(echo &quot;$json&quot; | jq -r '.branch')
+
+    if [ ! -d &quot;$REPOS_DIR/$project&quot; ]; then
+        git clone -q --depth 1 -b &quot;$branch&quot; &quot;$GIT_URL/$project&quot; &quot;$REPOS_DIR/$project&quot;
+
+        # This array will be used later to determine project build order.
+        projects+=(&quot;$project&quot;)
+    fi
+
+    pushd &quot;$REPOS_DIR/$project&quot;
+    git remote add gerrit &quot;$GIT_URL/$project&quot;
+    git review --cherrypick=&quot;$patch&quot;
+    popd
+done
+
+# This script should be a macro which re-inject's the projects variable back
+# into the build so a script later on can use it.
+echo &quot;DEPENDENCY_BUILD_ORDER=${projects[*]}&quot; &gt; &quot;$WORKSPACE/.dependency.properties&quot;
+</command>
+    </hudson.tasks.Shell>
+    <EnvInjectBuilder>
+      <info>
+        <propertiesFilePath>$WORKSPACE/.dependency.properties</propertiesFilePath>
+      </info>
+    </EnvInjectBuilder>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This file contains a list of variables that are generally useful in many
+# scripts. It is meant to be sourced in other scripts so that the variables can
+# be called.
+
+MAVEN_OPTIONS=&quot;$(echo --show-version \
+    --batch-mode \
+    -Djenkins \
+    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
+    -Dmaven.repo.local=/tmp/r \
+    -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)&quot;
+echo &quot;$MAVEN_OPTIONS&quot;
+
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# Builds projects provided via $DEPENDENCY_BUILD_ORDER list
+#
+# This runs a `mvn clean install` against all projects provided by a list. This
+# script is a companion script for the gerrit-fetch-dependencies script which
+# clones project repos to &quot;$WORKSPACE/.repos&quot;.
+
+# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -e -o pipefail
+set +u
+
+PROJECTS=($(echo &quot;$DEPENDENCY_BUILD_ORDER&quot;))
+REPOS_DIR=&quot;$WORKSPACE/.repos&quot;
+
+export MAVEN_OPTS
+
+for project in &quot;${PROJECTS[@]}&quot;; do
+    pushd &quot;$REPOS_DIR/$project&quot;
+    # Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
+    # shellcheck disable=SC2086
+    $MVN clean install \
+        -Pq \
+        -DskipTests=true \
+        --global-settings &quot;$GLOBAL_SETTINGS_FILE&quot; \
+        --settings &quot;$SETTINGS_FILE&quot; \
+        $MAVEN_PARAMS $MAVEN_OPTIONS
+    popd
+done
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This file contains a list of variables that are generally useful in many
+# scripts. It is meant to be sourced in other scripts so that the variables can
+# be called.
+
+MAVEN_OPTIONS=&quot;$(echo --show-version \
+    --batch-mode \
+    -Djenkins \
+    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
+    -Dmaven.repo.local=/tmp/r \
+    -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)&quot;
+echo &quot;$MAVEN_OPTIONS&quot;
+
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script builds a Maven project and deploys it into a staging repo which
+# can be used to deploy elsewhere later eg. Nexus staging / snapshot repos.
+
+# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -e -o pipefail
+set +u
+
+export MAVEN_OPTS
+
+# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
+# shellcheck disable=SC2086
+$MVN clean deploy \
+    --global-settings &quot;$GLOBAL_SETTINGS_FILE&quot; \
+    --settings &quot;$SETTINGS_FILE&quot; \
+    -DaltDeploymentRepository=staging::default::file:&quot;$WORKSPACE&quot;/m2repo \
+    $MAVEN_PARAMS $MAVEN_OPTIONS
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+set +e  # DO NOT cause build failure if any of the rm calls fail.
+
+rm &quot;$GLOBAL_SETTINGS_FILE&quot; &quot;$SETTINGS_FILE&quot;
+
+# In some cases we use the lf-provide-maven-settings macro to produce
+# a &quot;$HOME/.netrc&quot; file containing credentials. Remove that file here
+# too if it exists.
+rm &quot;$HOME/.netrc&quot;
+
+# DO NOT fail build if any of the above lines fail.
+exit 0
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers>
+    <hudson.plugins.findbugs.FindBugsPublisher plugin="findbugs">
+      <isRankActivated>false</isRankActivated>
+      <includePattern/>
+      <excludePattern/>
+      <healthy/>
+      <unHealthy/>
+      <thresholdLimit>low</thresholdLimit>
+      <pluginName>[FINDBUGS] </pluginName>
+      <defaultEncoding/>
+      <canRunOnFailed>false</canRunOnFailed>
+      <useStableBuildAsReference>false</useStableBuildAsReference>
+      <usePreviousBuildAsReference>false</usePreviousBuildAsReference>
+      <useDeltaValues>false</useDeltaValues>
+      <thresholds>
+        <unstableTotalAll/>
+        <unstableTotalHigh/>
+        <unstableTotalNormal/>
+        <unstableTotalLow/>
+        <failedTotalAll/>
+        <failedTotalHigh/>
+        <failedTotalNormal/>
+        <failedTotalLow/>
+      </thresholds>
+      <shouldDetectModules>false</shouldDetectModules>
+      <dontComputeNew>true</dontComputeNew>
+      <doNotResolveRelativePaths>false</doNotResolveRelativePaths>
+      <pattern/>
+    </hudson.plugins.findbugs.FindBugsPublisher>
+    <hudson.plugins.jacoco.JacocoPublisher>
+      <execPattern>**/**.exec</execPattern>
+      <classPattern>**/classes</classPattern>
+      <sourcePattern>**/src/main/java</sourcePattern>
+      <changeBuildStatus/>
+      <inclusionPattern/>
+      <exclusionPattern>**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**</exclusionPattern>
+      <maximumBranchCoverage>10</maximumBranchCoverage>
+      <minimumBranchCoverage>20</minimumBranchCoverage>
+      <maximumMethodCoverage>50</maximumMethodCoverage>
+      <minimumMethodCoverage>40</minimumMethodCoverage>
+    </hudson.plugins.jacoco.JacocoPublisher>
+    <org.jenkinsci.plugins.postbuildscript.PostBuildScript>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo &quot;---&gt; sysstat.sh&quot;
+set +e  # DON'T fail build if script fails.
+
+OS=$(facter operatingsystem)
+case &quot;$OS&quot; in
+    Ubuntu)
+        SYSSTAT_PATH=&quot;/var/log/sysstat&quot;
+
+        # Dont run the script when systat is not enabled by default
+        if ! grep --quiet 'ENABLED=&quot;true&quot;' &quot;/etc/default/sysstat&quot;; then
+            exit 0
+        fi
+    ;;
+    CentOS|RedHat)
+        SYSSTAT_PATH=&quot;/var/log/sa&quot;
+    ;;
+    *)
+        # nothing to do
+        exit 0
+    ;;
+esac
+
+SAR_DIR=&quot;$WORKSPACE/archives/sar-reports&quot;
+mkdir -p &quot;$SAR_DIR&quot;
+cp &quot;$SYSSTAT_PATH/&quot;* &quot;$_&quot;
+# convert sar data to ascii format
+while IFS=&quot;&quot; read -r s
+do
+    [ -f &quot;$s&quot; ] &amp;&amp; LC_TIME=POSIX sar -A -f &quot;$s&quot; &gt; &quot;$SAR_DIR/sar${s//[!0-9]/}&quot;
+done &lt; &lt;(find &quot;$SYSSTAT_PATH&quot; -name &quot;sa[0-9]*&quot; || true)
+
+# DON'T fail build if script fails.
+exit 0
+</command>
+        </hudson.tasks.Shell>
+        <org.jenkinsci.plugins.configfiles.builder.ConfigFileBuildStep plugin="config-file-provider">
+          <managedFiles>
+            <org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+              <fileId>jenkins-log-archives-settings</fileId>
+              <targetLocation/>
+              <variable>SETTINGS_FILE</variable>
+            </org.jenkinsci.plugins.configfiles.buildwrapper.ManagedFile>
+          </managedFiles>
+        </org.jenkinsci.plugins.configfiles.builder.ConfigFileBuildStep>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesContent>SERVER_ID=logs</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo &quot;---&gt; create-netrc.sh&quot;
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+NEXUS_URL=&quot;${NEXUS_URL:-$NEXUSPROXY}&quot;
+CREDENTIAL=$(xmlstarlet sel -N &quot;x=http://maven.apache.org/SETTINGS/1.0.0&quot; \
+    -t -m &quot;/x:settings/x:servers/x:server[x:id='${SERVER_ID}']&quot; \
+    -v x:username -o &quot;:&quot; -v x:password \
+    &quot;$SETTINGS_FILE&quot;)
+
+machine=$(echo &quot;$NEXUS_URL&quot; | awk -F/ '{print $3}')
+user=$(echo &quot;$CREDENTIAL&quot; | cut -f1 -d:)
+pass=$(echo &quot;$CREDENTIAL&quot; | cut -f2 -d:)
+
+echo &quot;machine $machine login $user password $pass&quot; &gt; ~/.netrc
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo &quot;---&gt; lftools-install.sh&quot;
+
+# Script to install lftools via a version passed in via lf-infra-parameters
+#
+# Required parameters:
+#
+#     LFTOOLS_VERSION: Passed in via lf-infra-parameters configuration. Can be
+#                      set to a strict version number like '1.2.3' or using
+#                      PEP-440 definitions.
+#
+#                      Examples:
+#                          &lt;1.0.0
+#                          &gt;=1.0.0,&lt;2.0.0
+#
+# By default a released version of lftools should always be used.
+# The purpose of the 2 variables below is so that lftools devs can test
+# unreleased versions of lftools. There are 2 methods to install a dev version
+# of lftools:
+#
+#     1) gerrit patch: Used to test a patch that has not yet been merged.
+#                      To do this set something like this:
+#                          LFTOOLS_MODE=gerrit
+#                          LFTOOLS_REFSPEC=refs/changes/96/5296/7
+#
+#     2) git branch: Used to install an lftools version from a specific branch.
+#                    To use this set the variables as follows:
+#                          LFTOOLS_MODE=git
+#                          LFTOOLS_REFSPEC=master
+#
+#     3) release : The intended use case and default setting.
+#                  Set LFTOOLS_MODE=release, in this case LFTOOLS_REFSPEC is unused.
+
+LFTOOLS_MODE=release  # release | git | gerrit
+LFTOOLS_REFSPEC=master
+
+# Ensure we fail the job if any steps fail.
+# DO NOT set -u as virtualenv's activate script has unbound variables
+set -e -o pipefail
+
+virtualenv --quiet &quot;/tmp/v/lftools&quot;
+# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091
+source &quot;/tmp/v/lftools/bin/activate&quot;
+pip install --quiet --upgrade pip
+
+case $LFTOOLS_MODE in
+    gerrit)
+        git clone https://gerrit.linuxfoundation.org/infra/releng/lftools.git /tmp/lftools
+        pushd /tmp/lftools
+        git fetch origin &quot;$LFTOOLS_REFSPEC&quot;
+        git checkout FETCH_HEAD
+        pip install --quiet --upgrade -r requirements.txt
+        pip install --quiet --upgrade -e .
+        popd
+        ;;
+
+    git)
+        pip install --quiet --upgrade git+https://gerrit.linuxfoundation.org/infra/releng/lftools.git@&quot;$BRANCH&quot;
+        ;;
+
+    release)
+        if [[ $LFTOOLS_VERSION =~ ^[0-9] ]]; then
+            LFTOOLS_VERSION=&quot;==$LFTOOLS_VERSION&quot;
+        fi
+
+        pip install --quiet --upgrade &quot;lftools${LFTOOLS_VERSION}&quot;
+        ;;
+esac
+
+lftools --version
+
+# pipdeptree prints out a lot of information because lftools pulls in many
+# dependencies. Let's only print it if we want to debug.
+# echo &quot;----&gt; Pip Dependency Tree&quot;
+# pip install --quiet --upgrade pipdeptree
+# pipdeptree
+
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo &quot;---&gt; logs-deploy.sh&quot;
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+set -x  # Trace commands for this script to make debugging easier.
+
+ARCHIVE_ARTIFACTS=&quot;${ARCHIVE_ARTIFACTS:-}&quot;
+LOGS_SERVER=&quot;${LOGS_SERVER:-None}&quot;
+
+if [ &quot;${LOGS_SERVER}&quot; == 'None' ]
+then
+    set +x # Disable trace since we no longer need it
+
+    echo &quot;WARNING: Logging server not set&quot;
+else
+    NEXUS_URL=&quot;${NEXUS_URL:-$NEXUSPROXY}&quot;
+    NEXUS_PATH=&quot;${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}&quot;
+    BUILD_URL=&quot;${BUILD_URL}&quot;
+
+    lftools deploy archives -p &quot;$ARCHIVE_ARTIFACTS&quot; &quot;$NEXUS_URL&quot; &quot;$NEXUS_PATH&quot; &quot;$WORKSPACE&quot;
+    lftools deploy logs &quot;$NEXUS_URL&quot; &quot;$NEXUS_PATH&quot; &quot;$BUILD_URL&quot;
+
+    set +x  # Disable trace since we no longer need it.
+
+    echo &quot;Build logs: &lt;a href=\&quot;$LOGS_SERVER/$NEXUS_PATH\&quot;&gt;$LOGS_SERVER/$NEXUS_PATH&lt;/a&gt;&quot;
+fi
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# Clear log credential files
+rm &quot;$SETTINGS_FILE&quot;
+rm ~/.netrc
+</command>
+        </hudson.tasks.Shell>
+        <hudson.plugins.descriptionsetter.DescriptionSetterBuilder>
+          <regexp>^Build logs: .*</regexp>
+        </hudson.plugins.descriptionsetter.DescriptionSetterBuilder>
+      </buildSteps>
+      <scriptOnlyIfSuccess>false</scriptOnlyIfSuccess>
+      <scriptOnlyIfFailure>false</scriptOnlyIfFailure>
+      <markBuildUnstable>false</markBuildUnstable>
+    </org.jenkinsci.plugins.postbuildscript.PostBuildScript>
+    <hudson.plugins.ws__cleanup.WsCleanup plugin="ws-cleanup">
+      <patterns>
+        <hudson.plugins.ws__cleanup.Pattern>
+          <pattern>**/*.jenkins-trigger</pattern>
+          <type>EXCLUDE</type>
+        </hudson.plugins.ws__cleanup.Pattern>
+      </patterns>
+      <deleteDirs>false</deleteDirs>
+      <cleanupMatrixParent>false</cleanupMatrixParent>
+      <externalDelete/>
+      <cleanWhenSuccess>true</cleanWhenSuccess>
+      <cleanWhenUnstable>true</cleanWhenUnstable>
+      <cleanWhenFailure>true</cleanWhenFailure>
+      <cleanWhenNotBuilt>true</cleanWhenNotBuilt>
+      <cleanWhenAborted>true</cleanWhenAborted>
+      <notFailBuild>true</notFailBuild>
+    </hudson.plugins.ws__cleanup.WsCleanup>
+  </publishers>
+  <buildWrappers>
+    <com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper/>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>60</timeoutMinutes>
+      <timeoutEnvVar>BUILD_TIMEOUT</timeoutEnvVar>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>0</timeoutPercentage>
+      <timeoutMinutesElasticDefault>3</timeoutMinutesElasticDefault>
+      <timeoutType>absolute</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <hudson.plugins.timestamper.TimestamperBuildWrapper/>
+    <com.cloudbees.jenkins.plugins.sshagent.SSHAgentBuildWrapper>
+      <user>test-credential</user>
+    </com.cloudbees.jenkins.plugins.sshagent.SSHAgentBuildWrapper>
+    <jenkins.plugins.openstack.compute.JCloudsOneOffSlave/>
+  </buildWrappers>
+</project>
index bb33d7a..ab0e894 100644 (file)
@@ -8,6 +8,7 @@
       - gerrit-maven-release
       - gerrit-maven-sonar
       - gerrit-maven-verify
+      - gerrit-maven-verify-dependencies
 
     project-name: gerrit-maven
     javadoc-path: org.example.project/$STREAM
index 4e0e047..50d9c6e 100644 (file)
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@ be available.
 For example:
 
 ```
+GERRIT_URL=https://git.opendaylight.org/gerrit
 GIT_URL=ssh://jenkins-$SILO@git.opendaylight.org:29418
 GIT_CLONE_URL=git@github.com:
 JENKINS_HOSTNAME=jenkins092
index 996ae5a..d56b25f 100644 (file)
@@ -3,6 +3,20 @@
 # BUILDERS #
 ############
 
+- builder:
+    name: lf-fetch-dependent-patches
+    # Fetches all patches provided via comment trigger
+    #
+    # This macro will fetch all patches provided via comment trigger and will
+    # create a list of projects from those patches via environment variable
+    # called DEPENDENCY_BUILD_ORDER which can be used if necessary to build
+    # projects in the specified order. The order is determined by first patch
+    # instance for a project in the patch list.
+    builders:
+      - shell: !include-raw: ../shell/gerrit-fetch-dependencies.sh
+      - inject:
+          properties-file: $WORKSPACE/.dependency.properties
+
 - builder:
     name: lf-infra-create-netrc
     # Macro to create a ~/.netrc file from a Maven settings.xml
index eaa4ef8..c6bfe1f 100644 (file)
@@ -11,6 +11,7 @@
       - gerrit-maven-clm
       - gerrit-maven-release
       - gerrit-maven-verify
+      - gerrit-maven-verify-dependencies
 
 - job-group:
     name: '{project-name}-github-maven-jobs'
             - ''
           github_pr_admin_list:
             - ''
+
+#############################
+# Maven Verify Dependencies #
+#############################
+
+- lf_maven_verify_dependencies: &lf_maven_verify_dependencies
+    name: lf-maven-verify-dependencies
+
+    # Verify job which runs mvn clean install to test a project build /w deps
+    #
+    # This job can be used to verify a patch in conjunction to all of the
+    # upstream patches it depends on. The user of this job can provide a list
+    # via comment trigger. The trigger is:
+    #
+    #     'recheck: SPACE_SEPERATED_LIST_OF_PATCHES'
+    #
+    # Required parameters:
+    #
+    #     :build-node: The node to run build on.
+    #     :jenkins-ssh-credential: Credential to use for SSH. (Generally should
+    #         be configured in defaults.yaml)
+    #     :mvn-settings: The name of settings file containing credentials for
+    #         the project.
+    #
+    # Optional parameters:
+    #
+    #     :branch: Git branch to fetch for the build. (default: master)
+    #     :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+    #     :build-timeout: Timeout in seconds before aborting build. (default: 60)
+    #     :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+    #     :java-version: Version of Java to use for the build. (default: openjdk8)
+    #     :mvn-global-settings: The name of the Maven global settings to use for
+    #         Maven configuration. (default: global-settings)
+    #     :mvn-opts: Sets MAVEN_OPTS. (default: '')
+    #     :mvn-params: Additional mvn parameters to pass to the cli. (default: '')
+    #     :mvn-version: Version of maven to use. (default: mvn33)
+    #     :stream: Keyword that can be used to represent a release code-name.
+    #         Often the same as the branch. (default: master)
+    #     :submodule-recursive: Whether to checkout submodules recursively.
+    #         (default: true)
+    #
+    #     :gerrit_verify_triggers: Override Gerrit Triggers.
+    #     :gerrit_trigger_file_paths: Override file paths which can be used to
+    #         filter which file modifications will trigger a build.
+
+    ######################
+    # Default parameters #
+    ######################
+
+    branch: master
+    build-days-to-keep: 7
+    build-timeout: 60
+    git-url: '$GIT_URL/$PROJECT'
+    java-version: openjdk8
+    mvn-global-settings: global-settings
+    mvn-opts: ''
+    mvn-params: '-Dstream=$STREAM'
+    mvn-version: mvn33
+    staging-profile-id: ''  # Unused by this job
+    stream: master
+    submodule-recursive: true
+
+    gerrit_verify_triggers:
+      - comment-added-contains-event:
+          comment-contains-value: 'recheck: [0-9 ]+'
+    gerrit_trigger_file_paths:
+      - compare-type: ANT
+        pattern: '**'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    concurrent: true
+
+    builders:
+      - lf-jacoco-nojava-workaround
+      - lf-maven-install:
+          mvn-version: '{mvn-version}'
+      - lf-provide-maven-settings:
+          global-settings-file: '{mvn-global-settings}'
+          settings-file: '{mvn-settings}'
+      - lf-fetch-dependent-patches
+      - shell: !include-raw-escape:
+          - ../shell/common-variables.sh
+          - ../shell/maven-build-deps.sh
+      - shell: !include-raw-escape:
+          - ../shell/common-variables.sh
+          - ../shell/maven-build.sh
+      - lf-provide-maven-settings-cleanup
+
+    publishers:
+      - findbugs
+      - lf-jacoco-report
+      - lf-infra-publish
+
+- job-template:
+    name: '{project-name}-maven-verify-deps-{stream}-{mvn-version}-{java-version}'
+    id: gerrit-maven-verify-dependencies
+    <<: *lf_maven_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_maven_verify_dependencies
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: gerrit
+
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_verify_triggers}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
diff --git a/shell/gerrit-fetch-dependencies.sh b/shell/gerrit-fetch-dependencies.sh
new file mode 100644 (file)
index 0000000..4870b10
--- /dev/null
@@ -0,0 +1,48 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# Fetches patches all projects provided by comment trigger
+#
+# Takes a list of Gerrit patches and fetches all projects and cherry-pick
+# patches for projects. The trigger is
+#     'recheck: SPACE_SEPERATED_LIST_OF_PATCHES'
+#
+# NOTE: This script assumes the user will provide the correct dependency order
+#       via the PATCHES list.
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+REPOS_DIR="$WORKSPACE/.repos"
+
+PATCHES=($(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'recheck:' | awk -F: '{print $2}'))
+
+projects=()
+for patch in $(echo "${PATCHES[@]}"); do
+    json=$(curl -s "$GERRIT_URL/changes/$patch" | sed -e "s/)]}'//")
+    project=$(echo "$json" | jq -r '.project')
+    branch=$(echo "$json" | jq -r '.branch')
+
+    if [ ! -d "$REPOS_DIR/$project" ]; then
+        git clone -q --depth 1 -b "$branch" "$GIT_URL/$project" "$REPOS_DIR/$project"
+
+        # This array will be used later to determine project build order.
+        projects+=("$project")
+    fi
+
+    pushd "$REPOS_DIR/$project"
+    git remote add gerrit "$GIT_URL/$project"
+    git review --cherrypick="$patch"
+    popd
+done
+
+# This script should be a macro which re-inject's the projects variable back
+# into the build so a script later on can use it.
+echo "DEPENDENCY_BUILD_ORDER=${projects[*]}" > "$WORKSPACE/.dependency.properties"
diff --git a/shell/maven-build-deps.sh b/shell/maven-build-deps.sh
new file mode 100644 (file)
index 0000000..fdb29b2
--- /dev/null
@@ -0,0 +1,39 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# Builds projects provided via $DEPENDENCY_BUILD_ORDER list
+#
+# This runs a `mvn clean install` against all projects provided by a list. This
+# script is a companion script for the gerrit-fetch-dependencies script which
+# clones project repos to "$WORKSPACE/.repos".
+
+# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -e -o pipefail
+set +u
+
+PROJECTS=($(echo "$DEPENDENCY_BUILD_ORDER"))
+REPOS_DIR="$WORKSPACE/.repos"
+
+export MAVEN_OPTS
+
+for project in "${PROJECTS[@]}"; do
+    pushd "$REPOS_DIR/$project"
+    # Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
+    # shellcheck disable=SC2086
+    $MVN clean install \
+        -Pq \
+        -DskipTests=true \
+        --global-settings "$GLOBAL_SETTINGS_FILE" \
+        --settings "$SETTINGS_FILE" \
+        $MAVEN_PARAMS $MAVEN_OPTIONS
+    popd
+done