From 34ab3807fb760dea4fdfc08b1692d49a243759f1 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 7 Jun 2017 21:09:51 -0400 Subject: [PATCH] Create Maven release job The Maven release job runs a build and deploys the artifacts into a local staging repository located at $WORKSPACE/m2repo which is then used to push to a Nexus staging repository later in the build process. - Adds a new Maven release job - Merges the maven-exec macro into a single lf-infra-maven-parameters macro. - Creates a common-variables.sh which can be sourced for useful common variables. - Store staging-repo number in archives staging-repo.txt for easy retrieval. Change-Id: I020d819ea59b809d06803c0dfd999ef002b201c4 Signed-off-by: Thanh Ha --- jjb/lf-macros.yaml | 100 +++++++++++++++++++++--------- jjb/lf-maven-jobs.yaml | 144 +++++++++++++++++++++++++++++++++++++++++++ shell/common-variables.sh | 22 +++++++ shell/maven-build.sh | 28 +++++++++ shell/maven-patch-release.sh | 27 ++++++++ shell/maven-stage.sh | 30 +++++++++ 6 files changed, 323 insertions(+), 28 deletions(-) create mode 100644 jjb/lf-maven-jobs.yaml create mode 100644 shell/common-variables.sh create mode 100644 shell/maven-build.sh create mode 100644 shell/maven-patch-release.sh create mode 100644 shell/maven-stage.sh diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index dcd79d3a..c2b3f2d8 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -1,5 +1,8 @@ --- -# BUILDERS +############ +# BUILDERS # +############ + - builder: name: lf-infra-create-netrc # Macro to create a ~/.netrc file from a Maven settings.xml @@ -51,21 +54,21 @@ builders: - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes' -# call maven-target builder with a goal of --version to force Jenkins to -# install the needed maven version - builder: name: lf-maven-install + # call maven-target builder with a goal of --version to force Jenkins to + # install the needed maven version builders: # Create a $HOME/.wgetrc to make the Maven download quiet. - shell: 'echo "quiet=on" > "$HOME/.wgetrc"' - maven-target: - maven-version: '{maven-version}' + maven-version: '{mvn-version}' goals: '--version' - shell: 'rm "$HOME/.wgetrc"' -# Push a global settings and user settings maven files - builder: name: lf-provide-maven-settings + # Push a global settings and user settings maven files builders: - config-file-provider: files: @@ -73,13 +76,32 @@ variable: 'GLOBAL_SETTINGS_FILE' - file-id: '{settings-file}' variable: 'SETTINGS_FILE' +- builder: + name: lf-provide-maven-settings-cleanup + # Clear maven settings files after we are done using them + builders: + - shell: | + #!/bin/bash + set +e # DO NOT cause build failure if any of the rm calls fail. + + rm "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE" + # In some cases we use the lf-provide-maven-settings macro to produce + # a "$HOME/.netrc" file containing credentials. Remove that file here + # too if it exists. + rm "$HOME/.netrc" + + # DO NOT fail build if any of the above lines fail. + exit 0 + +############## +# PARAMETERS # +############## -# PARAMETERS -# Standard parameters used in the LF CI environments. Gerrit variables are not -# used by GitHub projects, but defining them isn't harmful - parameter: name: lf-infra-parameters + # Standard parameters used in the LF CI environments. Gerrit variables are + # not used by GitHub projects, but defining them isn't harmful. parameters: - string: name: PROJECT @@ -88,9 +110,6 @@ Parameter to identify a Gerrit project. This is typically the project repo path as exists in Gerrit. For example: ofextensions/circuitsw - - (Deprecated) Please use GERRIT_PROJECT instead. A future version - of global-jjb will remove this variable. - string: name: GERRIT_PROJECT default: '{project}' @@ -118,10 +137,35 @@ Note that Gerrit will override this parameter automatically if a job is triggered by Gerrit. -# Useful parameters when working with TOX -# https://tox.readthedocs.io/ +- parameter: + name: lf-infra-maven-parameters + parameters: + - string: + name: MAVEN_OPTS + default: '{mvn-opts}' + description: | + Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m + - string: + name: MAVEN_PARAMS + default: '{mvn-params}' + description: | + Maven parameters to pass to the mvn command. + - string: + name: MVN + # Sets an env var for shell scripts to be able to call the dynamically + # installed maven without having to calculate the path themselves. + default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn' + description: 'Maven selector to be used by shell scripts' + - string: + name: STAGING_PROFILE_ID + default: '{staging-profile-id}' + description: | + Nexus staging profile ID. + - parameter: name: lf-infra-tox-parameters + # Useful parameters when working with TOX + # https://tox.readthedocs.io/ parameters: - string: name: TOX_DIR @@ -135,19 +179,10 @@ Tox environments to run build against. Example: docs,py2,py3 -# Set an env var for shell scripts to be able to call the dynamically installed -# maven without having to calculate the path themselves -- parameter: - name: lf-maven-exec - parameters: - - string: - name: MVN - # yamllint disable-line rule:line-length - default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn' - description: 'Maven selector to be used by shell scripts' - +############## +# PROPERTIES # +############## -# PROPERTIES - property: name: lf-infra-properties properties: @@ -158,7 +193,10 @@ # Do not allow artifacts to be stored in Jenkins. artifact-num-to-keep: 0 -# PUBLISHERS +############## +# PUBLISHERS # +############## + - publisher: name: lf-infra-publish # lf-infra macro to finish up a build. @@ -181,7 +219,10 @@ - '**/*.jenkins-trigger' fail-build: false -# SCM +####### +# SCM # +####### + - scm: name: lf-infra-gerrit-scm scm: @@ -210,7 +251,10 @@ recursive: '{submodule-recursive}' choosing-strategy: '{choosing-strategy}' -# WRAPPERS +############ +# WRAPPERS # +############ + - wrapper: name: lf-infra-wrappers wrappers: diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml new file mode 100644 index 00000000..4e8d9174 --- /dev/null +++ b/jjb/lf-maven-jobs.yaml @@ -0,0 +1,144 @@ +--- +# This file contains job templates for Maven projects. + +- job-group: + name: '{project-name}-maven-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci. + + jobs: + - gerrit-maven-release + +- job-group: + name: '{project-name}-github-maven-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci that is using github. + + jobs: + - github-maven-release + +#################### +# Anchors & Macros # +#################### +- lf_maven_release: &lf_maven_release + name: lf-maven-release + + # Produces a release candidate by creating a staging repo in Nexus. + # + # Runs a Maven build and deploys to $WORKSPACE/m2repo directory. This + # directory can then be reused later to deploy to Nexus. + # + # Required parameters: + # build-node: The node to run build on. + # jenkins-ssh-credential: Credential to use for SSH. (Generally should + # be configured in defaults.yaml) + # settings-file: The name of settings file containing credentials for + # the project. + # global-settings-file: The name of the Maven global settings to use for + # Maven configuration. + # staging-profile-id: Profile ID of the project's Nexus staging profile. + + project-type: freestyle + node: '{build-node}' + jdk: '{java-version}' + + ###################### + # Default parameters # + ###################### + + branch: master + build-timeout: 30 + git-url: '$GIT_URL/$PROJECT' + global-settings-file: global-settings + java-version: openjdk8 + mvn-opts: '' + mvn-params: '' + mvn-version: mvn33 + submodule-recursive: true + + ##################### + # Job Configuration # + ##################### + + properties: + - lf-infra-properties: + # Keep around for 30 days in case a release takes long to get approved. + build-days-to-keep: 30 + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + - lf-infra-maven-parameters: + mvn-opts: '{mvn-opts}' + mvn-params: '{mvn-params}' + mvn-version: '{mvn-version}' + staging-profile-id: '{staging-profile-id}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + builders: + - lf-jacoco-nojava-workaround + - lf-maven-install: + mvn-version: '{mvn-version}' + - lf-provide-maven-settings: + global-settings-file: '{global-settings-file}' + settings-file: '{settings-file}' + - lf-infra-create-netrc: + server-id: opendaylight-staging + - shell: !include-raw-escape: + - ../shell/lftools-install.sh + - ../shell/common-variables.sh + - ../shell/maven-patch-release.sh + - ../shell/maven-build.sh + - ../shell/maven-stage.sh + - lf-provide-maven-settings-cleanup + + publishers: + # TODO: Make email notification work. + # - lf-infra-email-notify: + # email-recipients: '{email-recipients}' + # email-prefix: '[releng]' + - lf-infra-publish + + +################# +# Job Templates # +################# + +- job-template: + name: '{project-name}-maven-release-{stream}' + id: gerrit-maven-release + <<: *lf_maven_release + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + +- job-template: + name: '{project-name}-maven-release-{stream}' + id: github-maven-release + <<: *lf_maven_release + + properties: + - github: + url: '${{GIT_URL}}{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '${{GIT_CLONE_URL}}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' diff --git a/shell/common-variables.sh b/shell/common-variables.sh new file mode 100644 index 00000000..fe602897 --- /dev/null +++ b/shell/common-variables.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This file contains a list of variables that are generally useful in many +# scripts. It is meant to be sourced in other scripts so that the variables can +# be called. + +MAVEN_OPTIONS="$(echo --show-version \ + --batch-mode \ + -Djenkins \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dmaven.repo.local=/tmp/r \ + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)" +echo "$MAVEN_OPTIONS" diff --git a/shell/maven-build.sh b/shell/maven-build.sh new file mode 100644 index 00000000..8bb45560 --- /dev/null +++ b/shell/maven-build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script builds a Maven project and deploys it into a staging repo which +# can be used to deploy elsewhere later eg. Nexus staging / snapshot repos. + +# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound. +# Ensure we fail the job if any steps fail. +set -e -o pipefail +set +u + +export MAVEN_OPTS + +# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters. +# shellcheck disable=SC2086 +$MVN clean deploy \ + --global-settings "$GLOBAL_SETTINGS_FILE" \ + --settings "$SETTINGS_FILE" \ + -DaltDeploymentRepository=staging::default::file:"$WORKSPACE"/m2repo \ + $MAVEN_PARAMS $MAVEN_OPTIONS diff --git a/shell/maven-patch-release.sh b/shell/maven-patch-release.sh new file mode 100644 index 00000000..f0b26c7a --- /dev/null +++ b/shell/maven-patch-release.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script removes the -SNAPSHOT from a project to prepare it for release. + +PATCH_DIR="$WORKSPACE/archives/patches" +mkdir -p "$PATCH_DIR" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +echo "$PROJECT" "$(git rev-parse --verify HEAD)" | tee -a "$PATCH_DIR/taglist.log" + +# Strip -SNAPSHOT from version to prepare release. +find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g' + +git commit -am "Release $PROJECT" +git format-patch --stdout "origin/$GERRIT_BRANCH" > "$PATCH_DIR/${PROJECT//\//-}.patch" +git bundle create "$PATCH_DIR/${PROJECT//\//-}.bundle" "origin/${GERRIT_BRANCH}..HEAD" diff --git a/shell/maven-stage.sh b/shell/maven-stage.sh new file mode 100644 index 00000000..09681eb4 --- /dev/null +++ b/shell/maven-stage.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script publishes artifacts to a staging repo in Nexus. +# +# $WORKSPACE/m2repo : Exists and used to deploy the staging repository. +# $NEXUS_URL : Jenkins global variable should be defined. +# $STAGING_PROFILE_ID : Provided by a job parameter. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +TMP_FILE="$(mktemp)" +lftools deploy nexus-stage "$NEXUS_URL" "$STAGING_PROFILE_ID" "$WORKSPACE/m2repo" | tee "$TMP_FILE" +staging_repo=$(sed -n -e 's/Staging repository \(.*\) created\./\1/p' "$TMP_FILE") + +# Store repo info to a file in archives +mkdir -p "$WORKSPACE/archives" +echo "$staging_repo" > "$WORSPACE/archives/staging-repo.txt" + +# Cleanup +rm "$TMP_FILE" -- 2.16.6