Create Maven release job

The Maven release job runs a build and deploys the artifacts into a
local staging repository located at $WORKSPACE/m2repo which is then
used to push to a Nexus staging repository later in the build process.

- Adds a new Maven release job
- Merges the maven-exec macro into a single lf-infra-maven-parameters
  macro.
- Creates a common-variables.sh which can be sourced for useful common
  variables.
- Store staging-repo number in archives staging-repo.txt for easy
  retrieval.

Change-Id: I020d819ea59b809d06803c0dfd999ef002b201c4
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml
index dcd79d3..c2b3f2d 100644 (file)
--- a/jjb/lf-macros.yaml
+++ b/jjb/lf-macros.yaml
@@ -1,5 +1,8 @@
 ---
 ---

-# BUILDERS
+############
+# BUILDERS #
+############
+

 - builder:
     name: lf-infra-create-netrc
     # Macro to create a ~/.netrc file from a Maven settings.xml
 - builder:
     name: lf-infra-create-netrc
     # Macro to create a ~/.netrc file from a Maven settings.xml


@@ -51,21 +54,21 @@
     builders:
       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
 
     builders:
       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
 

-# call maven-target builder with a goal of --version to force Jenkins to
-# install the needed maven version

 - builder:
     name: lf-maven-install
 - builder:
     name: lf-maven-install

+    # call maven-target builder with a goal of --version to force Jenkins to
+    # install the needed maven version

     builders:
       # Create a $HOME/.wgetrc to make the Maven download quiet.
       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
       - maven-target:
     builders:
       # Create a $HOME/.wgetrc to make the Maven download quiet.
       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
       - maven-target:

-          maven-version: '{maven-version}'
+          maven-version: '{mvn-version}'

           goals: '--version'
       - shell: 'rm "$HOME/.wgetrc"'
 
           goals: '--version'
       - shell: 'rm "$HOME/.wgetrc"'
 

-# Push a global settings and user settings maven files

 - builder:
     name: lf-provide-maven-settings
 - builder:
     name: lf-provide-maven-settings

+    # Push a global settings and user settings maven files

     builders:
       - config-file-provider:
           files:
     builders:
       - config-file-provider:
           files:


@@ -73,13 +76,32 @@
               variable: 'GLOBAL_SETTINGS_FILE'
             - file-id: '{settings-file}'
               variable: 'SETTINGS_FILE'
               variable: 'GLOBAL_SETTINGS_FILE'
             - file-id: '{settings-file}'
               variable: 'SETTINGS_FILE'

+- builder:
+    name: lf-provide-maven-settings-cleanup
+    # Clear maven settings files after we are done using them
+    builders:
+      - shell: |
+          #!/bin/bash
+          set +e  # DO NOT cause build failure if any of the rm calls fail.
+
+          rm "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE"

 
 

+          # In some cases we use the lf-provide-maven-settings macro to produce
+          # a "$HOME/.netrc" file containing credentials. Remove that file here
+          # too if it exists.
+          rm "$HOME/.netrc"
+
+          # DO NOT fail build if any of the above lines fail.
+          exit 0
+
+##############
+# PARAMETERS #
+##############

 
 

-# PARAMETERS
-# Standard parameters used in the LF CI environments. Gerrit variables are not
-# used by GitHub projects, but defining them isn't harmful

 - parameter:
     name: lf-infra-parameters
 - parameter:
     name: lf-infra-parameters

+    # Standard parameters used in the LF CI environments. Gerrit variables are
+    # not used by GitHub projects, but defining them isn't harmful.

     parameters:
       - string:
           name: PROJECT
     parameters:
       - string:
           name: PROJECT


@@ -88,9 +110,6 @@
               Parameter to identify a Gerrit project. This is typically the
               project repo path as exists in Gerrit.
               For example: ofextensions/circuitsw
               Parameter to identify a Gerrit project. This is typically the
               project repo path as exists in Gerrit.
               For example: ofextensions/circuitsw

-
-              (Deprecated) Please use GERRIT_PROJECT instead. A future version
-              of global-jjb will remove this variable.

       - string:
           name: GERRIT_PROJECT
           default: '{project}'
       - string:
           name: GERRIT_PROJECT
           default: '{project}'


@@ -118,10 +137,35 @@
               Note that Gerrit will override this parameter automatically if a
               job is triggered by Gerrit.
 
               Note that Gerrit will override this parameter automatically if a
               job is triggered by Gerrit.
 

-# Useful parameters when working with TOX
-# https://tox.readthedocs.io/
+- parameter:
+    name: lf-infra-maven-parameters
+    parameters:
+      - string:
+          name: MAVEN_OPTS
+          default: '{mvn-opts}'
+          description: |
+              Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
+      - string:
+          name: MAVEN_PARAMS
+          default: '{mvn-params}'
+          description: |
+              Maven parameters to pass to the mvn command.
+      - string:
+          name: MVN
+          # Sets an env var for shell scripts to be able to call the dynamically
+          # installed maven without having to calculate the path themselves.
+          default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn'
+          description: 'Maven selector to be used by shell scripts'
+      - string:
+          name: STAGING_PROFILE_ID
+          default: '{staging-profile-id}'
+          description: |
+              Nexus staging profile ID.
+

 - parameter:
     name: lf-infra-tox-parameters
 - parameter:
     name: lf-infra-tox-parameters

+    # Useful parameters when working with TOX
+    # https://tox.readthedocs.io/

     parameters:
       - string:
           name: TOX_DIR
     parameters:
       - string:
           name: TOX_DIR


@@ -135,19 +179,10 @@
               Tox environments to run build against.
               Example: docs,py2,py3
 
               Tox environments to run build against.
               Example: docs,py2,py3
 

-# Set an env var for shell scripts to be able to call the dynamically installed
-# maven without having to calculate the path themselves
-- parameter:
-    name: lf-maven-exec
-    parameters:
-      - string:
-          name: MVN
-          # yamllint disable-line rule:line-length
-          default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
-          description: 'Maven selector to be used by shell scripts'
-
+##############
+# PROPERTIES #
+##############

 
 

-# PROPERTIES

 - property:
     name: lf-infra-properties
     properties:
 - property:
     name: lf-infra-properties
     properties:


@@ -158,7 +193,10 @@
           # Do not allow artifacts to be stored in Jenkins.
           artifact-num-to-keep: 0
 
           # Do not allow artifacts to be stored in Jenkins.
           artifact-num-to-keep: 0
 

-# PUBLISHERS
+##############
+# PUBLISHERS #
+##############
+

 - publisher:
     name: lf-infra-publish
     # lf-infra macro to finish up a build.
 - publisher:
     name: lf-infra-publish
     # lf-infra macro to finish up a build.


@@ -181,7 +219,10 @@
             - '**/*.jenkins-trigger'
           fail-build: false
 
             - '**/*.jenkins-trigger'
           fail-build: false
 

-# SCM
+#######
+# SCM #
+#######
+

 - scm:
     name: lf-infra-gerrit-scm
     scm:
 - scm:
     name: lf-infra-gerrit-scm
     scm:


@@ -210,7 +251,10 @@
             recursive: '{submodule-recursive}'
           choosing-strategy: '{choosing-strategy}'
 
             recursive: '{submodule-recursive}'
           choosing-strategy: '{choosing-strategy}'
 

-# WRAPPERS
+############
+# WRAPPERS #
+############
+

 - wrapper:
     name: lf-infra-wrappers
     wrappers:
 - wrapper:
     name: lf-infra-wrappers
     wrappers:

diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml
new file mode 100644 (file)
index 0000000..4e8d917
--- /dev/null
+++ b/jjb/lf-maven-jobs.yaml
@@ -0,0 +1,144 @@
+---
+# This file contains job templates for Maven projects.
+
+- job-group:
+    name: '{project-name}-maven-jobs'
+
+    # This job group contains all the recommended jobs that should be deployed
+    # for any project ci.
+
+    jobs:
+      - gerrit-maven-release
+
+- job-group:
+    name: '{project-name}-github-maven-jobs'
+
+    # This job group contains all the recommended jobs that should be deployed
+    # for any project ci that is using github.
+
+    jobs:
+      - github-maven-release
+
+####################
+# Anchors & Macros #
+####################
+- lf_maven_release: &lf_maven_release
+    name: lf-maven-release
+
+    # Produces a release candidate by creating a staging repo in Nexus.
+    #
+    # Runs a Maven build and deploys to $WORKSPACE/m2repo directory. This
+    # directory can then be reused later to deploy to Nexus.
+    #
+    # Required parameters:
+    #     build-node:    The node to run build on.
+    #     jenkins-ssh-credential: Credential to use for SSH. (Generally should
+    #                             be configured in defaults.yaml)
+    #     settings-file: The name of settings file containing credentials for
+    #                    the project.
+    #     global-settings-file: The name of the Maven global settings to use for
+    #                           Maven configuration.
+    #     staging-profile-id: Profile ID of the project's Nexus staging profile.
+
+    project-type: freestyle
+    node: '{build-node}'
+    jdk: '{java-version}'
+
+    ######################
+    # Default parameters #
+    ######################
+
+    branch: master
+    build-timeout: 30
+    git-url: '$GIT_URL/$PROJECT'
+    global-settings-file: global-settings
+    java-version: openjdk8
+    mvn-opts: ''
+    mvn-params: ''
+    mvn-version: mvn33
+    submodule-recursive: true
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    properties:
+      - lf-infra-properties:
+          # Keep around for 30 days in case a release takes long to get approved.
+          build-days-to-keep: 30
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+      - lf-infra-maven-parameters:
+          mvn-opts: '{mvn-opts}'
+          mvn-params: '{mvn-params}'
+          mvn-version: '{mvn-version}'
+          staging-profile-id: '{staging-profile-id}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    builders:
+      - lf-jacoco-nojava-workaround
+      - lf-maven-install:
+          mvn-version: '{mvn-version}'
+      - lf-provide-maven-settings:
+          global-settings-file: '{global-settings-file}'
+          settings-file: '{settings-file}'
+      - lf-infra-create-netrc:
+          server-id: opendaylight-staging
+      - shell: !include-raw-escape:
+          - ../shell/lftools-install.sh
+          - ../shell/common-variables.sh
+          - ../shell/maven-patch-release.sh
+          - ../shell/maven-build.sh
+          - ../shell/maven-stage.sh
+      - lf-provide-maven-settings-cleanup
+
+    publishers:
+      # TODO: Make email notification work.
+      # - lf-infra-email-notify:
+      #     email-recipients: '{email-recipients}'
+      #     email-prefix: '[releng]'
+      - lf-infra-publish
+
+
+#################
+# Job Templates #
+#################
+
+- job-template:
+    name: '{project-name}-maven-release-{stream}'
+    id: gerrit-maven-release
+    <<: *lf_maven_release
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: gerrit
+
+- job-template:
+    name: '{project-name}-maven-release-{stream}'
+    id: github-maven-release
+    <<: *lf_maven_release
+
+    properties:
+      - github:
+          url: '${{GIT_URL}}{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '${{GIT_CLONE_URL}}{github-org}/{project}'
+          refspec: '+refs/pull/*:refs/remotes/origin/pr/*'
+          branch: '{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'

diff --git a/shell/common-variables.sh b/shell/common-variables.sh
new file mode 100644 (file)
index 0000000..fe60289
--- /dev/null
+++ b/shell/common-variables.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This file contains a list of variables that are generally useful in many
+# scripts. It is meant to be sourced in other scripts so that the variables can
+# be called.
+
+MAVEN_OPTIONS="$(echo --show-version \
+    --batch-mode \
+    -Djenkins \
+    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
+    -Dmaven.repo.local=/tmp/r \
+    -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)"
+echo "$MAVEN_OPTIONS"

diff --git a/shell/maven-build.sh b/shell/maven-build.sh
new file mode 100644 (file)
index 0000000..8bb4556
--- /dev/null
+++ b/shell/maven-build.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script builds a Maven project and deploys it into a staging repo which
+# can be used to deploy elsewhere later eg. Nexus staging / snapshot repos.
+
+# DO NOT enable -u because $MAVEN_PARAMS and $MAVEN_OPTIONS could be unbound.
+# Ensure we fail the job if any steps fail.
+set -e -o pipefail
+set +u
+
+export MAVEN_OPTS
+
+# Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
+# shellcheck disable=SC2086
+$MVN clean deploy \
+    --global-settings "$GLOBAL_SETTINGS_FILE" \
+    --settings "$SETTINGS_FILE" \
+    -DaltDeploymentRepository=staging::default::file:"$WORKSPACE"/m2repo \
+    $MAVEN_PARAMS $MAVEN_OPTIONS

diff --git a/shell/maven-patch-release.sh b/shell/maven-patch-release.sh
new file mode 100644 (file)
index 0000000..f0b26c7
--- /dev/null
+++ b/shell/maven-patch-release.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script removes the -SNAPSHOT from a project to prepare it for release.
+
+PATCH_DIR="$WORKSPACE/archives/patches"
+mkdir -p "$PATCH_DIR"
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+echo "$PROJECT" "$(git rev-parse --verify HEAD)" | tee -a "$PATCH_DIR/taglist.log"
+
+# Strip -SNAPSHOT from version to prepare release.
+find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
+
+git commit -am "Release $PROJECT"
+git format-patch --stdout "origin/$GERRIT_BRANCH" > "$PATCH_DIR/${PROJECT//\//-}.patch"
+git bundle create "$PATCH_DIR/${PROJECT//\//-}.bundle" "origin/${GERRIT_BRANCH}..HEAD"

diff --git a/shell/maven-stage.sh b/shell/maven-stage.sh
new file mode 100644 (file)
index 0000000..09681eb
--- /dev/null
+++ b/shell/maven-stage.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script publishes artifacts to a staging repo in Nexus.
+#
+# $WORKSPACE/m2repo   :  Exists and used to deploy the staging repository.
+# $NEXUS_URL          :  Jenkins global variable should be defined.
+# $STAGING_PROFILE_ID :  Provided by a job parameter.
+
+# Ensure we fail the job if any steps fail.
+set -eu -o pipefail
+
+TMP_FILE="$(mktemp)"
+lftools deploy nexus-stage "$NEXUS_URL" "$STAGING_PROFILE_ID" "$WORKSPACE/m2repo" | tee "$TMP_FILE"
+staging_repo=$(sed -n -e 's/Staging repository \(.*\) created\./\1/p' "$TMP_FILE")
+
+# Store repo info to a file in archives
+mkdir -p "$WORKSPACE/archives"
+echo "$staging_repo" > "$WORSPACE/archives/staging-repo.txt"
+
+# Cleanup
+rm "$TMP_FILE"