From: Thanh Ha Date: Fri, 28 Jul 2017 01:00:31 +0000 (+0000) Subject: Merge "Pass common maven options to deploy file builder" X-Git-Tag: v0.6.0~4 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=98f414a8e0c4a5977d05dd97b09399fcbeaae74d;hp=242c84d1804a9f1161dbd09d3c8147be1e6e7d99;p=releng%2Fglobal-jjb.git Merge "Pass common maven options to deploy file builder" --- diff --git a/README.md b/README.md index 8e76eed5..bc6fcc71 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,61 @@ Optional parameters: **branch**: is the git branch to build from. **jjb-version**: is the version of JJB to install in the build minion. +## Deploying packer-jobs + +The packer job group contains jobs to build custom minion images. The minimal +configuration needed to deploy the packer jobs is as follows which deploys the +**{project-name}-packer-jobs** job group as defined in **lf-ci-jobs.yaml**. + +ci-management.yaml: + +``` +- project: + name: packer-jobs + + jobs: + - '{project-name}-packer-jobs' + + project: ci-management + project-name: ci-management + branch: master + build-node: centos7-basebuild-2c-1g + + platforms: + - centos + - ubuntu-14.04 + - ubuntu-16.04 + + templates: + - devstack + - docker + - gbp + - java-builder + - mininet + + exclude: + - platforms: centos + templates: gbp + - platforms: centos + templates: mininet +``` + +Required parameters: + +**project**: is the project repo as defined in source control. +**project-name**: is a custom name to call the job in Jenkins. +**build-node**: is the name of the builder to use when building (Jenkins label). +**platforms**: is a list of supported platforms. +**templates**: is a list of supported templates. + +Optional parameters: + +**branch**: is the git branch to build from. +**packer-version**: is the version of packer to install in the build minion, +when packer is not available. +**exclude**: is a combination of platforms and templates which are not required +to build. + ## Deploying Python jobs We provide the following Python jobs templates: diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index bce59853..e0aee723 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -20,8 +20,34 @@ jjb-version: 1.6.2 jobs: - - 'github-jjb-merge' - - 'github-jjb-verify' + - github-jjb-merge + - github-jjb-verify + + +- job-group: + name: '{project-name}-packer-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci that is using packer. + + packer-version: 1.0.2 + + jobs: + - gerrit-packer-merge + - gerrit-packer-verify + + +- job-group: + name: '{project-name}-github-packer-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci that is using packer. + + packer-version: 1.0.2 + + jobs: + - github-packer-merge + - github-packer-verify #################### @@ -44,6 +70,19 @@ - compare-type: ANT pattern: '**/*.yaml' +- lf_packer_file_paths: &lf_packer_file_paths + name: lf-packer-file-paths + file-paths: + # Common files for all projects + - compare-type: ANT + pattern: 'packer/vars/{platforms}.json' + - compare-type: ANT + pattern: 'packer/templates/{templates}.json' + - compare-type: ANT + pattern: 'packer/provision/{templates}.sh' + - compare-type: ANT + pattern: 'packer/provision/lib/**' + - lf_jjb_merge_builders: &lf_jjb_merge_builders name: lf-jjb-merge-builders builders: @@ -63,6 +102,47 @@ - ../shell/jjb-check-unicode.sh - lf-infra-gpg-verify-git-signature +- lf_packer: &lf_packer_common + name: lf-packer-common + # Adds wrappers and parameters sections common to packer jobs. + + ###################### + # Default parameters # + ###################### + + build-timeout: 45 + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + parameters: + - lf-infra-parameters: + project: '{project}' + stream: '{stream}' + branch: '{branch}' + - lf-infra-packer-parameters: + packer-version: '{packer-version}' + +- lf_packer_merge_builders: &lf_packer_merge_builders + name: lf-packer-merge-builders + # Adds builders section common to merge packer jobs. + builders: + - lf-infra-packer-validate: + packer-version: '{packer-version}' + - lf-infra-packer-build: + platform: '{platforms}' + template: '{templates}' + packer-version: '{packer-version}' + +- lf_packer_verify_builders: &lf_packer_verify_builders + # Adds builders section common to verify packer jobs. + name: lf-packer-verify-builders + builders: + - lf-infra-packer-validate: + packer-version: '{packer-version}' + - parameter: name: lf-infra-jjb-parameters parameters: @@ -71,6 +151,15 @@ default: '{jjb-version}' description: Jenkins Job Builder version to download and install. +- parameter: + name: lf-infra-packer-parameters + parameters: + - string: + name: PACKER_VERSION + default: '{packer-version}' + description: Packer version to download and install. + + ################# # Job Templates # ################# @@ -278,3 +367,184 @@ - '' github_pr_admin_list: - '' + +- job-template: + name: '{project-name}-packer-merge-{platforms}-{templates}' + id: gerrit-packer-merge + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_merge_builders + + ###################### + # Default parameters # + ###################### + + git-url: '$GIT_URL/$GERRIT_PROJECT' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: remerge$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_packer_file_paths + +- job-template: + name: '{project-name}-packer-verify' + id: gerrit-packer-verify + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_verify_builders + + ###################### + # Default parameters # + ###################### + + build-timeout: 10 + git-url: '$GIT_URL/$GERRIT_PROJECT' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - patchset-created-event: + exclude-drafts: false + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: recheck$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_packer_file_paths + + +- job-template: + name: '{project-name}-packer-merge-{platforms}-{templates}' + id: github-packer-merge + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_merge_builders + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - github + - pollscm: + cron: '' + - lf-infra-github-pr-trigger: + trigger-phrase: '^remerge$' + only-trigger-phrase: true + status-context: 'Packer {platforms}-${templates} Merge' + permit-all: false + github-hooks: true + github-org: '{github-org}' + github_pr_whitelist: '{obj:github_pr_whitelist}' + github_pr_admin_list: '{obj:github_pr_admin_list}' + +- job-template: + name: '{project-name}-packer-verify' + id: github-packer-verify + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_verify_builders + + ###################### + # Default parameters # + ###################### + + build-timeout: 10 + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - lf-infra-github-pr-trigger: + trigger-phrase: '^recheck$' + only-trigger-phrase: false + status-context: 'Packer Verify' + permit-all: true + github-hooks: true + github-org: '' + github_pr_whitelist: + - '' + github_pr_admin_list: + - '' diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index c9c1fd02..7d3e1219 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -71,6 +71,40 @@ - description-setter: regexp: '^Build logs: .*' +- builder: + name: lf-infra-packer-build + builders: + - config-file-provider: + files: + - file-id: packer-cloud-env + variable: CLOUDENV + - inject: + properties-content: | + PACKER_PLATFORM={platform} + PACKER_TEMPLATE={template} + PACKER_VERSION={packer-version} + - shell: !include-raw-escape: + - ../shell/packer-install.sh + - ../shell/packer-build.sh + - shell: !include-raw: + - ../shell/packer-clear-credentials.sh + +- builder: + name: lf-infra-packer-validate + builders: + - config-file-provider: + files: + - file-id: 'packer-cloud-env' + variable: 'CLOUDENV' + - inject: + properties-content: | + PACKER_VERSION={packer-version} + - shell: !include-raw-escape: + - ../shell/packer-install.sh + - ../shell/packer-validate.sh + - shell: !include-raw: + - ../shell/packer-clear-credentials.sh + - builder: name: lf-infra-sysstat builders: @@ -152,6 +186,46 @@ # PARAMETERS # ############## +- parameter: + name: lf-infra-maven-parameters + parameters: + - string: + name: MAVEN_OPTS + default: '{mvn-opts}' + description: | + Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m + - string: + name: MAVEN_PARAMS + default: '{mvn-params}' + description: | + Maven parameters to pass to the mvn command. + - string: + name: MVN + # Sets an env var for shell scripts to be able to call the dynamically + # installed maven without having to calculate the path themselves. + default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn' + description: 'Maven selector to be used by shell scripts' + - string: + name: STAGING_PROFILE_ID + default: '{staging-profile-id}' + description: | + Nexus staging profile ID. + + +- parameter: + name: lf-infra-openstack-parameters + parameters: + - string: + name: OS_CLOUD + default: '{os-cloud}' + description: | + The name of a cloud configuration in clouds.yaml. OS_CLOUD is a + variable name that is significant to openstack client as a + environment variable. Please refer to the documentation for + further details. + https://docs.openstack.org/developer/python-openstackclient/ + + - parameter: name: lf-infra-parameters # Standard parameters used in the LF CI environments. Gerrit variables are @@ -198,30 +272,6 @@ Note that Gerrit will override this parameter automatically if a job is triggered by Gerrit. -- parameter: - name: lf-infra-maven-parameters - parameters: - - string: - name: MAVEN_OPTS - default: '{mvn-opts}' - description: | - Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m - - string: - name: MAVEN_PARAMS - default: '{mvn-params}' - description: | - Maven parameters to pass to the mvn command. - - string: - name: MVN - # Sets an env var for shell scripts to be able to call the dynamically - # installed maven without having to calculate the path themselves. - default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn' - description: 'Maven selector to be used by shell scripts' - - string: - name: STAGING_PROFILE_ID - default: '{staging-profile-id}' - description: | - Nexus staging profile ID. - parameter: name: lf-infra-tox-parameters diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml index 36edbdc9..3f6de2e8 100644 --- a/jjb/lf-maven-jobs.yaml +++ b/jjb/lf-maven-jobs.yaml @@ -18,7 +18,7 @@ # for any project ci that is using github. jobs: - - gerrit-maven-clm + - github-maven-clm - github-maven-release #################### @@ -34,10 +34,12 @@ # build-node: The node to run build on. # jenkins-ssh-credential: Credential to use for SSH. (Generally should # be configured in defaults.yaml) - # settings-file: The name of settings file containing credentials for - # the project. - # global-settings-file: The name of the Maven global settings to use for - # Maven configuration. + # mvn-settings: The name of settings file containing credentials for + # the project. + # + # Optional parameters: + # mvn-global-settings: The name of the Maven global settings to use for + # Maven configuration. (default: global-settings) project-type: freestyle node: '{build-node}' @@ -50,11 +52,12 @@ branch: master build-timeout: 15 git-url: '$GIT_URL/$PROJECT' - global-settings-file: global-settings java-version: openjdk8 + mvn-global-settings: global-settings mvn-opts: '' mvn-params: '' mvn-version: mvn33 + stream: master submodule-recursive: true ##################### @@ -70,19 +73,29 @@ - lf-infra-parameters: project: '{project}' branch: '{branch}' - - lf-infra-maven-parameters + stream: '{stream}' + - lf-infra-maven-parameters: + mvn-opts: '{mvn-opts}' + mvn-params: '{mvn-params}' + mvn-version: '{mvn-version}' + # Staging repos do not make sense for CLM jobs so set it blank. + staging-profile-id: '' wrappers: - lf-infra-wrappers: build-timeout: '{build-timeout}' jenkins-ssh-credential: '{jenkins-ssh-credential}' + triggers: + # Build weekly on Saturdays + - timed: 'H H * * 6' + builders: - lf-maven-install: mvn-version: '{mvn-version}' - lf-provide-maven-settings: - global-settings-file: '{global-settings-file}' - settings-file: '{settings-file}' + global-settings-file: '{mvn-global-settings}' + settings-file: '{mvn-settings}' - shell: !include-raw-escape: - ../shell/lftools-install.sh - ../shell/common-variables.sh @@ -107,11 +120,13 @@ # build-node: The node to run build on. # jenkins-ssh-credential: Credential to use for SSH. (Generally should # be configured in defaults.yaml) - # settings-file: The name of settings file containing credentials for - # the project. - # global-settings-file: The name of the Maven global settings to use for - # Maven configuration. + # mvn-settings: The name of settings file containing credentials for + # the project. # staging-profile-id: Profile ID of the project's Nexus staging profile. + # + # Optional parameters: + # mvn-global-settings: The name of the Maven global settings to use for + # Maven configuration. (default: global-settings) project-type: freestyle node: '{build-node}' @@ -124,8 +139,8 @@ branch: master build-timeout: 30 git-url: '$GIT_URL/$PROJECT' - global-settings-file: global-settings java-version: openjdk8 + mvn-global-settings: global-settings mvn-opts: '' mvn-params: '' mvn-version: mvn33 @@ -162,8 +177,8 @@ - lf-maven-install: mvn-version: '{mvn-version}' - lf-provide-maven-settings: - global-settings-file: '{global-settings-file}' - settings-file: '{settings-file}' + global-settings-file: '{mvn-global-settings}' + settings-file: '{mvn-settings}' - lf-infra-create-netrc: server-id: opendaylight-staging - shell: !include-raw-escape: diff --git a/shell/docker-login.sh b/shell/docker-login.sh index 4b8233d7..2c8a40bb 100644 --- a/shell/docker-login.sh +++ b/shell/docker-login.sh @@ -59,7 +59,7 @@ do_login() { docker login -u "$USER" -p "$PASS" -e "$2" "$1" } -if [ "${REGISTRY:-none}" != 'none' ] +if [ "${DOCKER_REGISTRY:-none}" != 'none' ] then for PORT in $REGISTRY_PORTS do diff --git a/shell/packer-build.sh b/shell/packer-build.sh new file mode 100644 index 00000000..714cd33d --- /dev/null +++ b/shell/packer-build.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done diff --git a/shell/packer-clear-credentials.sh b/shell/packer-clear-credentials.sh new file mode 100644 index 00000000..093b16b4 --- /dev/null +++ b/shell/packer-clear-credentials.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +exit 0 diff --git a/shell/packer-install.sh b/shell/packer-install.sh new file mode 100644 index 00000000..eb303f6e --- /dev/null +++ b/shell/packer-install.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi diff --git a/shell/packer-validate.sh b/shell/packer-validate.sh new file mode 100644 index 00000000..e19c973a --- /dev/null +++ b/shell/packer-validate.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*) +templates=(../packer/templates/*) + +for varfile in "${varfiles[@]}"; do + [[ "${varfile##*/}" =~ ^(cloud-env.*)$ ]] && continue + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done diff --git a/test.template b/test.template index 78a7c89f..f49d54e0 100644 --- a/test.template +++ b/test.template @@ -43,6 +43,20 @@ staging-profile-id: uuddlrlrba settings-file: gerrit-maven-project-settings +- project: + name: gerrit-packer-jobs + jobs: + - "{project-name}-packer-jobs" + + project-name: gerrit-ciman + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet + - project: name: gerrit-python-jobs jobs: @@ -70,10 +84,23 @@ staging-profile-id: uuddlrlrba settings-file: aproject-settings +- project: + name: github-packer-jobs + jobs: + - "{project-name}-github-packer-jobs" + + project-name: github-ciman + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet + - project: name: github-python-jobs jobs: - "{project-name}-github-python-jobs" project-name: github-python -