From 10821edc619b09e4623884b02900c39431f8a809 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Mon, 3 Jul 2017 21:54:03 +1000 Subject: [PATCH] Add generic packer templates - Packer jobs are designed to validate packer files and build custom images on the cloud. The packer templates can be used for and LF projects which clone global-jjb submodule. - The jobs check if packer binaries are unavailable and install packer on the build node. The default version of packer installed is 1.0.2 while more recent version can be defined in the job. - Update test templates according to new packer jobs introduced. Change-Id: I3657621e30d402655b84973ea4b4c48fa21cea2d Signed-off-by: Anil Belur Signed-off-by: Andrew Grimberg --- README.md | 58 +++++++++ jjb/lf-ci-jobs.yaml | 263 +++++++++++++++++++++++++++++++++++++- jjb/lf-macros.yaml | 102 +++++++++++---- shell/packer-build.sh | 39 ++++++ shell/packer-clear-credentials.sh | 17 +++ shell/packer-install.sh | 36 ++++++ shell/packer-validate.sh | 38 ++++++ test.template | 31 ++++- 8 files changed, 557 insertions(+), 27 deletions(-) create mode 100644 shell/packer-build.sh create mode 100644 shell/packer-clear-credentials.sh create mode 100644 shell/packer-install.sh create mode 100644 shell/packer-validate.sh diff --git a/README.md b/README.md index 8e76eed5..dbbf0d99 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,64 @@ Optional parameters: **branch**: is the git branch to build from. **jjb-version**: is the version of JJB to install in the build minion. +## Deploying packer-jobs + +The packer job group contains jobs to build custom minion images. The minimal +configuration needed to deploy the packer jobs is as follows which deploys the +**{project-name}-packer-jobs** job group as defined in **lf-ci-jobs.yaml**. + +ci-management.yaml: + +``` +- project: + name: packer-jobs + + jobs: + - '{project-name}-packer-jobs' + + project: ci-management + project-name: ci-management + branch: master + os-cloud-file-id: 'ci-managed-file-id' + build-node: centos7-basebuild-2c-1g + + platforms: + - centos + - ubuntu-14.04 + - ubuntu-16.04 + + templates: + - devstack + - docker + - gbp + - java-builder + - mininet + + exclude: + - platforms: centos + templates: gbp + - platforms: centos + templates: mininet +``` + +Required parameters: + +**project**: is the project repo as defined in source control. +**project-name**: is a custom name to call the job in Jenkins. +**build-node**: is the name of the builder to use when building (Jenkins label). +**os-cloud-file-id**: is the name of the managed file-id, which contains +credentials required for packer to spin up build nodes on the cloud provider. +**platforms**: is a list of supported platforms. +**templates**: is a list of supported templates. + +Optional parameters: + +**branch**: is the git branch to build from. +**packer-version**: is the version of packer to install in the build minion, +when packer is not available. +**exclude**: is a combination of platforms and templates which are not required +to build. + ## Deploying Python jobs We provide the following Python jobs templates: diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index bce59853..4191a9cc 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -20,8 +20,34 @@ jjb-version: 1.6.2 jobs: - - 'github-jjb-merge' - - 'github-jjb-verify' + - github-jjb-merge + - github-jjb-verify + + +- job-group: + name: '{project-name}-packer-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci that is using packer. + + packer-version: 1.0.2 + + jobs: + - gerrit-packer-merge + - gerrit-packer-verify + + +- job-group: + name: '{project-name}-github-packer-jobs' + + # This job group contains all the recommended jobs that should be deployed + # for any project ci that is using packer. + + packer-version: 1.0.2 + + jobs: + - github-packer-merge + - github-packer-verify #################### @@ -44,6 +70,19 @@ - compare-type: ANT pattern: '**/*.yaml' +- lf_packer_file_paths: &lf_packer_file_paths + name: lf-packer-file-paths + file-paths: + # Common files for all projects + - compare-type: ANT + pattern: 'packer/vars/{platforms}.json' + - compare-type: ANT + pattern: 'packer/templates/{templates}.json' + - compare-type: ANT + pattern: 'packer/provision/{templates}.sh' + - compare-type: ANT + pattern: 'packer/provision/lib/**' + - lf_jjb_merge_builders: &lf_jjb_merge_builders name: lf-jjb-merge-builders builders: @@ -63,6 +102,43 @@ - ../shell/jjb-check-unicode.sh - lf-infra-gpg-verify-git-signature +- lf_packer: &lf_packer_common + name: lf-packer-common + # Adds wrappers and parameters sections common to packer jobs. + wrappers: + - lf-infra-wrappers: + build-timeout: 60 + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + parameters: + - lf-infra-parameters: + project: '{project}' + stream: '{stream}' + branch: '{branch}' + - lf-infra-packer-parameters: + packer-version: '{packer-version}' + +- lf_packer_merge_builders: &lf_packer_merge_builders + name: lf-packer-merge-builders + # Adds builders section common to merge packer jobs. + builders: + - lf-infra-packer-validate: + os-cloud-file-id: '{os-cloud-file-id}' + packer-version: '{packer-version}' + - lf-infra-packer-build: + platform: '{platforms}' + template: '{templates}' + os-cloud-file-id: '{os-cloud-file-id}' + packer-version: '{packer-version}' + +- lf_packer_verify_builders: &lf_packer_verify_builders + # Adds builders section common to verify packer jobs. + name: lf-packer-verify-builders + builders: + - lf-infra-packer-validate: + os-cloud-file-id: '{os-cloud-file-id}' + packer-version: '{packer-version}' + - parameter: name: lf-infra-jjb-parameters parameters: @@ -71,6 +147,15 @@ default: '{jjb-version}' description: Jenkins Job Builder version to download and install. +- parameter: + name: lf-infra-packer-parameters + parameters: + - string: + name: PACKER_VERSION + default: '{packer-version}' + description: Packer version to download and install. + + ################# # Job Templates # ################# @@ -278,3 +363,177 @@ - '' github_pr_admin_list: - '' + +- job-template: + name: '{project-name}-packer-merge-{platforms}-{templates}' + id: gerrit-packer-merge + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_merge_builders + + ###################### + # Default parameters # + ###################### + + git-url: '$GIT_URL/$GERRIT_PROJECT' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: remerge$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_packer_file_paths + +- job-template: + name: '{project-name}-packer-verify' + id: gerrit-packer-verify + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_verify_builders + + ###################### + # Default parameters # + ###################### + + git-url: '$GIT_URL/$GERRIT_PROJECT' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - patchset-created-event: + exclude-drafts: false + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: recheck$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + <<: *lf_packer_file_paths + + +- job-template: + name: '{project-name}-packer-merge-{platforms}-{templates}' + id: github-packer-merge + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_merge_builders + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - github + - pollscm: + cron: '' + - lf-infra-github-pr-trigger: + trigger-phrase: '^remerge$' + only-trigger-phrase: true + status-context: 'JJB Merge' + permit-all: false + github-hooks: true + github-org: '{github-org}' + github_pr_whitelist: '{obj:github_pr_whitelist}' + github_pr_admin_list: '{obj:github_pr_admin_list}' + +- job-template: + name: '{project-name}-packer-verify' + id: github-packer-verify + concurrent: true + <<: *lf_jjb_verify_merge + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_verify_builders + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - lf-infra-github-pr-trigger: + trigger-phrase: '^recheck$' + only-trigger-phrase: false + status-context: 'JJB Verify' + permit-all: true + github-hooks: true + github-org: '' + github_pr_whitelist: + - '' + github_pr_admin_list: + - '' diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index b3e0222d..d0f596b4 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -71,6 +71,44 @@ - description-setter: regexp: '^Build logs: .*' +- builder: + name: lf-infra-packer-build + builders: + - config-file-provider: + files: + - file-id: '{os-cloud-file-id}' + target: '$HOME/.config/openstack/clouds.yaml' + - file-id: 'packer-cloud-env' + variable: 'CLOUDENV' + - inject: + properties-content: | + PACKER_PLATFORM={platform} + PACKER_TEMPLATE={template} + PACKER_VERSION={packer-version} + - shell: !include-raw-escape: + - ../shell/packer-install.sh + - ../shell/packer-build.sh + - shell: !include-raw: + - ../shell/packer-clear-credentials.sh + +- builder: + name: lf-infra-packer-validate + builders: + - config-file-provider: + files: + - file-id: '{os-cloud-file-id}' + target: '$HOME/.config/openstack/clouds.yaml' + - file-id: 'packer-cloud-env' + variable: 'CLOUDENV' + - inject: + properties-content: | + PACKER_VERSION={packer-version} + - shell: !include-raw-escape: + - ../shell/packer-install.sh + - ../shell/packer-validate.sh + - shell: !include-raw: + - ../shell/packer-clear-credentials.sh + - builder: name: lf-infra-sysstat builders: @@ -151,6 +189,46 @@ # PARAMETERS # ############## +- parameter: + name: lf-infra-maven-parameters + parameters: + - string: + name: MAVEN_OPTS + default: '{mvn-opts}' + description: | + Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m + - string: + name: MAVEN_PARAMS + default: '{mvn-params}' + description: | + Maven parameters to pass to the mvn command. + - string: + name: MVN + # Sets an env var for shell scripts to be able to call the dynamically + # installed maven without having to calculate the path themselves. + default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn' + description: 'Maven selector to be used by shell scripts' + - string: + name: STAGING_PROFILE_ID + default: '{staging-profile-id}' + description: | + Nexus staging profile ID. + + +- parameter: + name: lf-infra-openstack-parameters + parameters: + - string: + name: OS_CLOUD + default: '{os-cloud}' + description: | + The name of a cloud configuration in clouds.yaml. OS_CLOUD is a + variable name that is significant to openstack client as a + environment variable. Please refer to the documentation for + further details. + https://docs.openstack.org/developer/python-openstackclient/ + + - parameter: name: lf-infra-parameters # Standard parameters used in the LF CI environments. Gerrit variables are @@ -197,30 +275,6 @@ Note that Gerrit will override this parameter automatically if a job is triggered by Gerrit. -- parameter: - name: lf-infra-maven-parameters - parameters: - - string: - name: MAVEN_OPTS - default: '{mvn-opts}' - description: | - Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m - - string: - name: MAVEN_PARAMS - default: '{mvn-params}' - description: | - Maven parameters to pass to the mvn command. - - string: - name: MVN - # Sets an env var for shell scripts to be able to call the dynamically - # installed maven without having to calculate the path themselves. - default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn' - description: 'Maven selector to be used by shell scripts' - - string: - name: STAGING_PROFILE_ID - default: '{staging-profile-id}' - description: | - Nexus staging profile ID. - parameter: name: lf-infra-tox-parameters diff --git a/shell/packer-build.sh b/shell/packer-build.sh new file mode 100644 index 00000000..714cd33d --- /dev/null +++ b/shell/packer-build.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-build.sh" +# The script builds an image using packer +# $CLOUDENV : Provides the cloud credential file. +# $PACKER_PLATFORM : Provides the packer platform. +# $PACKER_TEMPLATE : Provides the packer temnplate. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +export PACKER_LOG="yes" && \ +export PACKER_LOG_PATH="$PACKER_BUILD_LOG" && \ + packer.io build -color=false \ + -var-file="$CLOUDENV" \ + -var-file="../packer/vars/$PACKER_PLATFORM.json" \ + "../packer/templates/$PACKER_TEMPLATE.json" + +# Retrive the list of cloud providers +clouds=($(jq -r '.builders[].name' "../packer/templates/$PACKER_TEMPLATE.json")) + +# Split public/private clouds logs +for cloud in "${clouds[@]}"; do + grep -e "$cloud" "$PACKER_BUILD_LOG" > "$PACKER_LOGS_DIR/packer-build_$cloud.log" 2>&1 +done diff --git a/shell/packer-clear-credentials.sh b/shell/packer-clear-credentials.sh new file mode 100644 index 00000000..099f5591 --- /dev/null +++ b/shell/packer-clear-credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-clear-credentials.sh" + +#!/bin/bash +set +e # DO NOT cause build failure if any of the rm calls fail. +rm "$CLOUDENV" +rm "$HOME/.config/openstack/clouds.yaml" +exit 0 diff --git a/shell/packer-install.sh b/shell/packer-install.sh new file mode 100644 index 00000000..eb303f6e --- /dev/null +++ b/shell/packer-install.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-install.sh" +# The script checks for the packer binaries and installs the binary +# if its not available + +# $PACKER_VERSION : Define a packer version passed as job paramter + +PACKER_VERSION="${PACKER_VERSION:-1.0.2}" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail +# Default packer binary made available on the build image +packer_bin="/usr/local/bin/packer.io" + +if hash "$packer_bin" 2>/dev/null; then + echo "packer.io command is available." +else + echo "packer.io command not is available. Installing packer ..." + # Installs Hashicorp's Packer binary, required for verify & merge packer jobs + pushd packer + wget "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" + mkdir -p "${WORKSPACE}/bin" + unzip "packer_${PACKER_VERSION}_linux_amd64.zip" -d ${WORKSPACE}/bin/ + # rename packer to avoid conflict with binary in cracklib + mv ${WORKSPACE}/bin/packer "${WORKSPACE}/bin/packer.io" + popd +fi diff --git a/shell/packer-validate.sh b/shell/packer-validate.sh new file mode 100644 index 00000000..e19c973a --- /dev/null +++ b/shell/packer-validate.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packer-validate.sh" +# The script validates an packers files. + +# $CLOUDENV : Provides the cloud credential file. + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +PACKER_LOGS_DIR="$WORKSPACE/archives/packer" +mkdir -p "$PACKER_LOGS_DIR" +export PATH="${WORKSPACE}/bin:$PATH" + +cd packer +varfiles=(../packer/vars/*) +templates=(../packer/templates/*) + +for varfile in "${varfiles[@]}"; do + [[ "${varfile##*/}" =~ ^(cloud-env.*)$ ]] && continue + for template in "${templates[@]}"; do + export PACKER_LOG="yes" && \ + export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" && \ + packer.io validate -var-file="$CLOUDENV" \ + -var-file="$varfile" "$template" + if [ $? -ne 0 ]; then + break + fi + done +done diff --git a/test.template b/test.template index 78a7c89f..c21533ff 100644 --- a/test.template +++ b/test.template @@ -43,6 +43,21 @@ staging-profile-id: uuddlrlrba settings-file: gerrit-maven-project-settings +- project: + name: gerrit-packer-jobs + jobs: + - "{project-name}-packer-jobs" + + project-name: gerrit-ciman + os-cloud-file-id: os-cloud-id + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet + - project: name: gerrit-python-jobs jobs: @@ -70,10 +85,24 @@ staging-profile-id: uuddlrlrba settings-file: aproject-settings +- project: + name: github-packer-jobs + jobs: + - "{project-name}-github-packer-jobs" + + project-name: github-ciman + os-cloud-file-id: os-cloud-id + platforms: + - centos + - ubuntu1604 + + templates: + - java-builder + - mininet + - project: name: github-python-jobs jobs: - "{project-name}-github-python-jobs" project-name: github-python - -- 2.16.6