From 907c06f064252914e44b92476ea4b12c07d99ebb Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Wed, 28 Jun 2023 12:20:36 +1000 Subject: [PATCH] Feat: Rewrite packer jobs to work with hcl2 format MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Rewrite packer jobs to work with HCL2 format. As of packer version 1.7.0 HCL2 is the preferred way to write Packer templates. HCL2 preserves existing workflows while leveraging HCL2’s advanced features like variable interpolation and configuration composability. Upgrade packer version to v1.9.1. JSON format templates are deprecated and no longer work with packer version > 1.9.x. Project specific templates require to be upgraded to HCL2 format. Support for '.json' templates will be removed from common-packer in subsequent releases. Therefore, the jobs are expected to work with older templates. Ref: https://gerrit.linuxfoundation.org/infra/c/releng/common-packer/+/71859 Issue: RELENG-4764 Change-Id: Ie591343ac87caca217ff7125a84f4b769bb3a40c Signed-off-by: Anil Belur --- jjb/lf-ci-jobs.yaml | 2 +- ...pdate-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml | 19 ++++++++++ shell/packer-build.sh | 40 ++++++++++++++++++---- shell/packer-validate.sh | 37 ++++++++++++++++---- 4 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 1dc14b2d..b61009aa 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -30,7 +30,7 @@ submodule-disable: false packer-builder: openstack packer-cloud-settings: packer-cloud-env - packer-version: 1.8.2 + packer-version: 1.9.1 ##################### # Job Configuration # diff --git a/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml b/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml new file mode 100644 index 00000000..58f20b11 --- /dev/null +++ b/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml @@ -0,0 +1,19 @@ +--- +prelude: > + As of packer version 1.7.0 HCL2 is the preferred way to write Packer + templates. HCL2 preserves existing workflows while leveraging HCL2’s + advanced features like variable interpolation and configuration + composability. +upgrade: + - | + Upgrade packer version to v1.9.1. JSON format templates are deprecated + and no longer work with packer version > 1.9.x. Project specific templates + require to be upgraded to HCL2 format. +deprecations: + - | + Support for '.json' templates will be removed from common-packer in + subsequent releases. Therefore, the jobs are expected to work with + older templates. +fixes: + - | + Rewrite packer jobs to work with HCL2 format. diff --git a/shell/packer-build.sh b/shell/packer-build.sh index b85c31a7..7dd2434b 100644 --- a/shell/packer-build.sh +++ b/shell/packer-build.sh @@ -11,31 +11,57 @@ echo "---> packer-build.sh" # The script builds an image using packer # $CLOUDENV : Provides the cloud credential file. +# $PACKER_BUILDER : Provides the packer cloud type. # $PACKER_PLATFORM : Provides the packer platform. -# $PACKER_TEMPLATE : Provides the packer temnplate. +# $PACKER_TEMPLATE : Provides the packer template. # Ensure we fail the job if any steps fail. set -eu -o pipefail +# Functions to compare semantic versions x.y.z +version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; } + PACKER_LOGS_DIR="$WORKSPACE/archives/packer" PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" mkdir -p "$PACKER_LOGS_DIR" export PATH="${WORKSPACE}/bin:$PATH" +template_file="${template_file:-}" cd packer +# Pick the correct format (hcl or json) based on packer version # Prioritize the project's own version of vars if available -platform_file="common-packer/vars/$PACKER_PLATFORM.json" -if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then - platform_file="vars/$PACKER_PLATFORM.json" +if version_ge "$PACKER_VERSION" "1.9.0"; then + platform_file="common-packer/vars/$PACKER_PLATFORM.pkrvars.hcl" + template_file="templates/$PACKER_TEMPLATE.pkr.hcl" + only="${PACKER_BUILDER}.${PACKER_TEMPLATE}" + + if [[ -f "vars/$PACKER_PLATFORM.pkrvars.hcl" ]]; then + platform_file="vars/$PACKER_PLATFORM.pkrvars.hcl" + fi +else + platform_file="common-packer/vars/$PACKER_PLATFORM.json" + template_file="templates/$PACKER_TEMPLATE.json" + only="${PACKER_BUILDER}" + + if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then + platform_file="vars/$PACKER_PLATFORM.json" + fi fi export PACKER_LOG="yes" export PACKER_LOG_PATH="$PACKER_BUILD_LOG" + +# download plugins only for HCL format +if [[ "${template_file#*.}" == "pkr.hcl" ]]; then + echo "packer init ${template_file} ..." + packer.io init "${template_file}" +fi + packer.io validate \ -var-file="$CLOUDENV" \ -var-file="$platform_file" \ - "templates/$PACKER_TEMPLATE.json" + "$template_file" set -x # If this is a Gerrit system, check patch comments for successful verify build. @@ -64,10 +90,10 @@ fi set +x packer.io build -color=false \ - -only "${PACKER_BUILDER}" \ + -only "$only" \ -var-file="$CLOUDENV" \ -var-file="$platform_file" \ - "templates/$PACKER_TEMPLATE.json" + "$template_file" # Extract image name from log and store value in the downstream job if [[ ${UPDATE_CLOUD_IMAGE} == 'true' ]]; then diff --git a/shell/packer-validate.sh b/shell/packer-validate.sh index 61e14313..86cf8de8 100644 --- a/shell/packer-validate.sh +++ b/shell/packer-validate.sh @@ -16,24 +16,47 @@ echo "---> packer-validate.sh" # Ensure we fail the job if any steps fail. set -eu -o pipefail +# Functions to compare semantic versions x.y.z +version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; } + PACKER_LOGS_DIR="$WORKSPACE/archives/packer" mkdir -p "$PACKER_LOGS_DIR" export PATH="${WORKSPACE}/bin:$PATH" - +set -x cd packer -varfiles=(vars/*.json common-packer/vars/*.json) -templates=(templates/*.json) + +if version_ge "$PACKER_VERSION" "1.9.0"; then + varfiles=(vars/*.pkrvars.hcl common-packer/vars/*.pkrvars.hcl) + templates=(templates/*.pkr.hcl) +else + varfiles=(vars/*.json common-packer/vars/*.json) + templates=(templates/*.json) +fi for varfile in "${varfiles[@]}"; do - # cloud-env.json is a file containing credentials which is pulled in via - # CLOUDENV variable so skip it here. Also handle the case where a project - # has not vars/*.json file. - if [[ "$varfile" == *"cloud-env.json"* ]] || [[ "$varfile" == 'vars/*.json' ]]; then + # cloud-env.{json,pkrvars.hcl} is a file containing credentials which is + # pulled in via CLOUDENV variable so skip it here. Also handle case + # where a project does not vars/*.{json,pkrvars.hcl} file. + if [[ "$varfile" == *"cloud-env.json"* ]] || \ + [[ "$varfile" == "vars/*.json" ]] || \ + [[ "$varfile" == *"cloud-env.pkrvars.hcl"* ]] || \ + [[ "$varfile" == *"cloud-env-aws.pkrvars.hcl"* ]] || \ + [[ "$varfile" == "vars/*.pkrvars.hcl" ]]; then continue fi echo "-----> Testing varfile: $varfile" for template in "${templates[@]}"; do + if [[ "$template" == *"variables.pkr.hcl"* ]] || \ + [[ "$template" == *"variables.auto.pkr.hcl"* ]]; then + continue + fi + + if [[ "${template#*.}" == "pkr.hcl" ]]; then + echo "packer init $template ..." + packer.io init "$template" + fi + export PACKER_LOG="yes" export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log" if output=$(packer.io validate -var-file="$CLOUDENV" -var-file="$varfile" "$template"); then -- 2.16.6