Feat: Rewrite packer jobs to work with hcl2 format

Rewrite packer jobs to work with HCL2 format.

As of packer version 1.7.0 HCL2 is the preferred way to write Packer
templates. HCL2 preserves existing workflows while leveraging HCL2’s
advanced features like variable interpolation and configuration
composability.

Upgrade packer version to v1.9.1. JSON format templates are
deprecated and no longer work with packer version > 1.9.x.
Project specific templates require to be upgraded to HCL2 format.

Support for '.json' templates will be removed from common-packer in
subsequent releases. Therefore, the jobs are expected to work with
older templates.

Ref: https://gerrit.linuxfoundation.org/infra/c/releng/common-packer/+/71859
Issue: RELENG-4764
Change-Id: Ie591343ac87caca217ff7125a84f4b769bb3a40c
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>

diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index 1dc14b2..b61009a 100644 (file)
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -30,7 +30,7 @@
     submodule-disable: false
     packer-builder: openstack
     packer-cloud-settings: packer-cloud-env
-    packer-version: 1.8.2
+    packer-version: 1.9.1
 
     #####################
     # Job Configuration #

diff --git a/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml b/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml
new file mode 100644 (file)
index 0000000..58f20b1
--- /dev/null
+++ b/releasenotes/notes/Update-packer-jobs-to-hcl2-c9c8292b27cc785f.yaml
@@ -0,0 +1,19 @@
+---
+prelude: >
+  As of packer version 1.7.0 HCL2 is the preferred way to write Packer
+  templates. HCL2 preserves existing workflows while leveraging HCL2’s
+  advanced features like variable interpolation and configuration
+  composability.
+upgrade:
+  - |
+    Upgrade packer version to v1.9.1. JSON format templates are deprecated
+    and no longer work with packer version > 1.9.x. Project specific templates
+    require to be upgraded to HCL2 format.
+deprecations:
+  - |
+    Support for '.json' templates will be removed from common-packer in
+    subsequent releases. Therefore, the jobs are expected to work with
+    older templates.
+fixes:
+  - |
+    Rewrite packer jobs to work with HCL2 format.

diff --git a/shell/packer-build.sh b/shell/packer-build.sh
index b85c31a..7dd2434 100644 (file)
--- a/shell/packer-build.sh
+++ b/shell/packer-build.sh
@@ -11,31 +11,57 @@
 echo "---> packer-build.sh"
 # The script builds an image using packer
 # $CLOUDENV            :  Provides the cloud credential file.
+# $PACKER_BUILDER      :  Provides the packer cloud type.
 # $PACKER_PLATFORM     :  Provides the packer platform.
-# $PACKER_TEMPLATE     :  Provides the packer temnplate.
+# $PACKER_TEMPLATE     :  Provides the packer template.
 
 # Ensure we fail the job if any steps fail.
 set -eu -o pipefail
 
+# Functions to compare semantic versions x.y.z
+version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
+
 PACKER_LOGS_DIR="$WORKSPACE/archives/packer"
 PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log"
 mkdir -p "$PACKER_LOGS_DIR"
 export PATH="${WORKSPACE}/bin:$PATH"
+template_file="${template_file:-}"
 
 cd packer
 
+# Pick the correct format (hcl or json) based on packer version
 # Prioritize the project's own version of vars if available
-platform_file="common-packer/vars/$PACKER_PLATFORM.json"
-if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then
-    platform_file="vars/$PACKER_PLATFORM.json"
+if version_ge "$PACKER_VERSION" "1.9.0"; then
+    platform_file="common-packer/vars/$PACKER_PLATFORM.pkrvars.hcl"
+    template_file="templates/$PACKER_TEMPLATE.pkr.hcl"
+    only="${PACKER_BUILDER}.${PACKER_TEMPLATE}"
+
+    if [[ -f "vars/$PACKER_PLATFORM.pkrvars.hcl" ]]; then
+        platform_file="vars/$PACKER_PLATFORM.pkrvars.hcl"
+    fi
+else
+    platform_file="common-packer/vars/$PACKER_PLATFORM.json"
+    template_file="templates/$PACKER_TEMPLATE.json"
+    only="${PACKER_BUILDER}"
+
+    if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then
+        platform_file="vars/$PACKER_PLATFORM.json"
+    fi
 fi
 
 export PACKER_LOG="yes"
 export PACKER_LOG_PATH="$PACKER_BUILD_LOG"
+
+# download plugins only for HCL format
+if [[ "${template_file#*.}" == "pkr.hcl" ]]; then
+    echo "packer init ${template_file} ..."
+    packer.io init "${template_file}"
+fi
+
 packer.io validate \
     -var-file="$CLOUDENV" \
     -var-file="$platform_file" \
-    "templates/$PACKER_TEMPLATE.json"
+    "$template_file"
 
 set -x
 # If this is a Gerrit system, check patch comments for successful verify build.
@@ -64,10 +90,10 @@ fi
 set +x
 
 packer.io build -color=false \
-    -only "${PACKER_BUILDER}" \
+    -only "$only" \
     -var-file="$CLOUDENV" \
     -var-file="$platform_file" \
-    "templates/$PACKER_TEMPLATE.json"
+    "$template_file"
 
 # Extract image name from log and store value in the downstream job
 if [[ ${UPDATE_CLOUD_IMAGE} == 'true' ]]; then

diff --git a/shell/packer-validate.sh b/shell/packer-validate.sh
index 61e1431..86cf8de 100644 (file)
--- a/shell/packer-validate.sh
+++ b/shell/packer-validate.sh
@@ -16,24 +16,47 @@ echo "---> packer-validate.sh"
 # Ensure we fail the job if any steps fail.
 set -eu -o pipefail
 
+# Functions to compare semantic versions x.y.z
+version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
+
 PACKER_LOGS_DIR="$WORKSPACE/archives/packer"
 mkdir -p "$PACKER_LOGS_DIR"
 export PATH="${WORKSPACE}/bin:$PATH"
-
+set -x
 cd packer
-varfiles=(vars/*.json common-packer/vars/*.json)
-templates=(templates/*.json)
+
+if version_ge "$PACKER_VERSION" "1.9.0"; then
+    varfiles=(vars/*.pkrvars.hcl common-packer/vars/*.pkrvars.hcl)
+    templates=(templates/*.pkr.hcl)
+else
+    varfiles=(vars/*.json common-packer/vars/*.json)
+    templates=(templates/*.json)
+fi
 
 for varfile in "${varfiles[@]}"; do
-    # cloud-env.json is a file containing credentials which is pulled in via
-    # CLOUDENV variable so skip it here. Also handle the case where a project
-    # has not vars/*.json file.
-    if [[ "$varfile" == *"cloud-env.json"* ]] || [[ "$varfile" == 'vars/*.json' ]]; then
+    # cloud-env.{json,pkrvars.hcl} is a file containing credentials which is
+    # pulled in via CLOUDENV variable so skip it here. Also handle case
+    # where a project does not vars/*.{json,pkrvars.hcl} file.
+    if [[ "$varfile" == *"cloud-env.json"* ]] || \
+       [[ "$varfile" == "vars/*.json" ]] || \
+       [[ "$varfile" == *"cloud-env.pkrvars.hcl"* ]] || \
+       [[ "$varfile" == *"cloud-env-aws.pkrvars.hcl"* ]] || \
+       [[ "$varfile" == "vars/*.pkrvars.hcl" ]]; then
         continue
     fi
 
     echo "-----> Testing varfile: $varfile"
     for template in "${templates[@]}"; do
+        if [[ "$template" == *"variables.pkr.hcl"* ]] || \
+           [[ "$template" == *"variables.auto.pkr.hcl"* ]]; then
+            continue
+        fi
+
+        if [[ "${template#*.}" == "pkr.hcl" ]]; then
+            echo "packer init $template ..."
+            packer.io init "$template"
+        fi
+
         export PACKER_LOG="yes"
         export PACKER_LOG_PATH="$PACKER_LOGS_DIR/packer-validate-${varfile##*/}-${template##*/}.log"
         if output=$(packer.io validate -var-file="$CLOUDENV" -var-file="$varfile" "$template"); then