Merge "Verify build nodes in YAML against config files"

diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index e4ef066..9ecd38f 100644 (file)
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -822,6 +822,8 @@
     ######################
 
     build-concurrent: true
+    check-build-node-labels: false
+    external-build-node-labels: ""
 
     gerrit_verify_triggers:
       - patchset-created-event:
@@ -843,6 +845,15 @@
       - lf-infra-jjbini
       - shell: !include-raw-escape:
           - ../shell/jjb-verify-job.sh
+      - conditional-step:
+          condition-kind: boolean-expression
+          condition-expression: "{check-build-node-labels}"
+          on-evaluation-failure: dont-run
+          steps:
+            - inject:
+                properties-content: EXTERNAL_LABELS="{external-build-node-labels}"
+            - shell: !include-raw-escape:
+                - ../shell/jjb-verify-build-nodes.sh
       - lf-infra-gpg-verify-git-signature
 
 - job-template:

diff --git a/releasenotes/notes/jjb-verify-build-nodes-ccb43bea721775983.yaml b/releasenotes/notes/jjb-verify-build-nodes-ccb43bea721775983.yaml
new file mode 100644 (file)
index 0000000..d89d87e
--- /dev/null
+++ b/releasenotes/notes/jjb-verify-build-nodes-ccb43bea721775983.yaml
@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Verify build nodes named in YAML files against config files.
+    Extend lf-jjb-verify anchor with boolean configuration variable
+    check-build-node-labels that guards a conditional build step.
+    If true, run script jjb-verify-build-nodes.sh to check build-node
+    labels in YAML files within the jjb subdirectory against labels
+    defined by config files in the jenkins cloud configuration
+    directory.  Disabled by default.  Projects should enable and
+    configure the job; e.g., for external build node labels.

diff --git a/shell/jjb-verify-build-nodes.sh b/shell/jjb-verify-build-nodes.sh
new file mode 100644 (file)
index 0000000..9483118
--- /dev/null
+++ b/shell/jjb-verify-build-nodes.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2020 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo "---> jjb-verify-build-nodes.sh"
+
+# Checks build-node labels used in ci-management templates and projects,
+# and prints file names with invalid values.
+#
+# Uses -l to get $HOME/.local/bin on path, where pip puts yq
+# Prereqs:
+# - Bash version 3+
+# - Python tool yq is installed; e.g., by python-tools-install.sh
+# - Working directory is a ci-management repo with subdirs
+#   jenkins-config/clouds/openstack and jjb
+# Environment variable:
+# - EXTERNAL_LABELS - a space-separated list of build-node labels
+#   for nodes not managed in the jenkins-config area (optional)
+
+set -eu -o pipefail
+
+# function to search an array for a value
+# $1 is value
+# $2 is array, passed via ${array[@]}
+isValueInArray () {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+# discover build node labels
+declare -a labels=()
+suffix=".cfg"
+while IFS= read -r ; do
+    file="$REPLY"
+    # valid files contain IMAGE_NAME; skip the cloud config file
+    if grep -q "IMAGE_NAME" "$file" && ! grep -q "CLOUD_CREDENTIAL_ID" "$file"; then
+        # file name is a valid label, without path prefix and suffix
+        name=$(basename -s "$suffix" "$file")
+        labels+=("$name")
+        # a file can define custom labels
+        if custom=$(grep "LABELS=" "$file" | cut -d= -f2); then
+            # TODO: confirm separator for multiple labels
+            read -r -a customarray <<< "$custom"
+            for c in "${customarray[@]}"; do
+                if ! isValueInArray "$c" "${labels[@]}"; then
+                    labels+=("$c")
+                fi
+            done
+        fi
+    fi
+done < <(find "jenkins-config/clouds/openstack" -name \*$suffix)
+echo "Found ${#labels[@]} configured label(s):"
+echo "${labels[@]}"
+declare -a externals=()
+if [[ -n ${EXTERNAL_LABELS:-} ]]; then
+    read -r -a externals <<< "$EXTERNAL_LABELS"
+    echo "Received ${#externals[@]} external label(s):"
+    echo "${externals[@]}"
+    labels=("${externals[@]}" "${labels[@]}")
+fi
+
+# check build node label uses
+errs=0
+while IFS= read -r ; do
+    file="$REPLY"
+    echo "Checking $file"
+    # this includes job-templates which may be annoying
+    nodes=$(yq 'recurse | ."build-node"? | values' "$file" | sort -u | tr -d '"')
+    for node in $nodes; do
+        # may be a yaml list; e.g., '[ foo, bar, baz ]'
+        node="${node//[\[\],]/}"
+        if [[ -n $node ]] && ! isValueInArray "$node" "${labels[@]}"; then
+            echo "ERROR: file $file uses unknown build-node $node"
+            errs=$((errs+1))
+        fi
+    done
+done < <(find "jjb" -name '*.yaml')
+
+echo "---> jjb-verify-build-nodes.sh ends"
+exit $errs