Merge "Fix rtd-verify.sh to better validate submodules"

diff --git a/releasenotes/notes/cmake-sonar-aaff1701b8ccdfa0.yaml b/releasenotes/notes/cmake-sonar-aaff1701b8ccdfa0.yaml
new file mode 100644 (file)
index 0000000..ab06027
--- /dev/null
+++ b/releasenotes/notes/cmake-sonar-aaff1701b8ccdfa0.yaml
@@ -0,0 +1,5 @@
+---
+deprecations:
+  - |
+    Cmake builds upload to SonarCloud. Fix build env to use openjdk11
+    sonarcloud upload will stop working with java8 on October 1st 2020

diff --git a/releasenotes/notes/fix-aws-metadata-d901348efb32b095.yaml b/releasenotes/notes/fix-aws-metadata-d901348efb32b095.yaml
new file mode 100644 (file)
index 0000000..cdf0c9b
--- /dev/null
+++ b/releasenotes/notes/fix-aws-metadata-d901348efb32b095.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fix the capture-instance-metadata script to cleanly exit on AWS nodes.

diff --git a/releasenotes/notes/fix_jenkins_image_verify-b2588bd58ddc2f83.yaml b/releasenotes/notes/fix_jenkins_image_verify-b2588bd58ddc2f83.yaml
new file mode 100644 (file)
index 0000000..7317e63
--- /dev/null
+++ b/releasenotes/notes/fix_jenkins_image_verify-b2588bd58ddc2f83.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    When evaluating jenkins-config management changes, if a system does not have
+    an openstack cloud defined, we should not fail the job. Instead we now
+    just skip that configuration and verification

diff --git a/releasenotes/notes/openstack-protect-in-use-images-cf8e4f722a1512ec.yaml b/releasenotes/notes/openstack-protect-in-use-images-cf8e4f722a1512ec.yaml
new file mode 100644 (file)
index 0000000..72f16e6
--- /dev/null
+++ b/releasenotes/notes/openstack-protect-in-use-images-cf8e4f722a1512ec.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    openstack-protect-in-use-images.sh
+    Projects that do not have images in jjb were failing this step of the build.
+    code now runs with set -eu -o pipefail for the duration of the script.
+    shellcheck disable comments no longer needed and removed.
+    Code now only merges arrays if non-empty.
+    Simplify dedupe array code.

diff --git a/shell/capture-instance-metadata.sh b/shell/capture-instance-metadata.sh
index 23a9f2b..a5914ad 100644 (file)
--- a/shell/capture-instance-metadata.sh
+++ b/shell/capture-instance-metadata.sh
@@ -17,10 +17,11 @@ if [ -n "${NOMAD_DC}" ]; then
     exit 0
 fi
 
-# detect if we're in AWS
-if grep -qi amazon /sys/devices/virtual/dmi/id/bios_vendor ; then
-    echo "INFO: Running in AWS, not capturing instance metadata"
-    exit 0
+# AWS not supported, exit
+cloudtype="$(jq -r .v1.datasource /run/cloud-init/result.json)"
+if [[ $cloudtype == "DataSourceEc2Local" ]]; then
+   echo "INFO: Running in AWS, not capturing instance metadata"
+   exit 0
 fi
 
 # Retrive OpenStack instace metadata APIs at this IP.

diff --git a/shell/cmake-sonar.sh b/shell/cmake-sonar.sh
index ccf17d5..e64ca69 100644 (file)
--- a/shell/cmake-sonar.sh
+++ b/shell/cmake-sonar.sh
@@ -20,6 +20,12 @@ make_opts="${MAKE_OPTS:-}"
 
 set -ex -o pipefail
 
+export SET_JDK_VERSION="${SET_JDK_VERSION:-openjdk11}"
+echo "$SET_JDK_VERSION"
+bash <(curl -s https://raw.githubusercontent.com/lfit/releng-global-jjb/master/shell/update-java-alternatives.sh)
+# shellcheck disable=SC1091
+source /tmp/java.env
+
 cd /tmp || exit 1
 wget -q -O /tmp/sonar-scan.zip \
     "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip"

diff --git a/shell/cmake-sonarqube.sh b/shell/cmake-sonarqube.sh
index 0dd2cf5..f34f7d7 100644 (file)
--- a/shell/cmake-sonarqube.sh
+++ b/shell/cmake-sonarqube.sh
@@ -25,6 +25,13 @@ wget -q -O bw.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip
 unzip -q bw.zip
 sudo mv build-wrapper-* /opt/build-wrapper
 
+
+export SET_JDK_VERSION="${SET_JDK_VERSION:-openjdk11}"
+echo "$SET_JDK_VERSION"
+bash <(curl -s https://raw.githubusercontent.com/lfit/releng-global-jjb/master/shell/update-java-alternatives.sh)
+# shellcheck disable=SC1091
+source /tmp/java.env
+
 mkdir -p "$build_dir"
 cd "$build_dir" || exit 1
 

diff --git a/shell/jenkins-verify-images.sh b/shell/jenkins-verify-images.sh
index d005c52..663dfb6 100755 (executable)
--- a/shell/jenkins-verify-images.sh
+++ b/shell/jenkins-verify-images.sh
@@ -40,18 +40,20 @@ verify_images()
 }
 
 echo "Verifying that cloud has a master configuration file"
-for cloud in jenkins-config/clouds/openstack/*; do
-  if [[ -f $cloud/cloud.cfg ]]; then
-    # Get the OS_CLOUD variable from cloud config
-    if ! os_cloud=$(grep -E "^OS_CLOUD=" "$cloud/cloud.cfg" | cut -d'=' -f2); then
-      os_cloud="vex"
+if [[ -d jenkins-config/clouds/openstack ]]; then
+  for cloud in jenkins-config/clouds/openstack/*; do
+    if [[ -f $cloud/cloud.cfg ]]; then
+      # Get the OS_CLOUD variable from cloud config
+      if ! os_cloud=$(grep -E "^OS_CLOUD=" "$cloud/cloud.cfg" | cut -d'=' -f2); then
+        os_cloud="vex"
+      fi
+      OS_CLOUD=$os_cloud verify_images "$cloud"
+    else
+      echo "ERROR: No cloud.cfg for $cloud"
+      error=true
     fi
-    OS_CLOUD=$os_cloud verify_images "$cloud"
-  else
-    echo "ERROR: No cloud.cfg for $cloud"
-    error=true
-  fi
-done
+  done
+fi
 
 if $error; then
   exit 1

diff --git a/shell/openstack-protect-in-use-images.sh b/shell/openstack-protect-in-use-images.sh
index a422cd8..e229906 100644 (file)
--- a/shell/openstack-protect-in-use-images.sh
+++ b/shell/openstack-protect-in-use-images.sh
@@ -15,22 +15,44 @@
 # getting purged by the image cleanup script.
 # This script assumes images prefixed with the string "ZZCI - " are ci-managed
 # images.
+set -eu -o pipefail
 echo "---> Protect in-use images"
-
 os_cloud="${OS_CLOUD:-vex}"
 
-set -eu -o pipefail
+images=()
+while read -r -d $'\n' ; do
+    images+=("$REPLY")
+done < <(grep -r IMAGE_NAME --include \*.cfg jenkins-config \
+         | awk -F'=' '{print $2}' \
+         | sort -u)
+
+jjbimages=()
+while read -r -d $'\n' ; do
+    jjbimages+=("$REPLY")
+done < <(grep -r 'ZZCI - ' --include \*.yaml jjb \
+         | awk -F": " '{print $3}' \
+         | sed -e "s:'::;s:'$::;/^$/d" -e 's/^"//' -e 's/"$//' \
+         | sort -u)
+
+if ! [[ ${#images[@]} -eq 0 ]]; then
+    echo "INFO: There are images to protect defined in jenkins-config."
+else
+    echo "ERROR: No images detected in the jenkins-config dir."
+    exit 1
+fi
 
-declare -a images
-images+=("$(grep -r IMAGE_NAME --include \*.cfg jenkins-config \
-    | awk -F'=' '{print $2}' | sort -u)")
-set +o pipefail  # Not all projects have images in YAML files and grep returns non-zero on 0 results
-# Ignore SC2179 since we do not want () to result in an empty array item.
-#shellcheck disable=SC2179
-images+=("$(grep -r 'ZZCI - ' --include \*.yaml jjb \
-    | awk -F": " '{print $3}' | sed -e "s:'::;s:'$::;/^$/d" -e 's/^"//' -e 's/"$//' | sort -u)")
-set -o pipefail
-readarray -t images <<< "$(for i in "${images[@]}"; do echo "$i"; done | sort -u)"
+if ! [[ ${#jjbimages[@]} -eq 0 ]]; then
+    echo "INFO: There are additional images to protect in the jjb dir."
+    images=("${images[@]}" "${jjbimages[@]}")
+    #dedupe
+    readarray -t images < <(printf '%s\n' "${images[@]}" | sort -u)
+fi
+
+
+echo "INFO: Protecting the following images:"
+for image in "${images[@]}"; do
+    echo "$image"
+done
 
 for image in "${images[@]}"; do
     os_image_protected=$(openstack --os-cloud "$os_cloud" \