Fix code for generating Build Config Tarball

Also cleanup log messages (reduce and refine)

Issue: RELENG-2542
Change-Id: I9d7a9fa9e083afb54069a3f5166c8f05ce20fe16
Signed-off-by: Tim Johnson <tijohnson@linuxfoundation.org>

diff --git a/releasenotes/notes/fix-config-tarball-a4bee31a90375126.yaml b/releasenotes/notes/fix-config-tarball-a4bee31a90375126.yaml
new file mode 100644 (file)
index 0000000..6992fc3
--- /dev/null
+++ b/releasenotes/notes/fix-config-tarball-a4bee31a90375126.yaml
@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    Currently a Job Config Tarball is added to the archive.  The directory
+    layout created by 'jenkins-jobs test' has changed and as a result the code
+    that sorted/reformated the directory and generated the tarball was broken.
+
+    The prefix path of the tarball is now 'job-configs'. The number of log
+    messages has been improved and greatly reduced. The tarball generated
+    contains directory names with spaces and upper-case characters.

diff --git a/shell/jjb-verify-job.sh b/shell/jjb-verify-job.sh
index d7a9e31..5ed12e2 100644 (file)
--- a/shell/jjb-verify-job.sh
+++ b/shell/jjb-verify-job.sh
@@ -10,8 +10,7 @@
 ##############################################################################
 echo "---> jjb-verify-job.sh"
 
-# Ensure we fail the job if any steps fail.
-set -eu -o pipefail
+set -eufo pipefail
 
 # shellcheck disable=SC1090
 source ~/lf-env.sh
@@ -21,23 +20,25 @@ lf-jjb-check-ascii
 
 lf-activate-venv jenkins-job-builder
 
-jenkins-jobs -l DEBUG test --recursive -o archives/job-configs --config-xml jjb/
+jenkins-jobs test --recursive -o archives/job-configs --config-xml jjb/
 
-# Sort job output into sub-directories. On large Jenkins systems that have
-# many jobs archiving so many files into the same directory makes NGINX return
-# the directory list slow.
-pushd archives/job-configs
-for letter in {a..z}
-do
-    if [[ $(ls "$letter"* > /dev/null 2>&1) -eq 0 ]]
-    then
-        mkdir "$letter"
-        find . -maxdepth 1 -type f -name "$letter*" -exec mv {} "$letter" \;
-    fi
-done
-popd
+# Enable 'globbing'
+set +f
+# NGINX is very sluggish with directories containing large numbers of objects
+# Add another directory level named {a..z} and move directories to them.
+# Directories beginning with {0..9} or {A..Z} are left at the top level.
+(   cd archives/job-configs
+    for letter in {a..z}; do
+        if ls -d $letter* > /dev/null 2>&1; then
+            mkdir .tmp
+            mv $letter* .tmp
+            mv .tmp $letter
+        fi
+    done
+)
 
-if [ -n "$(ls -A archives/job-configs)" ]; then
-    tar cJvf archives/job-configs.tar.xz archives/job-configs
-    rm -rf archives/job-configs
-fi
+(   cd archives
+    echo "INFO: Archiving $(find job-configs -name \*.xml | wc -l) job configurations"
+    tar -cJf job-configs.tar.xz job-configs
+    rm -rf job-configs
+)