From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Fri, 25 Aug 2017 17:54:33 +0000 (-0400)
Subject: Run tox envs separately and capture logs
X-Git-Tag: v0.9.0~4^2
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=74befe2e7c32b79d51891deb3d5e06a3ac446b42;p=releng%2Fglobal-jjb.git

Run tox envs separately and capture logs

Run the tox envlist as separate tox runs so that we can capture the logs
in separate files for easier troubleshooting.

This patch requires the vm to have the 'crudini' package installed.

Change-Id: I05e01668af6b84c38c10b24353a2157bfa8d5b23
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---

diff --git a/.jjb-test/expected-xml/gerrit-python-tox-verify-master b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
index 06ad1f18..93930444 100644
--- a/.jjb-test/expected-xml/gerrit-python-tox-verify-master
+++ b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
@@ -317,14 +317,42 @@ echo &quot;---&gt; tox-run.sh&quot;
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR=&quot;$WORKSPACE/archives/tox&quot;
+mkdir -p &quot;$ARCHIVE_TOX_DIR&quot;
+
 cd &quot;$WORKSPACE/$TOX_DIR&quot;
 
-if [ -n &quot;$TOX_ENVS&quot; ];
-then
-    tox -e &quot;$TOX_ENVS&quot;
+if [ -z &quot;$TOX_ENVS&quot; ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir=&quot;$1&quot;
+    local env=&quot;$2&quot;
+
+    echo &quot;-----&gt; Running tox $env&quot;
+    if ! tox -e $env &gt; &quot;$log_dir/tox-$env.log&quot;; then
+        echo &quot;$env&quot; &gt;&gt; &quot;$log_dir/failed-envs.log&quot;
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2&gt;/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% &quot;run_tox $ARCHIVE_TOX_DIR {}&quot; ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in &quot;${TOX_ENVS[@]}&quot;; do
+        run_tox &quot;$ARCHIVE_TOX_DIR&quot; &quot;$env&quot;
+    done
+fi
+
+if [ -f &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot; ]; then
+    failed_envs=($(cat &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot;))
+    echo &quot;ERROR: Failed the following builds: ${failed_envs[*]}&quot;
+    exit 1
 fi
+
+echo &quot;Completed tox runs.&quot;
 </command>
     </hudson.tasks.Shell>
   </builders>
diff --git a/.jjb-test/expected-xml/github-python-tox-verify-master b/.jjb-test/expected-xml/github-python-tox-verify-master
index 0dbf88d0..c2663421 100644
--- a/.jjb-test/expected-xml/github-python-tox-verify-master
+++ b/.jjb-test/expected-xml/github-python-tox-verify-master
@@ -279,14 +279,42 @@ echo &quot;---&gt; tox-run.sh&quot;
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR=&quot;$WORKSPACE/archives/tox&quot;
+mkdir -p &quot;$ARCHIVE_TOX_DIR&quot;
+
 cd &quot;$WORKSPACE/$TOX_DIR&quot;
 
-if [ -n &quot;$TOX_ENVS&quot; ];
-then
-    tox -e &quot;$TOX_ENVS&quot;
+if [ -z &quot;$TOX_ENVS&quot; ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir=&quot;$1&quot;
+    local env=&quot;$2&quot;
+
+    echo &quot;-----&gt; Running tox $env&quot;
+    if ! tox -e $env &gt; &quot;$log_dir/tox-$env.log&quot;; then
+        echo &quot;$env&quot; &gt;&gt; &quot;$log_dir/failed-envs.log&quot;
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2&gt;/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% &quot;run_tox $ARCHIVE_TOX_DIR {}&quot; ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in &quot;${TOX_ENVS[@]}&quot;; do
+        run_tox &quot;$ARCHIVE_TOX_DIR&quot; &quot;$env&quot;
+    done
+fi
+
+if [ -f &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot; ]; then
+    failed_envs=($(cat &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot;))
+    echo &quot;ERROR: Failed the following builds: ${failed_envs[*]}&quot;
+    exit 1
 fi
+
+echo &quot;Completed tox runs.&quot;
 </command>
     </hudson.tasks.Shell>
   </builders>
diff --git a/shell/tox-run.sh b/shell/tox-run.sh
index ef2481d0..e24e9740 100644
--- a/shell/tox-run.sh
+++ b/shell/tox-run.sh
@@ -14,11 +14,39 @@ echo "---> tox-run.sh"
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox"
+mkdir -p "$ARCHIVE_TOX_DIR"
+
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -n "$TOX_ENVS" ];
-then
-    tox -e "$TOX_ENVS"
+if [ -z "$TOX_ENVS" ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir="$1"
+    local env="$2"
+
+    echo "-----> Running tox $env"
+    if ! tox -e $env > "$log_dir/tox-$env.log"; then
+        echo "$env" >> "$log_dir/failed-envs.log"
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2>/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in "${TOX_ENVS[@]}"; do
+        run_tox "$ARCHIVE_TOX_DIR" "$env"
+    done
+fi
+
+if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
+    failed_envs=($(cat "$ARCHIVE_TOX_DIR/failed-envs.log"))
+    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
+    exit 1
 fi
+
+echo "Completed tox runs."