Run tox envs separately and capture logs

Run the tox envlist as separate tox runs so that we can capture the logs
in separate files for easier troubleshooting.

This patch requires the vm to have the 'crudini' package installed.

Change-Id: I05e01668af6b84c38c10b24353a2157bfa8d5b23
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/.jjb-test/expected-xml/gerrit-python-tox-verify-master b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
index 06ad1f1..9393044 100644 (file)
--- a/.jjb-test/expected-xml/gerrit-python-tox-verify-master
+++ b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
@@ -317,14 +317,42 @@ echo "---> tox-run.sh"
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox"
+mkdir -p "$ARCHIVE_TOX_DIR"
+
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -n "$TOX_ENVS" ];
-then
-    tox -e "$TOX_ENVS"
+if [ -z "$TOX_ENVS" ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir="$1"
+    local env="$2"
+
+    echo "-----> Running tox $env"
+    if ! tox -e $env > "$log_dir/tox-$env.log"; then
+        echo "$env" >> "$log_dir/failed-envs.log"
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2>/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in "${TOX_ENVS[@]}"; do
+        run_tox "$ARCHIVE_TOX_DIR" "$env"
+    done
+fi
+
+if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
+    failed_envs=($(cat "$ARCHIVE_TOX_DIR/failed-envs.log"))
+    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
+    exit 1
 fi
+
+echo "Completed tox runs."
 </command>
     </hudson.tasks.Shell>
   </builders>

diff --git a/.jjb-test/expected-xml/github-python-tox-verify-master b/.jjb-test/expected-xml/github-python-tox-verify-master
index 0dbf88d..c266342 100644 (file)
--- a/.jjb-test/expected-xml/github-python-tox-verify-master
+++ b/.jjb-test/expected-xml/github-python-tox-verify-master
@@ -279,14 +279,42 @@ echo "---> tox-run.sh"
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox"
+mkdir -p "$ARCHIVE_TOX_DIR"
+
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -n "$TOX_ENVS" ];
-then
-    tox -e "$TOX_ENVS"
+if [ -z "$TOX_ENVS" ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir="$1"
+    local env="$2"
+
+    echo "-----> Running tox $env"
+    if ! tox -e $env > "$log_dir/tox-$env.log"; then
+        echo "$env" >> "$log_dir/failed-envs.log"
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2>/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in "${TOX_ENVS[@]}"; do
+        run_tox "$ARCHIVE_TOX_DIR" "$env"
+    done
+fi
+
+if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
+    failed_envs=($(cat "$ARCHIVE_TOX_DIR/failed-envs.log"))
+    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
+    exit 1
 fi
+
+echo "Completed tox runs."
 </command>
     </hudson.tasks.Shell>
   </builders>

diff --git a/shell/tox-run.sh b/shell/tox-run.sh
index ef2481d..e24e974 100644 (file)
--- a/shell/tox-run.sh
+++ b/shell/tox-run.sh
@@ -14,11 +14,39 @@ echo "---> tox-run.sh"
 # DO NOT set -u as virtualenv's activate script has unbound variables
 set -e -o pipefail
 
+ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox"
+mkdir -p "$ARCHIVE_TOX_DIR"
+
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -n "$TOX_ENVS" ];
-then
-    tox -e "$TOX_ENVS"
+if [ -z "$TOX_ENVS" ]; then
+    TOX_ENVS=$(crudini --get tox.ini tox envlist)
+fi
+
+run_tox() {
+    local log_dir="$1"
+    local env="$2"
+
+    echo "-----> Running tox $env"
+    if ! tox -e $env > "$log_dir/tox-$env.log"; then
+        echo "$env" >> "$log_dir/failed-envs.log"
+    fi
+}
+
+TOX_ENVS=(${TOX_ENVS//,/ })
+if hash parallel 2>/dev/null; then
+    export -f run_tox
+    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
 else
-    tox
+    for env in "${TOX_ENVS[@]}"; do
+        run_tox "$ARCHIVE_TOX_DIR" "$env"
+    done
+fi
+
+if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
+    failed_envs=($(cat "$ARCHIVE_TOX_DIR/failed-envs.log"))
+    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
+    exit 1
 fi
+
+echo "Completed tox runs."