From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Thu, 4 Jan 2018 02:10:02 +0000 (-0500)
Subject: Use detox to run tox in parallel
X-Git-Tag: v0.14.0~2^2
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=f3ed64938de3f8efbc102cd9f16676b63af6cfeb;p=releng%2Fglobal-jjb.git

Use detox to run tox in parallel

Rather than hacking something together with parallel use a tool
supported by the tox project called detox.

Change-Id: I04fc3a083c420f71938b99f942b682093822ddda
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---

diff --git a/.jjb-test/expected-xml/gerrit-python-tox-verify-master b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
index 5573a887..497228ef 100644
--- a/.jjb-test/expected-xml/gerrit-python-tox-verify-master
+++ b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
@@ -296,7 +296,7 @@ virtualenv --quiet &quot;/tmp/v/tox&quot;
 source &quot;/tmp/v/tox/bin/activate&quot;
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo &quot;----&gt; Pip Dependency Tree&quot;
 pipdeptree
@@ -322,43 +322,17 @@ mkdir -p &quot;$ARCHIVE_TOX_DIR&quot;
 
 cd &quot;$WORKSPACE/$TOX_DIR&quot;
 
-if [ -z &quot;$TOX_ENVS&quot; ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir=&quot;$1&quot;
-    local env=&quot;$2&quot;
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo &quot;-----&gt; Running tox $env&quot;
-    if ! tox -e $env &gt; &quot;$log_dir/tox-$env.log&quot;; then
-        echo &quot;$env&quot; &gt;&gt; &quot;$log_dir/failed-envs.log&quot;
-    fi
-}
-
-IFS=&quot; &quot; read -r -a TOX_ENVS &lt;&lt;&lt; &quot;${TOX_ENVS//,/ }&quot;
-if hash parallel 2&gt;/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% &quot;run_tox $ARCHIVE_TOX_DIR {}&quot; ::: ${TOX_ENVS[*]}
+if [ ! -z &quot;$TOX_ENVS&quot; ]; then
+    detox -e &quot;$TOX_ENVS&quot;  | tee -a &quot;$ARCHIVE_TOX_DIR/detox.log&quot;
 else
-    for env in &quot;${TOX_ENVS[@]}&quot;; do
-        run_tox &quot;$ARCHIVE_TOX_DIR&quot; &quot;$env&quot;
-    done
+    detox | tee -a &quot;$ARCHIVE_TOX_DIR/detox.log&quot;
 fi
 
-if [ -f &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot; ]; then
-    mapfile -t failed_envs &lt; &lt;(cat &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot;)
-    for e in &quot;${failed_envs[@]}&quot;; do
-        echo &quot;cat $ARCHIVE_TOX_DIR/tox-$e.log&quot;
-        cat &quot;$ARCHIVE_TOX_DIR/tox-$e.log&quot;
-    done
-    echo &quot;ERROR: Failed the following builds: ${failed_envs[*]}&quot;
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo &quot;${TOX_ENVS//,/ }&quot;); do
+    cp -r &quot;.tox/$i/log&quot; &quot;$ARCHIVE_TOX_DIR/$i&quot;
+done
 
 echo &quot;Completed tox runs.&quot;
 </command>
diff --git a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
index 3473cdcf..5bc67eac 100644
--- a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
+++ b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
@@ -210,7 +210,7 @@ virtualenv --quiet &quot;/tmp/v/tox&quot;
 source &quot;/tmp/v/tox/bin/activate&quot;
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo &quot;----&gt; Pip Dependency Tree&quot;
 pipdeptree
diff --git a/.jjb-test/expected-xml/github-python-tox-verify-master b/.jjb-test/expected-xml/github-python-tox-verify-master
index 6d758b8c..9cd8ea30 100644
--- a/.jjb-test/expected-xml/github-python-tox-verify-master
+++ b/.jjb-test/expected-xml/github-python-tox-verify-master
@@ -258,7 +258,7 @@ virtualenv --quiet &quot;/tmp/v/tox&quot;
 source &quot;/tmp/v/tox/bin/activate&quot;
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo &quot;----&gt; Pip Dependency Tree&quot;
 pipdeptree
@@ -284,43 +284,17 @@ mkdir -p &quot;$ARCHIVE_TOX_DIR&quot;
 
 cd &quot;$WORKSPACE/$TOX_DIR&quot;
 
-if [ -z &quot;$TOX_ENVS&quot; ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir=&quot;$1&quot;
-    local env=&quot;$2&quot;
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo &quot;-----&gt; Running tox $env&quot;
-    if ! tox -e $env &gt; &quot;$log_dir/tox-$env.log&quot;; then
-        echo &quot;$env&quot; &gt;&gt; &quot;$log_dir/failed-envs.log&quot;
-    fi
-}
-
-IFS=&quot; &quot; read -r -a TOX_ENVS &lt;&lt;&lt; &quot;${TOX_ENVS//,/ }&quot;
-if hash parallel 2&gt;/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% &quot;run_tox $ARCHIVE_TOX_DIR {}&quot; ::: ${TOX_ENVS[*]}
+if [ ! -z &quot;$TOX_ENVS&quot; ]; then
+    detox -e &quot;$TOX_ENVS&quot;  | tee -a &quot;$ARCHIVE_TOX_DIR/detox.log&quot;
 else
-    for env in &quot;${TOX_ENVS[@]}&quot;; do
-        run_tox &quot;$ARCHIVE_TOX_DIR&quot; &quot;$env&quot;
-    done
+    detox | tee -a &quot;$ARCHIVE_TOX_DIR/detox.log&quot;
 fi
 
-if [ -f &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot; ]; then
-    mapfile -t failed_envs &lt; &lt;(cat &quot;$ARCHIVE_TOX_DIR/failed-envs.log&quot;)
-    for e in &quot;${failed_envs[@]}&quot;; do
-        echo &quot;cat $ARCHIVE_TOX_DIR/tox-$e.log&quot;
-        cat &quot;$ARCHIVE_TOX_DIR/tox-$e.log&quot;
-    done
-    echo &quot;ERROR: Failed the following builds: ${failed_envs[*]}&quot;
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo &quot;${TOX_ENVS//,/ }&quot;); do
+    cp -r &quot;.tox/$i/log&quot; &quot;$ARCHIVE_TOX_DIR/$i&quot;
+done
 
 echo &quot;Completed tox runs.&quot;
 </command>
diff --git a/.jjb-test/expected-xml/github-rtd-rtd-verify-master b/.jjb-test/expected-xml/github-rtd-rtd-verify-master
index c9731c17..3bf0043c 100644
--- a/.jjb-test/expected-xml/github-rtd-rtd-verify-master
+++ b/.jjb-test/expected-xml/github-rtd-rtd-verify-master
@@ -162,7 +162,7 @@ virtualenv --quiet &quot;/tmp/v/tox&quot;
 source &quot;/tmp/v/tox/bin/activate&quot;
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo &quot;----&gt; Pip Dependency Tree&quot;
 pipdeptree
diff --git a/shell/tox-install.sh b/shell/tox-install.sh
index 17bebecc..e6f26142 100644
--- a/shell/tox-install.sh
+++ b/shell/tox-install.sh
@@ -19,7 +19,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree
diff --git a/shell/tox-run.sh b/shell/tox-run.sh
index e8172b2b..97f8855b 100644
--- a/shell/tox-run.sh
+++ b/shell/tox-run.sh
@@ -19,42 +19,16 @@ mkdir -p "$ARCHIVE_TOX_DIR"
 
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -z "$TOX_ENVS" ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir="$1"
-    local env="$2"
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo "-----> Running tox $env"
-    if ! tox -e $env > "$log_dir/tox-$env.log"; then
-        echo "$env" >> "$log_dir/failed-envs.log"
-    fi
-}
-
-IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }"
-if hash parallel 2>/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
+if [ ! -z "$TOX_ENVS" ]; then
+    detox -e "$TOX_ENVS"  | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 else
-    for env in "${TOX_ENVS[@]}"; do
-        run_tox "$ARCHIVE_TOX_DIR" "$env"
-    done
+    detox | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 fi
 
-if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
-    mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log")
-    for e in "${failed_envs[@]}"; do
-        echo "cat $ARCHIVE_TOX_DIR/tox-$e.log"
-        cat "$ARCHIVE_TOX_DIR/tox-$e.log"
-    done
-    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo "${TOX_ENVS//,/ }"); do
+    cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i"
+done
 
 echo "Completed tox runs."