Use detox to run tox in parallel

Rather than hacking something together with parallel use a tool
supported by the tox project called detox.

Change-Id: I04fc3a083c420f71938b99f942b682093822ddda
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/.jjb-test/expected-xml/gerrit-python-tox-verify-master b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
index 5573a88..497228e 100644 (file)
--- a/.jjb-test/expected-xml/gerrit-python-tox-verify-master
+++ b/.jjb-test/expected-xml/gerrit-python-tox-verify-master
@@ -296,7 +296,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree
@@ -322,43 +322,17 @@ mkdir -p "$ARCHIVE_TOX_DIR"
 
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -z "$TOX_ENVS" ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir="$1"
-    local env="$2"
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo "-----> Running tox $env"
-    if ! tox -e $env > "$log_dir/tox-$env.log"; then
-        echo "$env" >> "$log_dir/failed-envs.log"
-    fi
-}
-
-IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }"
-if hash parallel 2>/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
+if [ ! -z "$TOX_ENVS" ]; then
+    detox -e "$TOX_ENVS"  | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 else
-    for env in "${TOX_ENVS[@]}"; do
-        run_tox "$ARCHIVE_TOX_DIR" "$env"
-    done
+    detox | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 fi
 
-if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
-    mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log")
-    for e in "${failed_envs[@]}"; do
-        echo "cat $ARCHIVE_TOX_DIR/tox-$e.log"
-        cat "$ARCHIVE_TOX_DIR/tox-$e.log"
-    done
-    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo "${TOX_ENVS//,/ }"); do
+    cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i"
+done
 
 echo "Completed tox runs."
 </command>

diff --git a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
index 3473cdc..5bc67ea 100644 (file)
--- a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
+++ b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master
@@ -210,7 +210,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree

diff --git a/.jjb-test/expected-xml/github-python-tox-verify-master b/.jjb-test/expected-xml/github-python-tox-verify-master
index 6d758b8..9cd8ea3 100644 (file)
--- a/.jjb-test/expected-xml/github-python-tox-verify-master
+++ b/.jjb-test/expected-xml/github-python-tox-verify-master
@@ -258,7 +258,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree
@@ -284,43 +284,17 @@ mkdir -p "$ARCHIVE_TOX_DIR"
 
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -z "$TOX_ENVS" ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir="$1"
-    local env="$2"
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo "-----> Running tox $env"
-    if ! tox -e $env > "$log_dir/tox-$env.log"; then
-        echo "$env" >> "$log_dir/failed-envs.log"
-    fi
-}
-
-IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }"
-if hash parallel 2>/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
+if [ ! -z "$TOX_ENVS" ]; then
+    detox -e "$TOX_ENVS"  | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 else
-    for env in "${TOX_ENVS[@]}"; do
-        run_tox "$ARCHIVE_TOX_DIR" "$env"
-    done
+    detox | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 fi
 
-if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
-    mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log")
-    for e in "${failed_envs[@]}"; do
-        echo "cat $ARCHIVE_TOX_DIR/tox-$e.log"
-        cat "$ARCHIVE_TOX_DIR/tox-$e.log"
-    done
-    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo "${TOX_ENVS//,/ }"); do
+    cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i"
+done
 
 echo "Completed tox runs."
 </command>

diff --git a/.jjb-test/expected-xml/github-rtd-rtd-verify-master b/.jjb-test/expected-xml/github-rtd-rtd-verify-master
index c9731c1..3bf0043 100644 (file)
--- a/.jjb-test/expected-xml/github-rtd-rtd-verify-master
+++ b/.jjb-test/expected-xml/github-rtd-rtd-verify-master
@@ -162,7 +162,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree

diff --git a/shell/tox-install.sh b/shell/tox-install.sh
index 17bebec..e6f2614 100644 (file)
--- a/shell/tox-install.sh
+++ b/shell/tox-install.sh
@@ -19,7 +19,7 @@ virtualenv --quiet "/tmp/v/tox"
 source "/tmp/v/tox/bin/activate"
 pip install --quiet --upgrade pip
 pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade tox argparse
+pip install --quiet --upgrade detox tox argparse
 
 echo "----> Pip Dependency Tree"
 pipdeptree

diff --git a/shell/tox-run.sh b/shell/tox-run.sh
index e8172b2..97f8855 100644 (file)
--- a/shell/tox-run.sh
+++ b/shell/tox-run.sh
@@ -19,42 +19,16 @@ mkdir -p "$ARCHIVE_TOX_DIR"
 
 cd "$WORKSPACE/$TOX_DIR"
 
-if [ -z "$TOX_ENVS" ]; then
-    TOX_ENVS=$(crudini --get tox.ini tox envlist)
-fi
-
-run_tox() {
-    local log_dir="$1"
-    local env="$2"
-
-    # Sleep a random 10 second interval to workaround tox sdist
-    # conflicts due to building in the same dist directory.
-    sleep $[ ( $RANDOM % 10 )  + 1 ]s
-
-    echo "-----> Running tox $env"
-    if ! tox -e $env > "$log_dir/tox-$env.log"; then
-        echo "$env" >> "$log_dir/failed-envs.log"
-    fi
-}
-
-IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }"
-if hash parallel 2>/dev/null; then
-    export -f run_tox
-    parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]}
+if [ ! -z "$TOX_ENVS" ]; then
+    detox -e "$TOX_ENVS"  | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 else
-    for env in "${TOX_ENVS[@]}"; do
-        run_tox "$ARCHIVE_TOX_DIR" "$env"
-    done
+    detox | tee -a "$ARCHIVE_TOX_DIR/detox.log"
 fi
 
-if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then
-    mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log")
-    for e in "${failed_envs[@]}"; do
-        echo "cat $ARCHIVE_TOX_DIR/tox-$e.log"
-        cat "$ARCHIVE_TOX_DIR/tox-$e.log"
-    done
-    echo "ERROR: Failed the following builds: ${failed_envs[*]}"
-    exit 1
-fi
+# Disable SC2116 as we want to echo a space separated list of TOX_ENVS
+# shellcheck disable=SC2116
+for i in $(echo "${TOX_ENVS//,/ }"); do
+    cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i"
+done
 
 echo "Completed tox runs."