From f3ed64938de3f8efbc102cd9f16676b63af6cfeb Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 3 Jan 2018 21:10:02 -0500 Subject: [PATCH] Use detox to run tox in parallel Rather than hacking something together with parallel use a tool supported by the tox project called detox. Change-Id: I04fc3a083c420f71938b99f942b682093822ddda Signed-off-by: Thanh Ha --- .../expected-xml/gerrit-python-tox-verify-master | 44 +++++----------------- .../expected-xml/gerrit-rtd-rtd-verify-master | 2 +- .../expected-xml/github-python-tox-verify-master | 44 +++++----------------- .../expected-xml/github-rtd-rtd-verify-master | 2 +- shell/tox-install.sh | 2 +- shell/tox-run.sh | 42 ++++----------------- 6 files changed, 29 insertions(+), 107 deletions(-) diff --git a/.jjb-test/expected-xml/gerrit-python-tox-verify-master b/.jjb-test/expected-xml/gerrit-python-tox-verify-master index 5573a887..497228ef 100644 --- a/.jjb-test/expected-xml/gerrit-python-tox-verify-master +++ b/.jjb-test/expected-xml/gerrit-python-tox-verify-master @@ -296,7 +296,7 @@ virtualenv --quiet "/tmp/v/tox" source "/tmp/v/tox/bin/activate" pip install --quiet --upgrade pip pip install --quiet --upgrade pipdeptree -pip install --quiet --upgrade tox argparse +pip install --quiet --upgrade detox tox argparse echo "----> Pip Dependency Tree" pipdeptree @@ -322,43 +322,17 @@ mkdir -p "$ARCHIVE_TOX_DIR" cd "$WORKSPACE/$TOX_DIR" -if [ -z "$TOX_ENVS" ]; then - TOX_ENVS=$(crudini --get tox.ini tox envlist) -fi - -run_tox() { - local log_dir="$1" - local env="$2" - - # Sleep a random 10 second interval to workaround tox sdist - # conflicts due to building in the same dist directory. - sleep $[ ( $RANDOM % 10 ) + 1 ]s - - echo "-----> Running tox $env" - if ! tox -e $env > "$log_dir/tox-$env.log"; then - echo "$env" >> "$log_dir/failed-envs.log" - fi -} - -IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }" -if hash parallel 2>/dev/null; then - export -f run_tox - parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]} +if [ ! -z "$TOX_ENVS" ]; then + detox -e "$TOX_ENVS" | tee -a "$ARCHIVE_TOX_DIR/detox.log" else - for env in "${TOX_ENVS[@]}"; do - run_tox "$ARCHIVE_TOX_DIR" "$env" - done + detox | tee -a "$ARCHIVE_TOX_DIR/detox.log" fi -if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then - mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log") - for e in "${failed_envs[@]}"; do - echo "cat $ARCHIVE_TOX_DIR/tox-$e.log" - cat "$ARCHIVE_TOX_DIR/tox-$e.log" - done - echo "ERROR: Failed the following builds: ${failed_envs[*]}" - exit 1 -fi +# Disable SC2116 as we want to echo a space separated list of TOX_ENVS +# shellcheck disable=SC2116 +for i in $(echo "${TOX_ENVS//,/ }"); do + cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i" +done echo "Completed tox runs." diff --git a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master index 3473cdcf..5bc67eac 100644 --- a/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master +++ b/.jjb-test/expected-xml/gerrit-rtd-rtd-verify-master @@ -210,7 +210,7 @@ virtualenv --quiet "/tmp/v/tox" source "/tmp/v/tox/bin/activate" pip install --quiet --upgrade pip pip install --quiet --upgrade pipdeptree -pip install --quiet --upgrade tox argparse +pip install --quiet --upgrade detox tox argparse echo "----> Pip Dependency Tree" pipdeptree diff --git a/.jjb-test/expected-xml/github-python-tox-verify-master b/.jjb-test/expected-xml/github-python-tox-verify-master index 6d758b8c..9cd8ea30 100644 --- a/.jjb-test/expected-xml/github-python-tox-verify-master +++ b/.jjb-test/expected-xml/github-python-tox-verify-master @@ -258,7 +258,7 @@ virtualenv --quiet "/tmp/v/tox" source "/tmp/v/tox/bin/activate" pip install --quiet --upgrade pip pip install --quiet --upgrade pipdeptree -pip install --quiet --upgrade tox argparse +pip install --quiet --upgrade detox tox argparse echo "----> Pip Dependency Tree" pipdeptree @@ -284,43 +284,17 @@ mkdir -p "$ARCHIVE_TOX_DIR" cd "$WORKSPACE/$TOX_DIR" -if [ -z "$TOX_ENVS" ]; then - TOX_ENVS=$(crudini --get tox.ini tox envlist) -fi - -run_tox() { - local log_dir="$1" - local env="$2" - - # Sleep a random 10 second interval to workaround tox sdist - # conflicts due to building in the same dist directory. - sleep $[ ( $RANDOM % 10 ) + 1 ]s - - echo "-----> Running tox $env" - if ! tox -e $env > "$log_dir/tox-$env.log"; then - echo "$env" >> "$log_dir/failed-envs.log" - fi -} - -IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }" -if hash parallel 2>/dev/null; then - export -f run_tox - parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]} +if [ ! -z "$TOX_ENVS" ]; then + detox -e "$TOX_ENVS" | tee -a "$ARCHIVE_TOX_DIR/detox.log" else - for env in "${TOX_ENVS[@]}"; do - run_tox "$ARCHIVE_TOX_DIR" "$env" - done + detox | tee -a "$ARCHIVE_TOX_DIR/detox.log" fi -if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then - mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log") - for e in "${failed_envs[@]}"; do - echo "cat $ARCHIVE_TOX_DIR/tox-$e.log" - cat "$ARCHIVE_TOX_DIR/tox-$e.log" - done - echo "ERROR: Failed the following builds: ${failed_envs[*]}" - exit 1 -fi +# Disable SC2116 as we want to echo a space separated list of TOX_ENVS +# shellcheck disable=SC2116 +for i in $(echo "${TOX_ENVS//,/ }"); do + cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i" +done echo "Completed tox runs." diff --git a/.jjb-test/expected-xml/github-rtd-rtd-verify-master b/.jjb-test/expected-xml/github-rtd-rtd-verify-master index c9731c17..3bf0043c 100644 --- a/.jjb-test/expected-xml/github-rtd-rtd-verify-master +++ b/.jjb-test/expected-xml/github-rtd-rtd-verify-master @@ -162,7 +162,7 @@ virtualenv --quiet "/tmp/v/tox" source "/tmp/v/tox/bin/activate" pip install --quiet --upgrade pip pip install --quiet --upgrade pipdeptree -pip install --quiet --upgrade tox argparse +pip install --quiet --upgrade detox tox argparse echo "----> Pip Dependency Tree" pipdeptree diff --git a/shell/tox-install.sh b/shell/tox-install.sh index 17bebecc..e6f26142 100644 --- a/shell/tox-install.sh +++ b/shell/tox-install.sh @@ -19,7 +19,7 @@ virtualenv --quiet "/tmp/v/tox" source "/tmp/v/tox/bin/activate" pip install --quiet --upgrade pip pip install --quiet --upgrade pipdeptree -pip install --quiet --upgrade tox argparse +pip install --quiet --upgrade detox tox argparse echo "----> Pip Dependency Tree" pipdeptree diff --git a/shell/tox-run.sh b/shell/tox-run.sh index e8172b2b..97f8855b 100644 --- a/shell/tox-run.sh +++ b/shell/tox-run.sh @@ -19,42 +19,16 @@ mkdir -p "$ARCHIVE_TOX_DIR" cd "$WORKSPACE/$TOX_DIR" -if [ -z "$TOX_ENVS" ]; then - TOX_ENVS=$(crudini --get tox.ini tox envlist) -fi - -run_tox() { - local log_dir="$1" - local env="$2" - - # Sleep a random 10 second interval to workaround tox sdist - # conflicts due to building in the same dist directory. - sleep $[ ( $RANDOM % 10 ) + 1 ]s - - echo "-----> Running tox $env" - if ! tox -e $env > "$log_dir/tox-$env.log"; then - echo "$env" >> "$log_dir/failed-envs.log" - fi -} - -IFS=" " read -r -a TOX_ENVS <<< "${TOX_ENVS//,/ }" -if hash parallel 2>/dev/null; then - export -f run_tox - parallel --jobs 200% "run_tox $ARCHIVE_TOX_DIR {}" ::: ${TOX_ENVS[*]} +if [ ! -z "$TOX_ENVS" ]; then + detox -e "$TOX_ENVS" | tee -a "$ARCHIVE_TOX_DIR/detox.log" else - for env in "${TOX_ENVS[@]}"; do - run_tox "$ARCHIVE_TOX_DIR" "$env" - done + detox | tee -a "$ARCHIVE_TOX_DIR/detox.log" fi -if [ -f "$ARCHIVE_TOX_DIR/failed-envs.log" ]; then - mapfile -t failed_envs < <(cat "$ARCHIVE_TOX_DIR/failed-envs.log") - for e in "${failed_envs[@]}"; do - echo "cat $ARCHIVE_TOX_DIR/tox-$e.log" - cat "$ARCHIVE_TOX_DIR/tox-$e.log" - done - echo "ERROR: Failed the following builds: ${failed_envs[*]}" - exit 1 -fi +# Disable SC2116 as we want to echo a space separated list of TOX_ENVS +# shellcheck disable=SC2116 +for i in $(echo "${TOX_ENVS//,/ }"); do + cp -r ".tox/$i/log" "$ARCHIVE_TOX_DIR/$i" +done echo "Completed tox runs." -- 2.16.6