From 0251b7fda7266760bd82a6e56a696160822a57a0 Mon Sep 17 00:00:00 2001
From: Tim Johnson <tijohnson@linuxfoundation.org>
Date: Tue, 9 Apr 2019 14:41:20 -0700
Subject: [PATCH] Reduce output in log files

Previously ran 'pip freeze' (> log) twice to get list of pip packages
installed during build.  Run 'pip list' (> file) twice and archive the
files and diffs. Configured 'logs-deploy.sh' to run as separate Jekins
script.

Issue: RELENG-1902
Change-Id: Ib0f99bd1eb125015467c7eaf91d26c76e92bffa0
Signed-off-by: Tim Johnson <tijohnson@linuxfoundation.org>
---
 jjb/lf-macros.yaml            |  1 +
 shell/logs-deploy.sh          | 24 ++++++--------------
 shell/python-tools-install.sh | 52 ++++++++++++++++++++++++++++++-------------
 3 files changed, 45 insertions(+), 32 deletions(-)

diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml
index 134ef13d..2d819240 100644
--- a/jjb/lf-macros.yaml
+++ b/jjb/lf-macros.yaml
@@ -76,6 +76,7 @@
           # Ensure python-tools are installed in case job template does not
           # call the lf-infra-pre-build macro.
           - ../shell/python-tools-install.sh
+      - shell: !include-raw:
           - ../shell/logs-deploy.sh
       - shell: !include-raw:
           - ../shell/logs-clear-credentials.sh
diff --git a/shell/logs-deploy.sh b/shell/logs-deploy.sh
index 114692e9..359642ad 100644
--- a/shell/logs-deploy.sh
+++ b/shell/logs-deploy.sh
@@ -13,25 +13,15 @@ echo "---> logs-deploy.sh"
 # Ensure we fail the job if any steps fail.
 set -eu -o pipefail
 
-set -x  # Trace commands for this script to make debugging easier.
-
-ARCHIVE_ARTIFACTS="${ARCHIVE_ARTIFACTS:-}"
-LOGS_SERVER="${LOGS_SERVER:-None}"
-
-if [ "${LOGS_SERVER}" == 'None' ]
-then
-    set +x # Disable trace since we no longer need it
-
+if [[ -z $LOGS_SERVER ]]; then
     echo "WARNING: Logging server not set"
 else
-    NEXUS_URL="${NEXUSPROXY:-$NEXUS_URL}"
-    NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}"
-    BUILD_URL="${BUILD_URL}"
-
-    lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE"
-    lftools deploy logs "$NEXUS_URL" "$NEXUS_PATH" "$BUILD_URL"
+    nexus_url="${NEXUSPROXY:-$NEXUS_URL}"
+    nexus_path="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}"
 
-    set +x  # Disable trace since we no longer need it.
+    lftools deploy archives -p "${ARCHIVE_ARTIFACTS:-}" "$nexus_url" \
+            "$nexus_path" "$WORKSPACE"
+    lftools deploy logs "$nexus_url" "$nexus_path" "${BUILD_URL:-}"
 
-    echo "Build logs: <a href=\"$LOGS_SERVER/$NEXUS_PATH\">$LOGS_SERVER/$NEXUS_PATH</a>"
+    echo "Build logs: <a href=\"$LOGS_SERVER/$nexus_path\">$LOGS_SERVER/$nexus_path</a>"
 fi
diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh
index d652763c..9eff4603 100644
--- a/shell/python-tools-install.sh
+++ b/shell/python-tools-install.sh
@@ -10,15 +10,40 @@
 ##############################################################################
 echo "---> python-tools-install.sh"
 
-set -eux -o pipefail
+set -eu -o pipefail
 
-REQUIREMENTS_FILE=$(mktemp /tmp/requirements-XXXX.txt)
+# Generate a list of 'pip' packages pre-build/post-build
+# During post-build, perform a diff on the two lists and copy files to archive directory
+echo "Listing pip packages"
+pip_list_pre=/tmp/pip-list-pre.txt
+pip_list_post=/tmp/pip-list-post.txt
+pip_list_diffs=/tmp/pip-list-diffs.txt
+if [[ -f $pip_list_pre ]]; then
+    pip list > $pip_list_post
+    echo "Compare pip packages before/after..."
+    if diff --suppress-common-lines $pip_list_pre $pip_list_post \
+            | tee $pip_list_diffs; then
+        echo "No diffs" | tee $pip_list_diffs
+    fi
+    mkdir -p $WORKSPACE/archives
+    cp $pip_list_pre $pip_list_post $pip_list_diffs $WORKSPACE/archives
+    rm -rf $pip_list_pre $pip_list_post $pip_list_diffs
+    ls $WORKSPACE/archives
+    # Would just like to 'exit 0' here but we can't because the
+    # log-deploy.sh script is 'appended' to this file and it would not
+    # be executed.
+else
+    pip list > $pip_list_pre
+    # These 'pip installs' only need to be executed during pre-build
 
-# Note: To test lftools master branch change the lftools configuration below in
-#       the requirements file from "lftools[openstack]~=#.##.#" to
-#       git+https://github.com/lfit/releng-lftools.git#egg=lftools[openstack]
+    requirements_file=$(mktemp /tmp/requirements-XXXX.txt)
 
-cat << EOF > "$REQUIREMENTS_FILE"
+    # Note: To test lftools master branch change the lftools configuration below in
+    #       the requirements file from "lftools[openstack]~=#.##.#" to
+    #       git+https://github.com/lfit/releng-lftools.git#egg=lftools[openstack]
+
+    echo "Generating Requirements File"
+    cat << 'EOF' > "$requirements_file"
 lftools[openstack]~=0.23.1
 python-heatclient~=1.16.1
 python-openstackclient~=3.16.0
@@ -26,12 +51,9 @@ dogpile.cache~=0.6.8  # Version 0.7.[01] seems to break openstackclient
 niet~=1.4.2 # Extract values from yaml
 EOF
 
-echo "Requirements file"
-echo "-----------------"
-cat "$REQUIREMENTS_FILE"
-
-# Use `python -m pip` to ensure we are using the latest version of pip
-python -m pip install --user --quiet --upgrade pip
-python -m pip install --user --quiet --upgrade setuptools
-python -m pip install --user --quiet --upgrade -r "$REQUIREMENTS_FILE"
-pip freeze
+    # Use `python -m pip` to ensure we are using the latest version of pip
+    python -m pip install --user --quiet --upgrade pip
+    python -m pip install --user --quiet --upgrade setuptools
+    python -m pip install --user --quiet --upgrade -r "$requirements_file"
+    rm -rf $requirements_file
+fi
-- 
2.16.6