From: Tim Johnson <tijohnson@linuxfoundation.org>
Date: Wed, 30 Oct 2019 21:29:52 +0000 (-0700)
Subject: Update Bash library
X-Git-Tag: v0.48.0^0
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=4ac79cd48227bbbfadbab995dc4535a61a7ee26f;p=releng%2Fglobal-jjb.git

Update Bash library

The lf-venv-activate() function has been deleted. The lf-venv-create()
function has been renamed lf-activate-venv(). The lf-activate-venv()
function now creates temporary venvs and updates the PATH.  Remove
required 'python' argument and '--python' flag. Add two new functions:
lf-git-validate-jira-urls() and lf-jjb-check-ascii().

Issue: RELENG-2527
Change-Id: I90fdbbc3d99dc78a5d0190e75997686d95ad671a
Signed-off-by: Tim Johnson <tijohnson@linuxfoundation.org>
---

diff --git a/jenkins-init-scripts/lf-env.sh b/jenkins-init-scripts/lf-env.sh
index b07bf818..ae28a19d 100644
--- a/jenkins-init-scripts/lf-env.sh
+++ b/jenkins-init-scripts/lf-env.sh
@@ -12,14 +12,10 @@
 
 # A library of functions for LF/Jenkins bash scripts. In the general case, these
 # functions should only use 'local' variables, and should NOT set
-# shell/environment variables. If you want to make a
-# variable available, provide a function that sets the variable: 'function
-# lf_set_foo() {foo=asdf;}'. Any scripts that need access to the variable can
-# call the 'set' function. This keeps the name-space pollution to a minimum.
-
-# Shell variables that are shared between functions
-
-_lf_done_file=".lf-done"
+# shell/environment variables. If you want to make a variable available, provide
+# a function that sets the variable: 'function lf_set_foo() {foo=asdf;}'. Any
+# scripts that need access to the variable can call the 'set' function. This
+# keeps the name-space pollution to a minimum.
 
 ################################################################################
 #
@@ -60,7 +56,8 @@ function lf-echo-stderr() { echo "$@" 1>&2; }
 #   other values will return false(2) and an error message.
 #
 # RETURN VALUES
-#   true(0), false(1) or false(2)
+#   OK: 0
+#   Fail: 1 or 2
 #
 ################################################################################
 
@@ -89,195 +86,182 @@ function lf-boolean()
 ################################################################################
 #
 # NAME
-#   lf-venv-activate()
+#   lf-activate-venv [-p|--python python] [package]...
 #
 # SYNOPSIS
 #   # shellcheck disable=SC1090
 #   source ~/lf-env.sh
 #
-#   lf-venv-activate python3
+#   lf-activate-venv tox tox-pyenv
+#   or
+#   lf-activate-venv jenkins-job-builder
+#   or
+#   lf-activate-venv lftools
+#   or
+#   lf-activate-venv --python 3.6 git-review
 #
 # DESCRIPTION
-#   This function will validate existance of 'python' venv. If it exists
-#   'path-to-venv/bin' will be prepended to the PATH.
+#   This function will create a new Python Virtual Environment (venv) and
+#   install the specified packages in the new venv.  The bin directory from the
+#   venv will be prepended to the PATH.
+#
+#   By default all packages are installed with '--upgrade-strategy eager'.
+#   The venv will always contain pip & virtualenv.
+#
+#   Some packages have a default version. If one of those packages is specified,
+#   the 'version' specifier will be added for the install. If the version is
+#   specified on the command line that version will be used.
+#   The following packages have default versions:
+#       Package                  Version
+#       jenkins-job-builder      $JJB_VERSION
+#
+#   If the --python option is specified, that python executable will be used to
+#   create the venv. The --python option must be in the PATH. The venv will be
+#   located in '/tmp/venv-####'.
 #
 # RETURN VALUES
-#   None
+#   OK: 0
+#   Fail: 1
 #
 ################################################################################
 
-function lf-venv-activate()
+function lf-activate-venv()
 {
-    if (( $# != 1 )); then
-        echo "${FUNCNAME[0]}(): Missing path operand"
+    local lf_tmp_venv
+    lf_tmp_venv=$(mktemp -d /tmp/venv-XXXX)
+    local python=python3
+    local options
+    options=$(getopt -o 'p:' -l 'python:' -n "${FUNCNAME[0]}" -- "$@" )
+    eval set -- "$options"
+    while true; do
+        case $1 in
+            -p|--python) python=$2; shift 2 ;;
+            --) shift; break ;;
+            *)  lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown switch '$1'." ; return 1 ;;
+        esac
+    done
+     if ! type $python > /dev/null; then
+        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python"
         return 1
     fi
-    local arg=$1
-    local venv
-    if [[ $arg =~ ^python ]] && type $arg > /dev/null ; then
-        venv=~/.venv${arg#python}
-    else
-        venv=$arg
-    fi
-    # Validate the path to a VENV
-    if [[ ! -f $venv/$_lf_done_file ]]; then
-        lf-echo-stderr "ERROR: Is '$venv' a Python Environment ?"
+
+    echo "${FUNCNAME[0]}(): Creating '$python' venv ($lf_tmp_venv)"
+
+    case $python in
+        python2*)
+        local pkg_list="$*"
+        # For Python2, just create venv and install pip
+        virtualenv -p $python $lf_tmp_venv || return 1
+        $lf_tmp_venv/bin/pip install --upgrade --quiet pip || return 1
+        if [[ -z $pkg_list ]]; then
+            echo "${FUNCNAME[0]}(): WARNING: No packages to install"
+            return 0
+        fi
+        echo "${FUNCNAME[0]}(): INFO: Installing: $pkg_list"
+        $lf_tmp_venv/bin/pip install --upgrade --quiet $pkg_list || return 1
+        ;;
+    python3*)
+        local pkg_list=""
+        # Add version specifier for some packages
+        for arg in "$@"; do
+            case $arg in
+                jenkins-job-builder) pkg_list+="jenkins-job-builder==${JJB_VERSION:-2.8.0} " ;;
+                *)                   pkg_list+="$arg " ;;
+            esac
+        done
+        $python -m venv $lf_tmp_venv || return 1
+        $lf_tmp_venv/bin/pip install --upgrade --quiet pip virtualenv || return 1
+        if [[ -z $pkg_list ]]; then
+            echo "${FUNCNAME[0]}(): WARNING: No packages to install"
+            return 0
+        fi
+        echo "${FUNCNAME[0]}(): INFO: Installing: $pkg_list"
+        echo "Installing: $pkg_list"
+        $lf_tmp_venv/bin/pip install --upgrade --quiet --upgrade-strategy eager \
+                             $pkg_list || return 1
+        ;;
+    *)
+        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python"
         return 1
-    fi
-    echo "${FUNCNAME[0]}(): Adding $venv/bin to PATH"
-    PATH=$venv/bin:$PATH
+        ;;
+    esac
+    echo "${FUNCNAME[0]}(): Adding $lf_tmp_venv/bin to PATH"
+    PATH=$lf_tmp_venv/bin:$PATH
+    return 0
 
-}  # ENd lf-venv-activate()
+}   # End lf-activate-venv()
 
 ################################################################################
 #
 # NAME
-#   lf-venv-create python [package]...
+#   lf-git-validate-jira-urls
 #
 # SYNOPSIS
 #   # shellcheck disable=SC1090
 #   source ~/lf-env.sh
 #
-#   lf-venv-create python3 tox tox-pyenv virtualenv
-#   lf-venv-create python3.6
+#   lf-git-validate-jira-urls
 #
 # DESCRIPTION
-#   This function will create/update a Python Virtual Environment (venv) based
-#   on the python specified. The 'python' argument must be in the PATH. The venv
-#   will be located in ~/.venv## where ## comes from the 'python' argument.
-#   I.E. python3 -> ~/.venv3. The resulting venv will be left 'read-only' to
-#   discourage the installation of any other packages (except by
-#   lf-venv-create()). By default, only versioned packages will be installed, so
-#   any required packages need to be specified.  By default the 'pip install
-#   --upgrade' will be run multiple times. Sometimes pip needs that to get the
-#   versioning correct.
+#   Check for JIRA URLS in the commit message
 #
 # RETURN VALUES
-#       None
+#   OK: 0
+#   Fail: 1
 #
 ################################################################################
 
-function lf-venv-create()
+function lf-git-validate-jira-urls()
 {
-    if (( $# < 1 )); then
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Required Arguments"
-        return 1
+    echo "Checking for JIRA URLs in commit message..."
+    # if JIRA_URL is not defined, nothing to do
+    if [[ -v JIRA_URL ]]; then
+        base_url=$(echo "$JIRA_URL" | awk -F'/' '{print $3}')
+        jira_link=$(git rev-list --format=%B --max-count=1 HEAD | grep -io "http[s]*://$base_url/" || true)
+        if [[ -n $jira_link ]]; then
+            lf-echo-error 'Remove JIRA URLs from commit message'
+            lf-echo-error 'Add jira references as: Issue: <JIRAKEY>-<ISSUE#>, instead of URLs'
+            return 1
+        fi
     fi
-    python=$1
-    shift
-    if ! type $python > /dev/null; then
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python"
-        return 1
-    fi
-    local pkg_list="$* "
-    local venv=~/.venv${python#python}
-    local suffix=$python-$$
-    local pip_log=/tmp/pip_log.$suffix
-    if [[ -f $venv/$_lf_done_file ]]; then
-        echo "Venv Already Exists: '$venv'"
-        return
-    fi
-    # Make sure noting is left over
-    if [[ -d $venv ]]; then
-        chmod -R +w $venv
-        rm -rf $venv
-    fi
-
-    echo "Creating '$python' venv ($venv)"
-
-    case $python in
-    python2*)
-        # For Python2, just create venv and install pip
-        virtualenv -p $python $venv > $pip_log || return 1
-        $venv/bin/pip install --upgrade pip > $pip_log || return 1
-        $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1
-        ;;
-    python3*)
-        # Include any packages that are tied to a specific version
-        pkg_list+="jenkins-job-builder==2.8.0 "
-        $python -m venv $venv > $pip_log
-        $venv/bin/pip install --upgrade pip > $pip_log || return 1
-        # Redirect errors for now
-        $venv/bin/pip install --upgrade $pkg_list >> $pip_log 2> /dev/null || return 1
-        # Generate list of packages
-        pkg_list=$($venv/bin/pip freeze | awk -F '=' '{print $1}') || return 1
-        # Update all packages, may need to run twice to get all versions
-        # synced up. Ignore exit status on first try
-        $venv/bin/pip install --upgrade $pkg_list >> $pip_log || true
-        echo "Running 'pip --upgrade' to validate..."
-        $venv/bin/pip install --upgrade $pkg_list >> $pip_log || return 1
-        ;;
-    *)
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python"
-        return 1
-        ;;
-    esac
-
-    touch $venv/$_lf_done_file
-    # Once this venv is created, make it read-only
-    chmod -R -w $venv
-    # Archive output of 'pip freeze'
-    mkdir -p $WORKSPACE/archives
-    $venv/bin/pip freeze > $WORKSPACE/archives/freeze-log-$python || return 1
-    rm -rf $pip_log
-
-}   # End lf-venv-create()
+    return 0
+}
 
 ################################################################################
 #
 # NAME
-#   lf-venv-add()
+#   lf-jjb-check-ascii()
 #
 # SYNOPSIS
 #   # shellcheck disable=SC1090
 #   source ~/lf-env.sh
 #
-#   lf-venv-add python3 pkg
-#   or
-#   lf-venv-add python2 pkg1 pkg2 pkg3
+#   lf-jjb-check-ascii
 #
 # DESCRIPTION
-#   This function will add one or more python packages to an existing venv.
-#   Attempts to add packages directly (pip) will result in errors because
-#   the venv does not have 'write' permission.
+#   Check for JJB YAML files containing non-printable ascii characters. This
+#   function must be run from the top of the global-jjb repo.
 #
 # RETURN VALUES
-#   None
+#   OK: 0
+#   Fail: 1
 #
 ################################################################################
 
-function lf-venv-add()
+function lf-jjb-check-ascii()
 {
-    if (( $# < 2 )); then
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Package argument"
+    if [[ ! -d "jjb" ]]; then
+        lf-echo-error "${FUNCNAME[0]}(): ERROR: missing jjb directory"
+        lf-echo-error "This function can only be run from top of global-jjb directory"
         return 1
     fi
-    python=$1
-    if ! type $python > /dev/null ; then
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python"
-        return 1
-    fi
-    shift
-    local venv=~/.venv${python#python}
-    if [[ ! -f $venv/$_lf_done_file ]]; then
-        lf-echo-stderr "${FUNCNAME[0]}(): ERROR: '$venv' is not a valid venv"
+    if LC_ALL=C grep -I -r '[^[:print:][:space:]]' jjb/; then
+        lf-echo-error "${FUNCNAME[0]}(): ERROR: Found YAML files containing non-printable characters."
         return 1
     fi
-    local pkg_list=$*
-    local pip_log=/tmp/pip_log-$$
-
-    echo "Installing '$pkg_list' into $venv"
-    chmod -R +w $venv
-    rm $venv/$_lf_done_file
-    $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1
-    pkg_list=$($venv/bin/pip freeze | awk -F '=' '{print $1}') || return 1
-    $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1
-    touch $venv/$_lf_done_file
-    chmod -R -w $venv
-    # Archive output of 'pip freeze'
-    $venv/bin/pip freeze > $WORKSPACE/archives/freeze-log-$python || return 1
-
-} # End lf-venv-add()
+    echo "${FUNCNAME[0]}(): INFO: All JJB YAML files contain only printable ASCII characters"
+    return 0
+}
 
 ################################################################################
 # Functions that assign Variables
diff --git a/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml b/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml
new file mode 100644
index 00000000..e7241aae
--- /dev/null
+++ b/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml
@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Created new function lf-activate-venv(). This function creates a venv in
+    /tmp and prepends the bin directory to the PATH. The 'pip install' command
+    now specifies: '--upgrade-strategy eager'. lf-activate-venv() supports an
+    optional --python flag to specify which python to use to create the venv,
+    the default is python3.
+
+    Two new functions: lf-git-validate-jira-urls() and lf-jjb-check-ascii().
+    They will be used to replace the git-validate-jira-urls.sh &
+    jjb-check-unicode.sh scripts at some point. For now, they are not being
+    used.
+other:
+  - |
+    The lf-venv-add(), lf-venv-create() & lf-venv-activate functions have been
+    removed. No-one is accessing it yet.