From e0b87a3264cfccc425c0dae54f8eda703e5d8492 Mon Sep 17 00:00:00 2001 From: Tim Johnson Date: Tue, 29 Oct 2019 12:00:48 -0700 Subject: [PATCH] Update lf-venv-create function Update the lf-venv-create() function to support a list of python packages to install in the new venv. Issue: RELENG-2502 Change-Id: Ic2e186e3d7db653f57b755974aeb1c63581df8f2 Signed-off-by: Tim Johnson --- jenkins-init-scripts/lf-env.sh | 115 +++++++++++---------- .../update-bash-library-86887cdc0675f956.yaml | 9 ++ 2 files changed, 67 insertions(+), 57 deletions(-) create mode 100644 releasenotes/notes/update-bash-library-86887cdc0675f956.yaml diff --git a/jenkins-init-scripts/lf-env.sh b/jenkins-init-scripts/lf-env.sh index 2acc4137..b07bf818 100644 --- a/jenkins-init-scripts/lf-env.sh +++ b/jenkins-init-scripts/lf-env.sh @@ -26,15 +26,15 @@ _lf_done_file=".lf-done" # Name: lf-echo-stderr # # SYNOPSIS -# source ~/lf-env.sh +# source ~/lf-env.sh # -# lf-echo-stderr "this entire" "string will be sent to stderr" +# lf-echo-stderr "this entire" "string will be sent to stderr" # # DESCRIPTION -# This function will echo all command line aruments to 'stderr' +# This function will echo all command line aruments to 'stderr' # # RETURN VALUE -# None +# None # ################################################################################ @@ -46,28 +46,28 @@ function lf-echo-stderr() { echo "$@" 1>&2; } # lf-boolean() # # SYNOPSIS -# # shellcheck disable=SC1090 -# source ~/lf-env.sh +# # shellcheck disable=SC1090 +# source ~/lf-env.sh # -# if lf-boolean $VAR; then -# echo "VAR is true" -# fi +# if lf-boolean $VAR; then +# echo "VAR is true" +# fi # # DESCRIPTION -# This function will return a BOOLEAN (true or false) based upon the value -# of VAR. The value of VAR will be mapped to lower case. If VAR maps to -# "true", return true(0). If VAR maps to "false", return false(1). Any -# other values will return false(2) and an error message. +# This function will return a BOOLEAN (true or false) based upon the value +# of VAR. The value of VAR will be mapped to lower case. If VAR maps to +# "true", return true(0). If VAR maps to "false", return false(1). Any +# other values will return false(2) and an error message. # # RETURN VALUES -# true(0), false(1) or false(2) +# true(0), false(1) or false(2) # ################################################################################ function lf-boolean() { if (( $# != 1 )); then - echo "ERROR: ${FUNCNAME[0]}() line: ${BASH_LINENO[0]} : Missing required argument" + echo "ERROR: ${FUNCNAME[0]}() line: ${BASH_LINENO[0]} : Missing Required Argument" return 1 fi local bool @@ -89,20 +89,20 @@ function lf-boolean() ################################################################################ # # NAME -# lf-venv-activate() +# lf-venv-activate() # # SYNOPSIS -# # shellcheck disable=SC1090 -# source ~/lf-env.sh +# # shellcheck disable=SC1090 +# source ~/lf-env.sh # -# lf-venv-activate python3 +# lf-venv-activate python3 # # DESCRIPTION -# This function will validate existance of 'python' venv. If it exists -# 'path-to-venv/bin' will be prepended to the PATH. +# This function will validate existance of 'python' venv. If it exists +# 'path-to-venv/bin' will be prepended to the PATH. # # RETURN VALUES -# None +# None # ################################################################################ @@ -132,23 +132,25 @@ function lf-venv-activate() ################################################################################ # # NAME -# lf-venv-create() +# lf-venv-create python [package]... # # SYNOPSIS -# # shellcheck disable=SC1090 -# source ~/lf-env.sh +# # shellcheck disable=SC1090 +# source ~/lf-env.sh # -# lf-venv-create python2 -# or -# lf-venv-create python3.6 +# lf-venv-create python3 tox tox-pyenv virtualenv +# lf-venv-create python3.6 # # DESCRIPTION -# This function will create a Python Virtual Environment based on the -# python specified. The 'python' argument must be in the PATH. The venv -# will be located in ~/.venv## where ## comes from the 'python' argument. -# I.E. python3 -> ~/.venv3. The resulting venv will be left 'read-only' to -# discourage the installation of any other packages (except by -# lf-venv-add()). +# This function will create/update a Python Virtual Environment (venv) based +# on the python specified. The 'python' argument must be in the PATH. The venv +# will be located in ~/.venv## where ## comes from the 'python' argument. +# I.E. python3 -> ~/.venv3. The resulting venv will be left 'read-only' to +# discourage the installation of any other packages (except by +# lf-venv-create()). By default, only versioned packages will be installed, so +# any required packages need to be specified. By default the 'pip install +# --upgrade' will be run multiple times. Sometimes pip needs that to get the +# versioning correct. # # RETURN VALUES # None @@ -157,15 +159,17 @@ function lf-venv-activate() function lf-venv-create() { - if (( $# != 1 )); then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Python argument" + if (( $# < 1 )); then + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Required Arguments" return 1 fi python=$1 + shift if ! type $python > /dev/null; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" return 1 fi + local pkg_list="$* " local venv=~/.venv${python#python} local suffix=$python-$$ local pip_log=/tmp/pip_log.$suffix @@ -186,25 +190,22 @@ function lf-venv-create() # For Python2, just create venv and install pip virtualenv -p $python $venv > $pip_log || return 1 $venv/bin/pip install --upgrade pip > $pip_log || return 1 + $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1 ;; python3*) - local pkg_list="git-review jenkins-job-builder lftools[openstack] " - pkg_list+="python-heatclient python-openstackclient " - pkg_list+="setuptools testresources tox yq" + # Include any packages that are tied to a specific version + pkg_list+="jenkins-job-builder==2.8.0 " $python -m venv $venv > $pip_log $venv/bin/pip install --upgrade pip > $pip_log || return 1 # Redirect errors for now $venv/bin/pip install --upgrade $pkg_list >> $pip_log 2> /dev/null || return 1 # Generate list of packages pkg_list=$($venv/bin/pip freeze | awk -F '=' '{print $1}') || return 1 - # Update all packages, usuaally need to run twice to get all versions - # correct. - local upgrade_cmd="$venv/bin/pip install --upgrade $pkg_list" - if $upgrade_cmd >> $pip_log 2>&1 > /dev/null ; then - echo -n "Running 'pip --upgrade' to validate..." - $upgrade_cmd >> $pip_log || return 1 - echo "..OK" - fi + # Update all packages, may need to run twice to get all versions + # synced up. Ignore exit status on first try + $venv/bin/pip install --upgrade $pkg_list >> $pip_log || true + echo "Running 'pip --upgrade' to validate..." + $venv/bin/pip install --upgrade $pkg_list >> $pip_log || return 1 ;; *) lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python" @@ -225,23 +226,23 @@ function lf-venv-create() ################################################################################ # # NAME -# lf-venv-add() +# lf-venv-add() # # SYNOPSIS -# # shellcheck disable=SC1090 -# source ~/lf-env.sh +# # shellcheck disable=SC1090 +# source ~/lf-env.sh # -# lf-venv-add python3 pkg -# or -# lf-venv-add python2 pkg1 pkg2 pkg3 +# lf-venv-add python3 pkg +# or +# lf-venv-add python2 pkg1 pkg2 pkg3 # # DESCRIPTION -# This function will add one or more python packages to an existing venv. -# Attempts to add packages directly (pip) will result in errors because -# the venv does not have 'write' permission. +# This function will add one or more python packages to an existing venv. +# Attempts to add packages directly (pip) will result in errors because +# the venv does not have 'write' permission. # # RETURN VALUES -# None +# None # ################################################################################ diff --git a/releasenotes/notes/update-bash-library-86887cdc0675f956.yaml b/releasenotes/notes/update-bash-library-86887cdc0675f956.yaml new file mode 100644 index 00000000..3238e137 --- /dev/null +++ b/releasenotes/notes/update-bash-library-86887cdc0675f956.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + By default, the lf-venv-create() function only installs/upgrades 'pip' and + and any packages that need to be tied to a specific version like. Currently + the only version specific package is: 'jenkins-job-builder==2.8.0'. The + current version of lf-venv-create() supports being called twice but that + functionality has NOT yet been thoroughly tested. RELENG-2508 has been + created to validate & optimize that functionality. -- 2.16.6