From 4ac79cd48227bbbfadbab995dc4535a61a7ee26f Mon Sep 17 00:00:00 2001 From: Tim Johnson Date: Wed, 30 Oct 2019 14:29:52 -0700 Subject: [PATCH] Update Bash library The lf-venv-activate() function has been deleted. The lf-venv-create() function has been renamed lf-activate-venv(). The lf-activate-venv() function now creates temporary venvs and updates the PATH. Remove required 'python' argument and '--python' flag. Add two new functions: lf-git-validate-jira-urls() and lf-jjb-check-ascii(). Issue: RELENG-2527 Change-Id: I90fdbbc3d99dc78a5d0190e75997686d95ad671a Signed-off-by: Tim Johnson --- jenkins-init-scripts/lf-env.sh | 268 ++++++++++----------- .../update-bash-library-aae95b6ccfb28c0d.yaml | 17 ++ 2 files changed, 143 insertions(+), 142 deletions(-) create mode 100644 releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml diff --git a/jenkins-init-scripts/lf-env.sh b/jenkins-init-scripts/lf-env.sh index b07bf818..ae28a19d 100644 --- a/jenkins-init-scripts/lf-env.sh +++ b/jenkins-init-scripts/lf-env.sh @@ -12,14 +12,10 @@ # A library of functions for LF/Jenkins bash scripts. In the general case, these # functions should only use 'local' variables, and should NOT set -# shell/environment variables. If you want to make a -# variable available, provide a function that sets the variable: 'function -# lf_set_foo() {foo=asdf;}'. Any scripts that need access to the variable can -# call the 'set' function. This keeps the name-space pollution to a minimum. - -# Shell variables that are shared between functions - -_lf_done_file=".lf-done" +# shell/environment variables. If you want to make a variable available, provide +# a function that sets the variable: 'function lf_set_foo() {foo=asdf;}'. Any +# scripts that need access to the variable can call the 'set' function. This +# keeps the name-space pollution to a minimum. ################################################################################ # @@ -60,7 +56,8 @@ function lf-echo-stderr() { echo "$@" 1>&2; } # other values will return false(2) and an error message. # # RETURN VALUES -# true(0), false(1) or false(2) +# OK: 0 +# Fail: 1 or 2 # ################################################################################ @@ -89,195 +86,182 @@ function lf-boolean() ################################################################################ # # NAME -# lf-venv-activate() +# lf-activate-venv [-p|--python python] [package]... # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # -# lf-venv-activate python3 +# lf-activate-venv tox tox-pyenv +# or +# lf-activate-venv jenkins-job-builder +# or +# lf-activate-venv lftools +# or +# lf-activate-venv --python 3.6 git-review # # DESCRIPTION -# This function will validate existance of 'python' venv. If it exists -# 'path-to-venv/bin' will be prepended to the PATH. +# This function will create a new Python Virtual Environment (venv) and +# install the specified packages in the new venv. The bin directory from the +# venv will be prepended to the PATH. +# +# By default all packages are installed with '--upgrade-strategy eager'. +# The venv will always contain pip & virtualenv. +# +# Some packages have a default version. If one of those packages is specified, +# the 'version' specifier will be added for the install. If the version is +# specified on the command line that version will be used. +# The following packages have default versions: +# Package Version +# jenkins-job-builder $JJB_VERSION +# +# If the --python option is specified, that python executable will be used to +# create the venv. The --python option must be in the PATH. The venv will be +# located in '/tmp/venv-####'. # # RETURN VALUES -# None +# OK: 0 +# Fail: 1 # ################################################################################ -function lf-venv-activate() +function lf-activate-venv() { - if (( $# != 1 )); then - echo "${FUNCNAME[0]}(): Missing path operand" + local lf_tmp_venv + lf_tmp_venv=$(mktemp -d /tmp/venv-XXXX) + local python=python3 + local options + options=$(getopt -o 'p:' -l 'python:' -n "${FUNCNAME[0]}" -- "$@" ) + eval set -- "$options" + while true; do + case $1 in + -p|--python) python=$2; shift 2 ;; + --) shift; break ;; + *) lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown switch '$1'." ; return 1 ;; + esac + done + if ! type $python > /dev/null; then + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" return 1 fi - local arg=$1 - local venv - if [[ $arg =~ ^python ]] && type $arg > /dev/null ; then - venv=~/.venv${arg#python} - else - venv=$arg - fi - # Validate the path to a VENV - if [[ ! -f $venv/$_lf_done_file ]]; then - lf-echo-stderr "ERROR: Is '$venv' a Python Environment ?" + + echo "${FUNCNAME[0]}(): Creating '$python' venv ($lf_tmp_venv)" + + case $python in + python2*) + local pkg_list="$*" + # For Python2, just create venv and install pip + virtualenv -p $python $lf_tmp_venv || return 1 + $lf_tmp_venv/bin/pip install --upgrade --quiet pip || return 1 + if [[ -z $pkg_list ]]; then + echo "${FUNCNAME[0]}(): WARNING: No packages to install" + return 0 + fi + echo "${FUNCNAME[0]}(): INFO: Installing: $pkg_list" + $lf_tmp_venv/bin/pip install --upgrade --quiet $pkg_list || return 1 + ;; + python3*) + local pkg_list="" + # Add version specifier for some packages + for arg in "$@"; do + case $arg in + jenkins-job-builder) pkg_list+="jenkins-job-builder==${JJB_VERSION:-2.8.0} " ;; + *) pkg_list+="$arg " ;; + esac + done + $python -m venv $lf_tmp_venv || return 1 + $lf_tmp_venv/bin/pip install --upgrade --quiet pip virtualenv || return 1 + if [[ -z $pkg_list ]]; then + echo "${FUNCNAME[0]}(): WARNING: No packages to install" + return 0 + fi + echo "${FUNCNAME[0]}(): INFO: Installing: $pkg_list" + echo "Installing: $pkg_list" + $lf_tmp_venv/bin/pip install --upgrade --quiet --upgrade-strategy eager \ + $pkg_list || return 1 + ;; + *) + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python" return 1 - fi - echo "${FUNCNAME[0]}(): Adding $venv/bin to PATH" - PATH=$venv/bin:$PATH + ;; + esac + echo "${FUNCNAME[0]}(): Adding $lf_tmp_venv/bin to PATH" + PATH=$lf_tmp_venv/bin:$PATH + return 0 -} # ENd lf-venv-activate() +} # End lf-activate-venv() ################################################################################ # # NAME -# lf-venv-create python [package]... +# lf-git-validate-jira-urls # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # -# lf-venv-create python3 tox tox-pyenv virtualenv -# lf-venv-create python3.6 +# lf-git-validate-jira-urls # # DESCRIPTION -# This function will create/update a Python Virtual Environment (venv) based -# on the python specified. The 'python' argument must be in the PATH. The venv -# will be located in ~/.venv## where ## comes from the 'python' argument. -# I.E. python3 -> ~/.venv3. The resulting venv will be left 'read-only' to -# discourage the installation of any other packages (except by -# lf-venv-create()). By default, only versioned packages will be installed, so -# any required packages need to be specified. By default the 'pip install -# --upgrade' will be run multiple times. Sometimes pip needs that to get the -# versioning correct. +# Check for JIRA URLS in the commit message # # RETURN VALUES -# None +# OK: 0 +# Fail: 1 # ################################################################################ -function lf-venv-create() +function lf-git-validate-jira-urls() { - if (( $# < 1 )); then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Required Arguments" - return 1 + echo "Checking for JIRA URLs in commit message..." + # if JIRA_URL is not defined, nothing to do + if [[ -v JIRA_URL ]]; then + base_url=$(echo "$JIRA_URL" | awk -F'/' '{print $3}') + jira_link=$(git rev-list --format=%B --max-count=1 HEAD | grep -io "http[s]*://$base_url/" || true) + if [[ -n $jira_link ]]; then + lf-echo-error 'Remove JIRA URLs from commit message' + lf-echo-error 'Add jira references as: Issue: -, instead of URLs' + return 1 + fi fi - python=$1 - shift - if ! type $python > /dev/null; then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" - return 1 - fi - local pkg_list="$* " - local venv=~/.venv${python#python} - local suffix=$python-$$ - local pip_log=/tmp/pip_log.$suffix - if [[ -f $venv/$_lf_done_file ]]; then - echo "Venv Already Exists: '$venv'" - return - fi - # Make sure noting is left over - if [[ -d $venv ]]; then - chmod -R +w $venv - rm -rf $venv - fi - - echo "Creating '$python' venv ($venv)" - - case $python in - python2*) - # For Python2, just create venv and install pip - virtualenv -p $python $venv > $pip_log || return 1 - $venv/bin/pip install --upgrade pip > $pip_log || return 1 - $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1 - ;; - python3*) - # Include any packages that are tied to a specific version - pkg_list+="jenkins-job-builder==2.8.0 " - $python -m venv $venv > $pip_log - $venv/bin/pip install --upgrade pip > $pip_log || return 1 - # Redirect errors for now - $venv/bin/pip install --upgrade $pkg_list >> $pip_log 2> /dev/null || return 1 - # Generate list of packages - pkg_list=$($venv/bin/pip freeze | awk -F '=' '{print $1}') || return 1 - # Update all packages, may need to run twice to get all versions - # synced up. Ignore exit status on first try - $venv/bin/pip install --upgrade $pkg_list >> $pip_log || true - echo "Running 'pip --upgrade' to validate..." - $venv/bin/pip install --upgrade $pkg_list >> $pip_log || return 1 - ;; - *) - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python" - return 1 - ;; - esac - - touch $venv/$_lf_done_file - # Once this venv is created, make it read-only - chmod -R -w $venv - # Archive output of 'pip freeze' - mkdir -p $WORKSPACE/archives - $venv/bin/pip freeze > $WORKSPACE/archives/freeze-log-$python || return 1 - rm -rf $pip_log - -} # End lf-venv-create() + return 0 +} ################################################################################ # # NAME -# lf-venv-add() +# lf-jjb-check-ascii() # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # -# lf-venv-add python3 pkg -# or -# lf-venv-add python2 pkg1 pkg2 pkg3 +# lf-jjb-check-ascii # # DESCRIPTION -# This function will add one or more python packages to an existing venv. -# Attempts to add packages directly (pip) will result in errors because -# the venv does not have 'write' permission. +# Check for JJB YAML files containing non-printable ascii characters. This +# function must be run from the top of the global-jjb repo. # # RETURN VALUES -# None +# OK: 0 +# Fail: 1 # ################################################################################ -function lf-venv-add() +function lf-jjb-check-ascii() { - if (( $# < 2 )); then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Missing Package argument" + if [[ ! -d "jjb" ]]; then + lf-echo-error "${FUNCNAME[0]}(): ERROR: missing jjb directory" + lf-echo-error "This function can only be run from top of global-jjb directory" return 1 fi - python=$1 - if ! type $python > /dev/null ; then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" - return 1 - fi - shift - local venv=~/.venv${python#python} - if [[ ! -f $venv/$_lf_done_file ]]; then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: '$venv' is not a valid venv" + if LC_ALL=C grep -I -r '[^[:print:][:space:]]' jjb/; then + lf-echo-error "${FUNCNAME[0]}(): ERROR: Found YAML files containing non-printable characters." return 1 fi - local pkg_list=$* - local pip_log=/tmp/pip_log-$$ - - echo "Installing '$pkg_list' into $venv" - chmod -R +w $venv - rm $venv/$_lf_done_file - $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1 - pkg_list=$($venv/bin/pip freeze | awk -F '=' '{print $1}') || return 1 - $venv/bin/pip install --upgrade $pkg_list > $pip_log || return 1 - touch $venv/$_lf_done_file - chmod -R -w $venv - # Archive output of 'pip freeze' - $venv/bin/pip freeze > $WORKSPACE/archives/freeze-log-$python || return 1 - -} # End lf-venv-add() + echo "${FUNCNAME[0]}(): INFO: All JJB YAML files contain only printable ASCII characters" + return 0 +} ################################################################################ # Functions that assign Variables diff --git a/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml b/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml new file mode 100644 index 00000000..e7241aae --- /dev/null +++ b/releasenotes/notes/update-bash-library-aae95b6ccfb28c0d.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Created new function lf-activate-venv(). This function creates a venv in + /tmp and prepends the bin directory to the PATH. The 'pip install' command + now specifies: '--upgrade-strategy eager'. lf-activate-venv() supports an + optional --python flag to specify which python to use to create the venv, + the default is python3. + + Two new functions: lf-git-validate-jira-urls() and lf-jjb-check-ascii(). + They will be used to replace the git-validate-jira-urls.sh & + jjb-check-unicode.sh scripts at some point. For now, they are not being + used. +other: + - | + The lf-venv-add(), lf-venv-create() & lf-venv-activate functions have been + removed. No-one is accessing it yet. -- 2.16.6