From 1a7660851f6e1a32ff2485b75042c65fa8c6e611 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Tue, 23 Aug 2022 14:02:18 +1000 Subject: [PATCH] Feat: Set python3 version from pyenv Use pyenv whih is the standard way to manage, set and use a python3 installation on the system. The required version of python3 for all jobs should be > 3.8.x, to avoid PyPI dependencies conflicts with outdated versions. However the lf-activate-venv() uses the system default version python installed through packages. This can cause warning and build failures that source lf-env.sh. Update lf-activate-venv to use pyenv versions of python3 installed through the lfit.python-install galaxy ansible role. Issue-ID: RELENG-4357 Change-Id: I821a86ac3b54f284e853316f0d782ad551249925 Signed-off-by: Anil Belur --- jenkins-init-scripts/lf-env.sh | 70 ++++++++++++++++++++-- .../notes/set-pyenv-python3-18154ae118e9e012.yaml | 13 ++++ shell/python-tools-install.sh | 16 +++++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/set-pyenv-python3-18154ae118e9e012.yaml diff --git a/jenkins-init-scripts/lf-env.sh b/jenkins-init-scripts/lf-env.sh index a1bd350c..97318e2b 100644 --- a/jenkins-init-scripts/lf-env.sh +++ b/jenkins-init-scripts/lf-env.sh @@ -167,10 +167,6 @@ lf-activate-venv () { return 1 ;; esac done - if ! type "$python" > /dev/null; then - lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" - return 1 - fi echo "${FUNCNAME[0]}(): INFO: Creating $python venv at $lf_venv" @@ -191,6 +187,18 @@ lf-activate-venv () { ;; python3*) local pkg_list="" + # Use pyenv for selecting the python version + if [[ -d "/opt/pyenv" ]]; then + # set_python_version = pyver "${python//[a-zA-Z]/}" + echo "---> Setting up pyenv" + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv versions + if command -v pyenv 1>/dev/null 2>&1; then + eval "$(pyenv init - --no-rehash)" + pyenv local $(lf-pyver "${python}") + fi + fi # Add version specifier for some packages for arg in "$@"; do case $arg in @@ -216,6 +224,12 @@ lf-activate-venv () { return 1 ;; esac + + if ! type "$python" > /dev/null; then + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" + return 1 + fi + if $set_path; then echo "${FUNCNAME[0]}(): INFO: Adding $lf_venv/bin to PATH" PATH=$lf_venv/bin:$PATH @@ -323,3 +337,51 @@ lf-set-maven-options () { -Dmaven.repo.local=/tmp/r \ -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r" } + +################################################################################ +# +# NAME +# lf-pyver [python-version X.Y] +# +# SYNOPSIS +# pyver 3.8 (outputs 3.8.13) +# or +# pyver 3.10 (outputs 3.10.6) +# or +# pyver 3 (outputs the most recent version 3.10.6) +# +# DESCRIPTION +# The function takes short python version in the format and X.Y and prints +# the semver format (X.Y.Z) of the version that has been installed on the system +# with pyenv. +# +# When the expected version is not installed, nothing is returned. +# +# RETURN VALUES +# OK: 0 +# Fail: 1 +# +################################################################################ + +lf-pyver() { + local py_version_xy="${1:-python3}" + local py_version_xyz="" + + pyenv versions | sed 's/[[:alpha:]|(|)|/||*|[:space:]]//g'| tr -d ' ' \ + > "/tmp/.pyenv_versions" + if [[ ! -s "/tmp/.pyenv_versions" ]]; then + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: pyenv not available" + return 1 + fi + + # strip out any prefix for (ex: 'python3.8' or 'v3.8') and match regex + # to the output return by pyenv + py_version_xyz=$(grep "^${py_version_xy//[a-zA-Z]/}\(\..*\)\?\.[0-9]\+$" \ + "/tmp/.pyenv_versions" | sort -V | tail -n 1;) + if [[ ! -n ${py_version_xyz} ]]; then + lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Not installed on host: ${py_version_xy}" + return 1 + fi + echo "${py_version_xyz}" + return 0 +} diff --git a/releasenotes/notes/set-pyenv-python3-18154ae118e9e012.yaml b/releasenotes/notes/set-pyenv-python3-18154ae118e9e012.yaml new file mode 100644 index 00000000..4cd6ca6d --- /dev/null +++ b/releasenotes/notes/set-pyenv-python3-18154ae118e9e012.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + Use pyenv whih is the standard way to manage, set and use a python3 + installation on the system. + + The required version of python3 for all jobs should be > 3.8.x, to + avoid PyPI dependencies conflicts with outdated versions. However, the + lf-activate-venv() uses the system default version python installed through + packages. This can cause warning and build failures that source lf-env.sh. + + Update lf-activate-venv to use pyenv versions of python3 installed through + the lfit.python-install galaxy ansible role. diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index 27b7aa4e..8251487d 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -12,6 +12,22 @@ echo "---> python-tools-install.sh" set -eufo pipefail +# Souce the python version from lf-env.sh if available. +python="python3.8.13" +if [[ -f ~/lf-env.sh ]]; then + source ~/lf-env.sh + lf-activate-venv --python "$python" lftools +elif [[ -d /opt/pyenv ]]; then + echo "---> Setting up pyenv" + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv versions + if command -v pyenv 1>/dev/null 2>&1; then + eval "$(pyenv init - --no-rehash)" + pyenv local "$python" + fi +fi + # This script will typically be called during pre-build & post-build. # Create the user venv during pre-build. if [[ ! -f /tmp/pre-build-complete ]]; then -- 2.16.6