Fix: Use lf-activate-venv to install openstack dep 78/70578/3 v0.80.2
authorAnil Belur <abelur@linuxfoundation.org>
Thu, 1 Sep 2022 03:03:24 +0000 (13:03 +1000)
committerAnil Belur <abelur@linuxfoundation.org>
Fri, 2 Sep 2022 09:06:18 +0000 (19:06 +1000)
Using python-tools-install.sh for the pre/post build is not recommended
approach for installing python dependencies since this installs the
dependencies with `--user` option (removed in I821a86ac3b54f284e8).

Instead use lf-activate-venv to setup an venv and pull in the required
dependencies and save the path of the virtualenv in a temp file that
can be checked before attempting to create a venv.

Issue-ID: RELENG-4357
Change-Id: I00bbf08921a3ddc6349e74e9cd9d6316b0876749
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>
16 files changed:
jenkins-init-scripts/lf-env.sh
releasenotes/notes/fix-openstack-pre-reqs-014ed3db94365608.yaml [new file with mode: 0644]
shell/jenkins-verify-images.sh
shell/logs-deploy.sh
shell/openstack-cleanup-old-images.sh
shell/openstack-cleanup-orphaned-k8s-clusters.sh
shell/openstack-cleanup-orphaned-ports.sh
shell/openstack-cleanup-orphaned-servers.sh
shell/openstack-cleanup-orphaned-stacks.sh
shell/openstack-cleanup-orphaned-volumes.sh
shell/openstack-kubernetes-create.sh
shell/openstack-protect-in-use-images.sh
shell/openstack-stack-copy-ssh-keys.sh
shell/openstack-stack-create.sh
shell/openstack-stack-delete.sh
shell/python-tools-install.sh

index a2b8cb2..114f0c1 100644 (file)
@@ -190,7 +190,7 @@ lf-activate-venv () {
         # Use pyenv for selecting the python version
         if [[ -d "/opt/pyenv" ]]; then
             # set_python_version = pyver "${python//[a-zA-Z]/}"
-            echo "---> Setting up pyenv"
+            echo "Setup pyenv:"
             export PYENV_ROOT="/opt/pyenv"
             export PATH="$PYENV_ROOT/bin:$PATH"
             pyenv versions
diff --git a/releasenotes/notes/fix-openstack-pre-reqs-014ed3db94365608.yaml b/releasenotes/notes/fix-openstack-pre-reqs-014ed3db94365608.yaml
new file mode 100644 (file)
index 0000000..a08afaa
--- /dev/null
@@ -0,0 +1,15 @@
+---
+issues:
+  - |
+    Error: openstack: command not found
+fixes:
+  - |
+    Use lf-activate-venv to install openstack deps
+
+    Using python-tools-install.sh for the pre/post build is not recommended
+    approach for installing python dependencies since this installs the
+    dependencies with `--user` option (removed in I821a86ac3b54f284e8).
+
+    Instead use lf-activate-venv to setup an venv and pull in the required
+    dependencies and save the path of the virtualenv in a temp file that
+    can be checked before attempting to create a venv.
index 483a51b..5db7fec 100755 (executable)
@@ -14,6 +14,19 @@ echo "---> jenkins-verify-images.sh"
 
 set -eux -o pipefail
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 python-openstackclient
+fi
 error=false
 
 verify_images()
index d797fa7..c28d92e 100644 (file)
@@ -12,7 +12,7 @@ echo "---> logs-deploy.sh"
 
 # shellcheck disable=SC1090
 source ~/lf-env.sh
-lf-activate-venv --python python3.8 lftools
+lf-activate-venv --python python3 lftools
 
 # Ensure we fail the job if any steps fail
 # Disable 'globbing'
index cbb2ef8..8d1ddd5 100644 (file)
 # Removes openstack images older than X days in the cloud
 echo "---> Cleanup old images"
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+set -x
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
+
 os_cloud="${OS_CLOUD:-vex}"
 os_image_cleanup_age="${OS_IMAGE_CLEANUP_AGE:-30}"
 
index b1f3923..33fcaa6 100644 (file)
@@ -58,6 +58,24 @@ cluster_in_jenkins() {
     return 1
 }
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 \
+        kubernetes \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient
+fi
+
 #########################
 ## FETCH ACTIVE BUILDS ##
 #########################
index 080b590..e1f8436 100644 (file)
 # Scans OpenStack for orphaned ports
 echo "---> Orphaned ports"
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
 os_cloud="${OS_CLOUD:-vex}"
 
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 \
+        python-heatclient \
+        python-openstackclient
+fi
+
 set -eux -o pipefail
 
 mapfile -t os_ports_ts < <(openstack --os-cloud "$os_cloud" port list \
index 629a3a0..4dad6ed 100644 (file)
@@ -47,6 +47,29 @@ minion_in_jenkins() {
     return 1
 }
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
 ##########################
 ## FETCH ACTIVE MINIONS ##
 ##########################
index cc150ac..187a83d 100644 (file)
@@ -56,6 +56,31 @@ stack_in_jenkins() {
 
     return 1
 }
+
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
+
 set -x
 #########################
 ## FETCH ACTIVE BUILDS ##
index 51b4e58..c5cc0b2 100644 (file)
@@ -15,6 +15,28 @@ os_cloud="${OS_CLOUD:-vex}"
 
 set -eux -o pipefail
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
 mapfile -t os_volumes < <(openstack --os-cloud "$os_cloud" volume list -f value -c ID --status Available)
 
 if [ ${#os_volumes[@]} -eq 0 ]; then
index 2f1a2b7..5aef45d 100755 (executable)
@@ -13,6 +13,30 @@ echo "---> Creating kubernetes cluster"
 
 set -eux -o pipefail
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
+
 os_cloud="${OS_CLOUD:-vex}"
 fixed_network="${FIXED_NETWORK}"
 fixed_subnet="${FIXED_SUBNET}"
index 468ea0d..0729440 100644 (file)
@@ -19,6 +19,23 @@ set -eu -o pipefail
 echo "---> Protect in-use images"
 os_cloud="${OS_CLOUD:-vex}"
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 \
+        python-heatclient \
+        python-openstackclient
+fi
+
 images=()
 while read -r -d $'\n' ; do
     images+=("$REPLY")
index 79d75d9..0509bb8 100644 (file)
@@ -50,6 +50,23 @@ copy_ssh_keys () {
     done
 }
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 \
+        python-heatclient \
+        python-openstackclient
+fi
+
 # IP Addresses are returned as a space separated list so word splitting is ok
 # shellcheck disable=SC2207
 ip_addresses=($(openstack --os-cloud "$os_cloud" stack show -f json -c outputs "$stack_name" |
index 478ac5b..0a2dc9f 100644 (file)
@@ -18,6 +18,29 @@ stack_parameters="$WORKSPACE/stack-parameters.yaml"
 
 set -eux -o pipefail
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 "cryptography<3.4" \
+        "lftools[openstack]" \
+        kubernetes \
+        "niet~=1.4.2" \
+        python-heatclient \
+        python-openstackclient \
+        python-magnumclient \
+        setuptools \
+        "openstacksdk<0.99" \
+        yq
+fi
 openstack --os-cloud "$os_cloud" limits show --absolute
 
 pushd "$stack_template_dir" || exit 1
index 79da58a..512ea99 100644 (file)
@@ -15,7 +15,19 @@ set -eufo pipefail
 # shellcheck disable=SC1090
 source ~/lf-env.sh
 
-lf-activate-venv lftools[openstack] python-openstackclient
+# Check if openstack venv was previously created
+if [ -f "/tmp/.os_lf_venv" ]; then
+    os_lf_venv=$(cat "/tmp/.os_lf_venv")
+fi
+
+if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
+    echo "Re-use existing venv: ${os_lf_venv}"
+    PATH=$os_lf_venv/bin:$PATH
+else
+    lf-activate-venv --python python3 lftools[openstack] \
+        python-heatclient \
+        python-openstackclient
+fi
 
 echo "INFO: Retrieving stack cost for: $OS_STACK_NAME"
 if ! lftools openstack --os-cloud "$OS_CLOUD" stack cost "$OS_STACK_NAME" > stack-cost; then
@@ -28,4 +40,3 @@ fi
 # Delete the stack even if the stack-cost script fails
 lftools openstack --os-cloud "$OS_CLOUD" stack delete "$OS_STACK_NAME" \
     | echo "INFO: $(cat)"
-
index c817360..e5d836b 100644 (file)
@@ -17,8 +17,10 @@ python="python3"
 if [[ -f ~/lf-env.sh ]]; then
     source ~/lf-env.sh
     lf-activate-venv --python "$python" lftools
+    # Save the virtualenv path
+    echo "$lf_venv" > "/tmp/.os_lf_venv"
 elif [[ -d /opt/pyenv ]]; then
-    echo "---> Setting up pyenv"
+    echo "Setup up pyenv"
     export PYENV_ROOT="/opt/pyenv"
     export PATH="$PYENV_ROOT/bin:$PATH"
     pyenv versions