X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=blobdiff_plain;f=shell%2Fopenstack-cleanup-orphaned-stacks.sh;h=e9299b53b04d2a012cc46fc3ceaeb8e9bf7a4d03;hb=4867c0799530acf13f7527c6a026879bdfb34e06;hp=8140adea55368940b7409b19015221dcec8e499c;hpb=a958793eac5df84be84d5b29850515aaa6d3be3b;p=releng%2Fglobal-jjb.git

diff --git a/shell/openstack-cleanup-orphaned-stacks.sh b/shell/openstack-cleanup-orphaned-stacks.sh
index 8140adea..e9299b53 100644
--- a/shell/openstack-cleanup-orphaned-stacks.sh
+++ b/shell/openstack-cleanup-orphaned-stacks.sh
@@ -22,7 +22,10 @@ stack_in_jenkins() {
 
     builds=()
     for jenkins in "${@:2}"; do
-        JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
+        PARAMS="tree=computer[executors[currentExecutable[url]],"
+        PARAMS=$PARAMS"oneOffExecutors[currentExecutable[url]]]"
+        PARAMS=$PARAMS"&xpath=//url&wrapper=builds"
+        JENKINS_URL="$jenkins/computer/api/json?$PARAMS"
         resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
         json_data=$(echo "$resp" | head -n1)
         status=$(echo "$resp" | awk 'END {print $NF}')
@@ -54,12 +57,56 @@ stack_in_jenkins() {
     return 1
 }
 
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+lf-activate-venv --python python3 "lftools[openstack]" \
+    kubernetes \
+    niet \
+    python-heatclient \
+    python-openstackclient \
+    python-magnumclient \
+    yq
+
+set -x
 #########################
 ## FETCH ACTIVE BUILDS ##
 #########################
+# Fetch COE cluster list before fetching active stacks. K8s cluster creates
+# stack that does not match JOB_NAME, therefore ignore them while processing
+# orphaned stacks and handle them separatly.
+# The stack naming scheme is limited in the source code to take only first 20
+# chars from the JOB_NAME, and the rest is randomly generated value for
+# uniqueness:
+# https://github.com/openstack/magnum/blob/master/magnum/drivers/heat/driver.py#L202-L212
+mapfile -t OS_COE_CLUSTERS_ID < <(openstack --os-cloud "${os_cloud}" coe cluster list \
+            -f value -c "uuid" -c "name" \
+            | grep -E '(DELETE_FAILED|UNKNOWN|UNHEALTHY)' | awk '{print $1}')
+
+echo "-----> Active clusters -> stacks"
+# mapfile -t OS_COE_STACKS_ID
+OS_COE_STACKS=()
+for cluster_id in "${OS_COE_CLUSTERS_ID[@]}"; do
+    # find active stacks id associated with the COE cluster
+    stack_id=$(openstack --os-cloud "${os_cloud}" coe cluster show "${cluster_id}" \
+                -f value -c "stack_id")
+    # get the stack name associated with the COE cluster
+    stack_name=$(openstack --os-cloud "${os_cloud}" stack show "${stack_id}" \
+                -f value -c "stack_name")
+    OS_COE_STACKS+=("${stack_id}")
+    echo "clusterid:${cluster_id} -> stackid:${stack_id} stack_name: ${stack_name}"
+done
+
+if [[ ${#OS_COE_STACKS[@]} -gt "0" ]]; then
+    echo "${OS_COE_STACKS[*]}"
+    echo "-----> Active COE cluster stacks"
+    for cstack in "${OS_COE_STACKS[@]}"; do
+        echo "$cstack"
+    done
+fi
+
 # Fetch stack list before fetching active builds to minimize race condition
 # where we might be try to delete stacks while jobs are trying to start
-
 mapfile -t OS_STACKS < <(openstack --os-cloud "$os_cloud" stack list \
             -f value -c "Stack Name" -c "Stack Status" \
             --property "stack_status=CREATE_COMPLETE" \
@@ -80,13 +127,17 @@ echo "-----> Delete orphaned stacks"
 
 # Search for stacks not in use by any active Jenkins systems and remove them.
 for STACK_NAME in "${OS_STACKS[@]}"; do
-    # jenkins_urls intentially needs globbing to be passed a separate params.
+    # Check for COE cluster stack is present
     # shellcheck disable=SC2153,SC2086
-    if stack_in_jenkins "$STACK_NAME" $jenkins_urls; then
+    if [[ ${#OS_COE_STACKS[@]} -gt "0" ]] && [[ ${OS_COE_STACKS[*]} =~ ${STACK_NAME} ]]; then
+        # Do not delete a stack linked to COE cluster, handle them separatly.
+        continue
+    # jenkins_urls intentially needs globbing to be passed a separate params.
+    elif stack_in_jenkins "$STACK_NAME" $jenkins_urls; then
         # No need to delete stacks if there exists an active build for them
         continue
     else
-        echo "Deleting orphaned stack: $STACK_NAME"
-        lftools openstack --os-cloud "$os_cloud" stack delete "$STACK_NAME"
+        echo "Deleting orphaned stack: ${STACK_NAME}"
+        lftools openstack --os-cloud "${os_cloud}" stack delete --force "${STACK_NAME}"
     fi
 done