2 # SPDX-License-Identifier: EPL-1.0
3 ##############################################################################
4 # Copyright (c) 2017, 2018 The Linux Foundation and others.
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Eclipse Public License v1.0
8 # which accompanies this distribution, and is available at
9 # http://www.eclipse.org/legal/epl-v10.html
10 ##############################################################################
11 # Scans OpenStack for orphaned stacks
12 echo "---> Orphaned stacks"
14 os_cloud="${OS_CLOUD:-vex}"
15 jenkins_urls="${JENKINS_URLS:-}"
18 # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
19 # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
24 for jenkins in "${@:2}"; do
25 PARAMS="tree=computer[executors[currentExecutable[url]],"
26 PARAMS=$PARAMS"oneOffExecutors[currentExecutable[url]]]"
27 PARAMS=$PARAMS"&xpath=//url&wrapper=builds"
28 JENKINS_URL="$jenkins/computer/api/json?$PARAMS"
29 resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
30 json_data=$(echo "$resp" | head -n1)
31 status=$(echo "$resp" | awk 'END {print $NF}')
33 if [ "$status" != 200 ]; then
34 >&2 echo "ERROR: Failed to fetch data from $JENKINS_URL with status code $status"
39 if [[ "${jenkins}" == *"jenkins."*".org" ]] || [[ "${jenkins}" == *"jenkins."*".io" ]]; then
42 silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}')
45 # We purposely want to wordsplit here to combine the arrays
46 # shellcheck disable=SC2206,SC2207
47 builds=(${builds[@]} $(echo "$json_data" | \
48 jq -r '.computer[].executors[].currentExecutable.url' \
49 | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}')
53 if [[ "${builds[*]}" =~ $STACK_NAME ]]; then
60 # shellcheck disable=SC1090
63 # Check if openstack venv was previously created
64 if [ -f "/tmp/.os_lf_venv" ]; then
65 os_lf_venv=$(cat "/tmp/.os_lf_venv")
68 if [ -d "${os_lf_venv}" ] && [ -f "${os_lf_venv}/bin/openstack" ]; then
69 echo "Re-use existing venv: ${os_lf_venv}"
70 PATH=$os_lf_venv/bin:$PATH
72 lf-activate-venv --python python3 "cryptography<3.4" \
73 "lftools[openstack]" \
77 python-openstackclient \
85 #########################
86 ## FETCH ACTIVE BUILDS ##
87 #########################
88 # Fetch COE cluster list before fetching active stacks. K8s cluster creates
89 # stack that does not match JOB_NAME, therefore ignore them while processing
90 # orphaned stacks and handle them separatly.
91 # The stack naming scheme is limited in the source code to take only first 20
92 # chars from the JOB_NAME, and the rest is randomly generated value for
94 # https://github.com/openstack/magnum/blob/master/magnum/drivers/heat/driver.py#L202-L212
95 mapfile -t OS_COE_CLUSTERS_ID < <(openstack --os-cloud "${os_cloud}" coe cluster list \
96 -f value -c "uuid" -c "name" \
97 | grep -E '(DELETE_FAILED|UNKNOWN|UNHEALTHY)' | awk '{print $1}')
99 echo "-----> Active clusters -> stacks"
100 # mapfile -t OS_COE_STACKS_ID
102 for cluster_id in "${OS_COE_CLUSTERS_ID[@]}"; do
103 # find active stacks id associated with the COE cluster
104 stack_id=$(openstack --os-cloud "${os_cloud}" coe cluster show "${cluster_id}" \
105 -f value -c "stack_id")
106 # get the stack name associated with the COE cluster
107 stack_name=$(openstack --os-cloud "${os_cloud}" stack show "${stack_id}" \
108 -f value -c "stack_name")
109 OS_COE_STACKS+=("${stack_id}")
110 echo "clusterid:${cluster_id} -> stackid:${stack_id} stack_name: ${stack_name}"
113 if [[ ${#OS_COE_STACKS[@]} -gt "0" ]]; then
114 echo "${OS_COE_STACKS[*]}"
115 echo "-----> Active COE cluster stacks"
116 for cstack in "${OS_COE_STACKS[@]}"; do
121 # Fetch stack list before fetching active builds to minimize race condition
122 # where we might be try to delete stacks while jobs are trying to start
123 mapfile -t OS_STACKS < <(openstack --os-cloud "$os_cloud" stack list \
124 -f value -c "Stack Name" -c "Stack Status" \
125 --property "stack_status=CREATE_COMPLETE" \
126 --property "stack_status=DELETE_FAILED" \
127 --property "stack_status=CREATE_FAILED" \
130 echo "-----> Active stacks"
131 for stack in "${OS_STACKS[@]}"; do
136 ##########################
137 ## DELETE UNUSED STACKS ##
138 ##########################
139 echo "-----> Delete orphaned stacks"
141 # Search for stacks not in use by any active Jenkins systems and remove them.
142 for STACK_NAME in "${OS_STACKS[@]}"; do
143 # Check for COE cluster stack is present
144 # shellcheck disable=SC2153,SC2086
145 if [[ ${#OS_COE_STACKS[@]} -gt "0" ]] && [[ ${OS_COE_STACKS[*]} =~ ${STACK_NAME} ]]; then
146 # Do not delete a stack linked to COE cluster, handle them separatly.
148 # jenkins_urls intentially needs globbing to be passed a separate params.
149 elif stack_in_jenkins "$STACK_NAME" $jenkins_urls; then
150 # No need to delete stacks if there exists an active build for them
153 echo "Deleting orphaned stack: ${STACK_NAME}"
154 lftools openstack --os-cloud "${os_cloud}" stack delete --force "${STACK_NAME}"