From ec94b22a64e710af14e6f4f862a4d7ccb3a8e030 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Tue, 13 Nov 2018 14:03:09 +0800 Subject: [PATCH] Migrate cleanup orphaned nodes cron script Generalize script for reuse in global-jjb. Issue: RELENG-1434 Change-Id: I7901e58dbc0136c339a2d488c9eef8d46cd5446a Signed-off-by: Thanh Ha --- .jjb-test/lf-ci-jobs/openstack-cron-full.yaml | 1 + docs/jjb/lf-ci-jobs.rst | 2 + jjb/lf-ci-jobs.yaml | 7 ++ ...igrate-heat-stale-servers-621af66b14c4a71e.yaml | 5 ++ shell/openstack-cleanup-orphaned-servers.sh | 82 ++++++++++++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml create mode 100644 shell/openstack-cleanup-orphaned-servers.sh diff --git a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml index 61edaf99..53188362 100644 --- a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml +++ b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml @@ -13,5 +13,6 @@ openstack-image-cleanup: false openstack-image-cleanup-age: 42 openstack-image-protect: false + openstack-server-cleanup: false openstack-stack-cleanup: false openstack-volume-cleanup: false diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index f86c0560..96f694af 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -577,6 +577,8 @@ containing the credentials for the cloud. removal. (default: 30) :openstack-image-protect: Whether or not to run the image protect script. (default: true) + :openstack-server-cleanup: Whether or not to run the server cleanup script. + (default: true) :openstack-stack-cleanup: Whether or not to run the stack cleanup script. (default: true) :openstack-volume-cleanup: Whether or not to run the volume cleanup script. diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 61e74f31..9b862bfe 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1242,6 +1242,7 @@ openstack-image-cleanup: true openstack-image-cleanup-age: 30 openstack-image-protect: true + openstack-server-cleanup: true openstack-stack-cleanup: true openstack-volume-cleanup: true stream: master @@ -1299,6 +1300,12 @@ condition-expression: '{openstack-stack-cleanup}' steps: - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-stacks.sh + # Servers + - conditional-step: + condition-kind: boolean-expression + condition-expression: '{openstack-server-cleanup}' + steps: + - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-servers.sh # Volumes - conditional-step: condition-kind: boolean-expression diff --git a/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml b/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml new file mode 100644 index 00000000..542c2690 --- /dev/null +++ b/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + The **openstack-cron** job now has the ability to remove orphaned + servers. diff --git a/shell/openstack-cleanup-orphaned-servers.sh b/shell/openstack-cleanup-orphaned-servers.sh new file mode 100644 index 00000000..629a3a0f --- /dev/null +++ b/shell/openstack-cleanup-orphaned-servers.sh @@ -0,0 +1,82 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017, 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Scans OpenStack for orphaned servers +echo "---> Orphaned servers" + +os_cloud="${OS_CLOUD:-vex}" +jenkins_urls="${JENKINS_URLS:-}" + +minion_in_jenkins() { + # Usage: minion_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...] + # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins. + + MINION="${1}" + + minions=() + for jenkins in "${@:2}"; do + JENKINS_URL="$jenkins/computer/api/json?tree=computer[displayName]" + resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL") + json_data=$(echo "$resp" | head -n1) + status=$(echo "$resp" | awk 'END {print $NF}') + + if [ "$status" != 200 ]; then + >&2 echo "ERROR: Failed to fetch data from $JENKINS_URL with status code $status" + >&2 echo "$resp" + exit 1 + fi + + # We purposely want to wordsplit here to combine the arrays + # shellcheck disable=SC2206,SC2207 + minions=(${minions[@]} $(echo "$json_data" | \ + jq -r '.computer[].displayName' | grep -v master) + ) + done + + if [[ "${minions[*]}" =~ $MINION ]]; then + return 0 + fi + + return 1 +} + +########################## +## FETCH ACTIVE MINIONS ## +########################## +# Fetch server list before fetching active minions to minimize race condition +# where we might be trying to delete servers while jobs are trying to start + +mapfile -t OS_SERVERS < <(openstack --os-cloud "$os_cloud" server list -f value -c "Name" | grep -E 'prd|snd') + +echo "-----> Active servers" +for server in "${OS_SERVERS[@]}"; do + echo "$server" +done + + +############################# +## DELETE ORPHANED SERVERS ## +############################# +echo "-----> Delete orphaned servers" + +# Search for servers not in use by any active Jenkins systems and remove them. +for server in "${OS_SERVERS[@]}"; do + # jenkins_urls intentially needs globbing to be passed a separate params. + # needs to be globbed. + # shellcheck disable=SC2153,SC2086 + if minion_in_jenkins "$server" $jenkins_urls; then + # No need to delete server if it is still attached to Jenkins + continue + else + echo "Deleting orphaned server: $server" + lftools openstack --os-cloud "$os_cloud" \ + server remove --minutes 15 "$server" + fi +done -- 2.16.6