From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Tue, 13 Nov 2018 06:03:09 +0000 (+0800)
Subject: Migrate cleanup orphaned nodes cron script
X-Git-Tag: v0.27.0~3
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=ec94b22a64e710af14e6f4f862a4d7ccb3a8e030;p=releng%2Fglobal-jjb.git

Migrate cleanup orphaned nodes cron script

Generalize script for reuse in global-jjb.

Issue: RELENG-1434
Change-Id: I7901e58dbc0136c339a2d488c9eef8d46cd5446a
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---

diff --git a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml
index 61edaf99..53188362 100644
--- a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml
+++ b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml
@@ -13,5 +13,6 @@
     openstack-image-cleanup: false
     openstack-image-cleanup-age: 42
     openstack-image-protect: false
+    openstack-server-cleanup: false
     openstack-stack-cleanup: false
     openstack-volume-cleanup: false
diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst
index f86c0560..96f694af 100644
--- a/docs/jjb/lf-ci-jobs.rst
+++ b/docs/jjb/lf-ci-jobs.rst
@@ -577,6 +577,8 @@ containing the credentials for the cloud.
         removal. (default: 30)
     :openstack-image-protect: Whether or not to run the image protect script.
         (default: true)
+    :openstack-server-cleanup: Whether or not to run the server cleanup script.
+        (default: true)
     :openstack-stack-cleanup: Whether or not to run the stack cleanup script.
         (default: true)
     :openstack-volume-cleanup: Whether or not to run the volume cleanup script.
diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index 61e74f31..9b862bfe 100644
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -1242,6 +1242,7 @@
     openstack-image-cleanup: true
     openstack-image-cleanup-age: 30
     openstack-image-protect: true
+    openstack-server-cleanup: true
     openstack-stack-cleanup: true
     openstack-volume-cleanup: true
     stream: master
@@ -1299,6 +1300,12 @@
           condition-expression: '{openstack-stack-cleanup}'
           steps:
             - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-stacks.sh
+      # Servers
+      - conditional-step:
+          condition-kind: boolean-expression
+          condition-expression: '{openstack-server-cleanup}'
+          steps:
+            - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-servers.sh
       # Volumes
       - conditional-step:
           condition-kind: boolean-expression
diff --git a/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml b/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml
new file mode 100644
index 00000000..542c2690
--- /dev/null
+++ b/releasenotes/notes/migrate-heat-stale-servers-621af66b14c4a71e.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    The **openstack-cron** job now has the ability to remove orphaned
+    servers.
diff --git a/shell/openstack-cleanup-orphaned-servers.sh b/shell/openstack-cleanup-orphaned-servers.sh
new file mode 100644
index 00000000..629a3a0f
--- /dev/null
+++ b/shell/openstack-cleanup-orphaned-servers.sh
@@ -0,0 +1,82 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017, 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# Scans OpenStack for orphaned servers
+echo "---> Orphaned servers"
+
+os_cloud="${OS_CLOUD:-vex}"
+jenkins_urls="${JENKINS_URLS:-}"
+
+minion_in_jenkins() {
+    # Usage: minion_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
+    # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
+
+    MINION="${1}"
+
+    minions=()
+    for jenkins in "${@:2}"; do
+        JENKINS_URL="$jenkins/computer/api/json?tree=computer[displayName]"
+        resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
+        json_data=$(echo "$resp" | head -n1)
+        status=$(echo "$resp" | awk 'END {print $NF}')
+
+        if [ "$status" != 200 ]; then
+            >&2 echo "ERROR: Failed to fetch data from $JENKINS_URL with status code $status"
+            >&2 echo "$resp"
+            exit 1
+        fi
+
+        # We purposely want to wordsplit here to combine the arrays
+        # shellcheck disable=SC2206,SC2207
+        minions=(${minions[@]} $(echo "$json_data" | \
+            jq -r '.computer[].displayName' | grep -v master)
+        )
+    done
+
+    if [[ "${minions[*]}" =~ $MINION ]]; then
+        return 0
+    fi
+
+    return 1
+}
+
+##########################
+## FETCH ACTIVE MINIONS ##
+##########################
+# Fetch server list before fetching active minions to minimize race condition
+# where we might be trying to delete servers while jobs are trying to start
+
+mapfile -t OS_SERVERS < <(openstack --os-cloud "$os_cloud" server list -f value -c "Name" | grep -E 'prd|snd')
+
+echo "-----> Active servers"
+for server in "${OS_SERVERS[@]}"; do
+    echo "$server"
+done
+
+
+#############################
+## DELETE ORPHANED SERVERS ##
+#############################
+echo "-----> Delete orphaned servers"
+
+# Search for servers not in use by any active Jenkins systems and remove them.
+for server in "${OS_SERVERS[@]}"; do
+    # jenkins_urls intentially needs globbing to be passed a separate params.
+    # needs to be globbed.
+    # shellcheck disable=SC2153,SC2086
+    if minion_in_jenkins "$server" $jenkins_urls; then
+        # No need to delete server if it is still attached to Jenkins
+        continue
+    else
+        echo "Deleting orphaned server: $server"
+        lftools openstack --os-cloud "$os_cloud" \
+            server remove --minutes 15 "$server"
+    fi
+done