From 0b174473134b72965f9642a32d24dc43279a014a Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Tue, 13 Nov 2018 11:39:45 +0800 Subject: [PATCH] Migrate cleanup-stale-stacks cron script Generalize script for use in global-jjb. Issue: RELENG-1432 Change-Id: I9c450c36b0f44a5583bee1932016e7cb3d5ef834 Signed-off-by: Thanh Ha --- .jjb-test/lf-ci-jobs.yaml | 8 ++ .jjb-test/lf-ci-jobs/openstack-cron-full.yaml | 5 ++ .jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml | 4 + docs/jjb/lf-ci-jobs.rst | 5 +- jjb/lf-ci-jobs.yaml | 14 +++- ...migrate-heat-stale-stacks-29c3abd7b989aac6.yaml | 13 ++++ shell/openstack-cleanup-orphaned-stacks.sh | 86 ++++++++++++++++++++++ 7 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/migrate-heat-stale-stacks-29c3abd7b989aac6.yaml create mode 100644 shell/openstack-cleanup-orphaned-stacks.sh diff --git a/.jjb-test/lf-ci-jobs.yaml b/.jjb-test/lf-ci-jobs.yaml index d80ae610..5e716d9f 100644 --- a/.jjb-test/lf-ci-jobs.yaml +++ b/.jjb-test/lf-ci-jobs.yaml @@ -22,6 +22,10 @@ project-name: gerrit-ciman + jenkins-urls: > + https://jenkins.example.org + https://jenkins.example.org/sandbox + - project: name: github-openstack-jobs jobs: @@ -29,6 +33,10 @@ project-name: github-ciman + jenkins-urls: > + https://jenkins.example.org + https://jenkins.example.org/sandbox + - project: name: gerrit-packer-jobs diff --git a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml index f0724fac..61edaf99 100644 --- a/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml +++ b/.jjb-test/lf-ci-jobs/openstack-cron-full.yaml @@ -5,8 +5,13 @@ - 'gerrit-openstack-cron' project-name: ciman-full + + jenkins-urls: > + https://jenkins.example.org + https://jenkins.example.org/sandbox openstack-cloud: example-cloud openstack-image-cleanup: false openstack-image-cleanup-age: 42 openstack-image-protect: false + openstack-stack-cleanup: false openstack-volume-cleanup: false diff --git a/.jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml b/.jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml index a6fc0b46..d59c1ba1 100644 --- a/.jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml +++ b/.jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml @@ -5,3 +5,7 @@ - 'gerrit-openstack-cron' project-name: ci-management + + jenkins-urls: > + https://jenkins.example.org + https://jenkins.example.org/sandbox diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 3ab71f27..f86c0560 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -560,13 +560,14 @@ containing the credentials for the cloud. :build-node: The node to run build on. :jenkins-ssh-credential: Credential to use for SSH. (Generally should be configured in defaults.yaml) + :jenkins-urls: URLs to Jenkins systems to check for active builds. :Optional parameters: :branch: Git branch to fetch for the build. (default: master) :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) :build-timeout: Timeout in minutes before aborting build. (default: 90) - :cron: Time when the packer image should be rebuilt (default: @daily) + :cron: Time when the packer image should be rebuilt (default: @hourly) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) :openstack-cloud: OS_CLOUD setting to pass to openstack client. (default: vex) @@ -576,6 +577,8 @@ containing the credentials for the cloud. removal. (default: 30) :openstack-image-protect: Whether or not to run the image protect script. (default: true) + :openstack-stack-cleanup: Whether or not to run the stack cleanup script. + (default: true) :openstack-volume-cleanup: Whether or not to run the volume cleanup script. (default: true) :stream: Keyword that can be used to represent a release code-name. diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index b6397753..61e74f31 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1234,7 +1234,7 @@ branch: master build-days-to-keep: 7 build-timeout: 10 - cron: '@daily' + cron: '@hourly' disable-job: false git-url: '$GIT_URL/$PROJECT' github-url: 'https://github.com' @@ -1242,6 +1242,7 @@ openstack-image-cleanup: true openstack-image-cleanup-age: 30 openstack-image-protect: true + openstack-stack-cleanup: true openstack-volume-cleanup: true stream: master submodule-timeout: 10 @@ -1265,6 +1266,11 @@ stream: '{stream}' branch: '{branch}' lftools-version: '{lftools-version}' + - string: + name: JENKINS_URLS + default: '{jenkins-urls}' + description: | + Space separated list of Jenkins URLs to check for active builds. wrappers: - lf-infra-wrappers: @@ -1287,6 +1293,12 @@ - lf-infra-pre-build - inject: properties-content: OS_CLOUD={openstack-cloud} + # Stacks + - conditional-step: + condition-kind: boolean-expression + condition-expression: '{openstack-stack-cleanup}' + steps: + - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-stacks.sh # Volumes - conditional-step: condition-kind: boolean-expression diff --git a/releasenotes/notes/migrate-heat-stale-stacks-29c3abd7b989aac6.yaml b/releasenotes/notes/migrate-heat-stale-stacks-29c3abd7b989aac6.yaml new file mode 100644 index 00000000..359c37f2 --- /dev/null +++ b/releasenotes/notes/migrate-heat-stale-stacks-29c3abd7b989aac6.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + The **openstack-cron** job now has the ability to remove orphaned + stacks. +upgrade: + - | + The **openstack-cron** job now requires a new parameter configured + ``jenkins-urls`` in order to use the job. +other: + - | + The **openstack-cron** job now runs every hour instead of daily. This is + because stack cleanup should happen more regularly. diff --git a/shell/openstack-cleanup-orphaned-stacks.sh b/shell/openstack-cleanup-orphaned-stacks.sh new file mode 100644 index 00000000..8f1851e1 --- /dev/null +++ b/shell/openstack-cleanup-orphaned-stacks.sh @@ -0,0 +1,86 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017, 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Scans OpenStack for orphaned stacks +echo "---> Orphaned stacks" + +os_cloud="${OS_CLOUD:-vex}" +jenkins_urls="${JENKINS_URLS:-}" + +stack_in_jenkins() { + # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...] + # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins. + + STACK_NAME="${1}" + + builds=() + for jenkins in "${@:2}"; do + JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds" + resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL") + json_data=$(echo "$resp" | head -n1) + #status=$(echo "$resp" | awk 'END {print $NF}') + + if [[ "${jenkins}" == *"jenkins."*".org" ]] || [[ "${jenkins}" == *"jenkins."*".io" ]]; then + silo="production" + else + silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}') + fi + export silo + # We purposely want to wordsplit here to combine the arrays + # shellcheck disable=SC2206,SC2207 + builds=(${builds[@]} $(echo "$json_data" | \ + jq -r '.computer[].executors[].currentExecutable.url' \ + | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}') + ) + done + + if [[ "${builds[*]}" =~ $STACK_NAME ]]; then + return 0 + fi + + return 1 +} + +######################### +## FETCH ACTIVE BUILDS ## +######################### +# Fetch stack list before fetching active builds to minimize race condition +# where we might be try to delete stacks while jobs are trying to start + +mapfile -t OS_STACKS < <(openstack --os-cloud "$os_cloud" stack list \ + -f value -c "Stack Name" -c "Stack Status" \ + --property "stack_status=CREATE_COMPLETE" \ + --property "stack_status=DELETE_FAILED" \ + --property "stack_status=CREATE_FAILED" \ + | awk '{print $1}') + +echo "-----> Active stacks" +for stack in "${OS_STACKS[@]}"; do + echo "$stack" +done + + +########################## +## DELETE UNUSED STACKS ## +########################## +echo "-----> Delete orphaned stacks" + +# Search for stacks not in use by any active Jenkins systems and remove them. +for STACK_NAME in "${OS_STACKS[@]}"; do + # jenkins_urls intentially needs globbing to be passed a separate params. + # shellcheck disable=SC2153,SC2086 + if stack_in_jenkins "$STACK_NAME" $jenkins_urls; then + # No need to delete stacks if there exists an active build for them + continue + else + echo "Deleting orphaned stack: $STACK_NAME" + lftools openstack --os-cloud "$os_cloud" stack delete "$STACK_NAME" + fi +done -- 2.16.6