From b8cc2d59ce5bd319a4c3d15e6fdab3ef6875c8f5 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Tue, 14 Jul 2020 14:10:43 +1000 Subject: [PATCH] Fix port clean up race condition Fix the race condition by checking the created_at timestamp and clean up ports that were created atleast 30 minutes before. Jira: RELENG-3062 Change-Id: I151d4d541e6858214fc49d3479cf932feb8b9acf Signed-off-by: Anil Belur --- ...rt-cleanup-race-condition-8746f634fd1709ae.yaml | 11 ++++++++++ shell/openstack-cleanup-orphaned-ports.sh | 24 +++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml diff --git a/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml b/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml new file mode 100644 index 00000000..76d04b65 --- /dev/null +++ b/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + Fix the race condition by checking the created_at timestamp and clean up + ports that were created at least 30 minutes before. + + There a race condition in the openstack-cron job that causes the script to + delete ports in DOWN state and are still in use by the VM, causing the + ODL CSIT jobs to fail. + + JSD ISSUE: `RELENG-3062 `_ diff --git a/shell/openstack-cleanup-orphaned-ports.sh b/shell/openstack-cleanup-orphaned-ports.sh index c47228f8..3030fdf2 100644 --- a/shell/openstack-cleanup-orphaned-ports.sh +++ b/shell/openstack-cleanup-orphaned-ports.sh @@ -15,13 +15,27 @@ os_cloud="${OS_CLOUD:-vex}" set -eux -o pipefail -mapfile -t os_ports < <(openstack --os-cloud "$os_cloud" port list -f value -c ID -c status | grep -E DOWN | awk '{print $1}') +mapfile -t os_ports_ts < <(openstack --os-cloud "$os_cloud" port list \ + -f value \ + -c ID \ + -c status \ + -c created_at \ + | grep -E DOWN \ + | awk -F' ' '{print $1 " " $3}') -if [ ${#os_ports[@]} -eq 0 ]; then +if [ ${#os_ports_ts[@]} -eq 0 ]; then echo "No orphaned ports found." else - for port in "${os_ports[@]}"; do - echo "Removing orphaned port $port" - openstack --os-cloud "$os_cloud" port delete "$port" + cutoff=$(date -d "30 minutes ago" +%s) + for port_ts in "${os_ports_ts[@]}"; do + created_at_isots="${port_ts#* }" + port_uuid="${port_ts% *}" + echo "checking port uuid: ${port_uuid} with TS: ${created_at_isots}" + created_at_uxts=$(date -d "${created_at_isots}" +"%s") + # Clean up ports where created_at > 30 minutes + if [[ "$created_at_uxts" -gt "$cutoff" ]]; then + echo "Removing orphaned port $port_uuid created_at ts > 30 minutes." + openstack --os-cloud "$os_cloud" port delete "$port_uuid" + fi done fi -- 2.16.6