Fix port clean up race condition

Fix the race condition by checking the created_at timestamp and
clean up ports that were created atleast 30 minutes before.

Jira: RELENG-3062
Change-Id: I151d4d541e6858214fc49d3479cf932feb8b9acf
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>

diff --git a/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml b/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml
new file mode 100644 (file)
index 0000000..76d04b6
--- /dev/null
+++ b/releasenotes/notes/fix-openstack-port-cleanup-race-condition-8746f634fd1709ae.yaml
@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    Fix the race condition by checking the created_at timestamp and clean up
+    ports that were created at least 30 minutes before.
+
+    There a race condition in the openstack-cron job that causes the script to
+    delete ports in DOWN state and are still in use by the VM, causing the
+    ODL CSIT jobs to fail.
+
+    JSD ISSUE: `RELENG-3062 <https://jira.linuxfoundation.org/browse/RELENG-3062>`_

diff --git a/shell/openstack-cleanup-orphaned-ports.sh b/shell/openstack-cleanup-orphaned-ports.sh
index c47228f..3030fdf 100644 (file)
--- a/shell/openstack-cleanup-orphaned-ports.sh
+++ b/shell/openstack-cleanup-orphaned-ports.sh
@@ -15,13 +15,27 @@ os_cloud="${OS_CLOUD:-vex}"
 
 set -eux -o pipefail
 
-mapfile -t os_ports < <(openstack --os-cloud "$os_cloud" port list -f value -c ID -c status | grep -E DOWN | awk '{print $1}')
+mapfile -t os_ports_ts < <(openstack --os-cloud "$os_cloud" port list \
+      -f value \
+      -c ID \
+      -c status \
+      -c created_at \
+      | grep -E DOWN \
+      | awk -F' ' '{print $1 " " $3}')
 
-if [ ${#os_ports[@]} -eq 0 ]; then
+if [ ${#os_ports_ts[@]} -eq 0 ]; then
     echo "No orphaned ports found."
 else
-    for port in "${os_ports[@]}"; do
-        echo "Removing orphaned port $port"
-        openstack --os-cloud "$os_cloud" port delete "$port"
+    cutoff=$(date -d "30 minutes ago"  +%s)
+    for port_ts in "${os_ports_ts[@]}"; do
+        created_at_isots="${port_ts#* }"
+        port_uuid="${port_ts% *}"
+        echo "checking port uuid: ${port_uuid} with TS: ${created_at_isots}"
+        created_at_uxts=$(date -d "${created_at_isots}" +"%s")
+        # Clean up ports where created_at > 30 minutes
+        if [[ "$created_at_uxts" -gt "$cutoff" ]]; then
+            echo "Removing orphaned port $port_uuid created_at ts > 30 minutes."
+            openstack --os-cloud "$os_cloud" port delete "$port_uuid"
+        fi
     done
 fi