[Kuttl] Add test for OVN db cluster pod deletions

test steps:

1. stand up a 3 replicas ovn db cluster (both nb & sb).
2. confirm pods are all up.
3. confirm that the pods established the mesh.
4. delete all pods.
5. check that new pods are respawned.
6. check that they re-established the mesh with the correct number of pods.

related Jira: OSPRH-6135

tests/kuttl/common/scripts/check_cluster_status.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# arguments: db-type: {nb, sb}, num-pods
+# Check arguments
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <db-type> <num-pods>"
+    exit 1
+fi
+
+DB_TYPE="$1"
+NUM_PODS="$2"
+POD_PREFIX="ovsdbserver-${DB_TYPE}"
+CTL_FILE="ovn${DB_TYPE}_db.ctl"
+if [ "$DB_TYPE" == "nb" ]; then
+    DB_NAME="OVN_Northbound"
+elif [ "$DB_TYPE" == "sb" ]; then
+    DB_NAME="OVN_Southbound"
+fi
+
+declare -a pods
+for i in $(seq 0 $((NUM_PODS-1))); do
+    pods+=("${POD_PREFIX}-${i}")
+done
+
+# check each pod replica
+for pod in "${pods[@]}"; do
+
+    echo "Checking status of $pod"
+    output=$(oc exec $pod -n $NAMESPACE -- bash -c "OVS_RUNDIR=/tmp ovs-appctl -t /tmp/$CTL_FILE cluster/status $DB_NAME")
+
+    # Example of part of output string that needs parsing:
+    # Status: cluster member
+    # Connections: ->0000 ->5476 <-5476 <-36cf
+    # Disconnections: 0
+    # Servers:
+    #     36cf (36cf at ssl:ovsdbserver-nb-0.ovsdbserver-nb.openstack.svc.cluster.local:6643) last msg 590 ms ago
+    #     85de (85de at ssl:ovsdbserver-nb-2.ovsdbserver-nb.openstack.svc.cluster.local:6643) (self)
+    #     5476 (5476 at ssl:ovsdbserver-nb-1.ovsdbserver-nb.openstack.svc.cluster.local:6643) last msg 12063993 ms ago
+
+    # Check if the pod is a cluster member
+    is_cluster_member=$(echo "$output" | grep -q "Status: cluster member"; echo $?)
+    if [ $is_cluster_member -ne 0 ]; then
+        exit 1
+    fi
+
+    # check if the pod is connected with all other pods
+    for server in "${pods[@]}"; do
+    echo "Checking if $server is mentioned in the output"
+    if ! echo "$output" | grep -q "$server"; then
+        exit 1
+    fi
+    done
+done

tests/kuttl/tests/ovn_db_delete/01-assert.yaml

@@ -0,0 +1 @@
+../../common/assert_sample_deployment.yaml

tests/kuttl/tests/ovn_db_delete/01-deploy-ovn.yaml

@@ -0,0 +1 @@
+../../common/deploy_ovn.yaml

tests/kuttl/tests/ovn_db_delete/02-assert.yaml

@@ -0,0 +1,47 @@
+#
+# Check for:
+#
+# - 1 OVNDBCluster CR
+# - 3 Pods for OVNDBCluster nb CR
+#
+
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  finalizers:
+  - OVNDBCluster
+  name: ovndbcluster-nb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ovsdbserver-nb
+spec:
+  replicas: 3
+status:
+  availableReplicas: 3
+---
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  finalizers:
+  - OVNDBCluster
+  name: ovndbcluster-sb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ovsdbserver-sb
+spec:
+  replicas: 3
+status:
+  availableReplicas: 3
+---

tests/kuttl/tests/ovn_db_delete/02-scale-ovndb.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-nb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":3}]'
+  - script: |
+      oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-sb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":3}]'

tests/kuttl/tests/ovn_db_delete/03-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 3
+      test $? -eq 0
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 3
+      test $? -eq 0

tests/kuttl/tests/ovn_db_delete/04-assert.yaml

@@ -0,0 +1 @@
+02-assert.yaml

tests/kuttl/tests/ovn_db_delete/04-delete-pods.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc delete pods -l service=ovsdbserver-nb
+  - script: |
+      oc delete pods -l service=ovsdbserver-sb

tests/kuttl/tests/ovn_db_delete/05-assert.yaml

@@ -0,0 +1 @@
+03-assert.yaml

tests/kuttl/tests/ovn_db_delete/06-assert.yaml

@@ -0,0 +1 @@
+02-assert.yaml

tests/kuttl/tests/ovn_db_delete/06-delete-pods-non-graceful.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc delete pods -l service=ovsdbserver-nb --force --grace-period=0
+  - script: |
+      oc delete pods -l service=ovsdbserver-sb --force --grace-period=0

tests/kuttl/tests/ovn_db_delete/07-assert.yaml

@@ -0,0 +1 @@
+03-assert.yaml

tests/kuttl/tests/ovn_db_delete/08-assert.yaml

@@ -0,0 +1,48 @@
+#
+# Check for:
+#
+# - 1 OVNDBCluster CR
+# - 1 Pod for OVNDBCluster nb CR
+# - 1 Pod for OVNDBCluster sb CR
+#
+
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  finalizers:
+  - OVNDBCluster
+  name: ovndbcluster-nb-sample
+spec:
+  replicas: 1
+status:
+  readyCount: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ovsdbserver-nb
+spec:
+  replicas: 1
+status:
+  availableReplicas: 1
+---
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  finalizers:
+  - OVNDBCluster
+  name: ovndbcluster-sb-sample
+spec:
+  replicas: 1
+status:
+  readyCount: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ovsdbserver-sb
+spec:
+  replicas: 1
+status:
+  availableReplicas: 1
+---

tests/kuttl/tests/ovn_db_delete/08-scale-down-ovndb.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-nb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":1}]'
+  - script: |
+      oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-sb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":1}]'

tests/kuttl/tests/ovn_db_delete/09-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 1
+      test $? -eq 0
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 1
+      test $? -eq 0

tests/kuttl/tests/ovn_db_delete/10-cleanup-ovn.yaml

@@ -0,0 +1 @@
+../../common/cleanup-ovn.yaml

tests/kuttl/tests/ovn_db_delete/10-errors.yaml

@@ -0,0 +1 @@
+../../common/errors_cleanup_ovn.yaml