From 9a94cc07d3e065fdcd3df172a3e41c0191afbc88 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Fri, 27 Oct 2023 22:28:00 +0000
Subject: [PATCH 1/4] test async delete in pipeline

---
 .../aks-swift/e2e-step-template.yaml          |  8 +++
 .../cilium-overlay-e2e-step-template.yaml     |  8 +++
 .../cilium/cilium-e2e-step-template.yaml      |  8 +++
 hack/scripts/async-delete-test.sh             | 49 +++++++++++++++++++
 .../manifests/cnsconfig/overlayconfigmap.yaml |  4 +-
 5 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100755 hack/scripts/async-delete-test.sh

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ed6a64ace8..5daab03336 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -110,6 +110,14 @@ steps:
     retryCountOnTaskFailure: 3
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
+  
+  - ${{ if eq( parameters['testDropgz'], true) }}:
+    - script: |
+        cd hack/scripts
+        chmod +x async-delete-test.sh
+        ./async-delete-test.sh
+      name: "testAsyncDelete"
+      displayName: "Verify Async Delete when CNS is down"
 
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 4a48a84844..0fb67e81df 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -152,6 +152,14 @@ steps:
     name: "servicesConformance"
     displayName: "Run Services Conformance Tests"
 
+  - ${{ if eq( parameters['testDropgz'], true) }}:
+    - script: |
+        cd hack/scripts
+        chmod +x async-delete-test.sh
+        ./async-delete-test.sh
+      name: "testAsyncDelete"
+      displayName: "Verify Async Delete when CNS is down"
+
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
index eb347089b5..47b4ebf406 100644
--- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -142,6 +142,14 @@ steps:
     name: "servicesConformance"
     displayName: "Run Services Conformance Tests"
 
+  - ${{ if eq( parameters['testDropgz'], true) }}:
+    - script: |
+        cd hack/scripts
+        chmod +x async-delete-test.sh
+        ./async-delete-test.sh
+      name: "testAsyncDelete"
+      displayName: "Verify Async Delete when CNS is down"
+
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status
diff --git a/hack/scripts/async-delete-test.sh b/hack/scripts/async-delete-test.sh
new file mode 100755
index 0000000000..72517a40c2
--- /dev/null
+++ b/hack/scripts/async-delete-test.sh
@@ -0,0 +1,49 @@
+echo "create busybox deployment and verify async delete"
+kubectl apply -f ../manifests/busybox.yaml
+kubectl rollout status deployment busybox
+
+echo "temporarily disable CNS daemonset and attempt busybox pod delete"
+kubectl -n kube-system patch daemonset azure-cns -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
+
+echo "delete busybox pod"
+for node in $(kubectl get nodes -o name);
+do
+    node_name="${node##*/}"
+    busybox_pod=$(kubectl get pods -l k8s-app=busybox -o wide | grep "$node_name" | awk '{print $1}')
+    if [ -z $busybox_pod  ]; then
+        continue
+    else
+        echo "wait 1 min for delete to processes and error to catch. expect a file to be written to var/run/azure-vnet/deleteIDs"
+        kubectl delete pod $busybox_pod 
+        sleep 60s
+        
+        echo "restore azure-cns pods"
+        kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
+        echo "wait 5s for cns to start back up"
+        sleep 5s
+
+        echo "check directory for pending delete"
+        cns_pod=$(kubectl get pods -l k8s-app=azure-cns -n kube-system -o wide | grep "$node_name" | awk '{print $1}')
+        file=$(kubectl exec -it $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs)
+        if [ -z $file ]; then
+            while [ -z $file ]; 
+            do
+                file=$(kubectl exec -i $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs)
+            done
+        fi
+        echo "pending deletes"
+        echo $file
+
+        echo "wait 30s for filesystem delete to occur"
+        sleep 30s
+        echo "check directory is now empty"
+        check_directory=$(kubectl exec -i $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs)
+        if [ -z $check_directory ]; then
+            echo "async delete success"
+            break
+        else
+            echo "##[error]async delete failure. file still exists in deleteIDs directory."
+        fi
+    fi
+done
+
diff --git a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml
index a4d5c379f0..f9959f9898 100644
--- a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml
+++ b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml
@@ -27,5 +27,7 @@ data:
       "ProgramSNATIPTables" : false,
       "EnableCNIConflistGeneration": true,
       "CNIConflistFilepath": "/etc/cni/net.d/05-cilium.conflist",
-      "CNIConflistScenario": "cilium"
+      "CNIConflistScenario": "cilium",
+      "EnableAsyncPodDelete": true,
+      "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs"
     }

From 6997a986b63ccb186b7df97d06ec1c3ad580146f Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Fri, 10 Nov 2023 17:38:29 +0000
Subject: [PATCH 2/4] test in swift stage only

---
 .../cilium-overlay/cilium-overlay-e2e-step-template.yaml  | 8 --------
 .../singletenancy/cilium/cilium-e2e-step-template.yaml    | 8 --------
 2 files changed, 16 deletions(-)

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
index 0fb67e81df..4a48a84844 100644
--- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
@@ -152,14 +152,6 @@ steps:
     name: "servicesConformance"
     displayName: "Run Services Conformance Tests"
 
-  - ${{ if eq( parameters['testDropgz'], true) }}:
-    - script: |
-        cd hack/scripts
-        chmod +x async-delete-test.sh
-        ./async-delete-test.sh
-      name: "testAsyncDelete"
-      displayName: "Verify Async Delete when CNS is down"
-
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
index 47b4ebf406..eb347089b5 100644
--- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
+++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -142,14 +142,6 @@ steps:
     name: "servicesConformance"
     displayName: "Run Services Conformance Tests"
 
-  - ${{ if eq( parameters['testDropgz'], true) }}:
-    - script: |
-        cd hack/scripts
-        chmod +x async-delete-test.sh
-        ./async-delete-test.sh
-      name: "testAsyncDelete"
-      displayName: "Verify Async Delete when CNS is down"
-
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status

From 983f7ee3751e0935cee68e29a98c1bbc7723fa04 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Fri, 10 Nov 2023 23:31:44 +0000
Subject: [PATCH 3/4] check cns patch is removed

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5daab03336..06376db00c 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -116,6 +116,9 @@ steps:
         cd hack/scripts
         chmod +x async-delete-test.sh
         ./async-delete-test.sh
+        if ! [-z $(kubectl -n kube-system get ds  azure-cns | grep non-existing)]; then
+          kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
+        fi
       name: "testAsyncDelete"
       displayName: "Verify Async Delete when CNS is down"
 

From 6a6979c22193efabb4d85f3ad566a063f2242579 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Sat, 11 Nov 2023 00:53:54 +0000
Subject: [PATCH 4/4] fix syntax in patch check

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 06376db00c..0563cafafe 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -116,7 +116,7 @@ steps:
         cd hack/scripts
         chmod +x async-delete-test.sh
         ./async-delete-test.sh
-        if ! [-z $(kubectl -n kube-system get ds  azure-cns | grep non-existing)]; then
+        if ! [ -z $(kubectl -n kube-system get ds  azure-cns | grep non-existing) ]; then
           kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
         fi
       name: "testAsyncDelete"