diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index ed6a64ace8..0563cafafe 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -110,6 +110,17 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" + + - ${{ if eq( parameters['testDropgz'], true) }}: + - script: | + cd hack/scripts + chmod +x async-delete-test.sh + ./async-delete-test.sh + if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then + kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' + fi + name: "testAsyncDelete" + displayName: "Verify Async Delete when CNS is down" - script: | ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ diff --git a/hack/scripts/async-delete-test.sh b/hack/scripts/async-delete-test.sh new file mode 100755 index 0000000000..72517a40c2 --- /dev/null +++ b/hack/scripts/async-delete-test.sh @@ -0,0 +1,49 @@ +echo "create busybox deployment and verify async delete" +kubectl apply -f ../manifests/busybox.yaml +kubectl rollout status deployment busybox + +echo "temporarily disable CNS daemonset and attempt busybox pod delete" +kubectl -n kube-system patch daemonset azure-cns -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}' + +echo "delete busybox pod" +for node in $(kubectl get nodes -o name); +do + node_name="${node##*/}" + busybox_pod=$(kubectl get pods -l k8s-app=busybox -o wide | grep "$node_name" | awk '{print $1}') + if [ -z $busybox_pod ]; then + continue + else + echo "wait 1 min for delete to processes and error to catch. expect a file to be written to var/run/azure-vnet/deleteIDs" + kubectl delete pod $busybox_pod + sleep 60s + + echo "restore azure-cns pods" + kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' + echo "wait 5s for cns to start back up" + sleep 5s + + echo "check directory for pending delete" + cns_pod=$(kubectl get pods -l k8s-app=azure-cns -n kube-system -o wide | grep "$node_name" | awk '{print $1}') + file=$(kubectl exec -it $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs) + if [ -z $file ]; then + while [ -z $file ]; + do + file=$(kubectl exec -i $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs) + done + fi + echo "pending deletes" + echo $file + + echo "wait 30s for filesystem delete to occur" + sleep 30s + echo "check directory is now empty" + check_directory=$(kubectl exec -i $cns_pod -n kube-system -- ls var/run/azure-vnet/deleteIDs) + if [ -z $check_directory ]; then + echo "async delete success" + break + else + echo "##[error]async delete failure. file still exists in deleteIDs directory." + fi + fi +done + diff --git a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml index a4d5c379f0..f9959f9898 100644 --- a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml @@ -27,5 +27,7 @@ data: "ProgramSNATIPTables" : false, "EnableCNIConflistGeneration": true, "CNIConflistFilepath": "/etc/cni/net.d/05-cilium.conflist", - "CNIConflistScenario": "cilium" + "CNIConflistScenario": "cilium", + "EnableAsyncPodDelete": true, + "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs" }