chore: wait for deployment to completely delete before proceeding

test/internal/kubernetes/utils.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/test/internal/retry"
 	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,8 +34,10 @@ const (
 	SubnetNameLabel        = "kubernetes.azure.com/podnetwork-subnet"
 
 	// RetryAttempts is the number of times to retry a test.
-	RetryAttempts = 90
-	RetryDelay    = 10 * time.Second
+	RetryAttempts       = 90
+	RetryDelay          = 10 * time.Second
+	DeleteRetryAttempts = 12
+	DeleteRetryDelay    = 5 * time.Second
 )
 
 var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file")
@@ -247,6 +250,19 @@ func WaitForPodDeployment(ctx context.Context, clientset *kubernetes.Clientset,
 	return errors.Wrapf(retrier.Do(ctx, checkPodDeploymentFn), "could not wait for deployment %s", deploymentName)
 }
 
+func WaitForDeploymentToDelete(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, d appsv1.Deployment) error {
+	assertDeploymentNotFound := func() error {
+		_, err := deploymentsClient.Get(ctx, d.Name, metav1.GetOptions{})
+		// only if the error is "isNotFound", do we say, the deployment is deleted
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Errorf(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err))
+	}
+	retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: DeleteRetryDelay}
+	return errors.Wrapf(retrier.Do(ctx, assertDeploymentNotFound), "could not assert deployment %s isNotFound", d.Name)
+}
+
 func WaitForPodDaemonset(ctx context.Context, clientset *kubernetes.Clientset, namespace, daemonsetName, podLabelSelector string) error {
 	podsClient := clientset.CoreV1().Pods(namespace)
 	daemonsetClient := clientset.AppsV1().DaemonSets(namespace)

test/internal/kubernetes/utils_delete.go

@@ -35,6 +35,9 @@ func MustDeleteDeployment(ctx context.Context, deployments typedappsv1.Deploymen
 			panic(errors.Wrap(err, "failed to delete deployment"))
 		}
 	}
+	if err := WaitForDeploymentToDelete(ctx, deployments, d); err != nil {
+		panic(errors.Wrap(err, "failed to wait for deployment to delete"))
+	}
 }
 
 func MustDeleteNamespace(ctx context.Context, clienset *kubernetes.Clientset, namespace string) {