From 3378f83f35d013ef49256cc13d256429b666da15 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 19 Oct 2023 10:53:58 -0700 Subject: [PATCH 1/3] chore: wait for deployment to completely delete before proceeding --- test/internal/kubernetes/utils.go | 19 +++++++++++++++++-- test/internal/kubernetes/utils_delete.go | 3 +++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index b04f1f2772..dbd4a3018f 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/test/internal/retry" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,8 +34,9 @@ const ( SubnetNameLabel = "kubernetes.azure.com/podnetwork-subnet" // RetryAttempts is the number of times to retry a test. - RetryAttempts = 90 - RetryDelay = 10 * time.Second + RetryAttempts = 90 + DeleteRetryAttempts = 6 + RetryDelay = 10 * time.Second ) var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file") @@ -247,6 +249,19 @@ func WaitForPodDeployment(ctx context.Context, clientset *kubernetes.Clientset, return errors.Wrapf(retrier.Do(ctx, checkPodDeploymentFn), "could not wait for deployment %s", deploymentName) } +func WaitForDeploymentToDelete(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, d appsv1.Deployment) error { + assertDeploymentNotFound := func() error { + _, err := deploymentsClient.Get(ctx, d.Name, metav1.GetOptions{}) + // only if the error is "isNotFound", do we say, the deployment is deleted + if apierrors.IsNotFound(err) { + return nil + } + return errors.New(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err)) + } + retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: RetryDelay} + return errors.Wrapf(retrier.Do(ctx, assertDeploymentNotFound), "could not assert deployment %s isNotFound", d.Name) +} + func WaitForPodDaemonset(ctx context.Context, clientset *kubernetes.Clientset, namespace, daemonsetName, podLabelSelector string) error { podsClient := clientset.CoreV1().Pods(namespace) daemonsetClient := clientset.AppsV1().DaemonSets(namespace) diff --git a/test/internal/kubernetes/utils_delete.go b/test/internal/kubernetes/utils_delete.go index d2ef27926f..acbdf8a375 100644 --- a/test/internal/kubernetes/utils_delete.go +++ b/test/internal/kubernetes/utils_delete.go @@ -35,6 +35,9 @@ func MustDeleteDeployment(ctx context.Context, deployments typedappsv1.Deploymen panic(errors.Wrap(err, "failed to delete deployment")) } } + if err := WaitForDeploymentToDelete(ctx, deployments, d); err != nil { + panic(errors.Wrap(err, "failed to wait for deployment to delete")) + } } func MustDeleteNamespace(ctx context.Context, clienset *kubernetes.Clientset, namespace string) { From 3ca5755edd069cf86c07ca4d78fdf81b9e1b7c16 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 19 Oct 2023 11:16:47 -0700 Subject: [PATCH 2/3] chore: feedbacak and lint --- test/internal/kubernetes/utils.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index dbd4a3018f..715ed1623c 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -35,8 +35,9 @@ const ( // RetryAttempts is the number of times to retry a test. RetryAttempts = 90 - DeleteRetryAttempts = 6 RetryDelay = 10 * time.Second + DeleteRetryAttempts = 12 + DeleteRetryDelay = 5 * time.Second ) var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file") @@ -256,7 +257,7 @@ func WaitForDeploymentToDelete(ctx context.Context, deploymentsClient typedappsv if apierrors.IsNotFound(err) { return nil } - return errors.New(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err)) + return errors.Errorf(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err)) } retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: RetryDelay} return errors.Wrapf(retrier.Do(ctx, assertDeploymentNotFound), "could not assert deployment %s isNotFound", d.Name) From 1a19571b1eb8be00a19e417c57c6f74ad58810de Mon Sep 17 00:00:00 2001 From: Paul Johnston <35265851+pjohnst5@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:38:03 -0700 Subject: [PATCH 3/3] chore: fix delay Co-authored-by: John Payne <89417863+jpayne3506@users.noreply.github.com> Signed-off-by: Paul Johnston <35265851+pjohnst5@users.noreply.github.com> --- test/internal/kubernetes/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index 715ed1623c..caefd465ab 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -259,7 +259,7 @@ func WaitForDeploymentToDelete(ctx context.Context, deploymentsClient typedappsv } return errors.Errorf(fmt.Sprintf("expected isNotFound error when getting deployment, but got %+v", err)) } - retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: RetryDelay} + retrier := retry.Retrier{Attempts: DeleteRetryAttempts, Delay: DeleteRetryDelay} return errors.Wrapf(retrier.Do(ctx, assertDeploymentNotFound), "could not assert deployment %s isNotFound", d.Name) }