trt-1538: Wait for monitor resources cleanup

openshift · May 3, 2024 · 2a07b9f · 2a07b9f
1 parent 9260d49
commit 2a07b9f
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 1 deletion.
diff --git a/pkg/cmd/openshift-tests/monitor/run/run_monitor_command.go b/pkg/cmd/openshift-tests/monitor/run/run_monitor_command.go
@@ -186,7 +186,7 @@ func (o *RunMonitorOptions) Run() error {
 
 	fmt.Fprintf(o.Out, "Monitor shutting down, this may take up to five minutes...\n")
 
-	cleanupContext, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	cleanupContext, cleanupCancel := context.WithTimeout(context.Background(), 20*time.Minute)
 	defer cleanupCancel()
 	// ignore the ResultState because we're interested in whether we collected, not whether what we collected passed.
 	if _, err := m.Stop(cleanupContext); err != nil {

diff --git a/pkg/monitortests/imageregistry/disruptionimageregistry/monitortest.go b/pkg/monitortests/imageregistry/disruptionimageregistry/monitortest.go
@@ -6,6 +6,9 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+
 	"github.com/openshift/origin/pkg/monitortestframework"
 
 	"github.com/openshift/origin/pkg/monitortestlibrary/disruptionlibrary"
@@ -149,12 +152,35 @@ func (w *availability) WriteContentToStorage(ctx context.Context, storageDir, ti
 	return w.notSupportedReason
 }
 
+func (w *availability) routeDeleted(ctx context.Context) (bool, error) {
+	_, err := w.routeClient.RouteV1().Routes("openshift-image-registry").Get(ctx, w.imageRegistryRoute.Name, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		return true, nil
+	}
+
+	if err != nil {
+		klog.Errorf("Error checking for deleted route: %s, %s", w.imageRegistryRoute.Name, err.Error())
+		return false, err
+	}
+
+	return false, nil
+}
+
 func (w *availability) Cleanup(ctx context.Context) error {
 	if w.imageRegistryRoute != nil {
 		err := w.routeClient.RouteV1().Routes("openshift-image-registry").Delete(ctx, w.imageRegistryRoute.Name, metav1.DeleteOptions{})
 		if err != nil {
 			return fmt.Errorf("failed to delete route: %w", err)
 		}
+
+		startTime := time.Now()
+		err = wait.PollUntilContextTimeout(ctx, 15*time.Second, 20*time.Minute, true, w.routeDeleted)
+		if err != nil {
+			return err
+		}
+
+		klog.Infof("Deleting route: %s took %.2f seconds", w.imageRegistryRoute.Name, time.Now().Sub(startTime).Seconds())
+
 	}
 
 	return w.notSupportedReason

diff --git a/pkg/monitortests/network/disruptionpodnetwork/monitortest.go b/pkg/monitortests/network/disruptionpodnetwork/monitortest.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
+
 	"github.com/openshift/library-go/pkg/operator/resource/resourceread"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -356,11 +357,34 @@ func (pna *podNetworkAvalibility) WriteContentToStorage(ctx context.Context, sto
 	return pna.notSupportedReason
 }
 
+func (pna *podNetworkAvalibility) namespaceDeleted(ctx context.Context) (bool, error) {
+	_, err := pna.kubeClient.CoreV1().Namespaces().Get(ctx, pna.namespaceName, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		return true, nil
+	}
+
+	if err != nil {
+		klog.Errorf("Error checking for deleted namespace: %s, %s", pna.namespaceName, err.Error())
+		return false, err
+	}
+
+	return false, nil
+}
+
 func (pna *podNetworkAvalibility) Cleanup(ctx context.Context) error {
 	if len(pna.namespaceName) > 0 && pna.kubeClient != nil {
 		if err := pna.kubeClient.CoreV1().Namespaces().Delete(ctx, pna.namespaceName, metav1.DeleteOptions{}); err != nil {
 			return err
 		}
+
+		startTime := time.Now()
+		err := wait.PollUntilContextTimeout(ctx, 15*time.Second, 20*time.Minute, true, pna.namespaceDeleted)
+		if err != nil {
+			return err
+		}
+
+		klog.Infof("Deleting namespace: %s took %.2f seconds", pna.namespaceName, time.Now().Sub(startTime).Seconds())
+
 	}
 	return nil
 }
diff --git a/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go b/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go
@@ -11,6 +11,9 @@ import (
 	"strconv"
 	"time"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/klog/v2"
+
 	"github.com/openshift/origin/pkg/monitortestframework"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -272,12 +275,35 @@ func (w *availability) WriteContentToStorage(ctx context.Context, storageDir, ti
 	return w.notSupportedReason
 }
 
+func (w *availability) namespaceDeleted(ctx context.Context) (bool, error) {
+	_, err := w.kubeClient.CoreV1().Namespaces().Get(ctx, w.namespaceName, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		return true, nil
+	}
+
+	if err != nil {
+		klog.Errorf("Error checking for deleted namespace: %s, %s", w.namespaceName, err.Error())
+		return false, err
+	}
+
+	return false, nil
+}
+
 func (w *availability) Cleanup(ctx context.Context) error {
 	if len(w.namespaceName) > 0 && w.kubeClient != nil {
 		if err := w.kubeClient.CoreV1().Namespaces().Delete(ctx, w.namespaceName, metav1.DeleteOptions{}); err != nil {
 			return err
 		}
 	}
+
+	startTime := time.Now()
+	err := wait.PollUntilContextTimeout(ctx, 15*time.Second, 20*time.Minute, true, w.namespaceDeleted)
+	if err != nil {
+		return err
+	}
+
+	klog.Infof("Deleting namespace: %s took %.2f seconds", w.namespaceName, time.Now().Sub(startTime).Seconds())
+
 	return nil
 }