Fix Kueue startup by waiting for webhooks server using probes (#1682)

* Wait for webhooks server using probes * Delete KueueReadyForTesting * revert the setting of healthz * Add a comment about the readyz proberevert the setting of healthz
kubernetes-sigs · Feb 2, 2024 · 1a76d03 · 1a76d03
1 parent 2fe9fff
commit 1a76d03
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 25 deletions.
diff --git a/cmd/experimental/podtaintstolerations/test/e2e/suite_test.go b/cmd/experimental/podtaintstolerations/test/e2e/suite_test.go
@@ -29,8 +29,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
-	kueuetest "sigs.k8s.io/kueue/pkg/util/testing"
-	"sigs.k8s.io/kueue/test/util"
 )
 
 var (
@@ -67,17 +65,7 @@ func CreateClientUsingCluster() client.Client {
 	return client
 }
 
-func KueueReadyForTesting(client client.Client) {
-	// To verify that webhooks are ready, let's create a simple resourceflavor
-	resourceKueue := kueuetest.MakeResourceFlavor("default").Obj()
-	gomega.Eventually(func() error {
-		return client.Create(context.Background(), resourceKueue)
-	}, Timeout, Interval).Should(gomega.Succeed())
-	util.ExpectResourceFlavorToBeDeleted(ctx, k8sClient, resourceKueue, true)
-}
-
 var _ = ginkgo.BeforeSuite(func() {
 	k8sClient = CreateClientUsingCluster()
 	ctx = context.Background()
-	KueueReadyForTesting(k8sClient)
 })
diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go
@@ -21,6 +21,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -308,7 +309,17 @@ func setupProbeEndpoints(mgr ctrl.Manager) {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+
+	// Wait for the webhook server to be listening before advertising the
+	// Kueue replica as ready. This allows users to wait with sending the first
+	// requests, requiring webhooks, until the Kueue deployment is available, so
+	// that the early requests are not rejected during the Kueue's startup.
+	// We wrap the call to GetWebhookServer in a closure to delay calling
+	// the function, otherwise a not fully-initialized webhook server (without
+	// ready certs) fails the start of the manager.
+	if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error {
+		return mgr.GetWebhookServer().StartedChecker()(req)
+	}); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}

diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh
@@ -65,6 +65,7 @@ function kind_load {
 function kueue_deploy {
     (cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG)
     kubectl apply --server-side -k test/e2e/config
+    kubectl wait --for=condition=available --timeout=3m deployment/kueue-controller-manager -n kueue-system
 }
 
 trap cleanup EXIT

diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go
@@ -30,8 +30,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
-	kueuetest "sigs.k8s.io/kueue/pkg/util/testing"
-	"sigs.k8s.io/kueue/test/util"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -69,17 +67,7 @@ func CreateClientUsingCluster() client.Client {
 	return client
 }
 
-func KueueReadyForTesting(client client.Client) {
-	// To verify that webhooks are ready, let's create a simple resourceflavor
-	resourceKueue := kueuetest.MakeResourceFlavor("default").Obj()
-	gomega.Eventually(func() error {
-		return client.Create(context.Background(), resourceKueue)
-	}, Timeout, Interval).Should(gomega.Succeed())
-	util.ExpectResourceFlavorToBeDeleted(ctx, k8sClient, resourceKueue, true)
-}
-
 var _ = ginkgo.BeforeSuite(func() {
 	k8sClient = CreateClientUsingCluster()
 	ctx = context.Background()
-	KueueReadyForTesting(k8sClient)
 })