acquire leader lease before serving

cmd/admission/admission.go

@@ -1,8 +1,12 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"github.com/hwameistor/hwameistor/pkg/local-storage/utils"
+	hookcfg "github.com/hwameistor/hwameistor/pkg/webhook/config"
 	"net/http"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -31,15 +35,29 @@ func main() {
 		log.Fatal("--cert-dir, --tls-cert-file, --tls-private-key-file is required")
 	}
 
-	mux := http.NewServeMux()
-	mux.Handle("/mutate", app.RegisterHwameiStorMutateWebhooks(*options))
-	server := &http.Server{
-		Addr:    ":18443",
-		Handler: mux,
+	runServer := func(_ context.Context) {
+		// initialize webhook config first
+		if err := hookcfg.CreateOrUpdateWebHookConfig(); err != nil {
+			log.WithError(err).Fatal("failed to init webhook config")
+		}
+
+		mux := http.NewServeMux()
+		mux.Handle("/mutate", app.RegisterHwameiStorMutateWebhooks(*options))
+		mux.Handle("/healthz", app.RegisterHwameiStorHealthz(*options))
+		server := &http.Server{
+			Addr:    ":18443",
+			Handler: mux,
+		}
+
+		log.Infof("admission server at %v, using tls-cert-file %s, tls-private-key-file %s", server.Addr, certPath, keyPath)
+		log.Fatal(server.ListenAndServeTLS(certPath, keyPath))
 	}
 
-	log.Infof("admission server at %v, using tls-cert-file %s, tls-private-key-file %s", server.Addr, certPath, keyPath)
-	log.Fatal(server.ListenAndServeTLS(certPath, keyPath))
+	// Acquired leader lease before proceeding
+	if err := utils.RunWithLease("" /*namespace*/, "" /*podName*/, fmt.Sprintf("hwameistor-admission-controller-master"), runServer); err != nil {
+		log.Error(err, "Failed to init cluster lease election")
+		os.Exit(1)
+	}
 }
 
 func setupLogging() {
@@ -54,6 +72,7 @@ func setupLogging() {
 	}
 
 	log.SetLevel(level)
+	log.Infof("log level set to %s", level.String())
 	log.SetFormatter(&log.JSONFormatter{
 		CallerPrettyfier: func(f *runtime.Frame) (string, string) {
 			s := strings.Split(f.Function, ".")

cmd/admission/app/mutate.go

@@ -137,12 +137,23 @@ func serverHwameiStorMutateFunc(w http.ResponseWriter, r *http.Request, o webhoo
 	}
 }
 
+func healthz(w http.ResponseWriter, r *http.Request, o webhook.ServerOption) {
+	log.WithField("request.path", r.URL.Path).Info("health check ok")
+	_, _ = w.Write([]byte("ok"))
+}
+
 func RegisterHwameiStorMutateWebhooks(o webhook.ServerOption) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		serverHwameiStorMutateFunc(w, r, o)
 	})
 }
 
+func RegisterHwameiStorHealthz(o webhook.ServerOption) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		healthz(w, r, o)
+	})
+}
+
 func init() {
 	webhook.AddToMutateHooks(scheduler.NewPatchSchedulerWebHook())
 	if err := hookcfg.CreateOrUpdateWebHookConfig(); err != nil {

go.mod

@@ -148,7 +148,7 @@ require (
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.13.1
 	github.com/prometheus/client_model v0.2.0 // indirect

helm/hwameistor/templates/admission-controller.yaml

@@ -42,6 +42,12 @@ spec:
           ports:
             - containerPort: 18443
               name: admission-api
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 18443
+              scheme: HTTPS
           volumeMounts:
             - name: admission-controller-tls-certs
               mountPath: /etc/webhook/certs