deploy webhook by yaml

cmd/webhook-manager/app/options/options.go

@@ -79,7 +79,7 @@ func (c *Config) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&c.WebhookNamespace, "webhook-namespace", "", "The namespace of this webhook")
 	fs.StringVar(&c.WebhookName, "webhook-service-name", "", "The name of this webhook")
 	fs.StringVar(&c.WebhookURL, "webhook-url", "", "The url of this webhook")
-	fs.StringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks")
+	fs.StringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks, if this parameter is modified, make sure corresponding webhook configurations are the same.")
 	fs.StringArrayVar(&c.SchedulerNames, "scheduler-name", []string{defaultSchedulerName}, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name")
 	fs.StringVar(&c.ConfigPath, "admission-conf", "", "The configmap file of this webhook")
 	fs.StringVar(&c.IgnoredNamespaces, "ignored-namespaces", defaultIgnoredNamespaces, "Comma-separated list of namespaces to be ignored by admission webhooks")

cmd/webhook-manager/app/server.go

@@ -78,11 +78,13 @@ func Run(config *options.Config) error {
 
 		klog.V(3).Infof("Registered '%s' as webhook.", service.Path)
 		http.HandleFunc(service.Path, service.Handler)
-
-		klog.V(3).Infof("Registered configuration for webhook <%s>", service.Path)
-		registerWebhookConfig(kubeClient, config, service, config.CaCertData)
 	})
 
+	if err = addCaCertForWebhook(kubeClient, config.CaCertData); err != nil {
+		return fmt.Errorf("failed to add caCert for webhook %v", err)
+	}
+	klog.V(3).Infof("Successfully added caCert for all webhooks")
+
 	webhookServeError := make(chan struct{})
 	stopChannel := make(chan os.Signal, 1)
 	signal.Notify(stopChannel, syscall.SIGTERM, syscall.SIGINT)

cmd/webhook-manager/app/util.go

@@ -17,93 +17,103 @@ limitations under the License.
 package app
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"regexp"
-	"strings"
+	"fmt"
+	"time"
 
 	v1 "k8s.io/api/admissionregistration/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog"
 
 	"volcano.sh/apis/pkg/client/clientset/versioned"
 	"volcano.sh/volcano/cmd/webhook-manager/app/options"
-	"volcano.sh/volcano/pkg/webhooks/router"
 )
 
-func registerWebhookConfig(kubeClient *kubernetes.Clientset, config *options.Config, service *router.AdmissionService, caBundle []byte) {
-	sideEffect := v1.SideEffectClassNoneOnDryRun
-	reviewVersions := []string{"v1"}
-	webhookLabelSelector := &metav1.LabelSelector{}
-	clientConfig := v1.WebhookClientConfig{
-		CABundle: caBundle,
+var (
+	validatingWebhooksName = []string{
+		"volcano-admission-service-jobs-validate",
+		"volcano-admission-service-pods-validate",
+		"volcano-admission-service-queues-validate",
 	}
-	if config.WebhookURL != "" {
-		url := config.WebhookURL + service.Path
-		clientConfig.URL = &url
-		klog.Infof("The URL of webhook manager is <%s>.", url)
+	mutatingWebhooksName = []string{
+		"volcano-admission-service-pods-mutate",
+		"volcano-admission-service-queues-mutate",
+		"volcano-admission-service-podgroups-mutate",
+		"volcano-admission-service-jobs-mutate",
 	}
-	if config.WebhookName != "" && config.WebhookNamespace != "" {
-		clientConfig.Service = &v1.ServiceReference{
-			Name:      config.WebhookName,
-			Namespace: config.WebhookNamespace,
-			Path:      &service.Path,
-		}
-		klog.Infof("The service of webhook manager is <%s/%s/%s>.",
-			config.WebhookName, config.WebhookNamespace, service.Path)
-	}
-	if config.IgnoredNamespaces != "" {
-		ignoredNamespaces := strings.Split(strings.TrimSpace(config.IgnoredNamespaces), ",")
-		klog.Infof("The ignored namespaces list of webhook manager is <%v>.",
-			ignoredNamespaces)
-		webhookLabelSelector = &metav1.LabelSelector{
-			MatchExpressions: []metav1.LabelSelectorRequirement{
-				{
-					Values:   ignoredNamespaces,
-					Operator: "NotIn",
-					Key:      "kubernetes.io/metadata.name",
-				},
-			},
-		}
-	}
-	if service.MutatingConfig != nil {
-		for i := range service.MutatingConfig.Webhooks {
-			service.MutatingConfig.Webhooks[i].SideEffects = &sideEffect
-			service.MutatingConfig.Webhooks[i].AdmissionReviewVersions = reviewVersions
-			service.MutatingConfig.Webhooks[i].ClientConfig = clientConfig
-			service.MutatingConfig.Webhooks[i].NamespaceSelector = webhookLabelSelector
-		}
+)
 
-		service.MutatingConfig.ObjectMeta.Name = webhookConfigName(config.WebhookName, service.Path)
+func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) error {
+	for _, mutatingWebhookName := range mutatingWebhooksName {
+		var mutatingWebhook *v1.MutatingWebhookConfiguration
+		webhookChanged := false
+		if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) {
+			mutatingWebhook, err = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					klog.Errorln(err)
+					return false, nil
+				}
+				return false, fmt.Errorf("failed to get mutating webhook %v", err)
+			}
+			return true, nil
+		}); err != nil {
+			return fmt.Errorf("failed to get mutating webhook %v", err)
+		}
 
-		if err := registerMutateWebhook(kubeClient, service.MutatingConfig); err != nil {
-			klog.Errorf("Failed to register mutating admission webhook (%s): %v",
-				service.Path, err)
-		} else {
-			klog.V(3).Infof("Registered mutating webhook for path <%s>.", service.Path)
+		for index := 0; index < len(mutatingWebhook.Webhooks); index++ {
+			if mutatingWebhook.Webhooks[index].ClientConfig.CABundle == nil ||
+				!bytes.Equal(mutatingWebhook.Webhooks[index].ClientConfig.CABundle, caBundle) {
+				mutatingWebhook.Webhooks[index].ClientConfig.CABundle = caBundle
+				webhookChanged = true
+			}
 		}
-	}
-	if service.ValidatingConfig != nil {
-		for i := range service.ValidatingConfig.Webhooks {
-			service.ValidatingConfig.Webhooks[i].SideEffects = &sideEffect
-			service.ValidatingConfig.Webhooks[i].AdmissionReviewVersions = reviewVersions
-			service.ValidatingConfig.Webhooks[i].ClientConfig = clientConfig
-			service.ValidatingConfig.Webhooks[i].NamespaceSelector = webhookLabelSelector
+		if webhookChanged {
+			if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), mutatingWebhook, metav1.UpdateOptions{}); err != nil {
+				return fmt.Errorf("failed to update mutating admission webhooks %v %v", mutatingWebhookName, err)
+			}
 		}
+	}
 
-		service.ValidatingConfig.ObjectMeta.Name = webhookConfigName(config.WebhookName, service.Path)
+	for _, validatingWebhookName := range validatingWebhooksName {
+		var validatingWebhook *v1.ValidatingWebhookConfiguration
+		webhookChanged := false
+		if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) {
+			validatingWebhook, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.TODO(), validatingWebhookName, metav1.GetOptions{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					klog.Errorln(err)
+					return false, nil
+				}
+				return false, fmt.Errorf("failed to get validating webhook %v", err)
+			}
+			return true, nil
+		}); err != nil {
+			return fmt.Errorf("failed to get validating webhook %v", err)
+		}
 
-		if err := registerValidateWebhook(kubeClient, service.ValidatingConfig); err != nil {
-			klog.Errorf("Failed to register validating admission webhook (%s): %v",
-				service.Path, err)
-		} else {
-			klog.V(3).Infof("Registered validating webhook for path <%s>.", service.Path)
+		for index := 0; index < len(validatingWebhook.Webhooks); index++ {
+			if validatingWebhook.Webhooks[index].ClientConfig.CABundle == nil ||
+				!bytes.Equal(validatingWebhook.Webhooks[index].ClientConfig.CABundle, caBundle) {
+				validatingWebhook.Webhooks[index].ClientConfig.CABundle = caBundle
+				webhookChanged = true
+			}
+		}
+		if webhookChanged {
+			if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), validatingWebhook, metav1.UpdateOptions{}); err != nil {
+				return fmt.Errorf("failed to update validating admission webhooks %v %v", validatingWebhookName, err)
+			}
 		}
 	}
+
+	return nil
 }
 
 // getKubeClient Get a clientset with restConfig.
@@ -164,58 +174,3 @@ func configTLS(config *options.Config, restConfig *rest.Config) *tls.Config {
 	klog.Fatal("tls: failed to find any tls config data")
 	return &tls.Config{}
 }
-
-func registerMutateWebhook(clientset *kubernetes.Clientset, hook *v1.MutatingWebhookConfiguration) error {
-	client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations()
-	existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{})
-	if err != nil && !apierrors.IsNotFound(err) {
-		return err
-	}
-	if err == nil && existing != nil {
-		klog.V(4).Infof("Updating MutatingWebhookConfiguration %v", hook)
-		existing.Webhooks = hook.Webhooks
-		if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil {
-			return err
-		}
-	} else {
-		klog.V(4).Infof("Creating MutatingWebhookConfiguration %v", hook)
-		if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func registerValidateWebhook(clientset *kubernetes.Clientset, hook *v1.ValidatingWebhookConfiguration) error {
-	client := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations()
-
-	existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{})
-	if err != nil && !apierrors.IsNotFound(err) {
-		return err
-	}
-	if err == nil && existing != nil {
-		existing.Webhooks = hook.Webhooks
-		klog.V(4).Infof("Updating ValidatingWebhookConfiguration %v", hook)
-		if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil {
-			return err
-		}
-	} else {
-		klog.V(4).Infof("Creating ValidatingWebhookConfiguration %v", hook)
-		if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func webhookConfigName(name, path string) string {
-	if name == "" {
-		name = "webhook"
-	}
-
-	re := regexp.MustCompile(`-+`)
-	raw := strings.Join([]string{name, strings.ReplaceAll(path, "/", "-")}, "-")
-	return re.ReplaceAllString(raw, "-")
-}

hack/generate-yaml.sh

@@ -118,6 +118,7 @@ ${HELM_BIN_DIR}/helm template ${VK_ROOT}/installer/helm/chart/volcano --namespac
       -s templates/scheduling_v1beta1_podgroup.yaml \
       -s templates/scheduling_v1beta1_queue.yaml \
       -s templates/nodeinfo_v1alpha1_numatopologies.yaml \
+      -s templates/webhooks.yaml \
       >> ${DEPLOYMENT_FILE}
 
 ${HELM_BIN_DIR}/helm template ${VK_ROOT}/installer/helm/chart/volcano --namespace volcano-monitoring \