webhook.go

/*
Copyright (c) Microsoft Corporation.
Licensed under the MIT license.
*/

package webhook

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"os"
	"path/filepath"
	"time"

	admv1 "k8s.io/api/admissionregistration/v1"
	admv1beta1 "k8s.io/api/admissionregistration/v1beta1"
	appsv1 "k8s.io/api/apps/v1"
	authorizationv1 "k8s.io/api/authorization/v1"
	autoscalingv1 "k8s.io/api/autoscaling/v1"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	discoveryv1 "k8s.io/api/discovery/v1"
	networkingv1 "k8s.io/api/networking/v1"
	policyv1 "k8s.io/api/policy/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	storagev1 "k8s.io/api/storage/v1"
	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1"

	fleetnetworkingv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"
	clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1"
	placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1"
	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
	"go.goms.io/fleet/cmd/hubagent/options"
	"go.goms.io/fleet/pkg/webhook/clusterresourceoverride"
	"go.goms.io/fleet/pkg/webhook/clusterresourceplacement"
	"go.goms.io/fleet/pkg/webhook/fleetresourcehandler"
	"go.goms.io/fleet/pkg/webhook/membercluster"
	"go.goms.io/fleet/pkg/webhook/pod"
	"go.goms.io/fleet/pkg/webhook/replicaset"
	"go.goms.io/fleet/pkg/webhook/resourceoverride"
)

const (
	fleetWebhookCertFileName      = "tls.crt"
	fleetWebhookKeyFileName       = "tls.key"
	fleetValidatingWebhookCfgName = "fleet-validating-webhook-configuration"
	fleetGuardRailWebhookCfgName  = "fleet-guard-rail-webhook-configuration"

	crdResourceName                      = "customresourcedefinitions"
	bindingResourceName                  = "bindings"
	configMapResourceName                = "configmaps"
	endPointResourceName                 = "endpoints"
	limitRangeResourceName               = "limitranges"
	persistentVolumeClaimsName           = "persistentvolumeclaims"
	podTemplateResourceName              = "podtemplates"
	replicationControllerResourceName    = "replicationcontrollers"
	resourceQuotaResourceName            = "resourcequotas"
	secretResourceName                   = "secrets"
	serviceAccountResourceName           = "serviceaccounts"
	servicesResourceName                 = "services"
	controllerRevisionResourceName       = "controllerrevisions"
	daemonSetResourceName                = "daemonsets"
	deploymentResourceName               = "deployments"
	statefulSetResourceName              = "statefulsets"
	localSubjectAccessReviewResourceName = "localsubjectaccessreviews"
	horizontalPodAutoScalerResourceName  = "horizontalpodautoscalers"
	cronJobResourceName                  = "cronjobs"
	jobResourceName                      = "jobs"
	workResourceName                     = "works"
	endPointSlicesResourceName           = "endpointslices"
	ingressResourceName                  = "ingresses"
	networkPolicyResourceName            = "networkpolicies"
	podDisruptionBudgetsResourceName     = "poddisruptionbudgets"
	roleResourceName                     = "roles"
	roleBindingResourceName              = "rolebindings"
	csiStorageCapacityResourceName       = "csistoragecapacities"
	memberClusterResourceName            = "memberclusters"
	internalMemberClusterResourceName    = "internalmemberclusters"
	endpointSliceExportResourceName      = "endpointsliceexports"
	endpointSliceImportResourceName      = "endpointsliceimports"
	internalServiceExportResourceName    = "internalserviceexports"
	internalServiceImportResourceName    = "internalserviceimports"
	namespaceResourceName                = "namespaces"
	replicaSetResourceName               = "replicasets"
	podResourceName                      = "pods"
	clusterResourceOverrideName          = "clusterresourceoverrides"
	resourceOverrideName                 = "resourceoverrides"
)

var (
	admissionReviewVersions = []string{admv1.SchemeGroupVersion.Version, admv1beta1.SchemeGroupVersion.Version}

	ignoreFailurePolicy = admv1.Ignore
	failFailurePolicy   = admv1.Fail
	sideEffortsNone     = admv1.SideEffectClassNone
	namespacedScope     = admv1.NamespacedScope
	clusterScope        = admv1.ClusterScope
	shortWebhookTimeout = ptr.To(int32(1))
	longWebhookTimeout  = ptr.To(int32(5))
)

var AddToManagerFuncs []func(manager.Manager) error
var AddToManagerFleetResourceValidator func(manager.Manager, []string, bool) error

// AddToManager adds all Controllers to the Manager
func AddToManager(m manager.Manager, whiteListedUsers []string, isFleetV1Beta1API bool) error {
	for _, f := range AddToManagerFuncs {
		if err := f(m); err != nil {
			return err
		}
	}
	return AddToManagerFleetResourceValidator(m, whiteListedUsers, isFleetV1Beta1API)
}

type Config struct {
	mgr manager.Manager

	// webhook server info
	serviceNamespace string
	serviceName      string
	servicePort      int32
	serviceURL       string

	// caPEM is a PEM encoded CA bundle which will be used to validate the webhook's server certificate.
	caPEM []byte

	clientConnectionType *options.WebhookClientConnectionType

	enableGuardRail bool
}

func NewWebhookConfig(mgr manager.Manager, webhookServiceName string, port int, clientConnectionType *options.WebhookClientConnectionType, certDir string, enableGuardRail bool) (*Config, error) {
	// We assume the Pod namespace should be passed to env through downward API in the Pod spec.
	namespace := os.Getenv("POD_NAMESPACE")
	if namespace == "" {
		return nil, errors.New("fail to obtain Pod namespace from POD_NAMESPACE")
	}
	w := Config{
		mgr:                  mgr,
		servicePort:          int32(port),
		serviceNamespace:     namespace,
		serviceName:          webhookServiceName,
		serviceURL:           fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", webhookServiceName, namespace, port),
		clientConnectionType: clientConnectionType,
		enableGuardRail:      enableGuardRail,
	}
	caPEM, err := w.genCertificate(certDir)
	if err != nil {
		return nil, err
	}
	w.caPEM = caPEM
	return &w, err
}

func (w *Config) Start(ctx context.Context) error {
	klog.V(2).InfoS("setting up webhooks in apiserver from the leader")
	if err := w.createFleetWebhookConfiguration(ctx); err != nil {
		klog.ErrorS(err, "unable to setup webhook configurations in apiserver")
		return err
	}
	return nil
}

// createFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook.
func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error {
	if err := w.createValidatingWebhookConfiguration(ctx, w.buildFleetValidatingWebhooks(), fleetValidatingWebhookCfgName); err != nil {
		return err
	}
	if w.enableGuardRail {
		if err := w.createValidatingWebhookConfiguration(ctx, w.buildFleetGuardRailValidatingWebhooks(), fleetGuardRailWebhookCfgName); err != nil {
			return err
		}
	}
	return nil
}

func (w *Config) createValidatingWebhookConfiguration(ctx context.Context, webhooks []admv1.ValidatingWebhook, configName string) error {
	validatingWebhookConfig := admv1.ValidatingWebhookConfiguration{
		ObjectMeta: metav1.ObjectMeta{
			Name: configName,
			Labels: map[string]string{
				"admissions.enforcer/disabled": "true",
			},
		},
		Webhooks: webhooks,
	}

	// We need to ensure this webhook configuration is garbage collected if Fleet is uninstalled from the cluster.
	// Since the fleet-system namespace is a prerequisite for core Fleet components, we bind to this namespace.
	if err := bindWebhookConfigToFleetSystem(ctx, w.mgr.GetClient(), &validatingWebhookConfig); err != nil {
		return err
	}

	if err := w.mgr.GetClient().Create(ctx, &validatingWebhookConfig); err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return err
		}
		klog.V(2).InfoS("validating webhook configuration exists, need to overwrite", "name", configName)
		// Here we simply use delete/create pattern to implement full overwrite
		if err := w.mgr.GetClient().Delete(ctx, &validatingWebhookConfig); err != nil {
			return err
		}
		if err = w.mgr.GetClient().Create(ctx, &validatingWebhookConfig); err != nil {
			return err
		}
		klog.V(2).InfoS("successfully overwritten validating webhook configuration", "name", configName)
		return nil
	}
	klog.V(2).InfoS("successfully created validating webhook configuration", "name", configName)
	return nil
}

// buildValidatingWebHooks returns a slice of fleet validating webhook objects.
func (w *Config) buildFleetValidatingWebhooks() []admv1.ValidatingWebhook {
	webHooks := []admv1.ValidatingWebhook{
		{
			Name:                    "fleet.pod.validating",
			ClientConfig:            w.createClientConfig(pod.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
					},
					Rule: createRule([]string{corev1.SchemeGroupVersion.Group}, []string{corev1.SchemeGroupVersion.Version}, []string{podResourceName}, &namespacedScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.clusterresourceplacementv1alpha1.validating",
			ClientConfig:            w.createClientConfig(clusterresourceplacement.V1Alpha1CRPValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
						admv1.Update,
					},
					Rule: createRule([]string{fleetv1alpha1.GroupVersion.Group}, []string{fleetv1alpha1.GroupVersion.Version}, []string{fleetv1alpha1.ClusterResourcePlacementResource}, &clusterScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.clusterresourceplacementv1beta1.validating",
			ClientConfig:            w.createClientConfig(clusterresourceplacement.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
						admv1.Update,
					},
					Rule: createRule([]string{placementv1beta1.GroupVersion.Group}, []string{placementv1beta1.GroupVersion.Version}, []string{placementv1beta1.ClusterResourcePlacementResource}, &clusterScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.replicaset.validating",
			ClientConfig:            w.createClientConfig(replicaset.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
					},
					Rule: createRule([]string{appsv1.SchemeGroupVersion.Group}, []string{appsv1.SchemeGroupVersion.Version}, []string{replicaSetResourceName}, &namespacedScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.membercluster.validating",
			ClientConfig:            w.createClientConfig(membercluster.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
						admv1.Update,
					},
					Rule: createRule([]string{clusterv1beta1.GroupVersion.Group}, []string{clusterv1beta1.GroupVersion.Version}, []string{memberClusterResourceName}, &clusterScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.clusterresourceoverride.validating",
			ClientConfig:            w.createClientConfig(clusterresourceoverride.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
						admv1.Update,
					},
					Rule: createRule([]string{placementv1alpha1.GroupVersion.Group}, []string{placementv1alpha1.GroupVersion.Version}, []string{clusterResourceOverrideName}, &clusterScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
		{
			Name:                    "fleet.resourceoverride.validating",
			ClientConfig:            w.createClientConfig(resourceoverride.ValidationPath),
			FailurePolicy:           &failFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: []admv1.OperationType{
						admv1.Create,
						admv1.Update,
					},
					Rule: createRule([]string{placementv1alpha1.GroupVersion.Group}, []string{placementv1alpha1.GroupVersion.Version}, []string{resourceOverrideName}, &namespacedScope),
				},
			},
			TimeoutSeconds: longWebhookTimeout,
		},
	}

	return webHooks
}

// buildFleetGuardRailValidatingWebhooks returns a slice of fleet guard rail validating webhook objects.
func (w *Config) buildFleetGuardRailValidatingWebhooks() []admv1.ValidatingWebhook {
	// MatchLabels/MatchExpressions values are ANDed to select resources.
	fleetMemberNamespaceSelector := &metav1.LabelSelector{
		MatchExpressions: []metav1.LabelSelectorRequirement{
			{
				Key:      placementv1beta1.FleetResourceLabelKey,
				Operator: metav1.LabelSelectorOpIn,
				Values:   []string{"true"},
			},
		},
	}
	fleetSystemNamespaceSelector := &metav1.LabelSelector{
		MatchExpressions: []metav1.LabelSelectorRequirement{
			{
				Key:      corev1.LabelMetadataName,
				Operator: metav1.LabelSelectorOpIn,
				Values:   []string{"fleet-system"},
			},
		},
	}
	kubeNamespaceSelector := &metav1.LabelSelector{
		MatchExpressions: []metav1.LabelSelectorRequirement{
			{
				Key:      corev1.LabelMetadataName,
				Operator: metav1.LabelSelectorOpIn,
				Values:   []string{"kube-system", "kube-public", "kube-node-lease"},
			},
		},
	}
	cudOperations := []admv1.OperationType{
		admv1.Create,
		admv1.Update,
		admv1.Delete,
	}
	cuOperations := []admv1.OperationType{
		admv1.Create,
		admv1.Update,
	}
	// we don't monitor lease to prevent the deadlock issue, we also don't monitor events.
	namespacedResourcesRules := []admv1.RuleWithOperations{
		// we want to monitor delete operations on all fleet/kube pre-fixed namespaced resources.
		{
			Operations: []admv1.OperationType{admv1.Delete},
			Rule:       createRule([]string{"*"}, []string{"*"}, []string{"*/*"}, &namespacedScope),
		},
		// TODO(ArvindThiru): not handling pods, replicasets as part of the fleet guard rail since they have validating webhooks, need to remove validating webhooks before adding these resources to fleet guard rail.
		{
			Operations: cuOperations,
			Rule: createRule([]string{corev1.SchemeGroupVersion.Group}, []string{corev1.SchemeGroupVersion.Version}, []string{bindingResourceName, configMapResourceName, endPointResourceName,
				limitRangeResourceName, persistentVolumeClaimsName, persistentVolumeClaimsName + "/status", podTemplateResourceName,
				replicationControllerResourceName, replicationControllerResourceName + "/status", resourceQuotaResourceName, resourceQuotaResourceName + "/status", secretResourceName,
				serviceAccountResourceName, servicesResourceName, servicesResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule: createRule([]string{appsv1.SchemeGroupVersion.Group}, []string{appsv1.SchemeGroupVersion.Version}, []string{controllerRevisionResourceName, daemonSetResourceName, daemonSetResourceName + "/status",
				deploymentResourceName, deploymentResourceName + "/status", statefulSetResourceName, statefulSetResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{authorizationv1.SchemeGroupVersion.Group}, []string{authorizationv1.SchemeGroupVersion.Version}, []string{localSubjectAccessReviewResourceName, localSubjectAccessReviewResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{autoscalingv1.SchemeGroupVersion.Group}, []string{autoscalingv1.SchemeGroupVersion.Version}, []string{horizontalPodAutoScalerResourceName, horizontalPodAutoScalerResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{batchv1.SchemeGroupVersion.Group}, []string{batchv1.SchemeGroupVersion.Version}, []string{cronJobResourceName, cronJobResourceName + "/status", jobResourceName, jobResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{discoveryv1.SchemeGroupVersion.Group}, []string{discoveryv1.SchemeGroupVersion.Version}, []string{endPointSlicesResourceName}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{networkingv1.SchemeGroupVersion.Group}, []string{networkingv1.SchemeGroupVersion.Version}, []string{ingressResourceName, ingressResourceName + "/status", networkPolicyResourceName, networkPolicyResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{policyv1.SchemeGroupVersion.Group}, []string{policyv1.SchemeGroupVersion.Version}, []string{podDisruptionBudgetsResourceName, podDisruptionBudgetsResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{rbacv1.SchemeGroupVersion.Group}, []string{rbacv1.SchemeGroupVersion.Version}, []string{roleResourceName, roleBindingResourceName}, &namespacedScope),
		},
		// rules for fleet namespaced resources.
		{
			Operations: cuOperations,
			Rule:       createRule([]string{storagev1.SchemeGroupVersion.Group}, []string{storagev1.SchemeGroupVersion.Version}, []string{csiStorageCapacityResourceName}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{fleetv1alpha1.GroupVersion.Group}, []string{fleetv1alpha1.GroupVersion.Version}, []string{internalMemberClusterResourceName, internalMemberClusterResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{clusterv1beta1.GroupVersion.Group}, []string{clusterv1beta1.GroupVersion.Version}, []string{internalMemberClusterResourceName, internalMemberClusterResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{placementv1beta1.GroupVersion.Group}, []string{placementv1beta1.GroupVersion.Version}, []string{workResourceName, workResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{workv1alpha1.GroupVersion.Group}, []string{workv1alpha1.GroupVersion.Version}, []string{workResourceName, workResourceName + "/status"}, &namespacedScope),
		},
		{
			Operations: cuOperations,
			Rule:       createRule([]string{fleetnetworkingv1alpha1.GroupVersion.Group}, []string{fleetnetworkingv1alpha1.GroupVersion.Version}, []string{endpointSliceExportResourceName, endpointSliceImportResourceName, internalServiceExportResourceName, internalServiceExportResourceName + "/status", internalServiceImportResourceName, internalServiceImportResourceName + "/status"}, &namespacedScope),
		},
	}
	guardRailWebhookConfigurations := []admv1.ValidatingWebhook{
		{
			Name:                    "fleet.customresourcedefinition.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: cudOperations,
					Rule:       createRule([]string{apiextensionsv1.SchemeGroupVersion.Group}, []string{apiextensionsv1.SchemeGroupVersion.Version}, []string{crdResourceName}, &clusterScope),
				},
			},
			TimeoutSeconds: shortWebhookTimeout,
		},
		{
			Name:                    "fleet.membercluster.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: cudOperations,
					Rule:       createRule([]string{clusterv1beta1.GroupVersion.Group}, []string{clusterv1beta1.GroupVersion.Version}, []string{memberClusterResourceName, memberClusterResourceName + "/status"}, &clusterScope),
				},
			},
			TimeoutSeconds: shortWebhookTimeout,
		},
		{
			Name:                    "fleet.v1alpha1.membercluster.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: cudOperations,
					Rule:       createRule([]string{fleetv1alpha1.GroupVersion.Group}, []string{fleetv1alpha1.GroupVersion.Version}, []string{memberClusterResourceName, memberClusterResourceName + "/status"}, &clusterScope),
				},
			},
			TimeoutSeconds: shortWebhookTimeout,
		},
		{
			Name:                    "fleet.fleetmembernamespacedresources.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			NamespaceSelector:       fleetMemberNamespaceSelector,
			Rules:                   namespacedResourcesRules,
			TimeoutSeconds:          shortWebhookTimeout,
		},
		{
			Name:                    "fleet.fleetsystemnamespacedresources.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			NamespaceSelector:       fleetSystemNamespaceSelector,
			Rules:                   namespacedResourcesRules,
			TimeoutSeconds:          shortWebhookTimeout,
		},
		{
			Name:                    "fleet.kubenamespacedresources.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			NamespaceSelector:       kubeNamespaceSelector,
			Rules:                   namespacedResourcesRules,
			TimeoutSeconds:          shortWebhookTimeout,
		},
		{
			Name:                    "fleet.namespace.guardrail.validating",
			ClientConfig:            w.createClientConfig(fleetresourcehandler.ValidationPath),
			FailurePolicy:           &ignoreFailurePolicy,
			SideEffects:             &sideEffortsNone,
			AdmissionReviewVersions: admissionReviewVersions,
			Rules: []admv1.RuleWithOperations{
				{
					Operations: cudOperations,
					Rule:       createRule([]string{corev1.SchemeGroupVersion.Group}, []string{corev1.SchemeGroupVersion.Version}, []string{namespaceResourceName}, &clusterScope),
				},
			},
			TimeoutSeconds: shortWebhookTimeout,
		},
	}

	return guardRailWebhookConfigurations
}

// createClientConfig generates the client configuration with either service ref or URL for the argued interface.
func (w *Config) createClientConfig(validationPath string) admv1.WebhookClientConfig {
	serviceRef := admv1.ServiceReference{
		Namespace: w.serviceNamespace,
		Name:      w.serviceName,
		Port:      ptr.To(w.servicePort),
	}
	serviceEndpoint := w.serviceURL + validationPath
	serviceRef.Path = ptr.To(validationPath)
	config := admv1.WebhookClientConfig{
		CABundle: w.caPEM,
	}
	switch *w.clientConnectionType {
	case options.Service:
		config.Service = &serviceRef
	case options.URL:
		config.URL = ptr.To(serviceEndpoint)
	}
	return config
}

// genCertificate generates the serving cerficiate for the webhook server.
func (w *Config) genCertificate(certDir string) ([]byte, error) {
	caPEM, certPEM, keyPEM, err := w.genSelfSignedCert()
	if err != nil {
		klog.ErrorS(err, "fail to generate self-signed cert")
		return nil, err
	}

	// generate certificate files (i.e., tls.crt and tls.key)
	if err := genCertAndKeyFile(certPEM, keyPEM, certDir); err != nil {
		klog.ErrorS(err, "fail to generate certificate and key files")
		return nil, err
	}
	return caPEM, nil
}

// genSelfSignedCert generates the self signed Certificate/Key pair
func (w *Config) genSelfSignedCert() (caPEMByte, certPEMByte, keyPEMByte []byte, err error) {
	// CA config
	ca := &x509.Certificate{
		SerialNumber: big.NewInt(2022),
		Subject: pkix.Name{
			CommonName:         "fleet.azure.com",
			OrganizationalUnit: []string{"Azure Kubernetes Service"},
			Organization:       []string{"Microsoft"},
			Locality:           []string{"Redmond"},
			Province:           []string{"Washington"},
			Country:            []string{"United States of America"},
		},
		NotBefore:             time.Now(),
		NotAfter:              time.Now().AddDate(10, 0, 0), // Set expiration time to be 10 years for now
		IsCA:                  true,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
	}

	// CA private key
	caPrvKey, err := rsa.GenerateKey(rand.Reader, 4096)
	if err != nil {
		return nil, nil, nil, err
	}

	// Self signed CA certificate
	caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrvKey.PublicKey, caPrvKey)
	if err != nil {
		return nil, nil, nil, err
	}

	// PEM encode CA cert
	caPEM := new(bytes.Buffer)
	if err := pem.Encode(caPEM, &pem.Block{
		Type:  "CERTIFICATE",
		Bytes: caBytes,
	}); err != nil {
		return nil, nil, nil, err
	}
	caPEMByte = caPEM.Bytes()

	dnsNames := []string{
		fmt.Sprintf("%s.%s.svc", w.serviceName, w.serviceNamespace),
		fmt.Sprintf("%s.%s.svc.cluster.local", w.serviceName, w.serviceNamespace),
	}
	// server cert config
	cert := &x509.Certificate{
		DNSNames:     dnsNames,
		SerialNumber: big.NewInt(2022),
		Subject: pkix.Name{
			CommonName:         fmt.Sprintf("%s.cert.server", w.serviceName),
			OrganizationalUnit: []string{"Azure Kubernetes Service"},
			Organization:       []string{"Microsoft"},
			Locality:           []string{"Redmond"},
			Province:           []string{"Washington"},
			Country:            []string{"United States of America"},
		},
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(10, 0, 0),
		SubjectKeyId: []byte{1, 2, 3, 4, 5},
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:     x509.KeyUsageDigitalSignature,
	}

	// server private key
	certPrvKey, err := rsa.GenerateKey(rand.Reader, 4096)
	if err != nil {
		return nil, nil, nil, err
	}

	// sign the server cert
	certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certPrvKey.PublicKey, caPrvKey)
	if err != nil {
		return nil, nil, nil, err
	}

	// PEM encode the  server cert and key
	certPEM := new(bytes.Buffer)
	if err := pem.Encode(certPEM, &pem.Block{
		Type:  "CERTIFICATE",
		Bytes: certBytes,
	}); err != nil {
		return nil, nil, nil, err
	}
	certPEMByte = certPEM.Bytes()

	certPrvKeyPEM := new(bytes.Buffer)
	if err := pem.Encode(certPrvKeyPEM, &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(certPrvKey),
	}); err != nil {
		return nil, nil, nil, err
	}
	keyPEMByte = certPrvKeyPEM.Bytes()
	return caPEMByte, certPEMByte, keyPEMByte, nil
}

// genCertAndKeyFile creates the serving certificate/key files for the webhook server
func genCertAndKeyFile(certData, keyData []byte, certDir string) error {
	// always remove first
	if err := os.RemoveAll(certDir); err != nil {
		return fmt.Errorf("fail to remove certificates: %w", err)
	}
	if err := os.MkdirAll(certDir, 0755); err != nil {
		return fmt.Errorf("could not create directory %q to store certificates: %w", certDir, err)
	}
	certPath := filepath.Join(certDir, fleetWebhookCertFileName)
	f, err := os.OpenFile(filepath.Clean(certPath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
	if err != nil {
		return fmt.Errorf("could not open %q: %w", certPath, err)
	}
	defer f.Close()
	certBlock, _ := pem.Decode(certData)
	if certBlock == nil {
		return fmt.Errorf("invalid certificate data")
	}
	if err := pem.Encode(f, certBlock); err != nil {
		return err
	}

	keyPath := filepath.Join(certDir, fleetWebhookKeyFileName)
	kf, err := os.OpenFile(filepath.Clean(keyPath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
	if err != nil {
		return fmt.Errorf("could not open %q: %w", keyPath, err)
	}

	keyBlock, _ := pem.Decode(keyData)
	if keyBlock == nil {
		return fmt.Errorf("invalid key data")
	}
	if err := pem.Encode(kf, keyBlock); err != nil {
		return err
	}
	klog.V(2).InfoS("successfully generate certificate and key files")
	return nil
}

// bindWebhookConfigToFleetSystem sets the OwnerReference of the argued ValidatingWebhookConfiguration to the cluster scoped fleet-system namespace.
func bindWebhookConfigToFleetSystem(ctx context.Context, k8Client client.Client, validatingWebhookConfig *admv1.ValidatingWebhookConfiguration) error {
	var fleetNs corev1.Namespace
	if err := k8Client.Get(ctx, client.ObjectKey{Name: "fleet-system"}, &fleetNs); err != nil {
		return err
	}

	ownerRef := metav1.OwnerReference{
		APIVersion:         fleetNs.GroupVersionKind().GroupVersion().String(),
		Kind:               fleetNs.Kind,
		Name:               fleetNs.GetName(),
		UID:                fleetNs.GetUID(),
		BlockOwnerDeletion: ptr.To(false),
	}

	validatingWebhookConfig.OwnerReferences = []metav1.OwnerReference{ownerRef}
	return nil
}

// createRule returns a admission rule using the arguments passed.
func createRule(apiGroups, apiVersions, resources []string, scopeType *admv1.ScopeType) admv1.Rule {
	return admv1.Rule{
		APIGroups:   apiGroups,
		APIVersions: apiVersions,
		Resources:   resources,
		Scope:       scopeType,
	}
}