feat: support multi domains for ingress webhooks. (#4491)

* feat: support multi domains for ingress webhooks.

Signed-off-by: yy <lingdie.yy@outlook.com>

* chore: delete domain env.

Signed-off-by: yy <lingdie.yy@outlook.com>

* chore: fix license.

Signed-off-by: yy <lingdie.yy@outlook.com>

---------

Signed-off-by: yy <lingdie.yy@outlook.com>

controllers/admission/api/v1/ingress_webhook.go

@@ -47,13 +47,12 @@ var ilog = logf.Log.WithName("ingress-validating-webhook")
 
 type IngressMutator struct {
 	client.Client
-	domain             string
+	Domains            DomainList
 	IngressAnnotations map[string]string
 }
 
 func (m *IngressMutator) SetupWithManager(mgr ctrl.Manager) error {
 	m.Client = mgr.GetClient()
-	m.domain = os.Getenv("DOMAIN")
 	return builder.WebhookManagedBy(mgr).
 		For(&netv1.Ingress{}).
 		WithDefaulter(m).
@@ -65,9 +64,12 @@ func (m *IngressMutator) Default(_ context.Context, obj runtime.Object) error {
 	if !ok {
 		return errors.New("obj convert Ingress is error")
 	}
-	if isUserNamespace(i.Namespace) && hasSubDomain(i, m.domain) {
-		ilog.Info("mutating ingress in user ns", "ingress namespace", i.Namespace, "ingress name", i.Name)
-		m.mutateUserIngressAnnotations(i)
+
+	for _, domain := range m.Domains {
+		if isUserNamespace(i.Namespace) && hasSubDomain(i, domain) {
+			ilog.Info("mutating ingress in user ns", "ingress namespace", i.Namespace, "ingress name", i.Name)
+			m.mutateUserIngressAnnotations(i)
+		}
 	}
 	return nil
 }
@@ -83,8 +85,8 @@ func (m *IngressMutator) mutateUserIngressAnnotations(i *netv1.Ingress) {
 
 type IngressValidator struct {
 	client.Client
-	domain string
-	cache  cache.Cache
+	Domains DomainList
+	cache   cache.Cache
 
 	IcpValidator *IcpValidator
 }
@@ -96,7 +98,6 @@ func (v *IngressValidator) SetupWithManager(mgr ctrl.Manager) error {
 
 	iv := IngressValidator{
 		Client: mgr.GetClient(),
-		domain: os.Getenv("DOMAIN"),
 		cache:  mgr.GetCache(),
 
 		IcpValidator: NewIcpValidator(
@@ -203,28 +204,27 @@ func (v *IngressValidator) validate(ctx context.Context, i *netv1.Ingress) error
 }
 
 func (v *IngressValidator) checkCname(i *netv1.Ingress, rule *netv1.IngressRule) error {
-	// check if ingress host is end with domain
-	if strings.HasSuffix(rule.Host, v.domain) {
-		ilog.Info("ingress host is end with "+v.domain+", skip validate", "ingress namespace", i.Namespace, "ingress name", i.Name)
-		return nil
-	}
-
-	// get cname and check if it is cname to domain
-	cname, err := net.LookupCNAME(rule.Host)
-	if err != nil {
-		ilog.Error(err, "can not verify ingress host "+rule.Host+", lookup cname error")
-		return err
-	}
-	// remove last dot
-	cname = strings.TrimSuffix(cname, ".")
-
-	// if cname is not end with domain, return error
-	if !strings.HasSuffix(cname, v.domain) {
-		ilog.Info("deny ingress host "+rule.Host+", cname is not end with "+v.domain, "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname)
-		return fmt.Errorf(code.MessageFormat, code.IngressFailedCnameCheck, "can not verify ingress host "+rule.Host+", cname is not end with "+v.domain)
+	for _, domain := range v.Domains {
+		// check if ingress host is end with domain
+		if strings.HasSuffix(rule.Host, domain) {
+			ilog.Info("ingress host is end with "+domain+", skip validate", "ingress namespace", i.Namespace, "ingress name", i.Name)
+			return nil
+		}
+		// get cname and check if it is cname to domain
+		cname, err := net.LookupCNAME(rule.Host)
+		if err != nil {
+			ilog.Error(err, "can not verify ingress host "+rule.Host+", lookup cname error")
+			return err
+		}
+		// remove last dot
+		cname = strings.TrimSuffix(cname, ".")
+		// if cname is not end with domain, return error
+		if strings.HasSuffix(cname, domain) {
+			ilog.Info("ingress host "+rule.Host+" is cname to "+cname+", pass checkCname validate", "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname)
+			return nil
+		}
 	}
-	ilog.Info("ingress host "+rule.Host+" is cname to "+cname+", pass checkCname validate", "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname)
-	return nil
+	return fmt.Errorf(code.MessageFormat, code.IngressFailedCnameCheck, "can not verify ingress host "+rule.Host+", cname is not end with any domains in "+strings.Join(v.Domains, ","))
 }
 
 func (v *IngressValidator) checkOwner(i *netv1.Ingress, rule *netv1.IngressRule) error {

controllers/admission/api/v1/types.go

@@ -0,0 +1,31 @@
+// Copyright © 2024 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"fmt"
+	"strings"
+)
+
+type DomainList []string
+
+func (s *DomainList) String() string {
+	return fmt.Sprintf("%v", *s)
+}
+
+func (s *DomainList) Set(value string) error {
+	*s = strings.Split(value, ",")
+	return nil
+}

controllers/admission/cmd/main.go

@@ -53,13 +53,14 @@ func main() {
 	var enableLeaderElection bool
 	var probeAddr string
 	var ingressAnnotationString string
+	var domains v1.DomainList
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&ingressAnnotationString, "ingress-mutating-annotations", "", "Ingress annotations: 'key1=value1,key2=value2'")
-
+	flag.Var(&domains, "domains", "Domains to be used for check ingress cname")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -68,6 +69,11 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	if len(domains) == 0 {
+		setupLog.Error(nil, "domains is empty")
+		os.Exit(1)
+	}
+
 	setupLog.Info("ingress annotations:", "annotation", ingressAnnotationString)
 	ingressAnnotations := make(map[string]string)
 	if ingressAnnotationString != "" {
@@ -109,13 +115,16 @@ func main() {
 		os.Exit(1)
 	}
 
-	if (&v1.IngressValidator{}).SetupWithManager(mgr) != nil {
+	if (&v1.IngressValidator{
+		Domains: domains,
+	}).SetupWithManager(mgr) != nil {
 		setupLog.Error(err, "unable to create ingress validator webhook")
 		os.Exit(1)
 	}
 
 	if (&v1.IngressMutator{
 		IngressAnnotations: ingressAnnotations,
+		Domains:            domains,
 	}).SetupWithManager(mgr) != nil {
 		setupLog.Error(err, "unable to create ingress mutator webhook")
 		os.Exit(1)

controllers/admission/deploy/Kubefile

@@ -5,11 +5,11 @@ USER 65532:65532
 COPY registry registry
 COPY manifests manifests
 
-ENV cloudDomain="cloud.sealos.io"
 
 # Example for ingressMutatingAnnotations
 # ENV ingressMutatingAnnotations="nginx.ingress.kubernetes.io/limit-connections=10,nginx.ingress.kubernetes.io/limit-rate-after=10m,nginx.ingress.kubernetes.io/limit-rate=100k,nginx.ingress.kubernetes.io/proxy-buffering=on"
 
+ENV domains="cloud.sealos.io,laf.dev,laf.run"
 ENV ingressMutatingAnnotations=""
 ENV ingressWebhookEnabled="true"
 ENV ingressWebhookFailurePolicy="Fail"

controllers/admission/deploy/manifests/deploy.yaml.tmpl

@@ -292,11 +292,10 @@ spec:
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --ingress-mutating-annotations={{ .ingressMutatingAnnotations }}
+        - --domains={{ .domains }}
         command:
         - /manager
         env:
-        - name: DOMAIN
-          value: '{{ .cloudDomain }}'
         - name: ICP_ENABLED
           value: '{{ .icpEnabled }}'
         - name: ICP_ENDPOINT