From f3de924d870cef4b81d6248e4a68aba4fcb2a4fc Mon Sep 17 00:00:00 2001 From: yy <56745951+lingdie@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:58:30 +0800 Subject: [PATCH] feat: support multi domains for ingress webhooks. (#4491) * feat: support multi domains for ingress webhooks. Signed-off-by: yy * chore: delete domain env. Signed-off-by: yy * chore: fix license. Signed-off-by: yy --------- Signed-off-by: yy --- .../admission/api/v1/ingress_webhook.go | 58 +++++++++---------- controllers/admission/api/v1/types.go | 31 ++++++++++ controllers/admission/cmd/main.go | 13 ++++- controllers/admission/deploy/Kubefile | 2 +- .../deploy/manifests/deploy.yaml.tmpl | 3 +- 5 files changed, 73 insertions(+), 34 deletions(-) create mode 100644 controllers/admission/api/v1/types.go diff --git a/controllers/admission/api/v1/ingress_webhook.go b/controllers/admission/api/v1/ingress_webhook.go index 0adc03e8a29..6a30a9f71ee 100644 --- a/controllers/admission/api/v1/ingress_webhook.go +++ b/controllers/admission/api/v1/ingress_webhook.go @@ -47,13 +47,12 @@ var ilog = logf.Log.WithName("ingress-validating-webhook") type IngressMutator struct { client.Client - domain string + Domains DomainList IngressAnnotations map[string]string } func (m *IngressMutator) SetupWithManager(mgr ctrl.Manager) error { m.Client = mgr.GetClient() - m.domain = os.Getenv("DOMAIN") return builder.WebhookManagedBy(mgr). For(&netv1.Ingress{}). WithDefaulter(m). @@ -65,9 +64,12 @@ func (m *IngressMutator) Default(_ context.Context, obj runtime.Object) error { if !ok { return errors.New("obj convert Ingress is error") } - if isUserNamespace(i.Namespace) && hasSubDomain(i, m.domain) { - ilog.Info("mutating ingress in user ns", "ingress namespace", i.Namespace, "ingress name", i.Name) - m.mutateUserIngressAnnotations(i) + + for _, domain := range m.Domains { + if isUserNamespace(i.Namespace) && hasSubDomain(i, domain) { + ilog.Info("mutating ingress in user ns", "ingress namespace", i.Namespace, "ingress name", i.Name) + m.mutateUserIngressAnnotations(i) + } } return nil } @@ -83,8 +85,8 @@ func (m *IngressMutator) mutateUserIngressAnnotations(i *netv1.Ingress) { type IngressValidator struct { client.Client - domain string - cache cache.Cache + Domains DomainList + cache cache.Cache IcpValidator *IcpValidator } @@ -96,7 +98,6 @@ func (v *IngressValidator) SetupWithManager(mgr ctrl.Manager) error { iv := IngressValidator{ Client: mgr.GetClient(), - domain: os.Getenv("DOMAIN"), cache: mgr.GetCache(), IcpValidator: NewIcpValidator( @@ -203,28 +204,27 @@ func (v *IngressValidator) validate(ctx context.Context, i *netv1.Ingress) error } func (v *IngressValidator) checkCname(i *netv1.Ingress, rule *netv1.IngressRule) error { - // check if ingress host is end with domain - if strings.HasSuffix(rule.Host, v.domain) { - ilog.Info("ingress host is end with "+v.domain+", skip validate", "ingress namespace", i.Namespace, "ingress name", i.Name) - return nil - } - - // get cname and check if it is cname to domain - cname, err := net.LookupCNAME(rule.Host) - if err != nil { - ilog.Error(err, "can not verify ingress host "+rule.Host+", lookup cname error") - return err - } - // remove last dot - cname = strings.TrimSuffix(cname, ".") - - // if cname is not end with domain, return error - if !strings.HasSuffix(cname, v.domain) { - ilog.Info("deny ingress host "+rule.Host+", cname is not end with "+v.domain, "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname) - return fmt.Errorf(code.MessageFormat, code.IngressFailedCnameCheck, "can not verify ingress host "+rule.Host+", cname is not end with "+v.domain) + for _, domain := range v.Domains { + // check if ingress host is end with domain + if strings.HasSuffix(rule.Host, domain) { + ilog.Info("ingress host is end with "+domain+", skip validate", "ingress namespace", i.Namespace, "ingress name", i.Name) + return nil + } + // get cname and check if it is cname to domain + cname, err := net.LookupCNAME(rule.Host) + if err != nil { + ilog.Error(err, "can not verify ingress host "+rule.Host+", lookup cname error") + return err + } + // remove last dot + cname = strings.TrimSuffix(cname, ".") + // if cname is not end with domain, return error + if strings.HasSuffix(cname, domain) { + ilog.Info("ingress host "+rule.Host+" is cname to "+cname+", pass checkCname validate", "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname) + return nil + } } - ilog.Info("ingress host "+rule.Host+" is cname to "+cname+", pass checkCname validate", "ingress namespace", i.Namespace, "ingress name", i.Name, "cname", cname) - return nil + return fmt.Errorf(code.MessageFormat, code.IngressFailedCnameCheck, "can not verify ingress host "+rule.Host+", cname is not end with any domains in "+strings.Join(v.Domains, ",")) } func (v *IngressValidator) checkOwner(i *netv1.Ingress, rule *netv1.IngressRule) error { diff --git a/controllers/admission/api/v1/types.go b/controllers/admission/api/v1/types.go new file mode 100644 index 00000000000..37b9aace4d9 --- /dev/null +++ b/controllers/admission/api/v1/types.go @@ -0,0 +1,31 @@ +// Copyright © 2024 sealos. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + "strings" +) + +type DomainList []string + +func (s *DomainList) String() string { + return fmt.Sprintf("%v", *s) +} + +func (s *DomainList) Set(value string) error { + *s = strings.Split(value, ",") + return nil +} diff --git a/controllers/admission/cmd/main.go b/controllers/admission/cmd/main.go index cf8d7134489..e3d350b6636 100644 --- a/controllers/admission/cmd/main.go +++ b/controllers/admission/cmd/main.go @@ -53,13 +53,14 @@ func main() { var enableLeaderElection bool var probeAddr string var ingressAnnotationString string + var domains v1.DomainList flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&ingressAnnotationString, "ingress-mutating-annotations", "", "Ingress annotations: 'key1=value1,key2=value2'") - + flag.Var(&domains, "domains", "Domains to be used for check ingress cname") opts := zap.Options{ Development: true, } @@ -68,6 +69,11 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + if len(domains) == 0 { + setupLog.Error(nil, "domains is empty") + os.Exit(1) + } + setupLog.Info("ingress annotations:", "annotation", ingressAnnotationString) ingressAnnotations := make(map[string]string) if ingressAnnotationString != "" { @@ -109,13 +115,16 @@ func main() { os.Exit(1) } - if (&v1.IngressValidator{}).SetupWithManager(mgr) != nil { + if (&v1.IngressValidator{ + Domains: domains, + }).SetupWithManager(mgr) != nil { setupLog.Error(err, "unable to create ingress validator webhook") os.Exit(1) } if (&v1.IngressMutator{ IngressAnnotations: ingressAnnotations, + Domains: domains, }).SetupWithManager(mgr) != nil { setupLog.Error(err, "unable to create ingress mutator webhook") os.Exit(1) diff --git a/controllers/admission/deploy/Kubefile b/controllers/admission/deploy/Kubefile index 4598458f1ae..a91fc554cea 100644 --- a/controllers/admission/deploy/Kubefile +++ b/controllers/admission/deploy/Kubefile @@ -5,11 +5,11 @@ USER 65532:65532 COPY registry registry COPY manifests manifests -ENV cloudDomain="cloud.sealos.io" # Example for ingressMutatingAnnotations # ENV ingressMutatingAnnotations="nginx.ingress.kubernetes.io/limit-connections=10,nginx.ingress.kubernetes.io/limit-rate-after=10m,nginx.ingress.kubernetes.io/limit-rate=100k,nginx.ingress.kubernetes.io/proxy-buffering=on" +ENV domains="cloud.sealos.io,laf.dev,laf.run" ENV ingressMutatingAnnotations="" ENV ingressWebhookEnabled="true" ENV ingressWebhookFailurePolicy="Fail" diff --git a/controllers/admission/deploy/manifests/deploy.yaml.tmpl b/controllers/admission/deploy/manifests/deploy.yaml.tmpl index e860bfb44bc..021b32e335c 100644 --- a/controllers/admission/deploy/manifests/deploy.yaml.tmpl +++ b/controllers/admission/deploy/manifests/deploy.yaml.tmpl @@ -292,11 +292,10 @@ spec: - --metrics-bind-address=127.0.0.1:8080 - --leader-elect - --ingress-mutating-annotations={{ .ingressMutatingAnnotations }} + - --domains={{ .domains }} command: - /manager env: - - name: DOMAIN - value: '{{ .cloudDomain }}' - name: ICP_ENABLED value: '{{ .icpEnabled }}' - name: ICP_ENDPOINT