From e61007a3a351fbda2af63f440543f348d3de1070 Mon Sep 17 00:00:00 2001 From: Ling Samuel Date: Thu, 21 Apr 2022 09:29:53 +0800 Subject: [PATCH 1/4] feat: support ApisixClusterConfig v2 Signed-off-by: Ling Samuel --- cmd/ingress/ingress.go | 1 + pkg/config/config.go | 44 +++--- pkg/config/config_test.go | 30 ++-- pkg/ingress/apisix_cluster_config.go | 54 ++++++-- pkg/ingress/controller.go | 23 +++- pkg/ingress/status.go | 17 +++ pkg/kube/apisix_cluster_config.go | 167 +++++++++++++++++++++++ pkg/kube/translation/global_rule.go | 23 +++- pkg/kube/translation/global_rule_test.go | 2 +- pkg/kube/translation/translator.go | 7 +- 10 files changed, 313 insertions(+), 55 deletions(-) create mode 100644 pkg/kube/apisix_cluster_config.go diff --git a/cmd/ingress/ingress.go b/cmd/ingress/ingress.go index ed4b2df897f..3ecfdd6cb38 100644 --- a/cmd/ingress/ingress.go +++ b/cmd/ingress/ingress.go @@ -162,6 +162,7 @@ For example, no available LB exists in the bare metal environment.`) cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ElectionID, "election-id", config.IngressAPISIXLeader, "election id used for campaign the controller leader") cmd.PersistentFlags().StringVar(&cfg.Kubernetes.IngressVersion, "ingress-version", config.IngressNetworkingV1, "the supported ingress api group version, can be \"networking/v1beta1\", \"networking/v1\" (for Kubernetes version v1.19.0 or higher) and \"extensions/v1beta1\"") cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixRouteVersion, "apisix-route-version", config.ApisixRouteV2beta3, "the supported apisixroute api group version, can be \"apisix.apache.org/v2beta2\" or \"apisix.apache.org/v2beta3\"") + cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixClusterConfigVersion, "apisix-cluster-config-version", config.ApisixV2beta3, "the supported ApisixClusterConfig api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"") cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.WatchEndpointSlices, "watch-endpointslices", false, "whether to watch endpointslices rather than endpoints") cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.EnableGatewayAPI, "enable-gateway-api", false, "whether to enable support for Gateway API") cmd.PersistentFlags().StringVar(&cfg.APISIX.DefaultClusterBaseURL, "default-apisix-cluster-base-url", "", "the base URL of admin api / manager api for the default APISIX cluster") diff --git a/pkg/config/config.go b/pkg/config/config.go index 99695fcd309..24cffb46a6b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -57,6 +57,10 @@ const ( ApisixRouteV2beta3 = "apisix.apache.org/v2beta3" // ApisixRouteV2 represents apisixroute.apisix.apache.org/v2 ApisixRouteV2 = "apisix.apache.org/v2" + // ApisixV2beta3 represents apisix.apache.org/v2beta3 + ApisixV2beta3 = "apisix.apache.org/v2beta3" + // ApisixV2 represents apisix.apache.org/v2 + ApisixV2 = "apisix.apache.org/v2" _minimalResyncInterval = 30 * time.Second @@ -83,16 +87,17 @@ type Config struct { // KubernetesConfig contains all Kubernetes related config items. type KubernetesConfig struct { - Kubeconfig string `json:"kubeconfig" yaml:"kubeconfig"` - ResyncInterval types.TimeDuration `json:"resync_interval" yaml:"resync_interval"` - AppNamespaces []string `json:"app_namespaces" yaml:"app_namespaces"` - NamespaceSelector []string `json:"namespace_selector" yaml:"namespace_selector"` - ElectionID string `json:"election_id" yaml:"election_id"` - IngressClass string `json:"ingress_class" yaml:"ingress_class"` - IngressVersion string `json:"ingress_version" yaml:"ingress_version"` - WatchEndpointSlices bool `json:"watch_endpoint_slices" yaml:"watch_endpoint_slices"` - ApisixRouteVersion string `json:"apisix_route_version" yaml:"apisix_route_version"` - EnableGatewayAPI bool `json:"enable_gateway_api" yaml:"enable_gateway_api"` + Kubeconfig string `json:"kubeconfig" yaml:"kubeconfig"` + ResyncInterval types.TimeDuration `json:"resync_interval" yaml:"resync_interval"` + AppNamespaces []string `json:"app_namespaces" yaml:"app_namespaces"` + NamespaceSelector []string `json:"namespace_selector" yaml:"namespace_selector"` + ElectionID string `json:"election_id" yaml:"election_id"` + IngressClass string `json:"ingress_class" yaml:"ingress_class"` + IngressVersion string `json:"ingress_version" yaml:"ingress_version"` + WatchEndpointSlices bool `json:"watch_endpoint_slices" yaml:"watch_endpoint_slices"` + ApisixRouteVersion string `json:"apisix_route_version" yaml:"apisix_route_version"` + ApisixClusterConfigVersion string `json:"apisix_cluster_config_version" yaml:"apisix_cluster_config_version"` + EnableGatewayAPI bool `json:"enable_gateway_api" yaml:"enable_gateway_api"` } // APISIXConfig contains all APISIX related config items. @@ -120,15 +125,16 @@ func NewDefaultConfig() *Config { KeyFilePath: "/etc/webhook/certs/key.pem", EnableProfiling: true, Kubernetes: KubernetesConfig{ - Kubeconfig: "", // Use in-cluster configurations. - ResyncInterval: types.TimeDuration{Duration: 6 * time.Hour}, - AppNamespaces: []string{v1.NamespaceAll}, - ElectionID: IngressAPISIXLeader, - IngressClass: IngressClass, - IngressVersion: IngressNetworkingV1, - ApisixRouteVersion: ApisixRouteV2beta3, - WatchEndpointSlices: false, - EnableGatewayAPI: false, + Kubeconfig: "", // Use in-cluster configurations. + ResyncInterval: types.TimeDuration{Duration: 6 * time.Hour}, + AppNamespaces: []string{v1.NamespaceAll}, + ElectionID: IngressAPISIXLeader, + IngressClass: IngressClass, + IngressVersion: IngressNetworkingV1, + ApisixRouteVersion: ApisixRouteV2beta3, + ApisixClusterConfigVersion: ApisixV2beta3, + WatchEndpointSlices: false, + EnableGatewayAPI: false, }, } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 17b55052ba3..1342038930b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -38,13 +38,14 @@ func TestNewConfigFromFile(t *testing.T) { KeyFilePath: "/etc/webhook/certs/key.pem", EnableProfiling: true, Kubernetes: KubernetesConfig{ - ResyncInterval: types.TimeDuration{Duration: time.Hour}, - Kubeconfig: "/path/to/foo/baz", - AppNamespaces: []string{""}, - ElectionID: "my-election-id", - IngressClass: IngressClass, - IngressVersion: IngressNetworkingV1, - ApisixRouteVersion: ApisixRouteV2beta3, + ResyncInterval: types.TimeDuration{Duration: time.Hour}, + Kubeconfig: "/path/to/foo/baz", + AppNamespaces: []string{""}, + ElectionID: "my-election-id", + IngressClass: IngressClass, + IngressVersion: IngressNetworkingV1, + ApisixRouteVersion: ApisixRouteV2beta3, + ApisixClusterConfigVersion: ApisixV2beta3, }, APISIX: APISIXConfig{ DefaultClusterName: "default", @@ -119,13 +120,14 @@ func TestConfigWithEnvVar(t *testing.T) { KeyFilePath: "/etc/webhook/certs/key.pem", EnableProfiling: true, Kubernetes: KubernetesConfig{ - ResyncInterval: types.TimeDuration{Duration: time.Hour}, - Kubeconfig: "", - AppNamespaces: []string{""}, - ElectionID: "my-election-id", - IngressClass: IngressClass, - IngressVersion: IngressNetworkingV1, - ApisixRouteVersion: ApisixRouteV2beta3, + ResyncInterval: types.TimeDuration{Duration: time.Hour}, + Kubeconfig: "", + AppNamespaces: []string{""}, + ElectionID: "my-election-id", + IngressClass: IngressClass, + IngressVersion: IngressNetworkingV1, + ApisixRouteVersion: ApisixRouteV2beta3, + ApisixClusterConfigVersion: ApisixV2beta3, }, APISIX: APISIXConfig{ DefaultClusterName: "default", diff --git a/pkg/ingress/apisix_cluster_config.go b/pkg/ingress/apisix_cluster_config.go index 7bed3d461e6..b4e21782274 100644 --- a/pkg/ingress/apisix_cluster_config.go +++ b/pkg/ingress/apisix_cluster_config.go @@ -16,6 +16,7 @@ package ingress import ( "context" + "github.com/apache/apisix-ingress-controller/pkg/kube" "time" "go.uber.org/zap" @@ -148,7 +149,7 @@ func (c *apisixClusterConfigController) sync(ctx context.Context, ev *types.Even } } - globalRule, err := c.controller.translator.TranslateClusterConfig(acc) + globalRule, err := c.controller.translator.TranslateClusterConfigV2beta3(acc) if err != nil { // TODO add status log.Errorw("failed to translate ApisixClusterConfig", @@ -209,6 +210,11 @@ func (c *apisixClusterConfigController) handleSyncErr(obj interface{}, err error } func (c *apisixClusterConfigController) onAdd(obj interface{}) { + acc, err := kube.NewApisixClusterConfig(obj) + if err != nil { + log.Errorw("found ApisixClusterConfig resource with bad type", zap.Error(err)) + return + } key, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { log.Errorf("found ApisixClusterConfig resource with bad meta key: %s", err.Error()) @@ -220,17 +226,28 @@ func (c *apisixClusterConfigController) onAdd(obj interface{}) { ) c.workqueue.Add(&types.Event{ - Type: types.EventAdd, - Object: key, + Type: types.EventAdd, + Object: kube.ApisixClusterConfigEvent{ + Key: key, + GroupVersion: acc.GroupVersion(), + }, }) c.controller.MetricsCollector.IncrEvents("clusterConfig", "add") } func (c *apisixClusterConfigController) onUpdate(oldObj, newObj interface{}) { - prev := oldObj.(*configv2beta3.ApisixClusterConfig) - curr := newObj.(*configv2beta3.ApisixClusterConfig) - if prev.ResourceVersion >= curr.ResourceVersion { + prev, err := kube.NewApisixClusterConfig(oldObj) + if err != nil { + log.Errorw("found ApisixClusterConfig resource with bad type", zap.Error(err)) + return + } + curr, err := kube.NewApisixClusterConfig(newObj) + if err != nil { + log.Errorw("found ApisixClusterConfig resource with bad type", zap.Error(err)) + return + } + if prev.ResourceVersion() >= curr.ResourceVersion() { return } key, err := cache.MetaNamespaceKeyFunc(newObj) @@ -244,21 +261,29 @@ func (c *apisixClusterConfigController) onUpdate(oldObj, newObj interface{}) { ) c.workqueue.Add(&types.Event{ - Type: types.EventUpdate, - Object: key, + Type: types.EventUpdate, + Object: kube.ApisixClusterConfigEvent{ + Key: key, + OldObject: prev, + GroupVersion: curr.GroupVersion(), + }, }) c.controller.MetricsCollector.IncrEvents("clusterConfig", "update") } func (c *apisixClusterConfigController) onDelete(obj interface{}) { - acc, ok := obj.(*configv2beta3.ApisixClusterConfig) - if !ok { + acc, err := kube.NewApisixClusterConfig(obj) + if err != nil { tombstone, ok := obj.(*cache.DeletedFinalStateUnknown) if !ok { return } - acc = tombstone.Obj.(*configv2beta3.ApisixClusterConfig) + acc, err = kube.NewApisixClusterConfig(tombstone) + if err != nil { + log.Errorw("found ApisixClusterConfig resource with bad type", zap.Error(err)) + return + } } key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) @@ -270,8 +295,11 @@ func (c *apisixClusterConfigController) onDelete(obj interface{}) { zap.Any("final state", acc), ) c.workqueue.Add(&types.Event{ - Type: types.EventDelete, - Object: key, + Type: types.EventDelete, + Object: kube.ApisixClusterConfigEvent{ + Key: key, + GroupVersion: acc.GroupVersion(), + }, Tombstone: acc, }) diff --git a/pkg/ingress/controller.go b/pkg/ingress/controller.go index 8721d3be70d..8e434403225 100644 --- a/pkg/ingress/controller.go +++ b/pkg/ingress/controller.go @@ -111,7 +111,7 @@ type Controller struct { apisixRouteInformer cache.SharedIndexInformer apisixTlsLister listersv2beta3.ApisixTlsLister apisixTlsInformer cache.SharedIndexInformer - apisixClusterConfigLister listersv2beta3.ApisixClusterConfigLister + apisixClusterConfigLister kube.ApisixClusterConfigLister apisixClusterConfigInformer cache.SharedIndexInformer apisixConsumerInformer cache.SharedIndexInformer apisixConsumerLister listersv2beta3.ApisixConsumerLister @@ -200,8 +200,9 @@ func NewController(cfg *config.Config) (*Controller, error) { func (c *Controller) initWhenStartLeading() { var ( - ingressInformer cache.SharedIndexInformer - apisixRouteInformer cache.SharedIndexInformer + ingressInformer cache.SharedIndexInformer + apisixRouteInformer cache.SharedIndexInformer + apisixClusterConfigInformer cache.SharedIndexInformer ) kubeFactory := c.kubeClient.NewSharedIndexInformerFactory() @@ -225,7 +226,10 @@ func (c *Controller) initWhenStartLeading() { ) c.apisixUpstreamLister = apisixFactory.Apisix().V2beta3().ApisixUpstreams().Lister() c.apisixTlsLister = apisixFactory.Apisix().V2beta3().ApisixTlses().Lister() - c.apisixClusterConfigLister = apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Lister() + c.apisixClusterConfigLister = kube.NewApisixClusterConfigLister( + apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Lister(), + apisixFactory.Apisix().V2().ApisixClusterConfigs().Lister(), + ) c.apisixConsumerLister = apisixFactory.Apisix().V2beta3().ApisixConsumers().Lister() c.apisixPluginConfigLister = kube.NewApisixPluginConfigLister( apisixFactory.Apisix().V2beta3().ApisixPluginConfigs().Lister(), @@ -261,13 +265,22 @@ func (c *Controller) initWhenStartLeading() { apisixRouteInformer = apisixFactory.Apisix().V2().ApisixRoutes().Informer() } + switch c.cfg.Kubernetes.ApisixClusterConfigVersion { + case config.ApisixV2beta3: + apisixClusterConfigInformer = apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Informer() + case config.ApisixV2: + apisixClusterConfigInformer = apisixFactory.Apisix().V2().ApisixClusterConfigs().Informer() + default: + panic(fmt.Errorf("unsupported ApisixClusterConfig version %v", c.cfg.Kubernetes.ApisixClusterConfigVersion)) + } + c.namespaceInformer = kubeFactory.Core().V1().Namespaces().Informer() c.podInformer = kubeFactory.Core().V1().Pods().Informer() c.svcInformer = kubeFactory.Core().V1().Services().Informer() c.ingressInformer = ingressInformer c.apisixRouteInformer = apisixRouteInformer c.apisixUpstreamInformer = apisixFactory.Apisix().V2beta3().ApisixUpstreams().Informer() - c.apisixClusterConfigInformer = apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Informer() + c.apisixClusterConfigInformer = apisixClusterConfigInformer c.secretInformer = kubeFactory.Core().V1().Secrets().Informer() c.apisixTlsInformer = apisixFactory.Apisix().V2beta3().ApisixTlses().Informer() c.apisixConsumerInformer = apisixFactory.Apisix().V2beta3().ApisixConsumers().Informer() diff --git a/pkg/ingress/status.go b/pkg/ingress/status.go index 67c00f556f6..852f1614ad8 100644 --- a/pkg/ingress/status.go +++ b/pkg/ingress/status.go @@ -18,6 +18,7 @@ package ingress import ( "context" "fmt" + configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" "net" "time" @@ -192,6 +193,22 @@ func (c *Controller) recordStatus(at interface{}, reason string, err error, stat ) } } + case *configv2.ApisixClusterConfig: + // set to status + if v.Status.Conditions == nil { + conditions := make([]metav1.Condition, 0) + v.Status.Conditions = conditions + } + if c.verifyGeneration(&v.Status.Conditions, condition) { + meta.SetStatusCondition(&v.Status.Conditions, condition) + if _, errRecord := client.ApisixV2().ApisixClusterConfigs(). + UpdateStatus(context.TODO(), v, metav1.UpdateOptions{}); errRecord != nil { + log.Errorw("failed to record status change for ApisixClusterConfig", + zap.Error(errRecord), + zap.String("name", v.Name), + ) + } + } case *networkingv1.Ingress: // set to status lbips, err := c.ingressLBStatusIPs() diff --git a/pkg/kube/apisix_cluster_config.go b/pkg/kube/apisix_cluster_config.go new file mode 100644 index 00000000000..0ba257fb8ab --- /dev/null +++ b/pkg/kube/apisix_cluster_config.go @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package kube + +import ( + "fmt" + "github.com/apache/apisix-ingress-controller/pkg/config" + configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" + listersv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2" + + configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3" + listersv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2beta3" +) + +// ApisixClusterConfigLister is an encapsulation for the lister of ApisixClusterConfig, +// it aims at to be compatible with different ApisixClusterConfig versions. +type ApisixClusterConfigLister interface { + // V2beta3 gets the ApisixClusterConfig in apisix.apache.org/v2beta3. + V2beta3(string) (ApisixClusterConfig, error) +} + +// ApisixClusterConfigInformer is an encapsulation for the informer of ApisixClusterConfig, +// it aims at to be compatible with different ApisixClusterConfig versions. +type ApisixClusterConfigInformer interface { + Run(chan struct{}) +} + +// ApisixClusterConfig is an encapsulation for ApisixClusterConfig resource with different +// versions, for now, they are apisix.apache.org/v1 and apisix.apache.org/v2alpha1 +type ApisixClusterConfig interface { + // GroupVersion returns the api group version of the + // real ApisixClusterConfig. + GroupVersion() string + // V2beta3 returns the ApisixClusterConfig in apisix.apache.org/v2beta3, the real + // ApisixClusterConfig must be in this group version, otherwise will panic. + V2beta3() *configv2beta3.ApisixClusterConfig + // V2 returns the ApisixClusterConfig in apisix.apache.org/v2, the real + // ApisixClusterConfig must be in this group version, otherwise will panic. + V2() *configv2.ApisixClusterConfig + // ResourceVersion returns the the resource version field inside + // the real ApisixClusterConfig. + ResourceVersion() string +} + +// ApisixClusterConfigEvent contains the ApisixClusterConfig key (namespace/name) +// and the group version message. +type ApisixClusterConfigEvent struct { + Key string + OldObject ApisixClusterConfig + GroupVersion string +} + +type apisixClusterConfig struct { + groupVersion string + v2beta3 *configv2beta3.ApisixClusterConfig + v2 *configv2.ApisixClusterConfig +} + +func (acc *apisixClusterConfig) V2beta3() *configv2beta3.ApisixClusterConfig { + if acc.groupVersion != config.ApisixV2beta3 { + panic("not a apisix.apache.org/v2beta3 apisixClusterConfig") + } + return acc.v2beta3 +} + +func (acc *apisixClusterConfig) V2() *configv2.ApisixClusterConfig { + if acc.groupVersion != config.ApisixV2 { + panic("not a apisix.apache.org/v2 apisixClusterConfig") + } + return acc.v2 +} + +func (acc *apisixClusterConfig) GroupVersion() string { + return acc.groupVersion +} + +func (acc *apisixClusterConfig) ResourceVersion() string { + if acc.groupVersion == config.ApisixV2beta3 { + return acc.V2beta3().ResourceVersion + } + return acc.V2().ResourceVersion +} + +type apisixClusterConfigLister struct { + v2beta3Lister listersv2beta3.ApisixClusterConfigLister + v2Lister listersv2.ApisixClusterConfigLister +} + +func (l *apisixClusterConfigLister) V2beta3(name string) (ApisixClusterConfig, error) { + acc, err := l.v2beta3Lister.Get(name) + if err != nil { + return nil, err + } + return &apisixClusterConfig{ + groupVersion: config.ApisixV2beta3, + v2beta3: acc, + }, nil +} + +func (l *apisixClusterConfigLister) V2(name string) (ApisixClusterConfig, error) { + acc, err := l.v2Lister.Get(name) + if err != nil { + return nil, err + } + return &apisixClusterConfig{ + groupVersion: config.ApisixV2, + v2: acc, + }, nil +} + +// MustNewApisixClusterConfig creates a kube.ApisixClusterConfig object according to the +// type of obj. +func MustNewApisixClusterConfig(obj interface{}) ApisixClusterConfig { + switch acc := obj.(type) { + case *configv2beta3.ApisixClusterConfig: + return &apisixClusterConfig{ + groupVersion: config.ApisixV2beta3, + v2beta3: acc, + } + case *configv2.ApisixClusterConfig: + return &apisixClusterConfig{ + groupVersion: config.ApisixV2, + v2: acc, + } + default: + panic("invalid ApisixClusterConfig type") + } +} + +// NewApisixClusterConfig creates a kube.ApisixClusterConfig object according to the +// type of obj. It returns nil and the error reason when the +// type assertion fails. +func NewApisixClusterConfig(obj interface{}) (ApisixClusterConfig, error) { + switch acc := obj.(type) { + case *configv2beta3.ApisixClusterConfig: + return &apisixClusterConfig{ + groupVersion: config.ApisixV2beta3, + v2beta3: acc, + }, nil + case *configv2.ApisixClusterConfig: + return &apisixClusterConfig{ + groupVersion: config.ApisixV2, + v2: acc, + }, nil + default: + return nil, fmt.Errorf("invalid ApisixClusterConfig type %T", acc) + } +} + +func NewApisixClusterConfigLister(v2beta3 listersv2beta3.ApisixClusterConfigLister, v2 listersv2.ApisixClusterConfigLister) ApisixClusterConfigLister { + return &apisixClusterConfigLister{ + v2beta3Lister: v2beta3, + v2Lister: v2, + } +} diff --git a/pkg/kube/translation/global_rule.go b/pkg/kube/translation/global_rule.go index 7fc1eaef41c..d59cd982d38 100644 --- a/pkg/kube/translation/global_rule.go +++ b/pkg/kube/translation/global_rule.go @@ -16,6 +16,7 @@ package translation import ( "github.com/apache/apisix-ingress-controller/pkg/id" + configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3" apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1" ) @@ -26,7 +27,27 @@ type skywalkingPluginConfig struct { SampleRatio float64 `json:"sample_ratio,omitempty"` } -func (t *translator) TranslateClusterConfig(acc *configv2beta3.ApisixClusterConfig) (*apisixv1.GlobalRule, error) { +func (t *translator) TranslateClusterConfigV2beta3(acc *configv2beta3.ApisixClusterConfig) (*apisixv1.GlobalRule, error) { + globalRule := &apisixv1.GlobalRule{ + ID: id.GenID(acc.Name), + Plugins: make(apisixv1.Plugins), + } + + if acc.Spec.Monitoring != nil { + if acc.Spec.Monitoring.Prometheus.Enable { + globalRule.Plugins["prometheus"] = &prometheusPluginConfig{} + } + if acc.Spec.Monitoring.Skywalking.Enable { + globalRule.Plugins["skywalking"] = &skywalkingPluginConfig{ + SampleRatio: acc.Spec.Monitoring.Skywalking.SampleRatio, + } + } + } + + return globalRule, nil +} + +func (t *translator) TranslateClusterConfigV2(acc *configv2.ApisixClusterConfig) (*apisixv1.GlobalRule, error) { globalRule := &apisixv1.GlobalRule{ ID: id.GenID(acc.Name), Plugins: make(apisixv1.Plugins), diff --git a/pkg/kube/translation/global_rule_test.go b/pkg/kube/translation/global_rule_test.go index 8a584ac4391..93cbc5fe013 100644 --- a/pkg/kube/translation/global_rule_test.go +++ b/pkg/kube/translation/global_rule_test.go @@ -44,7 +44,7 @@ func TestTranslateClusterConfig(t *testing.T) { }, }, } - gr, err := tr.TranslateClusterConfig(acc) + gr, err := tr.TranslateClusterConfigV2beta3(acc) assert.Nil(t, err, "translating ApisixClusterConfig") assert.Equal(t, gr.ID, id.GenID("qa-apisix"), "checking global_rule id") assert.Len(t, gr.Plugins, 2) diff --git a/pkg/kube/translation/translator.go b/pkg/kube/translation/translator.go index 69aeb31cd48..1ffe2b92e42 100644 --- a/pkg/kube/translation/translator.go +++ b/pkg/kube/translation/translator.go @@ -85,9 +85,12 @@ type Translator interface { TranslateRouteV2NotStrictly(*configv2.ApisixRoute) (*TranslateContext, error) // TranslateSSL translates the configv2beta3.ApisixTls object into the APISIX SSL resource. TranslateSSL(*configv2beta3.ApisixTls) (*apisixv1.Ssl, error) - // TranslateClusterConfig translates the configv2beta3.ApisixClusterConfig object into the APISIX + // TranslateClusterConfigV2beta3 translates the configv2beta3.ApisixClusterConfig object into the APISIX // Global Rule resource. - TranslateClusterConfig(*configv2beta3.ApisixClusterConfig) (*apisixv1.GlobalRule, error) + TranslateClusterConfigV2beta3(*configv2beta3.ApisixClusterConfig) (*apisixv1.GlobalRule, error) + // TranslateClusterConfigV2 translates the configv2.ApisixClusterConfig object into the APISIX + // Global Rule resource. + TranslateClusterConfigV2(*configv2.ApisixClusterConfig) (*apisixv1.GlobalRule, error) // TranslateApisixConsumer translates the configv2beta3.APisixConsumer object into the APISIX Consumer // resource. TranslateApisixConsumer(*configv2beta3.ApisixConsumer) (*apisixv1.Consumer, error) From 855667de25b60d18d5ee894593fb3669de7e0998 Mon Sep 17 00:00:00 2001 From: Ling Samuel Date: Sun, 24 Apr 2022 08:21:17 +0800 Subject: [PATCH 2/4] update Signed-off-by: Ling Samuel --- pkg/ingress/apisix_cluster_config.go | 234 +++++++++++++++++++-------- pkg/kube/apisix_cluster_config.go | 2 + 2 files changed, 169 insertions(+), 67 deletions(-) diff --git a/pkg/ingress/apisix_cluster_config.go b/pkg/ingress/apisix_cluster_config.go index b4e21782274..40b7f2b17ff 100644 --- a/pkg/ingress/apisix_cluster_config.go +++ b/pkg/ingress/apisix_cluster_config.go @@ -16,6 +16,8 @@ package ingress import ( "context" + "fmt" + "github.com/apache/apisix-ingress-controller/pkg/config" "github.com/apache/apisix-ingress-controller/pkg/kube" "time" @@ -27,7 +29,6 @@ import ( "k8s.io/client-go/util/workqueue" "github.com/apache/apisix-ingress-controller/pkg/apisix" - configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3" "github.com/apache/apisix-ingress-controller/pkg/log" "github.com/apache/apisix-ingress-controller/pkg/types" ) @@ -82,107 +83,206 @@ func (c *apisixClusterConfigController) runWorker(ctx context.Context) { } func (c *apisixClusterConfigController) sync(ctx context.Context, ev *types.Event) error { - key := ev.Object.(string) + event := ev.Object.(kube.ApisixClusterConfigEvent) + key := event.Key _, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { log.Errorf("found ApisixClusterConfig resource with invalid meta key %s: %s", key, err) return err } - acc, err := c.controller.apisixClusterConfigLister.Get(name) + + var multiVersioned kube.ApisixClusterConfig + switch event.GroupVersion { + case config.ApisixV2beta3: + multiVersioned, err = c.controller.apisixClusterConfigLister.V2beta3(name) + case config.ApisixV2: + multiVersioned, err = c.controller.apisixClusterConfigLister.V2(name) + default: + return fmt.Errorf("unsupported ApisixClusterConfig group version %s", event.GroupVersion) + } + if err != nil { if !k8serrors.IsNotFound(err) { - log.Errorf("failed to get ApisixClusterConfig %s: %s", key, err) + log.Errorw("failed to get ApisixClusterConfig", + zap.Error(err), + zap.String("key", key), + zap.String("version", event.GroupVersion), + ) return err } if ev.Type != types.EventDelete { - log.Warnf("ApisixClusterConfig %s was deleted before it can be delivered", key) + log.Warnw("ApisixClusterConfig was deleted before it can be delivered", + zap.String("key", key), + zap.String("version", event.GroupVersion), + ) return nil } } if ev.Type == types.EventDelete { - if acc != nil { + if multiVersioned != nil { // We still find the resource while we are processing the DELETE event, // that means object with same namespace and name was created, discarding // this stale DELETE event. log.Warnf("discard the stale ApisixClusterConfig delete event since the %s exists", key) return nil } - acc = ev.Tombstone.(*configv2beta3.ApisixClusterConfig) + multiVersioned = ev.Tombstone.(kube.ApisixClusterConfig) } - // Currently we don't handle multiple cluster, so only process - // the default apisix cluster. - if acc.Name != c.controller.cfg.APISIX.DefaultClusterName { - log.Infow("ignore non-default apisix cluster config", - zap.String("default_cluster_name", c.controller.cfg.APISIX.DefaultClusterName), - zap.Any("ApisixClusterConfig", acc), + switch event.GroupVersion { + case config.ApisixV2beta3: + acc := multiVersioned.V2beta3() + // Currently we don't handle multiple cluster, so only process + // the default apisix cluster. + if acc.Name != c.controller.cfg.APISIX.DefaultClusterName { + log.Infow("ignore non-default apisix cluster config", + zap.String("default_cluster_name", c.controller.cfg.APISIX.DefaultClusterName), + zap.Any("ApisixClusterConfig", acc), + ) + return nil + } + // Cluster delete is dangerous. + // TODO handle delete? + if ev.Type == types.EventDelete { + log.Error("ApisixClusterConfig delete event for default apisix cluster will be ignored") + return nil + } + + if acc.Spec.Admin != nil { + clusterOpts := &apisix.ClusterOptions{ + Name: acc.Name, + BaseURL: acc.Spec.Admin.BaseURL, + AdminKey: acc.Spec.Admin.AdminKey, + } + log.Infow("updating cluster", + zap.Any("opts", clusterOpts), + ) + // TODO we may first call AddCluster. + // Since now we already have the default cluster, we just call UpdateCluster. + if err := c.controller.apisix.UpdateCluster(ctx, clusterOpts); err != nil { + log.Errorw("failed to update cluster", + zap.String("cluster_name", acc.Name), + zap.Error(err), + zap.Any("opts", clusterOpts), + ) + c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) + c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) + return err + } + } + + globalRule, err := c.controller.translator.TranslateClusterConfigV2beta3(acc) + if err != nil { + // TODO add status + log.Errorw("failed to translate ApisixClusterConfig", + zap.Error(err), + zap.String("key", key), + zap.Any("object", acc), + ) + c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) + c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) + return err + } + log.Debugw("translated global_rule", + zap.Any("object", globalRule), ) + + // TODO multiple cluster support + if ev.Type == types.EventAdd { + _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Create(ctx, globalRule) + } else { + _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Update(ctx, globalRule) + } + if err != nil { + log.Errorw("failed to reflect global_rule changes to apisix cluster", + zap.Any("global_rule", globalRule), + zap.Any("cluster", acc.Name), + ) + c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) + c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) + return err + } + c.controller.recorderEvent(acc, corev1.EventTypeNormal, _resourceSynced, nil) + c.controller.recordStatus(acc, _resourceSynced, nil, metav1.ConditionTrue, acc.GetGeneration()) return nil - } - // Cluster delete is dangerous. - // TODO handle delete? - if ev.Type == types.EventDelete { - log.Error("ApisixClusterConfig delete event for default apisix cluster will be ignored") - return nil - } + case config.ApisixV2: + acc := multiVersioned.V2() + // Currently we don't handle multiple cluster, so only process + // the default apisix cluster. + if acc.Name != c.controller.cfg.APISIX.DefaultClusterName { + log.Infow("ignore non-default apisix cluster config", + zap.String("default_cluster_name", c.controller.cfg.APISIX.DefaultClusterName), + zap.Any("ApisixClusterConfig", acc), + ) + return nil + } + // Cluster delete is dangerous. + // TODO handle delete? + if ev.Type == types.EventDelete { + log.Error("ApisixClusterConfig delete event for default apisix cluster will be ignored") + return nil + } - if acc.Spec.Admin != nil { - clusterOpts := &apisix.ClusterOptions{ - Name: acc.Name, - BaseURL: acc.Spec.Admin.BaseURL, - AdminKey: acc.Spec.Admin.AdminKey, + if acc.Spec.Admin != nil { + clusterOpts := &apisix.ClusterOptions{ + Name: acc.Name, + BaseURL: acc.Spec.Admin.BaseURL, + AdminKey: acc.Spec.Admin.AdminKey, + } + log.Infow("updating cluster", + zap.Any("opts", clusterOpts), + ) + // TODO we may first call AddCluster. + // Since now we already have the default cluster, we just call UpdateCluster. + if err := c.controller.apisix.UpdateCluster(ctx, clusterOpts); err != nil { + log.Errorw("failed to update cluster", + zap.String("cluster_name", acc.Name), + zap.Error(err), + zap.Any("opts", clusterOpts), + ) + c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) + c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) + return err + } } - log.Infow("updating cluster", - zap.Any("opts", clusterOpts), - ) - // TODO we may first call AddCluster. - // Since now we already have the default cluster, we just call UpdateCluster. - if err := c.controller.apisix.UpdateCluster(ctx, clusterOpts); err != nil { - log.Errorw("failed to update cluster", - zap.String("cluster_name", acc.Name), + + globalRule, err := c.controller.translator.TranslateClusterConfigV2(acc) + if err != nil { + // TODO add status + log.Errorw("failed to translate ApisixClusterConfig", zap.Error(err), - zap.Any("opts", clusterOpts), + zap.String("key", key), + zap.Any("object", acc), ) c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) return err } - } - - globalRule, err := c.controller.translator.TranslateClusterConfigV2beta3(acc) - if err != nil { - // TODO add status - log.Errorw("failed to translate ApisixClusterConfig", - zap.Error(err), - zap.String("key", key), - zap.Any("object", acc), + log.Debugw("translated global_rule", + zap.Any("object", globalRule), ) - c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) - c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) - return err - } - log.Debugw("translated global_rule", - zap.Any("object", globalRule), - ) - // TODO multiple cluster support - if ev.Type == types.EventAdd { - _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Create(ctx, globalRule) - } else { - _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Update(ctx, globalRule) - } - if err != nil { - log.Errorw("failed to reflect global_rule changes to apisix cluster", - zap.Any("global_rule", globalRule), - zap.Any("cluster", acc.Name), - ) - c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) - c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) - return err + // TODO multiple cluster support + if ev.Type == types.EventAdd { + _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Create(ctx, globalRule) + } else { + _, err = c.controller.apisix.Cluster(acc.Name).GlobalRule().Update(ctx, globalRule) + } + if err != nil { + log.Errorw("failed to reflect global_rule changes to apisix cluster", + zap.Any("global_rule", globalRule), + zap.Any("cluster", acc.Name), + ) + c.controller.recorderEvent(acc, corev1.EventTypeWarning, _resourceSyncAborted, err) + c.controller.recordStatus(acc, _resourceSyncAborted, err, metav1.ConditionFalse, acc.GetGeneration()) + return err + } + c.controller.recorderEvent(acc, corev1.EventTypeNormal, _resourceSynced, nil) + c.controller.recordStatus(acc, _resourceSynced, nil, metav1.ConditionTrue, acc.GetGeneration()) + return nil + default: + return fmt.Errorf("unsupported ApisixClusterConfig group version %s", event.GroupVersion) } - c.controller.recorderEvent(acc, corev1.EventTypeNormal, _resourceSynced, nil) - c.controller.recordStatus(acc, _resourceSynced, nil, metav1.ConditionTrue, acc.GetGeneration()) - return nil } func (c *apisixClusterConfigController) handleSyncErr(obj interface{}, err error) { diff --git a/pkg/kube/apisix_cluster_config.go b/pkg/kube/apisix_cluster_config.go index 0ba257fb8ab..a73e7d779fe 100644 --- a/pkg/kube/apisix_cluster_config.go +++ b/pkg/kube/apisix_cluster_config.go @@ -29,6 +29,8 @@ import ( type ApisixClusterConfigLister interface { // V2beta3 gets the ApisixClusterConfig in apisix.apache.org/v2beta3. V2beta3(string) (ApisixClusterConfig, error) + // V2 gets the ApisixClusterConfig in apisix.apache.org/v2. + V2(string) (ApisixClusterConfig, error) } // ApisixClusterConfigInformer is an encapsulation for the informer of ApisixClusterConfig, From cafd576ef793259f6db7756300f9107e07c55dac Mon Sep 17 00:00:00 2001 From: Ling Samuel Date: Sun, 24 Apr 2022 09:05:56 +0800 Subject: [PATCH 3/4] fmt Signed-off-by: Ling Samuel --- pkg/ingress/apisix_cluster_config.go | 4 ++-- pkg/ingress/status.go | 2 +- pkg/kube/apisix_cluster_config.go | 4 ++-- test/e2e/suite-features/global_rule.go | 2 +- test/e2e/suite-ingress/ingress.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/ingress/apisix_cluster_config.go b/pkg/ingress/apisix_cluster_config.go index 40b7f2b17ff..ff558a9301f 100644 --- a/pkg/ingress/apisix_cluster_config.go +++ b/pkg/ingress/apisix_cluster_config.go @@ -17,8 +17,6 @@ package ingress import ( "context" "fmt" - "github.com/apache/apisix-ingress-controller/pkg/config" - "github.com/apache/apisix-ingress-controller/pkg/kube" "time" "go.uber.org/zap" @@ -29,6 +27,8 @@ import ( "k8s.io/client-go/util/workqueue" "github.com/apache/apisix-ingress-controller/pkg/apisix" + "github.com/apache/apisix-ingress-controller/pkg/config" + "github.com/apache/apisix-ingress-controller/pkg/kube" "github.com/apache/apisix-ingress-controller/pkg/log" "github.com/apache/apisix-ingress-controller/pkg/types" ) diff --git a/pkg/ingress/status.go b/pkg/ingress/status.go index 852f1614ad8..a5fcbd1b1b8 100644 --- a/pkg/ingress/status.go +++ b/pkg/ingress/status.go @@ -18,7 +18,6 @@ package ingress import ( "context" "fmt" - configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" "net" "time" @@ -33,6 +32,7 @@ import ( "k8s.io/client-go/tools/cache" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" configv2beta2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta2" configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3" "github.com/apache/apisix-ingress-controller/pkg/log" diff --git a/pkg/kube/apisix_cluster_config.go b/pkg/kube/apisix_cluster_config.go index a73e7d779fe..9eb8eceeae3 100644 --- a/pkg/kube/apisix_cluster_config.go +++ b/pkg/kube/apisix_cluster_config.go @@ -16,11 +16,11 @@ package kube import ( "fmt" + "github.com/apache/apisix-ingress-controller/pkg/config" configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2" - listersv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2" - configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3" + listersv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2" listersv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/listers/config/v2beta3" ) diff --git a/test/e2e/suite-features/global_rule.go b/test/e2e/suite-features/global_rule.go index 773e19b87a9..4e97741014c 100644 --- a/test/e2e/suite-features/global_rule.go +++ b/test/e2e/suite-features/global_rule.go @@ -19,10 +19,10 @@ import ( "net/http" "time" + "github.com/apache/apisix-ingress-controller/pkg/id" "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - "github.com/apache/apisix-ingress-controller/pkg/id" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) diff --git a/test/e2e/suite-ingress/ingress.go b/test/e2e/suite-ingress/ingress.go index 33ebfca1edd..aa8ad33a5a2 100644 --- a/test/e2e/suite-ingress/ingress.go +++ b/test/e2e/suite-ingress/ingress.go @@ -21,11 +21,11 @@ import ( "net/http" "time" + "github.com/apache/apisix-ingress-controller/pkg/id" "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" - "github.com/apache/apisix-ingress-controller/pkg/id" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) From baf3ff458a286fbd830c6bdcf533ef5ef8fee17b Mon Sep 17 00:00:00 2001 From: Ling Samuel Date: Tue, 10 May 2022 17:05:26 +0800 Subject: [PATCH 4/4] remove todo Signed-off-by: Ling Samuel --- pkg/ingress/apisix_cluster_config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/ingress/apisix_cluster_config.go b/pkg/ingress/apisix_cluster_config.go index ff558a9301f..ecc33d0e6d8 100644 --- a/pkg/ingress/apisix_cluster_config.go +++ b/pkg/ingress/apisix_cluster_config.go @@ -173,7 +173,6 @@ func (c *apisixClusterConfigController) sync(ctx context.Context, ev *types.Even globalRule, err := c.controller.translator.TranslateClusterConfigV2beta3(acc) if err != nil { - // TODO add status log.Errorw("failed to translate ApisixClusterConfig", zap.Error(err), zap.String("key", key), @@ -248,7 +247,6 @@ func (c *apisixClusterConfigController) sync(ctx context.Context, ev *types.Even globalRule, err := c.controller.translator.TranslateClusterConfigV2(acc) if err != nil { - // TODO add status log.Errorw("failed to translate ApisixClusterConfig", zap.Error(err), zap.String("key", key),