Skip to content

Commit

Permalink
test: add e2e tests and CRDs for ApisixUpstream v2 (#1147)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlinsRan
AlinsRan committed Jul 13, 2022
1 parent 6cf8bb7 commit 35ca03c
Show file tree
Hide file tree
Showing 22 changed files with 511 additions and 286 deletions.
6 changes: 1 addition & 5 deletions cmd/ingress/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,7 @@ For example, no available LB exists in the bare metal environment.`)
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.IngressClass, "ingress-class", config.IngressClass, "the class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.18.0 or higher or the annotation \"kubernetes.io/ingress.class\" (deprecated)")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ElectionID, "election-id", config.IngressAPISIXLeader, "election id used for campaign the controller leader")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.IngressVersion, "ingress-version", config.IngressNetworkingV1, "the supported ingress api group version, can be \"networking/v1beta1\", \"networking/v1\" (for Kubernetes version v1.19.0 or higher) and \"extensions/v1beta1\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixRouteVersion, "apisix-route-version", config.ApisixV2beta3, "the supported apisixroute api group version, can be \"apisix.apache.org/v2beta2\" or \"apisix.apache.org/v2beta3\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixPluginConfigVersion, "apisix-plugin-config-version", config.ApisixV2beta3, "the supported ApisixPluginConfig api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixTlsVersion, "apisix-tls-version", config.ApisixV2beta3, "the supported apisixtls api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixClusterConfigVersion, "apisix-cluster-config-version", config.ApisixV2beta3, "the supported ApisixClusterConfig api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixConsumerVersion, "apisix-consumer-version", config.ApisixV2beta3, "the supported ApisixConsumer api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixRouteVersion, "apisix-route-version", config.DefaultAPIVersion, "the supported apisixroute api group version, can be \"apisix.apache.org/v2beta2\" or \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.APIVersion, "api-version", config.DefaultAPIVersion, config.APIVersionDescribe)
cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.WatchEndpointSlices, "watch-endpointslices", false, "whether to watch endpointslices rather than endpoints")
cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.EnableGatewayAPI, "enable-gateway-api", false, "whether to enable support for Gateway API")
Expand Down
50 changes: 21 additions & 29 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,21 +91,17 @@ type Config struct {

// KubernetesConfig contains all Kubernetes related config items.
type KubernetesConfig struct {
Kubeconfig string `json:"kubeconfig" yaml:"kubeconfig"`
ResyncInterval types.TimeDuration `json:"resync_interval" yaml:"resync_interval"`
AppNamespaces []string `json:"app_namespaces" yaml:"app_namespaces"`
NamespaceSelector []string `json:"namespace_selector" yaml:"namespace_selector"`
ElectionID string `json:"election_id" yaml:"election_id"`
IngressClass string `json:"ingress_class" yaml:"ingress_class"`
IngressVersion string `json:"ingress_version" yaml:"ingress_version"`
WatchEndpointSlices bool `json:"watch_endpoint_slices" yaml:"watch_endpoint_slices"`
ApisixRouteVersion string `json:"apisix_route_version" yaml:"apisix_route_version"`
ApisixPluginConfigVersion string `json:"apisix_plugin_config_version" yaml:"apisix_plugin_config_version"`
ApisixConsumerVersion string `json:"apisix_consumer_version" yaml:"apisix_consumer_version"`
ApisixTlsVersion string `json:"apisix_tls_version" yaml:"apisix_tls_version"`
ApisixClusterConfigVersion string `json:"apisix_cluster_config_version" yaml:"apisix_cluster_config_version"`
APIVersion string `json:"api_version" yaml:"api_version"`
EnableGatewayAPI bool `json:"enable_gateway_api" yaml:"enable_gateway_api"`
Kubeconfig string `json:"kubeconfig" yaml:"kubeconfig"`
ResyncInterval types.TimeDuration `json:"resync_interval" yaml:"resync_interval"`
AppNamespaces []string `json:"app_namespaces" yaml:"app_namespaces"`
NamespaceSelector []string `json:"namespace_selector" yaml:"namespace_selector"`
ElectionID string `json:"election_id" yaml:"election_id"`
IngressClass string `json:"ingress_class" yaml:"ingress_class"`
IngressVersion string `json:"ingress_version" yaml:"ingress_version"`
WatchEndpointSlices bool `json:"watch_endpoint_slices" yaml:"watch_endpoint_slices"`
ApisixRouteVersion string `json:"apisix_route_version" yaml:"apisix_route_version"`
APIVersion string `json:"api_version" yaml:"api_version"`
EnableGatewayAPI bool `json:"enable_gateway_api" yaml:"enable_gateway_api"`
}

// APISIXConfig contains all APISIX related config items.
Expand Down Expand Up @@ -134,20 +130,16 @@ func NewDefaultConfig() *Config {
EnableProfiling: true,
ApisixResourceSyncInterval: types.TimeDuration{Duration: 300 * time.Second},
Kubernetes: KubernetesConfig{
Kubeconfig: "", // Use in-cluster configurations.
ResyncInterval: types.TimeDuration{Duration: 6 * time.Hour},
AppNamespaces: []string{v1.NamespaceAll},
ElectionID: IngressAPISIXLeader,
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixV2beta3,
ApisixPluginConfigVersion: ApisixV2beta3,
ApisixConsumerVersion: ApisixV2beta3,
ApisixTlsVersion: ApisixV2beta3,
ApisixClusterConfigVersion: ApisixV2beta3,
APIVersion: DefaultAPIVersion,
WatchEndpointSlices: false,
EnableGatewayAPI: false,
Kubeconfig: "", // Use in-cluster configurations.
ResyncInterval: types.TimeDuration{Duration: 6 * time.Hour},
AppNamespaces: []string{v1.NamespaceAll},
ElectionID: IngressAPISIXLeader,
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: DefaultAPIVersion,
APIVersion: DefaultAPIVersion,
WatchEndpointSlices: false,
EnableGatewayAPI: false,
},
}
}
Expand Down
40 changes: 16 additions & 24 deletions pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,14 @@ func TestNewConfigFromFile(t *testing.T) {
EnableProfiling: true,
ApisixResourceSyncInterval: types.TimeDuration{Duration: 200 * time.Second},
Kubernetes: KubernetesConfig{
ResyncInterval: types.TimeDuration{Duration: time.Hour},
Kubeconfig: "/path/to/foo/baz",
AppNamespaces: []string{""},
ElectionID: "my-election-id",
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixV2beta3,
ApisixPluginConfigVersion: ApisixV2beta3,
ApisixConsumerVersion: ApisixV2beta3,
ApisixTlsVersion: ApisixV2beta3,
ApisixClusterConfigVersion: ApisixV2beta3,
APIVersion: DefaultAPIVersion,
ResyncInterval: types.TimeDuration{Duration: time.Hour},
Kubeconfig: "/path/to/foo/baz",
AppNamespaces: []string{""},
ElectionID: "my-election-id",
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: DefaultAPIVersion,
APIVersion: DefaultAPIVersion,
},
APISIX: APISIXConfig{
DefaultClusterName: "default",
Expand Down Expand Up @@ -128,18 +124,14 @@ func TestConfigWithEnvVar(t *testing.T) {
EnableProfiling: true,
ApisixResourceSyncInterval: types.TimeDuration{Duration: 200 * time.Second},
Kubernetes: KubernetesConfig{
ResyncInterval: types.TimeDuration{Duration: time.Hour},
Kubeconfig: "",
AppNamespaces: []string{""},
ElectionID: "my-election-id",
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixV2beta3,
ApisixPluginConfigVersion: ApisixV2beta3,
ApisixConsumerVersion: ApisixV2beta3,
ApisixTlsVersion: ApisixV2beta3,
ApisixClusterConfigVersion: ApisixV2beta3,
APIVersion: DefaultAPIVersion,
ResyncInterval: types.TimeDuration{Duration: time.Hour},
Kubeconfig: "",
AppNamespaces: []string{""},
ElectionID: "my-election-id",
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: DefaultAPIVersion,
APIVersion: DefaultAPIVersion,
},
APISIX: APISIXConfig{
DefaultClusterName: "default",
Expand Down
43 changes: 18 additions & 25 deletions pkg/ingress/compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,31 +135,32 @@ func (c *Controller) CompareResources(ctx context.Context) error {
// todo ApisixUpstream and ApisixPluginConfig
// ApisixUpstream and ApisixPluginConfig should be synced with ApisixRoute resource

switch c.cfg.Kubernetes.ApisixTlsVersion {
switch c.cfg.Kubernetes.APIVersion {
case config.ApisixV2beta3:
retSSL, err := c.kubeClient.APISIXClient.ApisixV2beta3().ApisixTlses(ns).List(ctx, opts)
// ApisixConsumer
retConsumer, err := c.kubeClient.APISIXClient.ApisixV2beta3().ApisixConsumers(ns).List(ctx, opts)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
for _, s := range retSSL.Items {
ssl, err := c.translator.TranslateSSLV2Beta3(&s)
for _, con := range retConsumer.Items {
consumer, err := c.translator.TranslateApisixConsumerV2beta3(&con)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
sslMapK8S.Store(ssl.ID, ssl.ID)
consumerMapK8S.Store(consumer.Username, consumer.Username)
}
}
}
case config.ApisixV2:
retSSL, err := c.kubeClient.APISIXClient.ApisixV2().ApisixTlses(ns).List(ctx, opts)
// ApisixTls
retSSL, err := c.kubeClient.APISIXClient.ApisixV2beta3().ApisixTlses(ns).List(ctx, opts)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
for _, s := range retSSL.Items {
ssl, err := c.translator.TranslateSSLV2(&s)
ssl, err := c.translator.TranslateSSLV2Beta3(&s)
if err != nil {
log.Error(err.Error())
ctx.Done()
Expand All @@ -168,22 +169,15 @@ func (c *Controller) CompareResources(ctx context.Context) error {
}
}
}
default:
log.Errorw("failed to sync ApisixTls, unexpected version",
zap.String("version", c.cfg.Kubernetes.ApisixTlsVersion),
)
}

switch c.cfg.Kubernetes.ApisixConsumerVersion {
case config.ApisixV2beta3:
case config.ApisixV2:
// ApisixConsumer
retConsumer, err := c.kubeClient.APISIXClient.ApisixV2beta3().ApisixConsumers(ns).List(ctx, opts)
retConsumer, err := c.kubeClient.APISIXClient.ApisixV2().ApisixConsumers(ns).List(ctx, opts)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
for _, con := range retConsumer.Items {
consumer, err := c.translator.TranslateApisixConsumerV2beta3(&con)
consumer, err := c.translator.TranslateApisixConsumerV2(&con)
if err != nil {
log.Error(err.Error())
ctx.Done()
Expand All @@ -192,26 +186,25 @@ func (c *Controller) CompareResources(ctx context.Context) error {
}
}
}
case config.ApisixV2:
// ApisixConsumer
retConsumer, err := c.kubeClient.APISIXClient.ApisixV2().ApisixConsumers(ns).List(ctx, opts)
// ApisixTls
retSSL, err := c.kubeClient.APISIXClient.ApisixV2().ApisixTlses(ns).List(ctx, opts)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
for _, con := range retConsumer.Items {
consumer, err := c.translator.TranslateApisixConsumerV2(&con)
for _, s := range retSSL.Items {
ssl, err := c.translator.TranslateSSLV2(&s)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
consumerMapK8S.Store(consumer.Username, consumer.Username)
sslMapK8S.Store(ssl.ID, ssl.ID)
}
}
}
default:
log.Errorw("failed to sync ApisixConsumer, unexpected version",
zap.String("version", c.cfg.Kubernetes.ApisixConsumerVersion),
zap.String("version", c.cfg.Kubernetes.APIVersion),
)
}
}(key)
Expand Down
60 changes: 11 additions & 49 deletions pkg/ingress/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,22 +239,20 @@ func (c *Controller) initWhenStartLeading() {
switch c.cfg.Kubernetes.APIVersion {
case config.ApisixV2beta3:
apisixUpstreamInformer = apisixFactory.Apisix().V2beta3().ApisixUpstreams().Informer()
// to do list:
// ApisixRoute
// ApisixPluginConfig
// ApisixTls
// ApisixConsumer
// ApisixClusterConfig
// to do ApisixRoute
apisixPluginConfigInformer = apisixFactory.Apisix().V2beta3().ApisixPluginConfigs().Informer()
apisixTlsInformer = apisixFactory.Apisix().V2beta3().ApisixTlses().Informer()
apisixConsumerInformer = apisixFactory.Apisix().V2beta3().ApisixConsumers().Informer()
apisixClusterConfigInformer = apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Informer()
case config.ApisixV2:
apisixUpstreamInformer = apisixFactory.Apisix().V2().ApisixUpstreams().Informer()
// to do list:
// ApisixRoute
// ApisixPluginConfig
// ApisixTls
// ApisixConsumer
// ApisixClusterConfig
// to do ApisixRoute
apisixPluginConfigInformer = apisixFactory.Apisix().V2().ApisixPluginConfigs().Informer()
apisixTlsInformer = apisixFactory.Apisix().V2().ApisixTlses().Informer()
apisixConsumerInformer = apisixFactory.Apisix().V2().ApisixConsumers().Informer()
apisixClusterConfigInformer = apisixFactory.Apisix().V2().ApisixClusterConfigs().Informer()
default:
panic(fmt.Errorf("unsupported ApisixUpstream version %v", c.cfg.Kubernetes.APIVersion))
panic(fmt.Errorf("unsupported API version %v", c.cfg.Kubernetes.APIVersion))
}

if c.cfg.Kubernetes.IngressVersion == config.IngressNetworkingV1 {
Expand All @@ -276,42 +274,6 @@ func (c *Controller) initWhenStartLeading() {
panic(fmt.Errorf("unsupported ApisixRoute version %s", c.cfg.Kubernetes.ApisixRouteVersion))
}

switch c.cfg.Kubernetes.ApisixTlsVersion {
case config.ApisixV2beta3:
apisixTlsInformer = apisixFactory.Apisix().V2beta3().ApisixTlses().Informer()
case config.ApisixV2:
apisixTlsInformer = apisixFactory.Apisix().V2().ApisixTlses().Informer()
default:
panic(fmt.Errorf("unsupported ApisixTls version %s", c.cfg.Kubernetes.ApisixTlsVersion))
}

switch c.cfg.Kubernetes.ApisixClusterConfigVersion {
case config.ApisixV2beta3:
apisixClusterConfigInformer = apisixFactory.Apisix().V2beta3().ApisixClusterConfigs().Informer()
case config.ApisixV2:
apisixClusterConfigInformer = apisixFactory.Apisix().V2().ApisixClusterConfigs().Informer()
default:
panic(fmt.Errorf("unsupported ApisixClusterConfig version %v", c.cfg.Kubernetes.ApisixClusterConfigVersion))
}

switch c.cfg.Kubernetes.ApisixConsumerVersion {
case config.ApisixV2beta3:
apisixConsumerInformer = apisixFactory.Apisix().V2beta3().ApisixConsumers().Informer()
case config.ApisixV2:
apisixConsumerInformer = apisixFactory.Apisix().V2().ApisixConsumers().Informer()
default:
panic(fmt.Errorf("unsupported ApisixConsumer version %v", c.cfg.Kubernetes.ApisixConsumerVersion))
}

switch c.cfg.Kubernetes.ApisixPluginConfigVersion {
case config.ApisixV2beta3:
apisixPluginConfigInformer = apisixFactory.Apisix().V2beta3().ApisixPluginConfigs().Informer()
case config.ApisixV2:
apisixPluginConfigInformer = apisixFactory.Apisix().V2().ApisixPluginConfigs().Informer()
default:
panic(fmt.Errorf("unsupported ApisixPluginConfig version %v", c.cfg.Kubernetes.ApisixPluginConfigVersion))
}

c.podInformer = kubeFactory.Core().V1().Pods().Informer()
c.svcInformer = kubeFactory.Core().V1().Services().Informer()
c.ingressInformer = ingressInformer
Expand Down
2 changes: 1 addition & 1 deletion pkg/ingress/secret.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error {
}
sslMap := ssls.(*sync.Map)

switch c.controller.cfg.Kubernetes.ApisixTlsVersion {
switch c.controller.cfg.Kubernetes.APIVersion {
case config.ApisixV2beta3:
sslMap.Range(c.syncV2Beta3Handler(ctx, ev, sec, key))
case config.ApisixV2:
Expand Down
Loading

0 comments on commit 35ca03c

Please sign in to comment.