From fb06670539e3e6989d7491adc37c713446cbdd08 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Tue, 1 Aug 2023 14:33:39 +0200 Subject: [PATCH] feat(consumer group): allow assigning plugins to consumer groups --- CHANGELOG.md | 2 +- go.mod | 2 +- go.sum | 4 +- internal/dataplane/deckgen/deckgen.go | 3 + internal/dataplane/kongstate/kongstate.go | 20 ++ .../dataplane/kongstate/kongstate_test.go | 80 ++++- internal/dataplane/parser/parser.go | 12 +- internal/util/relations.go | 7 +- internal/util/relations_test.go | 16 + test/integration/consumer_group_test.go | 318 ++++++++++++++---- 10 files changed, 380 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2177e1287..fd183126f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,7 @@ Adding a new version? You'll need three changes: ### Added - **WIP** Introduce `KongConsumerGroup` CRD (supported by Kong Enterprise only) - [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419), [4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437) + [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419), [#4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437), [#4452](https://github.com/Kong/kubernetes-ingress-controller/pull/4452) - The ResponseHeaderModifier Gateway API filter is now supported and translated to the proper set of Kong plugins. [#4350](https://github.com/Kong/kubernetes-ingress-controller/pull/4350) diff --git a/go.mod b/go.mod index bf7681388f..3c12f9e010 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/jpillora/backoff v1.0.0 - github.com/kong/deck v1.25.0 + github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a github.com/kong/go-kong v0.46.0 github.com/kong/kubernetes-telemetry v0.1.0 github.com/kong/kubernetes-testing-framework v0.34.0 diff --git a/go.sum b/go.sum index dd39327763..9d24ccd18a 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kong/deck v1.25.0 h1:lcNAJQhoJmTjFHLt2DFcsXS9aqclauUxqelPxTKz9bU= -github.com/kong/deck v1.25.0/go.mod h1:1AUvNaqRHhwqPNsQ0yd2Rgg5LHqS4v/PqsaEHJ8dXwE= +github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a h1:XLOxVDB6T8PVM9iCYRWH5FNWidSCEtwOURJ5xaka8vs= +github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a/go.mod h1:4Mt9LLrDSd+6mQsWfykLAP2D3iWWxgfQ5X1HraswYyc= github.com/kong/go-kong v0.46.0 h1:9I6nlX63WymU5Sg+d13iZDVwpW5vXh8/v0zarU27dzI= github.com/kong/go-kong v0.46.0/go.mod h1:41Sot1N/n8UHBp+gE/6nOw3vuzoHbhMSyU/zOS7VzPE= github.com/kong/kubernetes-telemetry v0.1.0 h1:dyDhB2SUvxVaDAxaZtLWL7JGAbkgwlCUM1P8gR/rCl0= diff --git a/internal/dataplane/deckgen/deckgen.go b/internal/dataplane/deckgen/deckgen.go index f2017caaf3..4a6380164e 100644 --- a/internal/dataplane/deckgen/deckgen.go +++ b/internal/dataplane/deckgen/deckgen.go @@ -59,6 +59,9 @@ func PluginString(plugin file.FPlugin) string { if plugin.Consumer != nil && plugin.Consumer.ID != nil { result += *plugin.Consumer.ID } + if plugin.ConsumerGroup != nil && plugin.ConsumerGroup.ID != nil { + result += *plugin.ConsumerGroup.ID + } if plugin.Route != nil && plugin.Route.ID != nil { result += *plugin.Route.ID } diff --git a/internal/dataplane/kongstate/kongstate.go b/internal/dataplane/kongstate/kongstate.go index ce9f5bd596..90a0d1a8dd 100644 --- a/internal/dataplane/kongstate/kongstate.go +++ b/internal/dataplane/kongstate/kongstate.go @@ -243,6 +243,15 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations { relations.Consumer = append(relations.Consumer, identifier) pluginRels[pluginKey] = relations } + addConsumerGroupRelation := func(namespace, pluginName, identifier string) { + pluginKey := namespace + ":" + pluginName + relations, ok := pluginRels[pluginKey] + if !ok { + relations = util.ForeignRelations{} + } + relations.ConsumerGroup = append(relations.ConsumerGroup, identifier) + pluginRels[pluginKey] = relations + } addRouteRelation := func(namespace, pluginName, identifier string) { pluginKey := namespace + ":" + pluginName relations, ok := pluginRels[pluginKey] @@ -286,6 +295,14 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations { addConsumerRelation(c.K8sKongConsumer.Namespace, pluginName, *c.Username) } } + // consumer group + for _, cg := range ks.ConsumerGroups { + pluginList := annotations.ExtractKongPluginsFromAnnotations(cg.K8sKongConsumerGroup.GetAnnotations()) + for _, pluginName := range pluginList { + addConsumerGroupRelation(cg.K8sKongConsumerGroup.Namespace, pluginName, *cg.Name) + } + } + return pluginRels } @@ -338,6 +355,9 @@ func buildPlugins( if rel.Consumer != "" { plugin.Consumer = &kong.Consumer{ID: kong.String(rel.Consumer)} } + if rel.ConsumerGroup != "" { + plugin.ConsumerGroup = &kong.ConsumerGroup{ID: kong.String(rel.ConsumerGroup)} + } plugins = append(plugins, plugin) } } diff --git a/internal/dataplane/kongstate/kongstate_test.go b/internal/dataplane/kongstate/kongstate_test.go index a197e615e2..651be68dc1 100644 --- a/internal/dataplane/kongstate/kongstate_test.go +++ b/internal/dataplane/kongstate/kongstate_test.go @@ -22,6 +22,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/store" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1" + kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1" ) var kongConsumerTypeMeta = metav1.TypeMeta{ @@ -105,6 +106,32 @@ func TestGetPluginRelations(t *testing.T) { "ns1:bar": {Consumer: []string{"foo-consumer"}}, }, }, + { + name: "single consumer group annotation", + args: args{ + state: KongState{ + ConsumerGroups: []ConsumerGroup{ + { + ConsumerGroup: kong.ConsumerGroup{ + Name: kong.String("foo-consumer-group"), + }, + K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Annotations: map[string]string{ + annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar", + }, + }, + }, + }, + }, + }, + }, + want: map[string]util.ForeignRelations{ + "ns1:foo": {ConsumerGroup: []string{"foo-consumer-group"}}, + "ns1:bar": {ConsumerGroup: []string{"foo-consumer-group"}}, + }, + }, { name: "single service annotation", args: args{ @@ -211,7 +238,7 @@ func TestGetPluginRelations(t *testing.T) { }, }, { - name: "multiple consumers, routes and services", + name: "multiple consumers, consumer groups, routes and services", args: args{ state: KongState{ Consumers: []Consumer{ @@ -255,6 +282,47 @@ func TestGetPluginRelations(t *testing.T) { }, }, }, + ConsumerGroups: []ConsumerGroup{ + { + ConsumerGroup: kong.ConsumerGroup{ + Name: kong.String("foo-consumer-group"), + }, + K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Annotations: map[string]string{ + annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar", + }, + }, + }, + }, + { + ConsumerGroup: kong.ConsumerGroup{ + Name: kong.String("foo-consumer-group"), + }, + K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns2", + Annotations: map[string]string{ + annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar", + }, + }, + }, + }, + { + ConsumerGroup: kong.ConsumerGroup{ + Name: kong.String("bar-consumer-group"), + }, + K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns2", + Annotations: map[string]string{ + annotations.AnnotationPrefix + annotations.PluginsKey: "bar,baz", + }, + }, + }, + }, + }, Services: []Service{ { Service: kong.Service{ @@ -301,12 +369,12 @@ func TestGetPluginRelations(t *testing.T) { }, }, want: map[string]util.ForeignRelations{ - "ns1:foo": {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}}, - "ns1:bar": {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}}, + "ns1:foo": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}}, + "ns1:bar": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}}, "ns1:foobar": {Consumer: []string{"bar-consumer"}}, - "ns2:foo": {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route"}}, - "ns2:bar": {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route", "bar-route"}}, - "ns2:baz": {Route: []string{"bar-route"}}, + "ns2:foo": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Route: []string{"foo-route"}}, + "ns2:bar": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group", "bar-consumer-group"}, Route: []string{"foo-route", "bar-route"}}, + "ns2:baz": {Route: []string{"bar-route"}, ConsumerGroup: []string{"bar-consumer-group"}}, }, }, } diff --git a/internal/dataplane/parser/parser.go b/internal/dataplane/parser/parser.go index d00f3b601f..a4ffa91a85 100644 --- a/internal/dataplane/parser/parser.go +++ b/internal/dataplane/parser/parser.go @@ -232,18 +232,18 @@ func (p *Parser) BuildKongConfig() KongConfigBuildingResult { p.registerSuccessfullyParsedObject(&result.Consumers[i].K8sKongConsumer) } - // process annotation plugins - result.FillPlugins(p.logger, p.storer, p.failuresCollector) - for i := range result.Plugins { - p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent) - } - // process consumer groups result.FillConsumerGroups(p.logger, p.storer) for i := range result.ConsumerGroups { p.registerSuccessfullyParsedObject(&result.ConsumerGroups[i].K8sKongConsumerGroup) } + // process annotation plugins + result.FillPlugins(p.logger, p.storer, p.failuresCollector) + for i := range result.Plugins { + p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent) + } + // generate Certificates and SNIs ingressCerts := p.getCerts(ingressRules.SecretNameToSNIs) gatewayCerts := p.getGatewayCerts() diff --git a/internal/util/relations.go b/internal/util/relations.go index 71ab4a8af5..a0f9fc0cd5 100644 --- a/internal/util/relations.go +++ b/internal/util/relations.go @@ -1,11 +1,11 @@ package util type ForeignRelations struct { - Consumer, Route, Service []string + Consumer, ConsumerGroup, Route, Service []string } type Rel struct { - Consumer, Route, Service string + Consumer, ConsumerGroup, Route, Service string } func (relations *ForeignRelations) GetCombinations() []Rel { @@ -36,6 +36,9 @@ func (relations *ForeignRelations) GetCombinations() []Rel { } } } else { + for _, consumerGroup := range relations.ConsumerGroup { + cartesianProduct = append(cartesianProduct, Rel{ConsumerGroup: consumerGroup}) + } for _, service := range relations.Service { cartesianProduct = append(cartesianProduct, Rel{Service: service}) } diff --git a/internal/util/relations_test.go b/internal/util/relations_test.go index cdb635312a..1f9dbff416 100644 --- a/internal/util/relations_test.go +++ b/internal/util/relations_test.go @@ -37,6 +37,22 @@ func TestGetCombinations(t *testing.T) { }, }, }, + { + name: "plugins on consumer group only", + args: args{ + relations: ForeignRelations{ + ConsumerGroup: []string{"foo", "bar"}, + }, + }, + want: []Rel{ + { + ConsumerGroup: "foo", + }, + { + ConsumerGroup: "bar", + }, + }, + }, { name: "plugins on service only", args: args{ diff --git a/test/integration/consumer_group_test.go b/test/integration/consumer_group_test.go index a09af6aaf0..bfaf16ac2f 100644 --- a/test/integration/consumer_group_test.go +++ b/test/integration/consumer_group_test.go @@ -5,11 +5,17 @@ package integration import ( "context" "fmt" - "io" "net/http" + "strings" "testing" + "github.com/kong/kubernetes-testing-framework/pkg/clusters" + "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/kong/kubernetes-ingress-controller/v2/internal/annotations" @@ -17,9 +23,9 @@ import ( kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1" kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1" "github.com/kong/kubernetes-ingress-controller/v2/pkg/clientset" + "github.com/kong/kubernetes-ingress-controller/v2/test" "github.com/kong/kubernetes-ingress-controller/v2/test/consts" "github.com/kong/kubernetes-ingress-controller/v2/test/internal/helpers" - "github.com/kong/kubernetes-ingress-controller/v2/test/internal/testenv" ) func TestConsumerGroup(t *testing.T) { @@ -31,87 +37,267 @@ func TestConsumerGroup(t *testing.T) { ctx := context.Background() ns, cleaner := helpers.Setup(ctx, t, env) + d, s, i, p := deployMinimalSvcWithKeyAuth(ctx, t, ns.Name) + cleaner.Add(d) + cleaner.Add(s) + cleaner.Add(i) + cleaner.Add(p) + + addedHeader := header{ + K: "X-Test-Header", + V: "added", + } + // Use the same header key as plugin name. + pluginRespTrans := configurePlugin( + ctx, t, ns.Name, "response-transformer-advanced-1", "response-transformer-advanced", fmt.Sprintf( + `{ + "add": { + "headers": [ + "%s: %s" + ] + } + }`, + addedHeader.K, addedHeader.V, + ), + ) + cleaner.Add(pluginRespTrans) + + const rateLimitValue = 100 + pluginRateLimit := configurePlugin( + ctx, t, ns.Name, "rate-limiting-advanced-1", "rate-limiting-advanced", fmt.Sprintf( + `{ + "limit": [%d], + "window_size": [100], + "namespace": "test", + "sync_rate": -1, + "window_type": "fixed" + }`, + rateLimitValue, + ), + ) + cleaner.Add(pluginRateLimit) + + addHeaderGroup := configureConsumerGroupWithPlugins( + ctx, t, ns.Name, "test-consumer-group-1", pluginRespTrans.Name, + ) + cleaner.Add(addHeaderGroup) + rateLimitGroup := configureConsumerGroupWithPlugins( + ctx, t, ns.Name, "test-consumer-group-2", pluginRespTrans.Name, pluginRateLimit.Name, + ) + cleaner.Add(rateLimitGroup) + + rateLimitHeader := header{ + K: "RateLimit-Limit", + V: fmt.Sprintf("%d", rateLimitValue), + } + consumers := [...]struct { + Name string + ConsumerGroups []string + ExpectedHeaders []header + ForbiddenHeaderKeys []string + }{ + { + Name: "test-consumer-1", + ConsumerGroups: []string{addHeaderGroup.Name}, + ExpectedHeaders: []header{addedHeader}, + ForbiddenHeaderKeys: []string{rateLimitHeader.K}, + }, + { + Name: "test-consumer-2", + ConsumerGroups: []string{addHeaderGroup.Name, rateLimitGroup.Name}, + ExpectedHeaders: []header{addedHeader, rateLimitHeader}, + ForbiddenHeaderKeys: nil, + }, + { + Name: "test-consumer-3", + ConsumerGroups: nil, + ExpectedHeaders: nil, + ForbiddenHeaderKeys: []string{addedHeader.K, rateLimitHeader.K}, + }, + } + t.Log("creating consumers to be created") + for _, consumer := range consumers { + c, s := configureConsumerWithAPIKey( + ctx, t, ns.Name, consumer.Name, consumer.ConsumerGroups..., + ) + cleaner.Add(c) + cleaner.Add(s) + } + t.Log("checking if consumer has plugin configured correctly based on consumer group membership") + for _, consumer := range consumers { + require.Eventually(t, func() bool { + req := helpers.MustHTTPRequest(t, http.MethodGet, proxyURL, "/", map[string]string{ + "apikey": consumer.Name, + }) + resp, err := helpers.DefaultHTTPClient().Do(req) + if err != nil { + t.Logf("WARNING: consumer %q failed to make a request: %v", consumer.Name, err) + return false + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Logf("WARNING: consumer %q expected status code %d, got %d", consumer.Name, http.StatusOK, resp.StatusCode) + return false + } + for _, hk := range consumer.ForbiddenHeaderKeys { + if hv := resp.Header.Get(hk); hv != "" { + t.Logf("WARNING: consumer %q expected header %q to be empty, got %q", consumer.Name, hk, hv) + return false + } + } + for _, eh := range consumer.ExpectedHeaders { + if hv := resp.Header.Get(eh.K); hv != eh.V { + t.Logf("WARNING: consumer %q expected header %q to be %q, got %q", consumer.Name, eh.K, eh.V, hv) + return false + } + } + return true + }, ingressWait, waitTick) + } +} + +func deployMinimalSvcWithKeyAuth( + ctx context.Context, t *testing.T, namespace string, +) (*appsv1.Deployment, *corev1.Service, *netv1.Ingress, *kongv1.KongPlugin) { + const pluginKeyAuthName = "key-auth" + t.Logf("configuring plugin %q (to give consumers an identity)", pluginKeyAuthName) c, err := clientset.NewForConfig(env.Cluster().Config()) require.NoError(t, err) + pluginKeyAuth, err := c.ConfigurationV1().KongPlugins(namespace).Create( + ctx, + &kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: pluginKeyAuthName, + Annotations: map[string]string{ + annotations.IngressClassKey: consts.IngressClass, + }, + }, + PluginName: "key-auth", + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + t.Log("deploying a minimal HTTP container") + deployment := generators.NewDeploymentForContainer( + generators.NewContainer("echo", test.EchoImage, test.EchoHTTPPort), + ) + deployment, err = env.Cluster().Client().AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) + require.NoError(t, err) - consumerGroupNames := []string{ - "test-consumer-group-1", - "test-consumer-group-2", - } - for _, cgName := range consumerGroupNames { - t.Logf("configuring consumer group: %q", cgName) - cg, err := c.ConfigurationV1beta1().KongConsumerGroups(ns.Name).Create( - ctx, - &kongv1beta1.KongConsumerGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: cgName, - Annotations: map[string]string{ - annotations.IngressClassKey: consts.IngressClass, - }, + t.Logf("exposing deployment %q via service", deployment.Name) + service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeClusterIP) + _, err = env.Cluster().Client().CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("creating an ingress for service %q with plugin %q attached", service.Name, pluginKeyAuthName) + ingress := generators.NewIngressForService("/", map[string]string{ + annotations.IngressClassKey: consts.IngressClass, + annotations.AnnotationPrefix + annotations.StripPathKey: "true", + annotations.AnnotationPrefix + annotations.PluginsKey: pluginKeyAuthName, + }, service) + require.NoError(t, clusters.DeployIngress(ctx, env.Cluster(), namespace, ingress)) + return deployment, service, ingress, pluginKeyAuth +} + +func configurePlugin( + ctx context.Context, t *testing.T, namespace string, name string, pluginName string, cfg string, +) *kongv1.KongPlugin { + c, err := clientset.NewForConfig(env.Cluster().Config()) + require.NoError(t, err) + t.Logf("configuring plugin %q (%q)", name, pluginName) + pluginRespTrans, err := c.ConfigurationV1().KongPlugins(namespace).Create( + ctx, + &kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + annotations.IngressClassKey: consts.IngressClass, }, }, - metav1.CreateOptions{}, - ) - require.NoError(t, err) - cleaner.Add(cg) + PluginName: pluginName, + Config: apiextensionsv1.JSON{ + Raw: []byte(cfg), + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + return pluginRespTrans +} + +func configureConsumerGroupWithPlugins( + ctx context.Context, t *testing.T, namespace string, name string, pluginName ...string, +) *kongv1beta1.KongConsumerGroup { + c, err := clientset.NewForConfig(env.Cluster().Config()) + require.NoError(t, err) + a := map[string]string{ + annotations.IngressClassKey: consts.IngressClass, } + if plugins := strings.Join(pluginName, ","); plugins != "" { + a[annotations.AnnotationPrefix+annotations.PluginsKey] = plugins + t.Logf("configuring consumer group %q with attached plugins: %s", name, plugins) + } else { + t.Logf("configuring consumer group %q with no plugins attached", name) + } + cg, err := c.ConfigurationV1beta1().KongConsumerGroups(namespace).Create( + ctx, + &kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: a, + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + return cg +} - const consumerName = "test-consumer" - t.Logf("configuring consumer: %q", consumerName) - consumer, err := c.ConfigurationV1().KongConsumers(ns.Name).Create( +// configureConsumerWithAPIKey creates a consumer with a key-auth credential set to the consumer's name. +// Assign consumer to specified consumer groups. +func configureConsumerWithAPIKey( + ctx context.Context, t *testing.T, namespace string, name string, consumerGroups ...string, +) (*kongv1.KongConsumer, *corev1.Secret) { + t.Logf( + "creating a consumer: %q with api-key and consumer groups: %s configured", + name, strings.Join(consumerGroups, ","), + ) + secret, err := env.Cluster().Client().CoreV1().Secrets(namespace).Create( + ctx, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + StringData: map[string]string{ + "key": name, + "kongCredType": "key-auth", + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + c, err := clientset.NewForConfig(env.Cluster().Config()) + require.NoError(t, err) + consumer, err := c.ConfigurationV1().KongConsumers(namespace).Create( ctx, &kongv1.KongConsumer{ ObjectMeta: metav1.ObjectMeta{ - Name: consumerName, + Name: name, Annotations: map[string]string{ annotations.IngressClassKey: consts.IngressClass, }, }, - Username: consumerName, - ConsumerGroups: consumerGroupNames, + Username: name, + ConsumerGroups: consumerGroups, + Credentials: []string{name}, }, metav1.CreateOptions{}, ) require.NoError(t, err) - cleaner.Add(consumer) + return consumer, secret +} - for _, cgName := range consumerGroupNames { - t.Logf("validating that consumer %q was successfully added to previously configured consumer group: %q", consumerName, cgName) - require.Eventually(t, func() bool { - cgPath := fmt.Sprintf("/consumer_groups/%s/consumers/%s", cgName, consumerName) - var headers map[string]string - if testenv.DBMode() != testenv.DBModeOff { - cgPath = fmt.Sprintf( - "/%s/consumer_groups/%s/consumers/%s", consts.KongTestWorkspace, cgName, consumerName, - ) - headers = map[string]string{ - "Kong-Admin-Token": consts.KongTestPassword, - } - } - req := helpers.MustHTTPRequest(t, http.MethodGet, proxyAdminURL, cgPath, headers) - resp, err := helpers.DefaultHTTPClient().Do(req) - if err != nil { - t.Logf("WARNING: error while waiting for %s: %v", req.URL, err) - return false - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - t.Logf("WARNING: error while reading response from %s: %v", req.URL, err) - } - switch resp.StatusCode { - case http.StatusOK: - return true - case http.StatusForbidden: - t.Logf( - "WARNING: it seems Kong Gateway Enterprise hasn't got a valid license passed - from: %s received: %s with body: %s", - req.URL, resp.Status, body, - ) - return false - default: - t.Logf("WARNING: from: %s received unexpected: %s with body: %s", req.URL, resp.Status, body) - return false - } - }, ingressWait, waitTick) - } +type header struct { + K string + V string }