From 3eeea9ef09e0354498e655ec33bf7059664f82d1 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Fri, 14 Jul 2023 18:43:15 +0200 Subject: [PATCH] feat: Consumer Groups can be configured in Consumer --- CHANGELOG.md | 2 +- ...onfiguration.konghq.com_kongconsumers.yaml | 6 + .../single/all-in-one-dbless-enterprise.yaml | 6 + .../all-in-one-dbless-k4k8s-enterprise.yaml | 6 + .../all-in-one-dbless-konnect-enterprise.yaml | 6 + deploy/single/all-in-one-dbless-konnect.yaml | 6 + deploy/single/all-in-one-dbless-legacy.yaml | 6 + deploy/single/all-in-one-dbless.yaml | 6 + .../all-in-one-postgres-enterprise.yaml | 6 + deploy/single/all-in-one-postgres.yaml | 6 + docs/api-reference.md | 1 + internal/dataplane/deckgen/generate.go | 5 + internal/dataplane/kongstate/consumer.go | 4 +- internal/dataplane/kongstate/kongstate.go | 12 ++ .../configuration/v1/kongconsumer_types.go | 3 + .../configuration/v1/kongprotocol_types.go | 6 +- .../configuration/v1/zz_generated.deepcopy.go | 5 + test/integration/consumer_group_test.go | 109 +++++++++++------- 18 files changed, 155 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba047be3bc..b5e63c3b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,7 @@ Adding a new version? You'll need three changes: ### Added - **WIP** Introduce `KongConsumerGroup` CRD (supported by Kong Enterprise only) - [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387) + [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419) - The ResponseHeaderModifier Gateway API filter is now supported and translated to the proper set of Kong plugins. [#4350](https://github.com/Kong/kubernetes-ingress-controller/pull/4350) diff --git a/config/crd/bases/configuration.konghq.com_kongconsumers.yaml b/config/crd/bases/configuration.konghq.com_kongconsumers.yaml index a26cec3613..ff00c05ae9 100644 --- a/config/crd/bases/configuration.konghq.com_kongconsumers.yaml +++ b/config/crd/bases/configuration.konghq.com_kongconsumers.yaml @@ -37,6 +37,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless-enterprise.yaml b/deploy/single/all-in-one-dbless-enterprise.yaml index dbbe69320e..bd908a5bef 100644 --- a/deploy/single/all-in-one-dbless-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-enterprise.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml index 45ae166bc0..9081c3d7b7 100644 --- a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml index be3b95ade2..e14120e14b 100644 --- a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless-konnect.yaml b/deploy/single/all-in-one-dbless-konnect.yaml index 973bac1e9b..0179a23aad 100644 --- a/deploy/single/all-in-one-dbless-konnect.yaml +++ b/deploy/single/all-in-one-dbless-konnect.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless-legacy.yaml b/deploy/single/all-in-one-dbless-legacy.yaml index a4354aca46..1709df85f5 100644 --- a/deploy/single/all-in-one-dbless-legacy.yaml +++ b/deploy/single/all-in-one-dbless-legacy.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-dbless.yaml b/deploy/single/all-in-one-dbless.yaml index 32b6dfe9f6..e81c62b1d1 100644 --- a/deploy/single/all-in-one-dbless.yaml +++ b/deploy/single/all-in-one-dbless.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-postgres-enterprise.yaml b/deploy/single/all-in-one-postgres-enterprise.yaml index 619452b288..b826033a70 100644 --- a/deploy/single/all-in-one-postgres-enterprise.yaml +++ b/deploy/single/all-in-one-postgres-enterprise.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/deploy/single/all-in-one-postgres.yaml b/deploy/single/all-in-one-postgres.yaml index 9c34140e1b..582aa002a6 100644 --- a/deploy/single/all-in-one-postgres.yaml +++ b/deploy/single/all-in-one-postgres.yaml @@ -378,6 +378,12 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + consumer_groups: + description: ConsumerGroups are references to Consumer Groups (that Consumer + wants to be part of) provisioned in Kong. + items: + type: string + type: array credentials: description: Credentials are references to secrets containing a credential to be provisioned in Kong. diff --git a/docs/api-reference.md b/docs/api-reference.md index b31227afae..b8f086ce53 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -57,6 +57,7 @@ KongConsumer is the Schema for the kongconsumers API. | `username` _string_ | Username is a Kong cluster-unique username of the consumer. | | `custom_id` _string_ | CustomID is a Kong cluster-unique existing ID for the consumer - useful for mapping Kong with users in your existing database. | | `credentials` _string array_ | Credentials are references to secrets containing a credential to be provisioned in Kong. | +| `consumer_groups` _string array_ | ConsumerGroups are references to Consumer Groups (that Consumer wants to be part of) provisioned in Kong. | diff --git a/internal/dataplane/deckgen/generate.go b/internal/dataplane/deckgen/generate.go index e469ccc6f4..e829490b83 100644 --- a/internal/dataplane/deckgen/generate.go +++ b/internal/dataplane/deckgen/generate.go @@ -164,6 +164,11 @@ func ToDeckContent( continue } + for _, cg := range c.ConsumerGroups { + cg := cg + consumer.Groups = append(consumer.Groups, &cg) + } + for _, p := range c.Plugins { consumer.Plugins = append(consumer.Plugins, &file.FPlugin{Plugin: p}) } diff --git a/internal/dataplane/kongstate/consumer.go b/internal/dataplane/kongstate/consumer.go index 6e58a2bec6..0baa382e97 100644 --- a/internal/dataplane/kongstate/consumer.go +++ b/internal/dataplane/kongstate/consumer.go @@ -13,7 +13,9 @@ import ( // Consumer holds a Kong consumer and its plugins and credentials. type Consumer struct { kong.Consumer - Plugins []kong.Plugin + Plugins []kong.Plugin + ConsumerGroups []kong.ConsumerGroup + KeyAuths []*KeyAuth HMACAuths []*HMACAuth JWTAuths []*JWTAuth diff --git a/internal/dataplane/kongstate/kongstate.go b/internal/dataplane/kongstate/kongstate.go index adecef163d..15ea7ef941 100644 --- a/internal/dataplane/kongstate/kongstate.go +++ b/internal/dataplane/kongstate/kongstate.go @@ -73,6 +73,18 @@ func (ks *KongState) FillConsumersAndCredentials( c.K8sKongConsumer = *consumer c.Tags = util.GenerateTagsForObject(consumer) + // Get consumer groups + for _, cgName := range consumer.ConsumerGroups { + cg, err := s.GetKongConsumerGroup(consumer.Namespace, cgName) + if err != nil { + failuresCollector.PushResourceFailure(fmt.Sprintf("non-existing consumer group: %q", err), consumer) + continue + } + c.ConsumerGroups = append(c.ConsumerGroups, kong.ConsumerGroup{ + Name: &cg.Name, + }) + } + for _, cred := range consumer.Credentials { pushCredentialResourceFailures := func(message string) { failuresCollector.PushResourceFailure(fmt.Sprintf("credential %q failure: %s", cred, message), consumer) diff --git a/pkg/apis/configuration/v1/kongconsumer_types.go b/pkg/apis/configuration/v1/kongconsumer_types.go index 4915cfe1ff..c100409521 100644 --- a/pkg/apis/configuration/v1/kongconsumer_types.go +++ b/pkg/apis/configuration/v1/kongconsumer_types.go @@ -45,6 +45,9 @@ type KongConsumer struct { // Credentials are references to secrets containing a credential to be // provisioned in Kong. Credentials []string `json:"credentials,omitempty"` + // ConsumerGroups are references to Consumer Groups (that Consumer wants to be part of) + // provisioned in Kong. + ConsumerGroups []string `json:"consumer_groups,omitempty"` // Status represents the current status of the KongConsumer resource. Status KongConsumerStatus `json:"status,omitempty"` diff --git a/pkg/apis/configuration/v1/kongprotocol_types.go b/pkg/apis/configuration/v1/kongprotocol_types.go index 9c99dd4314..b2cb2b6750 100644 --- a/pkg/apis/configuration/v1/kongprotocol_types.go +++ b/pkg/apis/configuration/v1/kongprotocol_types.go @@ -6,7 +6,7 @@ package v1 // +kubebuilder:object:generate=true type KongProtocol string -// KongProtocolsToStrings converts a slice of KongProtocol to plain strings +// KongProtocolsToStrings converts a slice of KongProtocol to plain strings. func KongProtocolsToStrings(protocols []KongProtocol) (res []string) { for _, protocol := range protocols { res = append(res, string(protocol)) @@ -14,7 +14,7 @@ func KongProtocolsToStrings(protocols []KongProtocol) (res []string) { return } -// StringsToKongProtocols converts a slice of strings to KongProtocols +// StringsToKongProtocols converts a slice of strings to KongProtocols. func StringsToKongProtocols(strings []string) (res []KongProtocol) { for _, protocol := range strings { res = append(res, KongProtocol(protocol)) @@ -22,7 +22,7 @@ func StringsToKongProtocols(strings []string) (res []KongProtocol) { return } -// ProtocolSlice converts a slice of string to a slice of *KongProtocol +// ProtocolSlice converts a slice of string to a slice of *KongProtocol. func ProtocolSlice(elements ...string) []*KongProtocol { var res []*KongProtocol for _, element := range elements { diff --git a/pkg/apis/configuration/v1/zz_generated.deepcopy.go b/pkg/apis/configuration/v1/zz_generated.deepcopy.go index fb0bb1d996..2b9a6224a5 100644 --- a/pkg/apis/configuration/v1/zz_generated.deepcopy.go +++ b/pkg/apis/configuration/v1/zz_generated.deepcopy.go @@ -149,6 +149,11 @@ func (in *KongConsumer) DeepCopyInto(out *KongConsumer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ConsumerGroups != nil { + in, out := &in.ConsumerGroups, &out.ConsumerGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } in.Status.DeepCopyInto(&out.Status) } diff --git a/test/integration/consumer_group_test.go b/test/integration/consumer_group_test.go index a909c223e7..29a9639ea0 100644 --- a/test/integration/consumer_group_test.go +++ b/test/integration/consumer_group_test.go @@ -14,6 +14,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/annotations" "github.com/kong/kubernetes-ingress-controller/v2/internal/versions" + kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1" kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1" "github.com/kong/kubernetes-ingress-controller/v2/pkg/clientset" "github.com/kong/kubernetes-ingress-controller/v2/test/consts" @@ -33,57 +34,83 @@ func TestConsumerGroup(t *testing.T) { c, err := clientset.NewForConfig(env.Cluster().Config()) require.NoError(t, err) - const consumerGroupName = "test-consumer-group" - t.Logf("configuring Consumer Group: %q", consumerGroupName) - cg, err := c.ConfigurationV1beta1().KongConsumerGroups(ns.Name).Create( + consumerGroupNames := []string{ + "test-consumer-group-1", + "test-consumer-group-2", + } + for _, cgName := range consumerGroupNames { + t.Logf("configuring Consumer Group: %q", cgName) + cg, err := c.ConfigurationV1beta1().KongConsumerGroups(ns.Name).Create( + ctx, + &kongv1beta1.KongConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: cgName, + Annotations: map[string]string{ + annotations.IngressClassKey: consts.IngressClass, + }, + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + cleaner.Add(cg) + } + + const consumerName = "test-consumer" + consumer, err := c.ConfigurationV1().KongConsumers(ns.Name).Create( ctx, - &kongv1beta1.KongConsumerGroup{ + &kongv1.KongConsumer{ ObjectMeta: metav1.ObjectMeta{ - Name: consumerGroupName, - Namespace: ns.Name, + Name: consumerName, Annotations: map[string]string{ annotations.IngressClassKey: consts.IngressClass, }, }, + Username: consumerName, + ConsumerGroups: consumerGroupNames, }, metav1.CreateOptions{}, ) require.NoError(t, err) - cleaner.Add(cg) + cleaner.Add(consumer) - t.Logf("validating that Consumer Group: %q was successfully configured", consumerGroupName) - require.Eventually(t, func() bool { - cgPath := fmt.Sprintf("/consumer_groups/%s", consumerGroupName) - var headers map[string]string - if testenv.DBMode() != testenv.DBModeOff { - cgPath = fmt.Sprintf("/%s/consumer_groups/%s", consts.KongTestWorkspace, consumerGroupName) - headers = map[string]string{ - "Kong-Admin-Token": consts.KongTestPassword, + for _, cgName := range consumerGroupNames { + t.Logf("validating that Consumer %q was successfully added to previously configured Consumer Group: %q", consumerName, cgName) + require.Eventually(t, func() bool { + cgPath := fmt.Sprintf("/consumer_groups/%s/consumers/%s", cgName, consumerName) + var headers map[string]string + if testenv.DBMode() != testenv.DBModeOff { + cgPath = fmt.Sprintf( + "/%s/consumer_groups/%s/consumers/%s", consts.KongTestWorkspace, cgName, consumerName, + ) + headers = map[string]string{ + "Kong-Admin-Token": consts.KongTestPassword, + } + } + req := helpers.MustHTTPRequest(t, http.MethodGet, proxyAdminURL, cgPath, headers) + resp, err := helpers.DefaultHTTPClient().Do(req) + if err != nil { + t.Logf("WARNING: error while waiting for %s: %v", req.URL, err) + return false + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Logf("WARNING: error while reading response from %s: %v", req.URL, err) + } + switch resp.StatusCode { + case http.StatusOK: + return true + case http.StatusForbidden: + t.Logf( + "WARNING: it seems Kong Gateway Enterprise hasn't got a valid license passed - from: %s received: %s with body: %s", + req.URL, resp.Status, body, + ) + return false + default: + t.Logf("WARNING: from: %s received unexpected: %s with body: %s", req.URL, resp.Status, body) + return false } - } - req := helpers.MustHTTPRequest(t, http.MethodGet, proxyAdminURL, cgPath, headers) - resp, err := helpers.DefaultHTTPClient().Do(req) - if err != nil { - t.Logf("WARNING: error while waiting for %s: %v", resp.Request.URL, err) - return false - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - t.Logf("WARNING: error while reading response from %s: %v", resp.Request.URL, err) - } - switch resp.StatusCode { - case http.StatusOK: - return true - case http.StatusForbidden: - t.Logf( - "WARNING: it seems Kong Gateway Enterprise hasn't got a valid license passed - from: %s received: %s with body: %s", - resp.Request.URL, resp.Status, body, - ) - return false - default: - t.Logf("WARNING: from: %s received unexpected: %s with body: %s", resp.Request.URL, resp.Status, body) - return false - } - }, ingressWait, waitTick) + }, ingressWait, waitTick) + } }