feat(consumer group): allow assigning plugins to consumer groups

CHANGELOG.md

@@ -76,7 +76,7 @@ Adding a new version? You'll need three changes:
 ### Added
 
 - **WIP** Introduce `KongConsumerGroup` CRD (supported by Kong Enterprise only)
-  [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419), [4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437)
+  [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419), [#4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437), [#4452](https://github.com/Kong/kubernetes-ingress-controller/pull/4452)
 - The ResponseHeaderModifier Gateway API filter is now supported and translated
   to the proper set of Kong plugins.
   [#4350](https://github.com/Kong/kubernetes-ingress-controller/pull/4350)

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/jpillora/backoff v1.0.0
-	github.com/kong/deck v1.25.0
+	github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a
 	github.com/kong/go-kong v0.46.0
 	github.com/kong/kubernetes-telemetry v0.1.0
 	github.com/kong/kubernetes-testing-framework v0.34.0

go.sum

@@ -280,8 +280,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
-github.com/kong/deck v1.25.0 h1:lcNAJQhoJmTjFHLt2DFcsXS9aqclauUxqelPxTKz9bU=
-github.com/kong/deck v1.25.0/go.mod h1:1AUvNaqRHhwqPNsQ0yd2Rgg5LHqS4v/PqsaEHJ8dXwE=
+github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a h1:XLOxVDB6T8PVM9iCYRWH5FNWidSCEtwOURJ5xaka8vs=
+github.com/kong/deck v1.25.1-0.20230804143828-c8234139284a/go.mod h1:4Mt9LLrDSd+6mQsWfykLAP2D3iWWxgfQ5X1HraswYyc=
 github.com/kong/go-kong v0.46.0 h1:9I6nlX63WymU5Sg+d13iZDVwpW5vXh8/v0zarU27dzI=
 github.com/kong/go-kong v0.46.0/go.mod h1:41Sot1N/n8UHBp+gE/6nOw3vuzoHbhMSyU/zOS7VzPE=
 github.com/kong/kubernetes-telemetry v0.1.0 h1:dyDhB2SUvxVaDAxaZtLWL7JGAbkgwlCUM1P8gR/rCl0=

internal/dataplane/deckgen/deckgen.go

@@ -59,6 +59,9 @@ func PluginString(plugin file.FPlugin) string {
 	if plugin.Consumer != nil && plugin.Consumer.ID != nil {
 		result += *plugin.Consumer.ID
 	}
+	if plugin.ConsumerGroup != nil && plugin.ConsumerGroup.ID != nil {
+		result += *plugin.ConsumerGroup.ID
+	}
 	if plugin.Route != nil && plugin.Route.ID != nil {
 		result += *plugin.Route.ID
 	}

internal/dataplane/kongstate/kongstate.go

@@ -243,6 +243,15 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations {
 		relations.Consumer = append(relations.Consumer, identifier)
 		pluginRels[pluginKey] = relations
 	}
+	addConsumerGroupRelation := func(namespace, pluginName, identifier string) {
+		pluginKey := namespace + ":" + pluginName
+		relations, ok := pluginRels[pluginKey]
+		if !ok {
+			relations = util.ForeignRelations{}
+		}
+		relations.ConsumerGroup = append(relations.ConsumerGroup, identifier)
+		pluginRels[pluginKey] = relations
+	}
 	addRouteRelation := func(namespace, pluginName, identifier string) {
 		pluginKey := namespace + ":" + pluginName
 		relations, ok := pluginRels[pluginKey]
@@ -286,6 +295,14 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations {
 			addConsumerRelation(c.K8sKongConsumer.Namespace, pluginName, *c.Username)
 		}
 	}
+	// consumer group
+	for _, cg := range ks.ConsumerGroups {
+		pluginList := annotations.ExtractKongPluginsFromAnnotations(cg.K8sKongConsumerGroup.GetAnnotations())
+		for _, pluginName := range pluginList {
+			addConsumerGroupRelation(cg.K8sKongConsumerGroup.Namespace, pluginName, *cg.Name)
+		}
+	}
+
 	return pluginRels
 }
 
@@ -338,6 +355,9 @@ func buildPlugins(
 			if rel.Consumer != "" {
 				plugin.Consumer = &kong.Consumer{ID: kong.String(rel.Consumer)}
 			}
+			if rel.ConsumerGroup != "" {
+				plugin.ConsumerGroup = &kong.ConsumerGroup{ID: kong.String(rel.ConsumerGroup)}
+			}
 			plugins = append(plugins, plugin)
 		}
 	}

internal/dataplane/kongstate/kongstate_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
+	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1"
 )
 
 var kongConsumerTypeMeta = metav1.TypeMeta{
@@ -105,6 +106,32 @@ func TestGetPluginRelations(t *testing.T) {
 				"ns1:bar": {Consumer: []string{"foo-consumer"}},
 			},
 		},
+		{
+			name: "single consumer group annotation",
+			args: args{
+				state: KongState{
+					ConsumerGroups: []ConsumerGroup{
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns1",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: map[string]util.ForeignRelations{
+				"ns1:foo": {ConsumerGroup: []string{"foo-consumer-group"}},
+				"ns1:bar": {ConsumerGroup: []string{"foo-consumer-group"}},
+			},
+		},
 		{
 			name: "single service annotation",
 			args: args{
@@ -211,7 +238,7 @@ func TestGetPluginRelations(t *testing.T) {
 			},
 		},
 		{
-			name: "multiple consumers, routes and services",
+			name: "multiple consumers, consumer groups, routes and services",
 			args: args{
 				state: KongState{
 					Consumers: []Consumer{
@@ -255,6 +282,47 @@ func TestGetPluginRelations(t *testing.T) {
 							},
 						},
 					},
+					ConsumerGroups: []ConsumerGroup{
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns1",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns2",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("bar-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns2",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "bar,baz",
+									},
+								},
+							},
+						},
+					},
 					Services: []Service{
 						{
 							Service: kong.Service{
@@ -301,12 +369,12 @@ func TestGetPluginRelations(t *testing.T) {
 				},
 			},
 			want: map[string]util.ForeignRelations{
-				"ns1:foo":    {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}},
-				"ns1:bar":    {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}},
+				"ns1:foo":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}},
+				"ns1:bar":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}},
 				"ns1:foobar": {Consumer: []string{"bar-consumer"}},
-				"ns2:foo":    {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route"}},
-				"ns2:bar":    {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route", "bar-route"}},
-				"ns2:baz":    {Route: []string{"bar-route"}},
+				"ns2:foo":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Route: []string{"foo-route"}},
+				"ns2:bar":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group", "bar-consumer-group"}, Route: []string{"foo-route", "bar-route"}},
+				"ns2:baz":    {Route: []string{"bar-route"}, ConsumerGroup: []string{"bar-consumer-group"}},
 			},
 		},
 	}

internal/dataplane/parser/parser.go

@@ -232,18 +232,18 @@ func (p *Parser) BuildKongConfig() KongConfigBuildingResult {
 		p.registerSuccessfullyParsedObject(&result.Consumers[i].K8sKongConsumer)
 	}
 
-	// process annotation plugins
-	result.FillPlugins(p.logger, p.storer, p.failuresCollector)
-	for i := range result.Plugins {
-		p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent)
-	}
-
 	// process consumer groups
 	result.FillConsumerGroups(p.logger, p.storer)
 	for i := range result.ConsumerGroups {
 		p.registerSuccessfullyParsedObject(&result.ConsumerGroups[i].K8sKongConsumerGroup)
 	}
 
+	// process annotation plugins
+	result.FillPlugins(p.logger, p.storer, p.failuresCollector)
+	for i := range result.Plugins {
+		p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent)
+	}
+
 	// generate Certificates and SNIs
 	ingressCerts := p.getCerts(ingressRules.SecretNameToSNIs)
 	gatewayCerts := p.getGatewayCerts()

internal/util/relations.go

@@ -1,11 +1,11 @@
 package util
 
 type ForeignRelations struct {
-	Consumer, Route, Service []string
+	Consumer, ConsumerGroup, Route, Service []string
 }
 
 type Rel struct {
-	Consumer, Route, Service string
+	Consumer, ConsumerGroup, Route, Service string
 }
 
 func (relations *ForeignRelations) GetCombinations() []Rel {
@@ -36,6 +36,9 @@ func (relations *ForeignRelations) GetCombinations() []Rel {
 			}
 		}
 	} else {
+		for _, consumerGroup := range relations.ConsumerGroup {
+			cartesianProduct = append(cartesianProduct, Rel{ConsumerGroup: consumerGroup})
+		}
 		for _, service := range relations.Service {
 			cartesianProduct = append(cartesianProduct, Rel{Service: service})
 		}

internal/util/relations_test.go

@@ -37,6 +37,22 @@ func TestGetCombinations(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "plugins on consumer group only",
+			args: args{
+				relations: ForeignRelations{
+					ConsumerGroup: []string{"foo", "bar"},
+				},
+			},
+			want: []Rel{
+				{
+					ConsumerGroup: "foo",
+				},
+				{
+					ConsumerGroup: "bar",
+				},
+			},
+		},
 		{
 			name: "plugins on service only",
 			args: args{