feat(consumer group): allow assigning plugins to consumer groups (#4452)

CHANGELOG.md

@@ -75,8 +75,12 @@ Adding a new version? You'll need three changes:
 
 ### Added
 
-- **WIP** Introduce `KongConsumerGroup` CRD (supported by Kong Enterprise only)
-  [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325), [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387), [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419), [4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437)
+- Introduce `KongConsumerGroup` CRD (supported by Kong Enterprise only)
+  [#4325](https://github.com/Kong/kubernetes-ingress-controller/pull/4325)
+  [#4387](https://github.com/Kong/kubernetes-ingress-controller/pull/4387)
+  [#4419](https://github.com/Kong/kubernetes-ingress-controller/pull/4419)
+  [#4437](https://github.com/Kong/kubernetes-ingress-controller/pull/4437)
+  [#4452](https://github.com/Kong/kubernetes-ingress-controller/pull/4452)
 - The ResponseHeaderModifier Gateway API filter is now supported and translated
   to the proper set of Kong plugins.
   [#4350](https://github.com/Kong/kubernetes-ingress-controller/pull/4350)

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/jpillora/backoff v1.0.0
-	github.com/kong/deck v1.25.0
+	github.com/kong/deck v1.25.1-0.20230807064622-328ae943624a
 	github.com/kong/go-kong v0.46.0
 	github.com/kong/kubernetes-telemetry v0.1.0
 	github.com/kong/kubernetes-testing-framework v0.35.0
@@ -167,7 +167,7 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.6 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sourcegraph/conc v0.3.0
 	github.com/spf13/cast v1.5.1 // indirect

go.sum

@@ -280,8 +280,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
-github.com/kong/deck v1.25.0 h1:lcNAJQhoJmTjFHLt2DFcsXS9aqclauUxqelPxTKz9bU=
-github.com/kong/deck v1.25.0/go.mod h1:1AUvNaqRHhwqPNsQ0yd2Rgg5LHqS4v/PqsaEHJ8dXwE=
+github.com/kong/deck v1.25.1-0.20230807064622-328ae943624a h1:RWLUQo7offpoT4dWEVntAW1y6SLLxNIDHzaZYxSy0Nk=
+github.com/kong/deck v1.25.1-0.20230807064622-328ae943624a/go.mod h1:B6f2nPWzTld/aTvtxAC6lWFIcAr4FsG25fgycdzeZQY=
 github.com/kong/go-kong v0.46.0 h1:9I6nlX63WymU5Sg+d13iZDVwpW5vXh8/v0zarU27dzI=
 github.com/kong/go-kong v0.46.0/go.mod h1:41Sot1N/n8UHBp+gE/6nOw3vuzoHbhMSyU/zOS7VzPE=
 github.com/kong/kubernetes-telemetry v0.1.0 h1:dyDhB2SUvxVaDAxaZtLWL7JGAbkgwlCUM1P8gR/rCl0=
@@ -408,8 +408,8 @@ github.com/samber/mo v1.8.0/go.mod h1:BfkrCPuYzVG3ZljnZB783WIJIGk1mcZr9c9CPf8tAx
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
 github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
-github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08=
-github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU=
+github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
+github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -596,7 +596,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

internal/dataplane/deckgen/deckgen.go

@@ -59,6 +59,9 @@ func PluginString(plugin file.FPlugin) string {
 	if plugin.Consumer != nil && plugin.Consumer.ID != nil {
 		result += *plugin.Consumer.ID
 	}
+	if plugin.ConsumerGroup != nil && plugin.ConsumerGroup.ID != nil {
+		result += *plugin.ConsumerGroup.ID
+	}
 	if plugin.Route != nil && plugin.Route.ID != nil {
 		result += *plugin.Route.ID
 	}

internal/dataplane/kongstate/kongstate.go

@@ -243,6 +243,15 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations {
 		relations.Consumer = append(relations.Consumer, identifier)
 		pluginRels[pluginKey] = relations
 	}
+	addConsumerGroupRelation := func(namespace, pluginName, identifier string) {
+		pluginKey := namespace + ":" + pluginName
+		relations, ok := pluginRels[pluginKey]
+		if !ok {
+			relations = util.ForeignRelations{}
+		}
+		relations.ConsumerGroup = append(relations.ConsumerGroup, identifier)
+		pluginRels[pluginKey] = relations
+	}
 	addRouteRelation := func(namespace, pluginName, identifier string) {
 		pluginKey := namespace + ":" + pluginName
 		relations, ok := pluginRels[pluginKey]
@@ -286,6 +295,14 @@ func (ks *KongState) getPluginRelations() map[string]util.ForeignRelations {
 			addConsumerRelation(c.K8sKongConsumer.Namespace, pluginName, *c.Username)
 		}
 	}
+	// consumer group
+	for _, cg := range ks.ConsumerGroups {
+		pluginList := annotations.ExtractKongPluginsFromAnnotations(cg.K8sKongConsumerGroup.GetAnnotations())
+		for _, pluginName := range pluginList {
+			addConsumerGroupRelation(cg.K8sKongConsumerGroup.Namespace, pluginName, *cg.Name)
+		}
+	}
+
 	return pluginRels
 }
 
@@ -338,6 +355,9 @@ func buildPlugins(
 			if rel.Consumer != "" {
 				plugin.Consumer = &kong.Consumer{ID: kong.String(rel.Consumer)}
 			}
+			if rel.ConsumerGroup != "" {
+				plugin.ConsumerGroup = &kong.ConsumerGroup{ID: kong.String(rel.ConsumerGroup)}
+			}
 			plugins = append(plugins, plugin)
 		}
 	}

internal/dataplane/kongstate/kongstate_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
+	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1"
 )
 
 var kongConsumerTypeMeta = metav1.TypeMeta{
@@ -105,6 +106,32 @@ func TestGetPluginRelations(t *testing.T) {
 				"ns1:bar": {Consumer: []string{"foo-consumer"}},
 			},
 		},
+		{
+			name: "single consumer group annotation",
+			args: args{
+				state: KongState{
+					ConsumerGroups: []ConsumerGroup{
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns1",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: map[string]util.ForeignRelations{
+				"ns1:foo": {ConsumerGroup: []string{"foo-consumer-group"}},
+				"ns1:bar": {ConsumerGroup: []string{"foo-consumer-group"}},
+			},
+		},
 		{
 			name: "single service annotation",
 			args: args{
@@ -211,7 +238,7 @@ func TestGetPluginRelations(t *testing.T) {
 			},
 		},
 		{
-			name: "multiple consumers, routes and services",
+			name: "multiple consumers, consumer groups, routes and services",
 			args: args{
 				state: KongState{
 					Consumers: []Consumer{
@@ -255,6 +282,47 @@ func TestGetPluginRelations(t *testing.T) {
 							},
 						},
 					},
+					ConsumerGroups: []ConsumerGroup{
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns1",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("foo-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns2",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "foo,bar",
+									},
+								},
+							},
+						},
+						{
+							ConsumerGroup: kong.ConsumerGroup{
+								Name: kong.String("bar-consumer-group"),
+							},
+							K8sKongConsumerGroup: kongv1beta1.KongConsumerGroup{
+								ObjectMeta: metav1.ObjectMeta{
+									Namespace: "ns2",
+									Annotations: map[string]string{
+										annotations.AnnotationPrefix + annotations.PluginsKey: "bar,baz",
+									},
+								},
+							},
+						},
+					},
 					Services: []Service{
 						{
 							Service: kong.Service{
@@ -301,12 +369,12 @@ func TestGetPluginRelations(t *testing.T) {
 				},
 			},
 			want: map[string]util.ForeignRelations{
-				"ns1:foo":    {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}},
-				"ns1:bar":    {Consumer: []string{"foo-consumer"}, Service: []string{"foo-service"}},
+				"ns1:foo":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}},
+				"ns1:bar":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}},
 				"ns1:foobar": {Consumer: []string{"bar-consumer"}},
-				"ns2:foo":    {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route"}},
-				"ns2:bar":    {Consumer: []string{"foo-consumer"}, Route: []string{"foo-route", "bar-route"}},
-				"ns2:baz":    {Route: []string{"bar-route"}},
+				"ns2:foo":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Route: []string{"foo-route"}},
+				"ns2:bar":    {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group", "bar-consumer-group"}, Route: []string{"foo-route", "bar-route"}},
+				"ns2:baz":    {Route: []string{"bar-route"}, ConsumerGroup: []string{"bar-consumer-group"}},
 			},
 		},
 	}

internal/dataplane/parser/parser.go

@@ -232,18 +232,18 @@ func (p *Parser) BuildKongConfig() KongConfigBuildingResult {
 		p.registerSuccessfullyParsedObject(&result.Consumers[i].K8sKongConsumer)
 	}
 
-	// process annotation plugins
-	result.FillPlugins(p.logger, p.storer, p.failuresCollector)
-	for i := range result.Plugins {
-		p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent)
-	}
-
 	// process consumer groups
 	result.FillConsumerGroups(p.logger, p.storer)
 	for i := range result.ConsumerGroups {
 		p.registerSuccessfullyParsedObject(&result.ConsumerGroups[i].K8sKongConsumerGroup)
 	}
 
+	// process annotation plugins
+	result.FillPlugins(p.logger, p.storer, p.failuresCollector)
+	for i := range result.Plugins {
+		p.registerSuccessfullyParsedObject(result.Plugins[i].K8sParent)
+	}
+
 	// generate Certificates and SNIs
 	ingressCerts := p.getCerts(ingressRules.SecretNameToSNIs)
 	gatewayCerts := p.getGatewayCerts()

internal/util/relations.go

@@ -1,11 +1,11 @@
 package util
 
 type ForeignRelations struct {
-	Consumer, Route, Service []string
+	Consumer, ConsumerGroup, Route, Service []string
 }
 
 type Rel struct {
-	Consumer, Route, Service string
+	Consumer, ConsumerGroup, Route, Service string
 }
 
 func (relations *ForeignRelations) GetCombinations() []Rel {
@@ -36,6 +36,9 @@ func (relations *ForeignRelations) GetCombinations() []Rel {
 			}
 		}
 	} else {
+		for _, consumerGroup := range relations.ConsumerGroup {
+			cartesianProduct = append(cartesianProduct, Rel{ConsumerGroup: consumerGroup})
+		}
 		for _, service := range relations.Service {
 			cartesianProduct = append(cartesianProduct, Rel{Service: service})
 		}

internal/util/relations_test.go

@@ -37,6 +37,22 @@ func TestGetCombinations(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "plugins on consumer group only",
+			args: args{
+				relations: ForeignRelations{
+					ConsumerGroup: []string{"foo", "bar"},
+				},
+			},
+			want: []Rel{
+				{
+					ConsumerGroup: "foo",
+				},
+				{
+					ConsumerGroup: "bar",
+				},
+			},
+		},
 		{
 			name: "plugins on service only",
 			args: args{

internal/versions/versions.go

@@ -16,9 +16,7 @@ var (
 	PluginOrderingVersionCutoff = semver.Version{Major: 3}
 
 	// ConsumerGroupsVersionCutoff is the Kong version prior to the addition of Consumer Groups as first class citizens.
-	// TODO: Change to 3.4 once it's released. For now let's assume it's 3.3 (some features work), but for the full
-	// support it should be 3.4 (unreleased yet).
-	ConsumerGroupsVersionCutoff = semver.Version{Major: 3, Minor: 3}
+	ConsumerGroupsVersionCutoff = semver.Version{Major: 3, Minor: 4}
 
 	// MTLSCredentialVersionCutoff is the minimum Kong version that support mTLS credentials. This is a patch version
 	// because the original version of the mTLS credential was not compatible with KIC.