[NET-9500] Cleanup orphaned inline-certs and acl role/policy (#4067)

acceptance/go.mod

@@ -5,6 +5,7 @@ go 1.21.1
 toolchain go1.22.0
 
 require (
+	github.com/go-logr/logr v1.2.4
 	github.com/google/uuid v1.3.0
 	github.com/gruntwork-io/terratest v0.46.7
 	github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b
@@ -45,9 +46,7 @@ require (
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
 	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -121,8 +120,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.14.0 // indirect

acceptance/go.sum

@@ -20,9 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I=
 github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
-github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -457,14 +454,8 @@ go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1
 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
-go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
-go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
 go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -481,7 +472,6 @@ golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -584,7 +574,6 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -593,7 +582,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=

control-plane/api-gateway/binding/cleanup.go

@@ -0,0 +1,146 @@
+package binding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/go-logr/logr"
+	"github.com/hashicorp/consul/api"
+
+	"github.com/hashicorp/consul-k8s/control-plane/consul"
+)
+
+const (
+	oldACLRoleName   = "managed-gateway-acl-role"
+	oldACLPolicyName = "api-gateway-token-policy"
+)
+
+var sleepTime = 10 * time.Minute
+
+type Cleaner struct {
+	Logger       logr.Logger
+	ConsulConfig *consul.Config
+	ServerMgr    consul.ServerConnectionManager
+	AuthMethod   string
+}
+
+// Run periodically cleans up old ACL roles and policies as well as orphaned inline certificate config entries.
+// When it detects that there are no more inline-certificates and that the old ACL role and policy are not in use, it exits.
+func (c Cleaner) Run(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(sleepTime):
+		}
+
+		client, err := consul.NewClientFromConnMgr(c.ConsulConfig, c.ServerMgr)
+		if err != nil {
+			c.Logger.Error(err, "failed to create Consul client")
+			continue
+		}
+
+		aclsCleanedUp, err := c.cleanupACLRoleAndPolicy(client)
+		if err != nil {
+			c.Logger.Error(err, "failed to cleanup old ACL role and policy")
+		}
+
+		if aclsCleanedUp {
+			c.Logger.Info("Cleanup complete")
+			return
+		}
+	}
+}
+
+// cleanupACLRoleAndPolicy deletes the old shared gateway ACL role and policy if they exist.
+func (c Cleaner) cleanupACLRoleAndPolicy(client *api.Client) (bool, error) {
+	existingRules, _, err := client.ACL().BindingRuleList(c.AuthMethod, &api.QueryOptions{})
+	if err != nil {
+		if err.Error() == "Unexpected response code: 401 (ACL support disabled)" {
+			return true, nil
+		}
+		return false, fmt.Errorf("failed to list binding rules: %w", err)
+	}
+
+	oldBindingRules := make(map[string]*api.ACLBindingRule)
+
+	// here we need to find binding rules with the old name that have a matching selector to the new gateway specific binding rule
+	// so we first get all the old rules and put them into a map and then ensure we can delete the old rule by finding the new rule that replaces it
+	// by matching the selector
+	for _, rule := range existingRules {
+		if rule.BindName == oldACLRoleName {
+			oldBindingRules[rule.Selector] = rule
+		}
+	}
+
+	rulesToDelete := mapset.NewSet[string]()
+
+	for _, rule := range existingRules {
+		if ruleToDelete, ok := oldBindingRules[rule.Selector]; ok && rule.BindName != oldACLRoleName {
+			rulesToDelete.Add(ruleToDelete.ID)
+		}
+	}
+
+	var mErr error
+	deletedRuleCount := 0
+	for ruleID := range rulesToDelete.Iter() {
+		_, err := client.ACL().BindingRuleDelete(ruleID, &api.WriteOptions{})
+		if ignoreNotFoundError(err) != nil {
+			mErr = errors.Join(mErr, fmt.Errorf("failed to delete binding rule: %w", err))
+		} else {
+			c.Logger.Info("Deleted unused binding rule", "id", ruleID)
+			deletedRuleCount++
+		}
+	}
+
+	if mErr != nil {
+		return false, mErr
+	}
+
+	if deletedRuleCount != len(oldBindingRules) {
+		return false, nil
+	}
+
+	role, _, err := client.ACL().RoleReadByName(oldACLRoleName, &api.QueryOptions{})
+	if ignoreNotFoundError(err) != nil {
+		return false, fmt.Errorf("failed to get role: %w", err)
+	}
+
+	if role != nil {
+		_, err = client.ACL().RoleDelete(role.ID, &api.WriteOptions{})
+		if err != nil {
+			return false, fmt.Errorf("failed to delete role: %w", err)
+		}
+		c.Logger.Info("Deleted unused ACL role", "id", role.ID)
+	}
+
+	policy, _, err := client.ACL().PolicyReadByName(oldACLPolicyName, &api.QueryOptions{})
+	if ignoreNotFoundError(err) != nil {
+		return false, fmt.Errorf("failed to get policy: %w", err)
+	}
+
+	if policy != nil {
+		_, err = client.ACL().PolicyDelete(policy.ID, &api.WriteOptions{})
+		if err != nil {
+			return false, fmt.Errorf("failed to delete policy: %w", err)
+		}
+		c.Logger.Info("Deleted unused ACL policy", "id", policy.ID)
+	}
+
+	return true, nil
+}
+
+func ignoreNotFoundError(err error) error {
+	if err == nil {
+		return nil
+	}
+	if strings.Contains(err.Error(), "Unexpected response code: 404") {
+		return nil
+	}
+
+	return err
+}