Skip to content

Commit

Permalink
[NET-9500] Cleanup orphaned inline-certs and acl role/policy (#4067) (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
jm96441n committed Jun 13, 2024
1 parent ae9e497 commit e78dd07
Show file tree
Hide file tree
Showing 8 changed files with 380 additions and 19 deletions.
5 changes: 1 addition & 4 deletions acceptance/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21.1
toolchain go1.22.0

require (
github.com/go-logr/logr v1.2.4
github.com/google/uuid v1.3.0
github.com/gruntwork-io/terratest v0.46.7
github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b
Expand Down Expand Up @@ -45,9 +46,7 @@ require (
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.3 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
Expand Down Expand Up @@ -121,8 +120,6 @@ require (
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.25.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/mod v0.14.0 // indirect
Expand Down
12 changes: 0 additions & 12 deletions acceptance/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I=
github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -457,14 +454,8 @@ go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1
go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand All @@ -481,7 +472,6 @@ golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
Expand Down Expand Up @@ -584,7 +574,6 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
Expand All @@ -593,7 +582,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
Expand Down
146 changes: 146 additions & 0 deletions control-plane/api-gateway/binding/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package binding

import (
"context"
"errors"
"fmt"
"strings"
"time"

mapset "github.com/deckarep/golang-set/v2"
"github.com/go-logr/logr"
"github.com/hashicorp/consul/api"

"github.com/hashicorp/consul-k8s/control-plane/consul"
)

const (
oldACLRoleName = "managed-gateway-acl-role"
oldACLPolicyName = "api-gateway-token-policy"
)

var sleepTime = 10 * time.Minute

type Cleaner struct {
Logger logr.Logger
ConsulConfig *consul.Config
ServerMgr consul.ServerConnectionManager
AuthMethod string
}

// Run periodically cleans up old ACL roles and policies as well as orphaned inline certificate config entries.
// When it detects that there are no more inline-certificates and that the old ACL role and policy are not in use, it exits.
func (c Cleaner) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-time.After(sleepTime):
}

client, err := consul.NewClientFromConnMgr(c.ConsulConfig, c.ServerMgr)
if err != nil {
c.Logger.Error(err, "failed to create Consul client")
continue
}

aclsCleanedUp, err := c.cleanupACLRoleAndPolicy(client)
if err != nil {
c.Logger.Error(err, "failed to cleanup old ACL role and policy")
}

if aclsCleanedUp {
c.Logger.Info("Cleanup complete")
return
}
}
}

// cleanupACLRoleAndPolicy deletes the old shared gateway ACL role and policy if they exist.
func (c Cleaner) cleanupACLRoleAndPolicy(client *api.Client) (bool, error) {
existingRules, _, err := client.ACL().BindingRuleList(c.AuthMethod, &api.QueryOptions{})
if err != nil {
if err.Error() == "Unexpected response code: 401 (ACL support disabled)" {
return true, nil
}
return false, fmt.Errorf("failed to list binding rules: %w", err)
}

oldBindingRules := make(map[string]*api.ACLBindingRule)

// here we need to find binding rules with the old name that have a matching selector to the new gateway specific binding rule
// so we first get all the old rules and put them into a map and then ensure we can delete the old rule by finding the new rule that replaces it
// by matching the selector
for _, rule := range existingRules {
if rule.BindName == oldACLRoleName {
oldBindingRules[rule.Selector] = rule
}
}

rulesToDelete := mapset.NewSet[string]()

for _, rule := range existingRules {
if ruleToDelete, ok := oldBindingRules[rule.Selector]; ok && rule.BindName != oldACLRoleName {
rulesToDelete.Add(ruleToDelete.ID)
}
}

var mErr error
deletedRuleCount := 0
for ruleID := range rulesToDelete.Iter() {
_, err := client.ACL().BindingRuleDelete(ruleID, &api.WriteOptions{})
if ignoreNotFoundError(err) != nil {
mErr = errors.Join(mErr, fmt.Errorf("failed to delete binding rule: %w", err))
} else {
c.Logger.Info("Deleted unused binding rule", "id", ruleID)
deletedRuleCount++
}
}

if mErr != nil {
return false, mErr
}

if deletedRuleCount != len(oldBindingRules) {
return false, nil
}

role, _, err := client.ACL().RoleReadByName(oldACLRoleName, &api.QueryOptions{})
if ignoreNotFoundError(err) != nil {
return false, fmt.Errorf("failed to get role: %w", err)
}

if role != nil {
_, err = client.ACL().RoleDelete(role.ID, &api.WriteOptions{})
if err != nil {
return false, fmt.Errorf("failed to delete role: %w", err)
}
c.Logger.Info("Deleted unused ACL role", "id", role.ID)
}

policy, _, err := client.ACL().PolicyReadByName(oldACLPolicyName, &api.QueryOptions{})
if ignoreNotFoundError(err) != nil {
return false, fmt.Errorf("failed to get policy: %w", err)
}

if policy != nil {
_, err = client.ACL().PolicyDelete(policy.ID, &api.WriteOptions{})
if err != nil {
return false, fmt.Errorf("failed to delete policy: %w", err)
}
c.Logger.Info("Deleted unused ACL policy", "id", policy.ID)
}

return true, nil
}

func ignoreNotFoundError(err error) error {
if err == nil {
return nil
}
if strings.Contains(err.Error(), "Unexpected response code: 404") {
return nil
}

return err
}
Loading

0 comments on commit e78dd07

Please sign in to comment.