Skip to content

Commit

Permalink
Support ACL replication
Browse files Browse the repository at this point in the history 
- Adds flag -enable-acl-replication that turns on ACL replication mode.
  In this mode we will not bootstrap ACLs since we expect replication
  to be running.
- Adds flag -acl-replication-token for setting the ACL replication
  token. This token is used by secondary dc's to create ACL policies.
  This token can also be set by the ACL_REPLICATION_TOKEN environment
  variable.
- Modifies various policies and tokens to only be applicable to the
  local datacenter. These policies should have been only local before.
- If running in a secondary DC, append the datacenter name to the
  policy name. This is required because policies must be globally
  unique.
- Note: we aren't sharing policies between datacenters because
  each server-acl-init could modify the policy depending on
  its local config.
  • Loading branch information
@lkysow
lkysow committed Mar 6, 2020
1 parent 9f636b6 commit de7b01c
Show file tree
Hide file tree
Showing 5 changed files with 643 additions and 89 deletions.
132 changes: 105 additions & 27 deletions subcommand/server-acl-init/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ import (
"k8s.io/client-go/kubernetes"
)

// aclReplicationTokenEnvVar is the name of the environment variable that
// can be set in place of the -acl-replication-token flag.
const aclReplicationTokenEnvVar = "ACL_REPLICATION_TOKEN"

type Command struct {
UI cli.Ui

Expand All @@ -40,6 +44,8 @@ type Command struct {
flagCreateSnapshotAgentToken bool
flagCreateMeshGatewayToken bool
flagCreateACLReplicationToken bool
flagEnableACLReplication bool
flagACLReplicationToken string
flagConsulCACert string
flagConsulTLSServerName string
flagUseHTTPS bool
Expand Down Expand Up @@ -126,6 +132,12 @@ func (c *Command) init() {
c.flags.StringVar(&c.flagInjectK8SNSMirroringPrefix, "inject-k8s-namespace-mirroring-prefix", "",
"[Enterprise Only] Prefix that will be added to all k8s namespaces mirrored into Consul by Connect inject "+
"if mirroring is enabled.")
c.flags.BoolVar(&c.flagEnableACLReplication, "enable-acl-replication", false,
fmt.Sprintf(
"Enables ACL replication. If true, -acl-replication-token or the %s environment variable must be set",
aclReplicationTokenEnvVar))
c.flags.StringVar(&c.flagACLReplicationToken, "acl-replication-token", "",
fmt.Sprintf("Token to be used for ACL replication. Can also be set via the %s environment variable", aclReplicationTokenEnvVar))
c.flags.StringVar(&c.flagTimeout, "timeout", "10m",
"How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m")
c.flags.StringVar(&c.flagLogLevel, "log-level", "info",
Expand Down Expand Up @@ -183,6 +195,17 @@ func (c *Command) Run(args []string) int {
if c.flagReleaseName != "" {
c.flagServerLabelSelector = fmt.Sprintf("app=consul,component=server,release=%s", c.flagReleaseName)
}
var aclReplicationToken string
if c.flagEnableACLReplication {
if os.Getenv(aclReplicationTokenEnvVar) == "" && c.flagACLReplicationToken == "" {
c.UI.Error(fmt.Sprintf("if -enable-acl-replication is true, -acl-replication-token or the environment variable %s must be set", aclReplicationTokenEnvVar))
return 1
} else if c.flagACLReplicationToken != "" {
aclReplicationToken = c.flagACLReplicationToken
} else {
aclReplicationToken = os.Getenv(aclReplicationTokenEnvVar)
}
}

var cancel context.CancelFunc
c.cmdTimeout, cancel = context.WithTimeout(context.Background(), timeout)
Expand Down Expand Up @@ -234,30 +257,41 @@ func (c *Command) Run(args []string) int {
return 1
}

// Check if we've already been bootstrapped.
bootTokenSecretName := c.withPrefix("bootstrap-acl-token")
bootstrapToken, err := c.getBootstrapToken(bootTokenSecretName)
if err != nil {
c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err))
return 1
}

var updateServerPolicy bool
if bootstrapToken != "" {
c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName))

// Mark that we should update the server ACL policy in case
// there are namespace related config changes. Because of the
// organization of the server token creation code, the policy
// otherwise won't be updated.
updateServerPolicy = true
var bootstrapToken string

if c.flagEnableACLReplication {
// If ACL replication is enabled, we don't need to ACL bootstrap the servers
// since they will be performing replication.
// We can use the replication token as our bootstrap token because it
// has permissions to create policies and tokens.
c.Log.Info("ACL replication is enabled so skipping ACL bootstrapping")
bootstrapToken = aclReplicationToken
} else {
c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping")
bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme)
// Check if we've already been bootstrapped.
bootTokenSecretName := c.withPrefix("bootstrap-acl-token")
bootstrapToken, err = c.getBootstrapToken(bootTokenSecretName)
if err != nil {
c.Log.Error(err.Error())
c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err))
return 1
}

if bootstrapToken != "" {
c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName))

// Mark that we should update the server ACL policy in case
// there are namespace related config changes. Because of the
// organization of the server token creation code, the policy
// otherwise won't be updated.
updateServerPolicy = true
} else {
c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping")
bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}
}

// For all of the next operations we'll need a Consul client.
Expand All @@ -281,6 +315,13 @@ func (c *Command) Run(args []string) int {
return 1
}

consulDC, err := c.consulDatacenter(consulClient)
if err != nil {
c.Log.Error("Error getting datacenter name", "err", err)
return 1
}
c.Log.Info("Current datacenter", "datacenter", consulDC)

// With the addition of namespaces, the ACL policies associated
// with the server tokens may need to be updated if Enterprise Consul
// users upgrade to 1.7+. This updates the policy if the bootstrap
Expand Down Expand Up @@ -338,14 +379,20 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("client", agentRules, consulClient)
err = c.createLocalACL("client", agentRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagAllowDNS {
// The DNS policy is attached to the anonymous token.
// If performing ACL replication, we assume that the primary datacenter
// has already created the DNS policy and attached it to the anonymous
// token. We don't want to modify the DNS policy in secondary datacenters
// because it is global and we can't create separate tokens for each
// secondary datacenter because the anonymous token is global.
if c.flagAllowDNS && !c.flagEnableACLReplication {
err := c.configureDNSPolicies(consulClient)
if err != nil {
c.Log.Error(err.Error())
Expand All @@ -360,7 +407,7 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("catalog-sync", syncRules, consulClient)
err = c.createLocalACL("catalog-sync", syncRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -374,23 +421,23 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("connect-inject", injectRules, consulClient)
err = c.createLocalACL("connect-inject", injectRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagCreateEntLicenseToken {
err := c.createACL("enterprise-license", entLicenseRules, consulClient)
err := c.createLocalACL("enterprise-license", entLicenseRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagCreateSnapshotAgentToken {
err := c.createACL("client-snapshot-agent", snapshotAgentRules, consulClient)
err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -404,7 +451,9 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("mesh-gateway", meshGatewayRules, consulClient)
// Mesh gateways require a global policy/token because they must
// discover services in other datacenters.
err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -425,7 +474,7 @@ func (c *Command) Run(args []string) int {
c.Log.Error("Error templating acl replication token rules", "err", err)
return 1
}
err = c.createACL("acl-replication", rules, consulClient)
err = c.createGlobalACL("acl-replication", rules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand Down Expand Up @@ -511,3 +560,32 @@ Usage: consul-k8s server-acl-init [options]
and safe to run multiple times.
`

// consulDatacenter returns the current datacenter name using the
// /agent/self API endpoint.
func (c *Command) consulDatacenter(client *api.Client) (string, error) {
var agentCfg map[string]map[string]interface{}
err := c.untilSucceeds("calling /agent/self to get datacenter",
func() error {
var opErr error
agentCfg, opErr = client.Agent().Self()
return opErr
})
if err != nil {
return "", err
}
if _, ok := agentCfg["Config"]; !ok {
return "", fmt.Errorf("/agent/self response did not contain Config key: %s", agentCfg)
}
if _, ok := agentCfg["Config"]["Datacenter"]; !ok {
return "", fmt.Errorf("/agent/self response did not contain Config.Datacenter key: %s", agentCfg)
}
dc, ok := agentCfg["Config"]["Datacenter"].(string)
if !ok {
return "", fmt.Errorf("could not cast Config.Datacenter as string: %s", agentCfg)
}
if dc == "" {
return "", fmt.Errorf("value of Config.Datacenter was empty string: %s", agentCfg)
}
return dc, nil
}
Loading

0 comments on commit de7b01c

Please sign in to comment.