Skip to content

Commit

Permalink
Support ACL replication
Browse files Browse the repository at this point in the history 
- Adds flag -acl-replication-token-file for setting the ACL replication
  token. This token is used by secondary dc's to create ACL policies.
  If set, this flag turns on ACL replication mode.
  In this mode we will not bootstrap ACLs since we expect replication
  to be running.
- Modifies various policies and tokens to only be applicable to the
  local datacenter. These policies should have been only local before.
- If running in a secondary DC, append the datacenter name to the
  policy name. This is required because policies must be globally
  unique.
- Note: we aren't sharing policies between datacenters because
  each server-acl-init could modify the policy depending on
  its local config.
- Adds agent:read permissions to the replication token which is
  needed to get the current datacenter.
  • Loading branch information
@lkysow
lkysow committed Mar 16, 2020
1 parent 5867ade commit 9cdda6c
Show file tree
Hide file tree
Showing 5 changed files with 591 additions and 89 deletions.
124 changes: 97 additions & 27 deletions subcommand/server-acl-init/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"errors"
"flag"
"fmt"
"io/ioutil"
"os"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -40,6 +42,7 @@ type Command struct {
flagCreateSnapshotAgentToken bool
flagCreateMeshGatewayToken bool
flagCreateACLReplicationToken bool
flagACLReplicationTokenFile string
flagConsulCACert string
flagConsulTLSServerName string
flagUseHTTPS bool
Expand Down Expand Up @@ -126,6 +129,8 @@ func (c *Command) init() {
c.flags.StringVar(&c.flagInjectK8SNSMirroringPrefix, "inject-k8s-namespace-mirroring-prefix", "",
"[Enterprise Only] Prefix that will be added to all k8s namespaces mirrored into Consul by Connect inject "+
"if mirroring is enabled.")
c.flags.StringVar(&c.flagACLReplicationTokenFile, "acl-replication-token-file", "",
"Path to file containing ACL token to be used for ACL replication. If set, ACL replication is enabled.")
c.flags.StringVar(&c.flagTimeout, "timeout", "10m",
"How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m")
c.flags.StringVar(&c.flagLogLevel, "log-level", "info",
Expand Down Expand Up @@ -183,6 +188,16 @@ func (c *Command) Run(args []string) int {
if c.flagReleaseName != "" {
c.flagServerLabelSelector = fmt.Sprintf("app=consul,component=server,release=%s", c.flagReleaseName)
}
var aclReplicationToken string
if c.flagACLReplicationTokenFile != "" {
// Load the ACL replication token from file.
tokenBytes, err := ioutil.ReadFile(c.flagACLReplicationTokenFile)
if err != nil {
c.UI.Error(fmt.Sprintf("Unable to read ACL replication token from file %q: %s", c.flagACLReplicationTokenFile, err))
return 1
}
aclReplicationToken = strings.TrimSpace(string(tokenBytes))
}

var cancel context.CancelFunc
c.cmdTimeout, cancel = context.WithTimeout(context.Background(), timeout)
Expand Down Expand Up @@ -234,30 +249,41 @@ func (c *Command) Run(args []string) int {
return 1
}

// Check if we've already been bootstrapped.
bootTokenSecretName := c.withPrefix("bootstrap-acl-token")
bootstrapToken, err := c.getBootstrapToken(bootTokenSecretName)
if err != nil {
c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err))
return 1
}

var updateServerPolicy bool
if bootstrapToken != "" {
c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName))

// Mark that we should update the server ACL policy in case
// there are namespace related config changes. Because of the
// organization of the server token creation code, the policy
// otherwise won't be updated.
updateServerPolicy = true
var bootstrapToken string

if c.flagACLReplicationTokenFile != "" {
// If ACL replication is enabled, we don't need to ACL bootstrap the servers
// since they will be performing replication.
// We can use the replication token as our bootstrap token because it
// has permissions to create policies and tokens.
c.Log.Info("ACL replication is enabled so skipping ACL bootstrapping")
bootstrapToken = aclReplicationToken
} else {
c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping")
bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme)
// Check if we've already been bootstrapped.
bootTokenSecretName := c.withPrefix("bootstrap-acl-token")
bootstrapToken, err = c.getBootstrapToken(bootTokenSecretName)
if err != nil {
c.Log.Error(err.Error())
c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err))
return 1
}

if bootstrapToken != "" {
c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName))

// Mark that we should update the server ACL policy in case
// there are namespace related config changes. Because of the
// organization of the server token creation code, the policy
// otherwise won't be updated.
updateServerPolicy = true
} else {
c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping")
bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}
}

// For all of the next operations we'll need a Consul client.
Expand All @@ -281,6 +307,13 @@ func (c *Command) Run(args []string) int {
return 1
}

consulDC, err := c.consulDatacenter(consulClient)
if err != nil {
c.Log.Error("Error getting datacenter name", "err", err)
return 1
}
c.Log.Info("Current datacenter", "datacenter", consulDC)

// With the addition of namespaces, the ACL policies associated
// with the server tokens may need to be updated if Enterprise Consul
// users upgrade to 1.7+. This updates the policy if the bootstrap
Expand Down Expand Up @@ -338,14 +371,20 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("client", agentRules, consulClient)
err = c.createLocalACL("client", agentRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagAllowDNS {
// The DNS policy is attached to the anonymous token.
// If performing ACL replication, we assume that the primary datacenter
// has already created the DNS policy and attached it to the anonymous
// token. We don't want to modify the DNS policy in secondary datacenters
// because it is global and we can't create separate tokens for each
// secondary datacenter because the anonymous token is global.
if c.flagAllowDNS && c.flagACLReplicationTokenFile == "" {
err := c.configureDNSPolicies(consulClient)
if err != nil {
c.Log.Error(err.Error())
Expand All @@ -360,7 +399,7 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("catalog-sync", syncRules, consulClient)
err = c.createLocalACL("catalog-sync", syncRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -374,23 +413,23 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("connect-inject", injectRules, consulClient)
err = c.createLocalACL("connect-inject", injectRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagCreateEntLicenseToken {
err := c.createACL("enterprise-license", entLicenseRules, consulClient)
err := c.createLocalACL("enterprise-license", entLicenseRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
}
}

if c.flagCreateSnapshotAgentToken {
err := c.createACL("client-snapshot-agent", snapshotAgentRules, consulClient)
err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -404,7 +443,9 @@ func (c *Command) Run(args []string) int {
return 1
}

err = c.createACL("mesh-gateway", meshGatewayRules, consulClient)
// Mesh gateways require a global policy/token because they must
// discover services in other datacenters.
err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand All @@ -425,7 +466,7 @@ func (c *Command) Run(args []string) int {
c.Log.Error("Error templating acl replication token rules", "err", err)
return 1
}
err = c.createACL("acl-replication", rules, consulClient)
err = c.createGlobalACL("acl-replication", rules, consulDC, consulClient)
if err != nil {
c.Log.Error(err.Error())
return 1
Expand Down Expand Up @@ -511,3 +552,32 @@ Usage: consul-k8s server-acl-init [options]
and safe to run multiple times.
`

// consulDatacenter returns the current datacenter name using the
// /agent/self API endpoint.
func (c *Command) consulDatacenter(client *api.Client) (string, error) {
var agentCfg map[string]map[string]interface{}
err := c.untilSucceeds("calling /agent/self to get datacenter",
func() error {
var opErr error
agentCfg, opErr = client.Agent().Self()
return opErr
})
if err != nil {
return "", err
}
if _, ok := agentCfg["Config"]; !ok {
return "", fmt.Errorf("/agent/self response did not contain Config key: %s", agentCfg)
}
if _, ok := agentCfg["Config"]["Datacenter"]; !ok {
return "", fmt.Errorf("/agent/self response did not contain Config.Datacenter key: %s", agentCfg)
}
dc, ok := agentCfg["Config"]["Datacenter"].(string)
if !ok {
return "", fmt.Errorf("could not cast Config.Datacenter as string: %s", agentCfg)
}
if dc == "" {
return "", fmt.Errorf("value of Config.Datacenter was empty string: %s", agentCfg)
}
return dc, nil
}
Loading

0 comments on commit 9cdda6c

Please sign in to comment.