Less scary AWS detection

chart/flux/README.md

@@ -225,6 +225,7 @@ The following tables lists the configurable parameters of the Weave Flux chart a
 | `registry.ecr.region`                             | `None`                                               | Restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned
 | `registry.ecr.includeId`                          | `None`                                               | Restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned
 | `registry.ecr.excludeId`                          | `602401143452`                                       | Do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account
+| `registry.ecr.require`                            | `false`                                              | Refuse to start if the AWS API is not available
 | `registry.acr.enabled`                            | `false`                                              | Mount `azure.json` via HostPath into the Flux Pod, enabling Flux to use AKS's service principal for ACR authentication
 | `registry.acr.hostPath`                           | `/etc/kubernetes/azure.json`                         | Alternative location of `azure.json` on the host
 | `registry.dockercfg.enabled`                      | `false`                                              | Mount `config.json` via Secret into the Flux Pod, enabling Flux to use a custom docker config file

chart/flux/templates/deployment.yaml

@@ -159,6 +159,9 @@ spec:
           {{- if .Values.registry.ecr.excludeId }}
           - --registry-ecr-exclude-id={{ .Values.registry.ecr.excludeId }}
           {{- end }}
+          {{- if .Values.registry.ecr.require }}
+          - --registry-require=ecr
+          {{- end }}
           {{- if .Values.registry.dockercfg.enabled }}
           - --docker-config=/dockercfg/config.json
           {{- end }}

chart/flux/values.yaml

@@ -154,6 +154,7 @@ registry:
     region:
     includeId:
     excludeId:
+    require: false
   # Azure ACR settings
   acr:
     enabled: false

cmd/fluxd/main.go

@@ -60,13 +60,30 @@ const (
 	defaultGitSyncTag     = "flux-sync"
 	defaultGitNotesRef    = "flux"
 	defaultGitSkipMessage = "\n\n[ci skip]"
+
+	RequireECR = "ecr"
+)
+
+var (
+	RequireValues = []string{RequireECR}
 )
 
 func optionalVar(fs *pflag.FlagSet, value ssh.OptionalValue, name, usage string) ssh.OptionalValue {
 	fs.Var(value, name, usage)
 	return value
 }
 
+type stringset []string
+
+func (set stringset) has(possible string) bool {
+	for _, s := range set {
+		if s == possible {
+			return true
+		}
+	}
+	return false
+}
+
 func main() {
 	// Flag domain.
 	fs := pflag.NewFlagSet("default", pflag.ContinueOnError)
@@ -121,10 +138,12 @@ func main() {
 		registryExcludeImage = fs.StringSlice("registry-exclude-image", []string{"k8s.gcr.io/*"}, "do not scan images that match these glob expressions; the default is to exclude the 'k8s.gcr.io/*' images")
 
 		// AWS authentication
-		registryAWSRegions         = fs.StringSlice("registry-ecr-region", nil, "restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned")
-		registryAWSAccountIDs      = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned")
+		registryAWSRegions         = fs.StringSlice("registry-ecr-region", nil, "include just these AWS regions when scanning images in ECR; when not supplied, the cluster's region will included if it can be detected through the AWS API")
+		registryAWSAccountIDs      = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if not supplied, all account IDs that aren't excluded may be scanned")
 		registryAWSBlockAccountIDs = fs.StringSlice("registry-ecr-exclude-id", []string{registry.EKS_SYSTEM_ACCOUNT}, "do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account")
 
+		registryRequire = fs.StringSlice("registry-require", nil, fmt.Sprintf(`exit with an error if auto-authentication with any of the given registries is not possible (possible values: {%s})`, strings.Join(RequireValues, ",")))
+
 		// k8s-secret backed ssh keyring configuration
 		k8sSecretName            = fs.String("k8s-secret-name", "flux-git-deploy", "name of the k8s secret used to store the private SSH key")
 		k8sSecretVolumeMountPath = fs.String("k8s-secret-volume-mount-path", "/etc/fluxd/ssh", "mount location of the k8s secret storing the private SSH key")
@@ -223,6 +242,15 @@ func main() {
 		}
 	}
 
+	possiblyRequired := stringset(RequireValues)
+	for _, r := range *registryRequire {
+		if !possiblyRequired.has(r) {
+			logger.Log("err", fmt.Sprintf("--registry-required value %q is not in possible values {%s}", r, strings.Join(RequireValues, ",")))
+			os.Exit(1)
+		}
+	}
+	mandatoryRegistry := stringset(*registryRequire)
+
 	// Mechanical components.
 
 	// When we can receive from this channel, it indicates that we
@@ -357,12 +385,16 @@ func main() {
 			AccountIDs: *registryAWSAccountIDs,
 			BlockIDs:   *registryAWSBlockAccountIDs,
 		}
-		credsWithAWSAuth, err := registry.ImageCredsWithAWSAuth(imageCreds, log.With(logger, "component", "aws"), awsConf)
-		if err != nil {
-			logger.Log("warning", "AWS authorization not used; pre-flight check failed")
-		} else {
-			imageCreds = credsWithAWSAuth
+
+		awsPreflight, credsWithAWSAuth := registry.ImageCredsWithAWSAuth(imageCreds, log.With(logger, "component", "aws"), awsConf)
+		if mandatoryRegistry.has(RequireECR) {
+			if err := awsPreflight(); err != nil {
+				logger.Log("error", "AWS API required (due to --registry-required=ecr), but not available", "err", err)
+				os.Exit(1)
+			}
 		}
+		imageCreds = credsWithAWSAuth
+
 		if *dockerConfig != "" {
 			credsWithDefaults, err := registry.ImageCredsWithDefaults(imageCreds, *dockerConfig)
 			if err != nil {

registry/aws.go

@@ -1,5 +1,10 @@
 package registry
 
+import (
+	"fmt"
+	"sync"
+)
+
 // References:
 //  - https://github.com/bzon/ecr-k8s-secret-creator
 //  - https://github.com/kubernetes/kubernetes/blob/master/pkg/credentialprovider/aws/aws_credentials.go
@@ -29,6 +34,8 @@ const (
 	EKS_SYSTEM_ACCOUNT = "602401143452"
 )
 
+// AWSRegistryConfig supplies constraints for scanning AWS (ECR) image
+// registries. Fields may be left empty.
 type AWSRegistryConfig struct {
 	Regions    []string
 	AccountIDs []string
@@ -44,41 +51,97 @@ func contains(strs []string, str string) bool {
 	return false
 }
 
+// ImageCredsWithAWSAuth wraps an image credentials func with another
+// that adds two capabilities:
+//
+//   - it will include or exclude images from ECR accounts and regions
+//     according to the config given; and,
+//
+//   - if it can reach the AWS API, it will obtain credentials for ECR
+//     accounts from it, automatically refreshing them when necessary.
+//
+// It also returns a "pre-flight check" that can be used to verify
+// that the AWS API is available while starting up.
+//
 // ECR registry URLs look like this:
 //
 //     <account-id>.dkr.ecr.<region>.amazonaws.com
 //
 // i.e., they can differ in the account ID and in the region. It's
 // possible to refer to any registry from any cluster (although, being
-// AWS, there will be a cost incurred).
+// AWS, there will be a cost incurred). The config supplied can
+// restrict based on the region:
+//
+//  - if a region or regions are supplied, exactly those regions shall
+//    be included;
+//  - if no region is supplied, but it can be detected, the detected
+//    region is included
+//  - if no region is supplied _or_ detected, no region is included
+//
+//  .. and on the account ID:
+//
+//  - if account IDs to include are supplied, only those are included
+//    - otherwise, all account IDs are included
+//    - the supplied list may be empty
+//  with the exception
+//  - if account IDs to _exclude_ are supplied, those shall be not be
+//    included
+func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config AWSRegistryConfig) (func() error, func() ImageCreds) {
+	// only ever do the preflight check once; all subsequent calls
+	// will succeed trivially, so the first caller should pay
+	// attention to the return value.
+	var preflightOnce sync.Once
+	// it's possible to fail the pre-flight check, but still apply the
+	// constraints given in the config. `okToUseAWS` is true if using
+	// the AWS API to get credentials is expected to work.
+	var okToUseAWS bool
 
-func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config AWSRegistryConfig) (func() ImageCreds, error) {
-	awsCreds := NoCredentials()
+	preflight := func() error {
+		var preflightErr error
+		preflightOnce.Do(func() {
+
+			defer func() {
+				logger.Log("info", "restricting ECR registry scans",
+					"regions", fmt.Sprintf("%v", config.Regions),
+					"include-ids", fmt.Sprintf("%v", config.AccountIDs),
+					"exclude-ids", fmt.Sprintf("%v", config.BlockIDs))
+			}()
 
-	if len(config.Regions) == 0 {
-		// this forces the AWS SDK to load config, so we can get the default region
-		sess := session.Must(session.NewSessionWithOptions(session.Options{
-			SharedConfigState: session.SharedConfigEnable,
-		}))
-		clusterRegion := *sess.Config.Region
-		if clusterRegion == "" {
-			// no region set in config; in that case, use the EC2 metadata service to find where we are running.
+			// This forces the AWS SDK to load config, so we can get
+			// the default region if it's there.
+			sess := session.Must(session.NewSessionWithOptions(session.Options{
+				SharedConfigState: session.SharedConfigEnable,
+			}))
+			// Always try to connect to the metadata service, so we
+			// can fail fast if it's not available.
 			ec2 := ec2metadata.New(sess)
-			instanceRegion, err := ec2.Region()
+			metadataRegion, err := ec2.Region()
 			if err != nil {
-				logger.Log("warn", "no AWS region configured, or detected as cluster region", "err", err)
-				return nil, err
+				preflightErr = err
+				if config.Regions == nil {
+					config.Regions = []string{}
+				}
+				return
 			}
-			clusterRegion = instanceRegion
-		}
-		logger.Log("info", "detected cluster region", "region", clusterRegion)
-		config.Regions = []string{clusterRegion}
+
+			okToUseAWS = true
+
+			if config.Regions == nil {
+				clusterRegion := *sess.Config.Region
+				regionSource := "local config"
+				if clusterRegion == "" {
+					// no region set in config; in that case, use what we got from the EC2 metadata service
+					clusterRegion = metadataRegion
+					regionSource = "EC2 metadata service"
+				}
+				logger.Log("info", "detected cluster region", "source", regionSource, "region", clusterRegion)
+				config.Regions = []string{clusterRegion}
+			}
+		})
+		return preflightErr
 	}
 
-	logger.Log("info", "restricting ECR registry scans",
-		"regions", strings.Join(config.Regions, ", "),
-		"include-ids", strings.Join(config.AccountIDs, ", "),
-		"exclude-ids", strings.Join(config.BlockIDs, ", "))
+	awsCreds := NoCredentials()
 
 	// this has the expiry time from the last request made per region. We request new tokens whenever
 	//  - we don't have credentials for the particular registry URL
@@ -93,15 +156,13 @@ func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config A
 
 	// should this registry be scanned?
 	var shouldScan func(string, string) bool
-	if len(config.AccountIDs) == 0 {
+	if config.AccountIDs == nil {
 		shouldScan = func(region, accountID string) bool {
 			return contains(config.Regions, region) && !contains(config.BlockIDs, accountID)
 		}
 	} else {
 		shouldScan = func(region, accountID string) bool {
-			return contains(config.Regions, region) &&
-				contains(config.AccountIDs, accountID) &&
-				!contains(config.BlockIDs, accountID)
+			return contains(config.Regions, region) && contains(config.AccountIDs, accountID) && !contains(config.BlockIDs, accountID)
 		}
 	}
 
@@ -146,35 +207,49 @@ func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config A
 		return nil
 	}
 
-	return func() ImageCreds {
+	lookupECR := func() ImageCreds {
 		imageCreds := lookup()
 
 		for name, creds := range imageCreds {
 			domain := name.Domain
 			if strings.HasSuffix(domain, ecrHostSuffix) {
 				bits := strings.Split(domain, ".")
-				if len(bits) != 6 {
+				if len(bits) != 6 || bits[1] != "dkr" || bits[2] != "ecr" {
 					logger.Log("warning", "AWS registry domain not in expected format <account-id>.dkr.ecr.<region>.amazonaws.com", "domain", domain)
 					continue
 				}
 				accountID := bits[0]
 				region := bits[3]
 
+				// Before deciding whether an image is included, we need to establish the included regions,
+				// and whether we can use the AWS API to get credentials. But we don't need to log any problem
+				// that arises _unless_ there's an image that ends up being included in the scanning.
+				preflightErr := preflight()
+
 				if !shouldScan(region, accountID) {
 					delete(imageCreds, name)
 					continue
 				}
-				if err := ensureCreds(domain, region, accountID, time.Now()); err != nil {
-					logger.Log("warning", "unable to ensure credentials for ECR", "domain", domain, "err", err)
+
+				if preflightErr != nil {
+					logger.Log("warning", "AWS auth implied by ECR image, but AWS API is not available. You can ignore this if you are providing credentials some other way (e.g., through imagePullSecrets)", "image", name.String(), "err", preflightErr)
+				}
+
+				if okToUseAWS {
+					if err := ensureCreds(domain, region, accountID, time.Now()); err != nil {
+						logger.Log("warning", "unable to ensure credentials for ECR", "domain", domain, "err", err)
+					}
+					newCreds := NoCredentials()
+					newCreds.Merge(awsCreds)
+					newCreds.Merge(creds)
+					imageCreds[name] = newCreds
 				}
-				newCreds := NoCredentials()
-				newCreds.Merge(awsCreds)
-				newCreds.Merge(creds)
-				imageCreds[name] = newCreds
 			}
 		}
 		return imageCreds
-	}, nil
+	}
+
+	return preflight, lookupECR
 }
 
 func allAccountIDsInRegion(hosts []string, region string) []string {

site/daemon.md

@@ -81,6 +81,7 @@ fluxd requires setup and offers customization though a multitude of flags.
 | --registry-ecr-region                            | `[]`                     | Allow these AWS regions when scanning images from ECR (multiple values allowed); defaults to the detected cluster region
 | --registry-ecr-include-id                        | `[]`                     | Include these AWS account ID(s) when scanning images in ECR (multiple values allowed); empty means allow all, unless excluded
 | --registry-ecr-exclude-id                        | `[<EKS SYSTEM ACCOUNT>]` | Exclude these AWS account ID(s) when scanning ECR (multiple values allowed); defaults to the EKS system account, so system images will not be scanned
+| --registry-require                               | `[]`                  | exit with an error if the given services are not available. Useful for escalating misconfiguration or outages that might otherwise go undetected. Presently supported values: {`ecr`} |
 | **k8s-secret backed ssh keyring configuration**
 | --k8s-secret-name                                | `flux-git-deploy`        | name of the k8s secret used to store the private SSH key
 | --k8s-secret-volume-mount-path                   | `/etc/fluxd/ssh`         | mount location of the k8s secret storing the private SSH key