Skip to content
This repository has been archived by the owner on Nov 1, 2022. It is now read-only.

Commit

Permalink
Separate ECR auto-auth from ECR include/exclude
Browse files Browse the repository at this point in the history 
It is useful to be able to constrain the image repos scanned, even
when you don't need or want to use the AWS authentication.

This commit makes the constraints operate independently of AWS
authentication, by using a "pre-flight check" to determine whether the
AWS auth should be used, but applying the constraints either way. The
preflight check is also used to exit from main() if
`--registry-required=ecr` is set and the AWS API is not available.
  • Loading branch information
@squaremo
squaremo committed Apr 3, 2019
1 parent ea55b2d commit 599a792
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 50 deletions.
15 changes: 7 additions & 8 deletions cmd/fluxd/main.go
View file
Expand Up @@ -138,8 +138,8 @@ func main() {
registryExcludeImage = fs.StringSlice("registry-exclude-image", []string{"k8s.gcr.io/*"}, "do not scan images that match these glob expressions; the default is to exclude the 'k8s.gcr.io/*' images")

// AWS authentication
registryAWSRegions = fs.StringSlice("registry-ecr-region", nil, "restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned")
registryAWSAccountIDs = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned")
registryAWSRegions = fs.StringSlice("registry-ecr-region", nil, "include just these AWS regions when scanning images in ECR; when not supplied, the cluster's region will included if it can be detected through the AWS API")
registryAWSAccountIDs = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if not supplied, all account IDs that aren't excluded may be scanned")
registryAWSBlockAccountIDs = fs.StringSlice("registry-ecr-exclude-id", []string{registry.EKS_SYSTEM_ACCOUNT}, "do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account")

registryRequire = fs.StringSlice("registry-require", nil, fmt.Sprintf(`exit with an error if auto-authentication with any of the given registries is not possible (possible values: {%s})`, strings.Join(RequireValues, ",")))
Expand Down Expand Up @@ -382,16 +382,15 @@ func main() {
AccountIDs: *registryAWSAccountIDs,
BlockIDs: *registryAWSBlockAccountIDs,
}
credsWithAWSAuth, err := registry.ImageCredsWithAWSAuth(imageCreds, log.With(logger, "component", "aws"), awsConf)
if err != nil {
if mandatoryRegistry.has(RequireECR) {

awsPreflight, credsWithAWSAuth := registry.ImageCredsWithAWSAuth(imageCreds, log.With(logger, "component", "aws"), awsConf)
if mandatoryRegistry.has(RequireECR) {
if err := awsPreflight(); err != nil {
logger.Log("error", "AWS API required (due to --registry-required=ecr), but not available", "err", err)
os.Exit(1)
}
logger.Log("warning", "AWS authorization not used; pre-flight check failed")
} else {
imageCreds = credsWithAWSAuth
}
imageCreds = credsWithAWSAuth

if *dockerConfig != "" {
credsWithDefaults, err := registry.ImageCredsWithDefaults(imageCreds, *dockerConfig)
Expand Down
155 changes: 113 additions & 42 deletions registry/aws.go
View file
@@ -1,5 +1,10 @@
package registry

import (
"fmt"
"sync"
)

// References:
// - https://github.com/bzon/ecr-k8s-secret-creator
// - https://github.com/kubernetes/kubernetes/blob/master/pkg/credentialprovider/aws/aws_credentials.go
Expand Down Expand Up @@ -29,6 +34,8 @@ const (
EKS_SYSTEM_ACCOUNT = "602401143452"
)

// AWSRegistryConfig supplies constraints for scanning AWS (ECR) image
// registries. Fields may be left empty.
type AWSRegistryConfig struct {
Regions []string
AccountIDs []string
Expand All @@ -44,46 +51,98 @@ func contains(strs []string, str string) bool {
return false
}

// ImageCredsWithAWSAuth wraps an image credentials func with another
// that adds two capabilities:
//
// - it will include or exclude images from ECR accounts and regions
// according to the config given; and,
//
// - if it can reach the AWS API, it will obtain credentials for ECR
// accounts from it, automatically refreshing them when necessary.
//
// It also returns a "pre-flight check" that can be used to verify
// that the AWS API is available while starting up.
//
// ECR registry URLs look like this:
//
// <account-id>.dkr.ecr.<region>.amazonaws.com
//
// i.e., they can differ in the account ID and in the region. It's
// possible to refer to any registry from any cluster (although, being
// AWS, there will be a cost incurred).
// AWS, there will be a cost incurred). The config supplied can
// restrict based on the region:
//
// - if a region or regions are supplied, exactly those regions shall
// be included;
// - if no region is supplied, but it can be detected, the detected
// region is included
// - if no region is supplied _or_ detected, no region is included
//
// .. and on the account ID:
//
// - if account IDs to include are supplied, only those are included
// - otherwise, all account IDs are included
// - the supplied list may be empty
// with the exception
// - if account IDs to _exclude_ are supplied, those shall be not be
// included
func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config AWSRegistryConfig) (func() error, func() ImageCreds) {
// only ever do the preflight check once; all subsequent calls
// will succeed trivially, so the first caller should pay
// attention to the return value.
var preflightOnce sync.Once
// it's possible to fail the pre-flight check, but still apply the
// constraints given in the config. `okToUseAWS` is true if using
// the AWS API to get credentials is expected to work.
var okToUseAWS bool

func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config AWSRegistryConfig) (func() ImageCreds, error) {
awsCreds := NoCredentials()
preflight := func() error {
var preflightErr error
preflightOnce.Do(func() {

// This forces the AWS SDK to load config, so we can get the
// default region if it's there
sess := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
// Always try to connect to the metadata service, so we can fail
// fast if it's not available.
ec2 := ec2metadata.New(sess)
metadataRegion, err := ec2.Region()
if err != nil {
return nil, err
}
defer func() {
logger.Log("info", "restricting ECR registry scans",
"regions", fmt.Sprintf("%v", config.Regions),
"include-ids", fmt.Sprintf("%v", config.AccountIDs),
"exclude-ids", fmt.Sprintf("%v", config.BlockIDs))
}()

if len(config.Regions) == 0 {
clusterRegion := *sess.Config.Region
regionSource := "local config"
if clusterRegion == "" {
// no region set in config; in that case, use what we got from the EC2 metadata service
clusterRegion = metadataRegion
regionSource = "EC2 metadata service"
}
logger.Log("info", "detected cluster region", "source", regionSource, "region", clusterRegion)
config.Regions = []string{clusterRegion}
// This forces the AWS SDK to load config, so we can get
// the default region if it's there.
sess := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
// Always try to connect to the metadata service, so we
// can fail fast if it's not available.
ec2 := ec2metadata.New(sess)
metadataRegion, err := ec2.Region()
if err != nil {
preflightErr = err
if config.Regions == nil {
config.Regions = []string{}
}
return
}

okToUseAWS = true

if config.Regions == nil {
clusterRegion := *sess.Config.Region
regionSource := "local config"
if clusterRegion == "" {
// no region set in config; in that case, use what we got from the EC2 metadata service
clusterRegion = metadataRegion
regionSource = "EC2 metadata service"
}
logger.Log("info", "detected cluster region", "source", regionSource, "region", clusterRegion)
config.Regions = []string{clusterRegion}
}

})
return preflightErr
}

logger.Log("info", "restricting ECR registry scans",
"regions", strings.Join(config.Regions, ", "),
"include-ids", strings.Join(config.AccountIDs, ", "),
"exclude-ids", strings.Join(config.BlockIDs, ", "))
awsCreds := NoCredentials()

// this has the expiry time from the last request made per region. We request new tokens whenever
// - we don't have credentials for the particular registry URL
Expand All @@ -98,15 +157,13 @@ func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config A

// should this registry be scanned?
var shouldScan func(string, string) bool
if len(config.AccountIDs) == 0 {
if config.AccountIDs == nil {
shouldScan = func(region, accountID string) bool {
return contains(config.Regions, region) && !contains(config.BlockIDs, accountID)
}
} else {
shouldScan = func(region, accountID string) bool {
return contains(config.Regions, region) &&
contains(config.AccountIDs, accountID) &&
!contains(config.BlockIDs, accountID)
return contains(config.Regions, region) && contains(config.AccountIDs, accountID) && !contains(config.BlockIDs, accountID)
}
}

Expand Down Expand Up @@ -151,35 +208,49 @@ func ImageCredsWithAWSAuth(lookup func() ImageCreds, logger log.Logger, config A
return nil
}

return func() ImageCreds {
lookupECR := func() ImageCreds {
imageCreds := lookup()

for name, creds := range imageCreds {
domain := name.Domain
if strings.HasSuffix(domain, ecrHostSuffix) {
bits := strings.Split(domain, ".")
if len(bits) != 6 {
if len(bits) != 6 || bits[1] != "dkr" || bits[2] != "ecr" {
logger.Log("warning", "AWS registry domain not in expected format <account-id>.dkr.ecr.<region>.amazonaws.com", "domain", domain)
continue
}
accountID := bits[0]
region := bits[3]

// Before deciding whether an image is included, we need to establish the included regions,
// and whether we can use the AWS API to get credentials. But we don't need to log any problem
// that arises _unless_ there's an image that ends up being included in the scanning.
preflightErr := preflight()

if !shouldScan(region, accountID) {
delete(imageCreds, name)
continue
}
if err := ensureCreds(domain, region, accountID, time.Now()); err != nil {
logger.Log("warning", "unable to ensure credentials for ECR", "domain", domain, "err", err)

if preflightErr != nil {
logger.Log("warning", "AWS auth implied by ECR image, but AWS API is not available. You can ignore this if you are providing credentials some other way (e.g., through imagePullSecrets)", "image", name.String(), "err", preflightErr)
}

if okToUseAWS {
if err := ensureCreds(domain, region, accountID, time.Now()); err != nil {
logger.Log("warning", "unable to ensure credentials for ECR", "domain", domain, "err", err)
}
newCreds := NoCredentials()
newCreds.Merge(awsCreds)
newCreds.Merge(creds)
imageCreds[name] = newCreds
}
newCreds := NoCredentials()
newCreds.Merge(awsCreds)
newCreds.Merge(creds)
imageCreds[name] = newCreds
}
}
return imageCreds
}, nil
}

return preflight, lookupECR
}

func allAccountIDsInRegion(hosts []string, region string) []string {
Expand Down

0 comments on commit 599a792

Please sign in to comment.