AWS: Allow cross-region image pulling with ECR

pkg/cloudprovider/providers/aws/aws.go

@@ -42,7 +42,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/cloudprovider"
-	"k8s.io/kubernetes/pkg/credentialprovider/aws"
+	aws_credentials "k8s.io/kubernetes/pkg/credentialprovider/aws"
 	"k8s.io/kubernetes/pkg/types"
 
 	"github.com/golang/glog"
@@ -566,21 +566,7 @@ func getAvailabilityZone(metadata EC2Metadata) (string, error) {
 }
 
 func isRegionValid(region string) bool {
-	regions := [...]string{
-		"us-east-1",
-		"us-west-1",
-		"us-west-2",
-		"eu-west-1",
-		"eu-central-1",
-		"ap-southeast-1",
-		"ap-southeast-2",
-		"ap-northeast-1",
-		"ap-northeast-2",
-		"cn-north-1",
-		"us-gov-west-1",
-		"sa-east-1",
-	}
-	for _, r := range regions {
+	for _, r := range aws_credentials.AWSRegions {
 		if r == region {
 			return true
 		}

pkg/credentialprovider/aws/aws_credentials.go

@@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package aws_credentials
+package credentials
 
 import (
 	"encoding/base64"
+	"fmt"
 	"strings"
 	"time"
 
@@ -26,23 +27,40 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/credentialprovider"
 )
 
-var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"}
+// AWSRegions is the complete list of regions known to the AWS cloudprovider
+// and credentialprovider.
+var AWSRegions = [...]string{
+	"us-east-1",
+	"us-west-1",
+	"us-west-2",
+	"eu-west-1",
+	"eu-central-1",
+	"ap-southeast-1",
+	"ap-southeast-2",
+	"ap-northeast-1",
+	"ap-northeast-2",
+	"cn-north-1",
+	"us-gov-west-1",
+	"sa-east-1",
+}
+
+const registryURLTemplate = "*.dkr.ecr.%s.amazonaws.com"
 
 // awsHandlerLogger is a handler that logs all AWS SDK requests
 // Copied from cloudprovider/aws/log_handler.go
 func awsHandlerLogger(req *request.Request) {
 	service := req.ClientInfo.ServiceName
+	region := req.Config.Region
 
 	name := "?"
 	if req.Operation != nil {
 		name = req.Operation.Name
 	}
 
-	glog.V(4).Infof("AWS request: %s %s", service, name)
+	glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region)
 }
 
 // An interface for testing purposes.
@@ -59,56 +77,101 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI
 	return p.svc.GetAuthorizationToken(input)
 }
 
+// lazyEcrProvider is a DockerConfigProvider that creates on demand an
+// ecrProvider for a given region and then proxies requests to it.
+type lazyEcrProvider struct {
+	region         string
+	regionURL      string
+	actualProvider *credentialprovider.CachingDockerConfigProvider
+}
+
+var _ credentialprovider.DockerConfigProvider = &lazyEcrProvider{}
+
 // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens
 // from AWS to access ECR.
 type ecrProvider struct {
-	getter tokenGetter
+	region    string
+	regionURL string
+	getter    tokenGetter
 }
 
+var _ credentialprovider.DockerConfigProvider = &ecrProvider{}
+
+// Init creates a lazy provider for each AWS region, in order to support
+// cross-region ECR access. They have to be lazy because it's unlikely, but not
+// impossible, that we'll use more than one.
 // Not using the package init() function: this module should be initialized only
 // if using the AWS cloud provider. This way, we avoid timeouts waiting for a
 // non-existent provider.
 func Init() {
-	credentialprovider.RegisterCredentialProvider("aws-ecr-key",
-		&credentialprovider.CachingDockerConfigProvider{
-			Provider: &ecrProvider{},
-			// Refresh credentials a little earlier before they expire
+	for _, region := range AWSRegions {
+		credentialprovider.RegisterCredentialProvider("aws-ecr-"+region,
+			&lazyEcrProvider{
+				region:    region,
+				regionURL: fmt.Sprintf(registryURLTemplate, region),
+			})
+	}
+
+}
+
+// Enabled implements DockerConfigProvider.Enabled for the lazy provider.
+// Since we perform no checks/work of our own and actualProvider is only created
+// later at image pulling time (if ever), always return true.
+func (p *lazyEcrProvider) Enabled() bool {
+	return true
+}
+
+// LazyProvide implements DockerConfigProvider.LazyProvide. It will be called
+// by the client when attempting to pull an image and it will create the actual
+// provider only when we actually need it the first time.
+func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry {
+	if p.actualProvider == nil {
+		glog.V(2).Infof("Creating ecrProvider for %s", p.region)
+		p.actualProvider = &credentialprovider.CachingDockerConfigProvider{
+			Provider: newEcrProvider(p.region, nil),
+			// Refresh credentials a little earlier than expiration time
 			Lifetime: 11*time.Hour + 55*time.Minute,
-		})
+		}
+		if !p.actualProvider.Enabled() {
+			return nil
+		}
+	}
+	entry := p.actualProvider.Provide()[p.regionURL]
+	return &entry
+}
+
+// Provide implements DockerConfigProvider.Provide, creating dummy credentials.
+// Client code will call Provider.LazyProvide() at image pulling time.
+func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig {
+	entry := credentialprovider.DockerConfigEntry{
+		Provider: p,
+	}
+	cfg := credentialprovider.DockerConfig{}
+	cfg[p.regionURL] = entry
+	return cfg
+}
+
+func newEcrProvider(region string, getter tokenGetter) *ecrProvider {
+	return &ecrProvider{
+		region:    region,
+		regionURL: fmt.Sprintf(registryURLTemplate, region),
+		getter:    getter,
+	}
 }
 
 // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation.
 // For now, it gets activated only if AWS was chosen as the cloud provider.
 // TODO: figure how to enable it manually for deployments that are not on AWS but still
 // use ECR somehow?
 func (p *ecrProvider) Enabled() bool {
-	provider, err := cloudprovider.GetCloudProvider("aws", nil)
-	if err != nil {
-		glog.Errorf("while initializing AWS cloud provider %v", err)
-		return false
-	}
-	if provider == nil {
-		return false
-	}
-
-	zones, ok := provider.Zones()
-	if !ok {
-		glog.Errorf("couldn't get Zones() interface")
-		return false
-	}
-	zone, err := zones.GetZone()
-	if err != nil {
-		glog.Errorf("while getting zone %v", err)
-		return false
-	}
-	if zone.Region == "" {
-		glog.Errorf("Region information is empty")
+	if p.region == "" {
+		glog.Errorf("Called ecrProvider.Enabled() with no region set")
 		return false
 	}
 
 	getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{
 		Credentials: nil,
-		Region:      &zone.Region,
+		Region:      &p.region,
 	}))}
 	getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{
 		Name: "k8s/logger",
@@ -158,10 +221,10 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig {
 				Email: "not@val.id",
 			}
 
-			// Add our entry for each of the supported container registry URLs
-			for _, k := range registryUrls {
-				cfg[k] = entry
-			}
+			glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region)
+			// Add our config entry for this region's registry URLs
+			cfg[p.regionURL] = entry
+
 		}
 	}
 	return cfg

pkg/credentialprovider/aws/aws_credentials_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package aws_credentials
+package credentials
 
 import (
 	"encoding/base64"
@@ -58,17 +58,18 @@ func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationToken
 
 func TestEcrProvide(t *testing.T) {
 	registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com"
-	otherRegistries := []string{"private.registry.com",
+	otherRegistries := []string{
+		"private.registry.com",
 		"gcr.io",
 	}
 	image := "foo/bar"
 
-	provider := &ecrProvider{
-		getter: &testTokenGetter{
+	provider := newEcrProvider("lala-land-1",
+		&testTokenGetter{
 			user:     user,
 			password: password,
-			endpoint: registry},
-	}
+			endpoint: registry,
+		})
 
 	keyring := &credentialprovider.BasicDockerKeyring{}
 	keyring.Add(provider.Provide())