Add support for EC2 IMDS endpoint from environment variable

CHANGELOG_PENDING.md

@@ -3,3 +3,6 @@
 ### SDK Enhancements
 
 ### SDK Bugs
+* `aws/ec2metadata`: Add support for EC2 IMDS endpoint from environment variable ([#3504](https://github.com/aws/aws-sdk-go/pull/3504))
+  * Adds support for specifying a custom EC2 IMDS endpoint from the environment variable, `AWS_EC2_METADATA_SERVICE_ENDPOINT`.
+  * The `aws/session#Options` struct also has a new field, `EC2IMDSEndpoint`. This field can be used to configure the custom endpoint of the EC2 IMDS client. The option only applies to EC2 IMDS clients created after the Session with `EC2IMDSEndpoint` is specified.

aws/defaults/defaults_test.go

@@ -120,7 +120,7 @@ func TestDefaultEC2RoleProvider(t *testing.T) {
 	if ec2Provider == nil {
 		t.Fatalf("expect provider not to be nil, but was")
 	}
-	if e, a := "http://169.254.169.254/latest", ec2Provider.Client.Endpoint; e != a {
+	if e, a := "http://169.254.169.254", ec2Provider.Client.Endpoint; e != a {
 		t.Errorf("expect %q endpoint, got %q", e, a)
 	}
 }

aws/ec2metadata/api.go

@@ -20,7 +20,7 @@ func (c *EC2Metadata) getToken(ctx aws.Context, duration time.Duration) (tokenOu
 	op := &request.Operation{
 		Name:       "GetToken",
 		HTTPMethod: "PUT",
-		HTTPPath:   "/api/token",
+		HTTPPath:   "/latest/api/token",
 	}
 
 	var output tokenOutput
@@ -62,7 +62,7 @@ func (c *EC2Metadata) GetMetadataWithContext(ctx aws.Context, p string) (string,
 	op := &request.Operation{
 		Name:       "GetMetadata",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/meta-data", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/meta-data", p),
 	}
 	output := &metadataOutput{}
 
@@ -88,7 +88,7 @@ func (c *EC2Metadata) GetUserDataWithContext(ctx aws.Context) (string, error) {
 	op := &request.Operation{
 		Name:       "GetUserData",
 		HTTPMethod: "GET",
-		HTTPPath:   "/user-data",
+		HTTPPath:   "/latest/user-data",
 	}
 
 	output := &metadataOutput{}
@@ -113,7 +113,7 @@ func (c *EC2Metadata) GetDynamicDataWithContext(ctx aws.Context, p string) (stri
 	op := &request.Operation{
 		Name:       "GetDynamicData",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/dynamic", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/dynamic", p),
 	}
 
 	output := &metadataOutput{}

aws/ec2metadata/api_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/awstesting/unit"
+	"github.com/aws/aws-sdk-go/internal/sdktesting"
 )
 
 const instanceIdentityDocument = `{
@@ -106,22 +107,22 @@ func newTestServer(t *testing.T, testType testType, testServer *testServer) *htt
 	switch testType {
 	case SecureTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.secureGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.secureGetLatestHandler)
+		mux.HandleFunc("/", testServer.secureGetLatestHandler)
 	case InsecureTestType:
 		mux.HandleFunc("/latest/api/token", testServer.insecureGetTokenHandler)
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case BadRequestTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.badRequestGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.badRequestGetLatestHandler)
+		mux.HandleFunc("/", testServer.badRequestGetLatestHandler)
 	case ServerErrorForTokenTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.serverErrorGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case pageNotFoundForTokenTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.pageNotFoundGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case pageNotFoundWith401TestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.pageNotFoundGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.unauthorizedGetLatestHandler)
+		mux.HandleFunc("/", testServer.unauthorizedGetLatestHandler)
 
 	}
 
@@ -204,17 +205,17 @@ func (opListProvider *operationListProvider) addToOperationPerformedList(r *requ
 }
 
 func TestEndpoint(t *testing.T) {
+	restoreEnvFn := sdktesting.StashEnv()
+	defer restoreEnvFn()
+
 	c := ec2metadata.New(unit.Session)
 	op := &request.Operation{
 		Name:       "GetMetadata",
 		HTTPMethod: "GET",
-		HTTPPath:   path.Join("/", "meta-data", "testpath"),
+		HTTPPath:   path.Join("/latest", "meta-data", "testpath"),
 	}
 
 	req := c.NewRequest(op, nil, nil)
-	if e, a := "http://169.254.169.254/latest", req.ClientInfo.Endpoint; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
 	if e, a := "http://169.254.169.254/latest/meta-data/testpath", req.HTTPRequest.URL.String(); e != a {
 		t.Errorf("expect %v, got %v", e, a)
 	}
@@ -289,7 +290,9 @@ func TestGetMetadata(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			resp, err := c.GetMetadata("some/path")
@@ -340,7 +343,9 @@ func TestGetUserData_Error(t *testing.T) {
 	}))
 
 	defer server.Close()
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	resp, err := c.GetUserData()
 	if err == nil {
@@ -425,7 +430,9 @@ func TestGetRegion(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			resp, err := c.Region()
@@ -494,7 +501,9 @@ func TestMetadataIAMInfo_success(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			iamInfo, err := c.IAMInfo()
@@ -570,7 +579,9 @@ func TestMetadataIAMInfo_failure(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			iamInfo, err := c.IAMInfo()
@@ -675,7 +686,9 @@ func TestEC2RoleProviderInstanceIdentity(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 			doc, err := c.GetInstanceIdentityDocument()
 
@@ -719,7 +732,9 @@ func TestEC2MetadataRetryFailure(t *testing.T) {
 	server := httptest.NewServer(mux)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	c.Handlers.AfterRetry.PushBack(func(i *request.Request) {
 		t.Logf("%v received, retrying operation %v", i.HTTPResponse.StatusCode, i.Operation.Name)
@@ -774,7 +789,9 @@ func TestEC2MetadataRetryOnce(t *testing.T) {
 
 	server := httptest.NewServer(mux)
 	defer server.Close()
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	// Handler on client that logs if retried
 	c.Handlers.AfterRetry.PushBack(func(i *request.Request) {
@@ -807,7 +824,9 @@ func TestEC2Metadata_Concurrency(t *testing.T) {
 	server := newTestServer(t, SecureTestType, ts)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	var wg sync.WaitGroup
 	wg.Add(10)
@@ -838,11 +857,13 @@ func TestRequestOnMetadata(t *testing.T) {
 	server := newTestServer(t, SecureTestType, ts)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	req := c.NewRequest(&request.Operation{
 		Name:            "Ec2Metadata request",
 		HTTPMethod:      "GET",
-		HTTPPath:        "/latest",
+		HTTPPath:        "/latest/foo",
 		Paginator:       nil,
 		BeforePresignFn: nil,
 	}, nil, nil)
@@ -878,7 +899,9 @@ func TestExhaustiveRetryToFetchToken(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")
@@ -930,7 +953,9 @@ func TestExhaustiveRetryWith401(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")
@@ -991,7 +1016,9 @@ func TestRequestTimeOut(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	// for test, change the timeout to 100 ms
 	c.Config.HTTPClient.Timeout = 100 * time.Millisecond
 
@@ -1068,7 +1095,9 @@ func TestTokenExpiredBehavior(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")

aws/ec2metadata/service.go

@@ -5,13 +5,18 @@
 // variable "AWS_EC2_METADATA_DISABLED=true". This environment variable set to
 // true instructs the SDK to disable the EC2 Metadata client. The client cannot
 // be used while the environment variable is set to true, (case insensitive).
+//
+// The endpoint of the EC2 IMDS client can be configured via the environment
+// variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
+// Session. See aws/session#Options.EC2IMDSEndpoint for more details.
 package ec2metadata
 
 import (
 	"bytes"
 	"errors"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -69,6 +74,9 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *EC2Metadata {
 // a client when not using a session. Generally using just New with a session
 // is preferred.
 //
+// Will remove the URL path from the endpoint provided to ensure the EC2 IMDS
+// client is able to communicate with the EC2 IMDS API.
+//
 // If an unmodified HTTP client is provided from the stdlib default, or no client
 // the EC2RoleProvider's EC2Metadata HTTP client's timeout will be shortened.
 // To disable this set Config.EC2MetadataDisableTimeoutOverride to false. Enabled by default.
@@ -86,6 +94,15 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
 		cfg.MaxRetries = aws.Int(2)
 	}
 
+	if u, err := url.Parse(endpoint); err == nil {
+		// Remove path from the endpoint since it will be added by requests.
+		// This is an artifact of the SDK adding `/latest` to the endpoint for
+		// EC2 IMDS, but this is now moved to the operation definition.
+		u.Path = ""
+		u.RawPath = ""
+		endpoint = u.String()
+	}
+
 	svc := &EC2Metadata{
 		Client: client.New(
 			cfg,

aws/ec2metadata/service_test.go

@@ -1,3 +1,5 @@
+// +build go1.7
+
 package ec2metadata_test
 
 import (
@@ -89,9 +91,7 @@ func TestClientDisableIMDS(t *testing.T) {
 
 	os.Setenv("AWS_EC2_METADATA_DISABLED", "true")
 
-	svc := ec2metadata.New(unit.Session, &aws.Config{
-		LogLevel: aws.LogLevel(aws.LogDebugWithHTTPBody),
-	})
+	svc := ec2metadata.New(unit.Session)
 	resp, err := svc.GetUserData()
 	if err == nil {
 		t.Fatalf("expect error, got none")
@@ -109,6 +109,37 @@ func TestClientDisableIMDS(t *testing.T) {
 	}
 }
 
+func TestClientStripPath(t *testing.T) {
+	cases := map[string]struct {
+		Endpoint string
+		Expect   string
+	}{
+		"no change": {
+			Endpoint: "http://example.aws",
+			Expect:   "http://example.aws",
+		},
+		"strip path": {
+			Endpoint: "http://example.aws/foo",
+			Expect:   "http://example.aws",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			restoreEnvFn := sdktesting.StashEnv()
+			defer restoreEnvFn()
+
+			svc := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(c.Endpoint),
+			})
+
+			if e, a := c.Expect, svc.ClientInfo.Endpoint; e != a {
+				t.Errorf("expect %v endpoint, got %v", e, a)
+			}
+		})
+	}
+}
+
 func runEC2MetadataClients(t *testing.T, cfg *aws.Config, atOnce int) {
 	var wg sync.WaitGroup
 	wg.Add(atOnce)