From 0965dfc5fbdc2cb574fd304e0e32bb2463e2eb5a Mon Sep 17 00:00:00 2001 From: ihamburglar Date: Mon, 6 Jun 2022 13:01:39 -0700 Subject: [PATCH] user-data Support (#171) * initial user-data commit * small changes in support of userdata PR #171 * Initial userdata test commit * some minor changes in support of userdata PR. Fix tests. * additional userdata test fixes. Unified userdata values. * updated comment Co-authored-by: Jim Shaver Co-authored-by: Jim Shaver --- pkg/cmd/cmdutil/cmdutil.go | 2 + pkg/config/config.go | 1 + .../aemm-metadata-default-values.json | 8 ++ pkg/config/defaults/defaults.go | 2 +- pkg/config/types.go | 18 ++++ pkg/config/userdata.go | 85 +++++++++++++++++++ pkg/mock/handlers/handlers.go | 28 ++++-- pkg/mock/userdata/userdata.go | 74 ++++++++++++++++ pkg/server/httpserver.go | 8 ++ test/e2e/cmd/userdata-test | 59 +++++++++++++ test/e2e/testdata/aemm-config-integ.json | 8 ++ .../e2e/testdata/output/aemm-config-used.json | 6 +- 12 files changed, 291 insertions(+), 8 deletions(-) create mode 100644 pkg/config/userdata.go create mode 100644 pkg/mock/userdata/userdata.go create mode 100755 test/e2e/cmd/userdata-test diff --git a/pkg/cmd/cmdutil/cmdutil.go b/pkg/cmd/cmdutil/cmdutil.go index 2d972d7..e5ecc5d 100644 --- a/pkg/cmd/cmdutil/cmdutil.go +++ b/pkg/cmd/cmdutil/cmdutil.go @@ -26,6 +26,7 @@ import ( "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/imdsv2" "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/spot" "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/static" + "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/userdata" "github.com/aws/amazon-ec2-metadata-mock/pkg/server" "github.com/spf13/cobra" @@ -91,6 +92,7 @@ func RegisterHandlers(cmd *cobra.Command, config cfg.Config) { static.RegisterHandlers(config) dynamic.RegisterHandlers(config) + userdata.RegisterHandlers(config) // paths without explicit handler bindings will fallback to CatchAllHandler server.HandleFuncPrefix("/", handlers.CatchAllHandler) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7adf65a..888fbc0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -119,6 +119,7 @@ func LoadConfigForRoot(configFileFlagName string, cmdDefaults map[string]interfa LoadConfigFromDefaults(cmdDefaults) SetMetadataDefaults(defaults.GetDefaultValues()) SetDynamicDefaults(defaults.GetDefaultValues()) + SetUserdataDefaults(defaults.GetDefaultValues()) SetServerCfgDefaults() // read in config using viper diff --git a/pkg/config/defaults/aemm-metadata-default-values.json b/pkg/config/defaults/aemm-metadata-default-values.json index c72d39a..ac3fb83 100644 --- a/pkg/config/defaults/aemm-metadata-default-values.json +++ b/pkg/config/defaults/aemm-metadata-default-values.json @@ -175,5 +175,13 @@ "instance-identity-signature": "TesTTKmBbj+DUw6ut6BOr4mFGpax/k6BhIbsotUHvSIhqv7oKqwB4HZhgGP2Gvcxtz5m3QGUbnwI\nhy33GWxjn7+qfZ/GUeZB1Ilc+3rW/P9G/tGxIB3HtqB6q2J6B4DOh6CJiH+BnrHazGW+bJD406Nz\neP9n/rGEGGm0cGEbbeB=", "fws-instance-monitoring": "disabled" } + }, + "userdata": { + "paths": { + "userdata": "/latest/user-data" + }, + "values": { + "userdata": "1234,john,reboot,true|4512,richard,|173,,," + } } } diff --git a/pkg/config/defaults/defaults.go b/pkg/config/defaults/defaults.go index 1d0aa41..3f1908e 100644 --- a/pkg/config/defaults/defaults.go +++ b/pkg/config/defaults/defaults.go @@ -1,4 +1,4 @@ -// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/pkg/config/types.go b/pkg/config/types.go index 7824275..b3687c7 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -24,6 +24,8 @@ import ( type Config struct { // ----- metadata config ----- // Metadata Metadata `mapstructure:"metadata"` + // ----- userdata config ----- // + Userdata Userdata `mapstructure:"userdata"` // ----- CLI config ----- // // config keys that are also cli flags @@ -57,6 +59,12 @@ type Metadata struct { Values Values `mapstructure:"values"` } +// Userdata represents userdata config used by the mock (Json values in metadata-config.json) +type Userdata struct { + Paths UserdataPaths `mapstructure:"paths"` + Values UserdataValues `mapstructure:"values"` +} + // Dynamic represents metadata config used by the mock (Json values in metadata-config.json) type Dynamic struct { Paths DynamicPaths `mapstructure:"paths"` @@ -194,6 +202,16 @@ type Values struct { TagsInstanceTest string `mapstructure:"tags-instance-test"` } +// UserdataPaths represents EC2 userdata paths +type UserdataPaths struct { + Userdata string `mapstructure:"userdata"` +} + +// UserdataValues represents EC2 userdata paths +type UserdataValues struct { + Userdata string `mapstructure:"userdata"` +} + // DynamicPaths represents EC2 dynamic paths type DynamicPaths struct { InstanceIdentityDocument string `mapstructure:"instance-identity-document"` diff --git a/pkg/config/userdata.go b/pkg/config/userdata.go new file mode 100644 index 0000000..c5cf3f4 --- /dev/null +++ b/pkg/config/userdata.go @@ -0,0 +1,85 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +import ( + "encoding/json" + + "github.com/spf13/pflag" +) + +var ( + udCfgPrefix = "userdata." + udPathsCfgPrefix = udCfgPrefix + "paths." + udValuesCfgPrefix = udCfgPrefix + "values." + + // mapping of udValue KEYS to udPath KEYS requiring path substitutions on override + udValueToPlaceholderPathsKeyMap = map[string][]string{} + + // supported URL paths to run a mock + udPathsDefaults = map[string]interface{}{} + + // values in mock responses + udValuesDefaults = map[string]interface{}{} +) + +// GetCfgUdValPrefix returns the prefix to use to access userdata values in config +func GetCfgUdValPrefix() string { + return udCfgPrefix + "." + udValuesCfgPrefix + "." +} + +// GetCfgUdPathsPrefix returns the prefix to use to access userdata values in config +func GetCfgUdPathsPrefix() string { + return udCfgPrefix + "." + udPathsCfgPrefix + "." +} + +// BindUserdataCfg binds a flag that represents a userdata value to configuration +func BindUserdataCfg(flag *pflag.Flag) { + bindFlagWithKeyPrefix(flag, udValuesCfgPrefix) +} + +// SetUserdataDefaults sets config defaults for userdata paths and values +func SetUserdataDefaults(jsonWithDefaults []byte) { + // Unmarshal to map to preserve keys for Paths and Values + var defaultsMap map[string]interface{} + json.Unmarshal(jsonWithDefaults, &defaultsMap) + udPaths := defaultsMap["userdata"].(map[string]interface{})["paths"].(map[string]interface{}) + + udValues := defaultsMap["userdata"].(map[string]interface{})["values"].(map[string]interface{}) + + for k, v := range udPaths { + newKey := udPathsCfgPrefix + k + // ex: "userdata": "/latest/user-data" + udPathsDefaults[newKey] = v + } + + for k, v := range udValues { + newKey := udValuesCfgPrefix + k + // ex: "userdata": "1234,john,reboot,true|4512,richard,|173,,," + udValuesDefaults[newKey] = v + } + + LoadConfigFromDefaults(udPathsDefaults) + LoadConfigFromDefaults(udValuesDefaults) +} + +// GetUserdataDefaults returns config defaults for userdata paths and values +func GetUserdataDefaults() (map[string]interface{}, map[string]interface{}) { + return udPathsDefaults, udValuesDefaults +} + +// GetUserdataValueToPlaceholderPathsKeyMap returns collection of userdata values that are substituted into paths +func GetUserdataValueToPlaceholderPathsKeyMap() map[string][]string { + return udValueToPlaceholderPathsKeyMap +} diff --git a/pkg/mock/handlers/handlers.go b/pkg/mock/handlers/handlers.go index c780db5..0c4c045 100644 --- a/pkg/mock/handlers/handlers.go +++ b/pkg/mock/handlers/handlers.go @@ -21,6 +21,7 @@ import ( "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/dynamic" "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/static" + "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/userdata" "github.com/aws/amazon-ec2-metadata-mock/pkg/server" ) @@ -35,11 +36,12 @@ var ( routeLookupTable = make(map[string][]string) supportedVersions = []string{"latest"} - supportedCategories = []string{"dynamic", "meta-data"} + supportedCategories = []string{"dynamic", "meta-data", "user-data"} // trimmedRoutes represents the list of routes served by the http server without "latest/meta-data/" prefix - trimmedRoutes []string - trimmedRoutesDynamic []string + trimmedRoutes []string + trimmedRoutesDynamic []string + trimmedRoutesUserdata []string ) // CatchAllHandler returns subpath listings, if available; 404 status code otherwise @@ -62,6 +64,10 @@ func CatchAllHandler(res http.ResponseWriter, req *http.Request) { trimmedRoute = strings.TrimPrefix(trimmedRoute, dynamic.ServicePath+"/") log.Println("dynamic prefix detected..trimming: ", trimmedRoute) routes = trimmedRoutesDynamic + } else if strings.HasPrefix(trimmedRoute, userdata.ServicePath) { + trimmedRoute = strings.TrimPrefix(trimmedRoute, userdata.ServicePath+"/") + log.Println("userdata prefix detected..trimming: ", trimmedRoute) + routes = trimmedRoutesUserdata } else { server.ReturnNotFoundResponse(res) return @@ -124,6 +130,8 @@ func ListRoutesHandler(res http.ResponseWriter, req *http.Request) { // these paths do not use routeLookupTable due to inconsistency of trailing "/" with IMDS switch req.URL.Path { + case userdata.ServicePath: + server.FormatAndReturnOctetResponse(res, strings.Join(trimmedRoutesUserdata, "\n")+"\n") case static.ServicePath: server.FormatAndReturnTextResponse(res, strings.Join(trimmedRoutes, "\n")+"\n") case dynamic.ServicePath: @@ -142,15 +150,24 @@ func formatRoutes() { var trimmedRoute string for _, route := range server.Routes { if strings.HasPrefix(route, dynamic.ServicePath) { - // Omit /latest/dynamic + // Omit /latest/dynamic and /latest/user-data trimmedRoute = strings.TrimPrefix(route, dynamic.ServicePath) // Omit empty paths and "/" if len(trimmedRoute) >= shortestRouteLength { trimmedRoute = strings.TrimPrefix(trimmedRoute, "/") trimmedRoutesDynamic = append(trimmedRoutesDynamic, trimmedRoute) } + } else if strings.HasPrefix(route, userdata.ServicePath) { + // Omit /latest/dynamic and /latest/meta-data + trimmedRoute = strings.TrimPrefix(route, userdata.ServicePath) + // Omit empty paths and "/" + if len(trimmedRoute) >= shortestRouteLength { + trimmedRoute = strings.TrimPrefix(trimmedRoute, "/") + trimmedRoutesUserdata = append(trimmedRoutesUserdata, trimmedRoute) + } + } else { - // Omit /latest/meta-data + // Omit /latest/meta-data and /latest/user-data trimmedRoute = strings.TrimPrefix(route, static.ServicePath) // Omit empty paths and "/" if len(trimmedRoute) >= shortestRouteLength { @@ -161,4 +178,5 @@ func formatRoutes() { } sort.Sort(sort.StringSlice(trimmedRoutes)) sort.Sort(sort.StringSlice(trimmedRoutesDynamic)) + sort.Sort(sort.StringSlice(trimmedRoutesUserdata)) } diff --git a/pkg/mock/userdata/userdata.go b/pkg/mock/userdata/userdata.go new file mode 100644 index 0000000..d81b908 --- /dev/null +++ b/pkg/mock/userdata/userdata.go @@ -0,0 +1,74 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package userdata + +import ( + "log" + "net/http" + "reflect" + + cfg "github.com/aws/amazon-ec2-metadata-mock/pkg/config" + "github.com/aws/amazon-ec2-metadata-mock/pkg/mock/imdsv2" + "github.com/aws/amazon-ec2-metadata-mock/pkg/server" +) + +var ( + supportedPaths = make(map[string]interface{}) + response interface{} + // ServicePath defines the userdata service path + ServicePath = "/latest/user-data" +) + +// Handler processes http requests +func Handler(res http.ResponseWriter, req *http.Request) { + log.Println("Received request to mock userdata:", req.URL.Path) + + if val, ok := supportedPaths[req.URL.Path]; ok { + + response = val + } else { + response = "Something went wrong with: " + req.URL.Path + } + server.FormatAndReturnOctetResponse(res, response.(string)) + +} + +// RegisterHandlers registers handlers for userdata paths +func RegisterHandlers(config cfg.Config) { + pathValues := reflect.ValueOf(config.Userdata.Paths) + udValues := reflect.ValueOf(config.Userdata.Values) + // Iterate over fields in config.Userdata.Paths to + // determine intersections with config.Userdata.Values. + // Intersections represent which paths and values to bind. + for i := 0; i < pathValues.NumField(); i++ { + pathFieldName := pathValues.Type().Field(i).Name + udValueFieldName := udValues.FieldByName(pathFieldName) + if udValueFieldName.IsValid() { + path := pathValues.Field(i).Interface().(string) + value := udValueFieldName.Interface() + if path != "" && value != nil { + // Ex: "/latest/meta-data/instance-id" : "i-1234567890abcdef0" + supportedPaths[path] = value + if config.Imdsv2Required { + server.HandleFunc(path, imdsv2.ValidateToken(Handler)) + } else { + + server.HandleFunc(path, Handler) + } + } else { + log.Printf("There was an issue registering path %v with udValue: %v", path, value) + } + } + } +} diff --git a/pkg/server/httpserver.go b/pkg/server/httpserver.go index db9ea51..c3b1671 100644 --- a/pkg/server/httpserver.go +++ b/pkg/server/httpserver.go @@ -120,6 +120,14 @@ func FormatAndReturnTextResponse(res http.ResponseWriter, data string) { return } +// FormatAndReturnOctetResponse formats the given data into an octet stream and returns the response +func FormatAndReturnOctetResponse(res http.ResponseWriter, data string) { + res.Header().Set("Content-Type", "application/octet-stream") + res.Write([]byte(data)) + log.Println("Returned octet stream response successfully.") + return +} + // FormatAndReturnJSONTextResponse formats the given data into JSON and returns a plaintext response func FormatAndReturnJSONTextResponse(res http.ResponseWriter, data interface{}) { res.Header().Set("Content-Type", "text/plain") diff --git a/test/e2e/cmd/userdata-test b/test/e2e/cmd/userdata-test new file mode 100755 index 0000000..8cfd72f --- /dev/null +++ b/test/e2e/cmd/userdata-test @@ -0,0 +1,59 @@ +#! /usr/bin/env bash + +set -euo pipefail + +TEST_CONFIG_FILE="$SCRIPTPATH/testdata/aemm-config-integ.json" + +USERDATA_DEFAULT="1234,john,reboot,true|4512,richard,|173,,," +USERDATA_OVERRIDDEN="1234" + +ROOT_PATH="http://$HOSTNAME:$AEMM_PORT" +USERDATA_TEST_PATH="$ROOT_PATH/latest/user-data" + +function test_userdata_defaults() { + pid=$1 + test_url=$2 + test_name=$3 + tput setaf $BLUE + health_check $test_url + TOKEN=$(get_v2Token $MAX_TOKEN_TTL $AEMM_PORT) + + actual_userdata=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $USERDATA_TEST_PATH) + assert_value "$actual_userdata" $USERDATA_DEFAULT "userdata $test_name" + + clean_up $pid +} + +function test_userdata_overrides() { + pid=$1 + test_url=$2 + test_name=$3 + tput setaf $BLUE + health_check $test_url + TOKEN=$(get_v2Token $MAX_TOKEN_TTL $AEMM_PORT) + + updated_userdata=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $USERDATA_TEST_PATH) + assert_value "$updated_userdata" $USERDATA_OVERRIDDEN "userdata $test_name" + + clean_up $pid +} + +tput setaf $BLUE +echo "======================================================================================================" +echo "🥑 Starting userdata integration tests $METADATA_VERSION" +echo "======================================================================================================" + + +# userdata data defaults +start_cmd=$(create_cmd $METADATA_VERSION --port $AEMM_PORT) +$start_cmd & +USERDATA_PID=$! +test_userdata_defaults $USERDATA_PID $USERDATA_TEST_PATH $USERDATA_DEFAULT + +# userdata data overrides +start_cmd=$(create_cmd $METADATA_VERSION --port $AEMM_PORT -c $TEST_CONFIG_FILE) +$start_cmd & +USERDATA_PID=$! +test_userdata_overrides $USERDATA_PID $USERDATA_TEST_PATH $USERDATA_OVERRIDDEN + +exit $EXIT_CODE_TO_RETURN diff --git a/test/e2e/testdata/aemm-config-integ.json b/test/e2e/testdata/aemm-config-integ.json index bf92248..18c821a 100644 --- a/test/e2e/testdata/aemm-config-integ.json +++ b/test/e2e/testdata/aemm-config-integ.json @@ -31,5 +31,13 @@ "instanceType": "m4.xlarge", "region": "us-east-1" } } + }, + "userdata": { + "paths": { + "userdata": "/latest/user-data" + }, + "values": { + "userdata": "1234" + } } } diff --git a/test/e2e/testdata/output/aemm-config-used.json b/test/e2e/testdata/output/aemm-config-used.json index c52400a..847c7ad 100644 --- a/test/e2e/testdata/output/aemm-config-used.json +++ b/test/e2e/testdata/output/aemm-config-used.json @@ -95,7 +95,8 @@ "services-domain": "/latest/meta-data/services/domain", "services-partition": "/latest/meta-data/services/partition", "spot": "/latest/meta-data/spot/instance-action", - "spot-termination-time": "/latest/meta-data/spot/termination-time" + "spot-termination-time": "/latest/meta-data/spot/termination-time", + "userdata": "/latest/user-data" }, "values": { "ami-id": "ami-0a887e401f7654935", @@ -173,7 +174,8 @@ "reservation-id": "r-046cb3eca3e201d2f", "security-groups": "ura-launch-wizard-harry-1", "services-domain": "amazonaws.com", - "services-partition": "aws" + "services-partition": "aws", + "userdata": "1234,john,reboot,true|4512,richard,|173,,," } }, "mock-delay-sec": 0,