diff --git a/README.md b/README.md index 72f3f5c..d7cb130 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,11 @@ except they explicitly `export` the environment variables. We source these envir run the development versions of our applications. It can sometimes be difficult to understand: -1. Whether all the environment variables we *expect* to be defined in production actually have been. + +1. Whether all the environment variables we _expect_ to be defined in production actually have been. 2. What the particular value of a production environment actually is. 3. What the differences are between our expectations and the actual environment variables in a running -application process. + application process. We are building and maintaining `checkenv` to make it easier for us to diagnose and fix issues with application configuration via environment variables. We stand in solidarity with anyone else who @@ -40,3 +41,24 @@ binary which supports your needs. There is currently no need to support runtime plugins. Since doing so would make this program a lot more complicated, we have decided to forego runtime plugin functionality for now. + +## Usage + +```bash +./checkenv plugins +``` + +Available plugins: + +- env - Provides the environment variables defined in the checkenv process. +- file - Provides the environment variables defined in the env file with the given path. +- proc - Provides the environment variables set for the process with the given pid. +- aws_ssm - Provides environment variables defined in AWS Systems Manager Parameter Store. + +### aws_ssm plugin + +In order to fetch parameters with tags `Product` = `test` and `Node` = `true` with `export ` prefix execute following command + +```bash +./checkenv show -export aws_ssm+Product:test,Node:true +``` diff --git a/aws_ssm.go b/aws_ssm.go new file mode 100644 index 0000000..9c913b6 --- /dev/null +++ b/aws_ssm.go @@ -0,0 +1,38 @@ +// checkenv plugin that provides environment variables defined in AWS System Manager Parameter Store. + +package main + +import ( + "context" + + "github.com/bugout-dev/checkenv/aws_ssm" +) + +func AWSSystemsManagerParameterStoreProvider(filter string) (map[string]string, error) { + environment := make(map[string]string) + + // Convert string of tags for filter to key:value structure + filterTags := aws_ssm.ParseFilterTags(filter) + + ctx := context.Background() + + api := aws_ssm.InitAWSClient(ctx) + + keys := aws_ssm.FetchKeysOfParameters(ctx, api, filterTags) + + // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) + // and fetch values for required parameters + keyChunks := aws_ssm.GenerateChunks(keys, 10) + parameters := aws_ssm.FetchParameters(ctx, api, keyChunks) + + for _, parameter := range parameters { + environment[parameter.Name] = parameter.Value + } + + return environment, nil +} + +func init() { + helpString := "Provides environment variables defined in AWS Systems Manager Parameter Store." + RegisterPlugin("aws_ssm", helpString, noop, AWSSystemsManagerParameterStoreProvider) +} diff --git a/aws_ssm/aws_ssm.go b/aws_ssm/aws_ssm.go new file mode 100644 index 0000000..14e5566 --- /dev/null +++ b/aws_ssm/aws_ssm.go @@ -0,0 +1,56 @@ +/* +Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go +*/ +package aws_ssm + +import ( + "context" + "log" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" +) + +// AWSSystemsManagerParameterStoreAPI defines the interface +// for the GetParameters and DescribeParameters function. +// We use this interface to test the function using a mocked service +type AWSSystemsManagerParameterStoreAPI interface { + GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.GetParametersOutput, error) + + DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.DescribeParametersOutput, error) +} + +// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter +// Inputs: +// c: is the context of the method call, which includes the AWS Region +// api: is the interface that defines the method call +// input: defines the input arguments to the service call +// Output: +// If success, a GetParametersOutput object containing the result of the service call and nil +// Otherwise, nil and an error from the call to GetParameters +func ExecGetParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { + return api.GetParameters(c, input) +} + +func ExecDescribeParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { + return api.DescribeParameters(c, input) +} + +// Load the Shared AWS Configuration (~/.aws/config) +func InitAWSClient(ctx context.Context) *ssm.Client { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + log.Fatalln("Failed loading AWS Configuration", err) + } + client := ssm.NewFromConfig(cfg) + + return client +} diff --git a/aws_ssm/data.go b/aws_ssm/data.go new file mode 100644 index 0000000..646efe5 --- /dev/null +++ b/aws_ssm/data.go @@ -0,0 +1,13 @@ +package aws_ssm + +// Parameter structure for storing final result from AWS SSM +type Parameter struct { + Name string + Value string +} + +// Tags for filter defined by user +type FilterTag struct { + Name string + Value string +} diff --git a/aws_ssm/data_test.json b/aws_ssm/data_test.json new file mode 100644 index 0000000..f413197 --- /dev/null +++ b/aws_ssm/data_test.json @@ -0,0 +1,44 @@ +[ + { + "Name": "/wrong/dev/y1", + "Value": "w1", + "Tags": [ + { + "Product": "wrong" + } + ] + }, + { + "Name": "/test/dev/t1", + "Value": "q1", + "Tags": [ + { + "Product": "test" + } + ] + }, + { + "Name": "/test/dev/t2", + "Value": "q2", + "Tags": [ + { + "Product": "test", + "Application": "dev" + } + ] + }, + { + "Name": "/test/dev/t3", + "Value": "q3", + "Tags": [] + }, + { + "Name": "/test/dev/t4", + "Value": "q4", + "Tags": [ + { + "Product": "wrong" + } + ] + } +] \ No newline at end of file diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go new file mode 100644 index 0000000..dad09c8 --- /dev/null +++ b/aws_ssm/parameters.go @@ -0,0 +1,141 @@ +package aws_ssm + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// Fetch values for parameters +// Inputs: +// chunks: list of lists with parameter key values +func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStoreAPI, chunks [][]string) []Parameter { + var parameters []Parameter + + for _, chunk := range chunks { + getInput := &ssm.GetParametersInput{ + Names: chunk, + } + results, err := ExecGetParameters(ctx, api, getInput) + if err != nil { + log.Fatal(err) + } + + for _, p := range results.Parameters { + parameter := Parameter{ + Name: *p.Name, Value: *p.Value, + } + parameters = append(parameters, parameter) + } + } + log.Println("Retrieved values for parameters") + + return parameters +} + +// Fetch list of parameter keys from AWS with defined filters +func FetchKeysOfParameters( + ctx context.Context, + api AWSSystemsManagerParameterStoreAPI, + filterTags []FilterTag, +) []string { + var parameters []string + + // Set parameter filters + parameterFilters := []types.ParameterStringFilter{} + for _, ft := range filterTags { + filterKey := fmt.Sprintf("tag:%s", ft.Name) + parameterFilters = append(parameterFilters, types.ParameterStringFilter{ + Key: &filterKey, + Values: []string{ft.Value}, + }) + } + describeInput := &ssm.DescribeParametersInput{ + MaxResults: 10, + ParameterFilters: parameterFilters, + } + + // CHECKENV_AWS_FETCH_LOOP_LIMIT by default set to 10, + // it is allows to load 100 parameters from AWS and it is + // a limiter to prevent loading too many parameters without + // control during passing erroneous filters + var err error + var fetchLoopLimit int + fetchLoopLimitStr := os.Getenv("CHECKENV_AWS_FETCH_LOOP_LIMIT") + if fetchLoopLimitStr != "" { + fetchLoopLimit, err = strconv.Atoi(fetchLoopLimitStr) + } + if fetchLoopLimitStr == "" || err != nil { + fetchLoopLimit = 10 + } + + n := 0 + for { + // Fetch list of parameter keys + results, err := ExecDescribeParameters(ctx, api, describeInput) + if err != nil { + log.Fatal(err) + } + for _, p := range results.Parameters { + parameters = append(parameters, *p.Name) + } + + // If there are no more parameters break + if results.NextToken == nil { + break + } + describeInput.NextToken = results.NextToken + + n++ + if n >= fetchLoopLimit { + log.Fatal("To many iterations over DescribeParameters loop") + } + } + log.Printf("Retrieved %d parameters", len(parameters)) + + return parameters +} + +// Split list of reports on nested lists +func GenerateChunks(flatSlice []string, chunkSize int) [][]string { + if len(flatSlice) == 0 { + return [][]string{} + } + + chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) + + for i, v := range flatSlice { + if i%chunkSize == 0 { + chunks = append(chunks, make([]string, 0, chunkSize)) + } + chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) + } + + return chunks +} + +// ParseFilterTags convert string from user input to key value structure +func ParseFilterTags(filterTagsStr string) []FilterTag { + var filterTags []FilterTag + + filterTagsSlice := strings.Split(filterTagsStr, ",") + for _, t := range filterTagsSlice { + tagNameValue := strings.Split(t, ":") + if len(tagNameValue) != 2 || len(tagNameValue[0]) == 0 || len(tagNameValue[1]) == 0 { + log.Printf("Unable to parse tag name and value: %s", t) + continue + } + filterTags = append(filterTags, FilterTag{ + Name: tagNameValue[0], + Value: tagNameValue[1], + }) + } + + return filterTags +} diff --git a/aws_ssm/parameters_test.go b/aws_ssm/parameters_test.go new file mode 100644 index 0000000..5a0d15a --- /dev/null +++ b/aws_ssm/parameters_test.go @@ -0,0 +1,50 @@ +package aws_ssm + +import ( + "reflect" + "testing" +) + +func TestGenerateChunks(t *testing.T) { + var cases = []struct { + flatSlice []string + chunkSIze int + expected [][]string + }{ + {[]string{}, 1, [][]string{}}, + {[]string{}, 2, [][]string{}}, + {[]string{"val-1", "val-2"}, 2, [][]string{{"val-1", "val-2"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 1, [][]string{{"val-1"}, {"val-2"}, {"val-3"}, {"val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 2, [][]string{{"val-1", "val-2"}, {"val-3", "val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5", "val-6"}, 3, [][]string{{"val-1", "val-2", "val-3"}, {"val-4", "val-5", "val-6"}}}, + } + for _, c := range cases { + chunks := GenerateChunks(c.flatSlice, c.chunkSIze) + if !reflect.DeepEqual(chunks, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, chunks) + t.Fail() + } + } +} + +func TestFilterTags(t *testing.T) { + var emptyFilterTags []FilterTag + var cases = []struct { + filterTagsStr string + expected []FilterTag + }{ + {"Product", emptyFilterTags}, + {"Product:", emptyFilterTags}, + {":test", emptyFilterTags}, + {":", emptyFilterTags}, + {"Product:test", []FilterTag{{Name: "Product", Value: "test"}}}, + {"Product:test,Node:true", []FilterTag{{Name: "Product", Value: "test"}, {Name: "Node", Value: "true"}}}, + } + for _, c := range cases { + filterTags := ParseFilterTags(c.filterTagsStr) + if !reflect.DeepEqual(filterTags, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, filterTags) + t.Fatal() + } + } +} diff --git a/checkenv.go b/checkenv.go index f159463..d66f0d2 100644 --- a/checkenv.go +++ b/checkenv.go @@ -64,6 +64,7 @@ func main() { showFlags := flag.NewFlagSet("show", flag.ExitOnError) showHelp := showFlags.Bool("h", false, "Use this flag if you want help with this command") showFlags.BoolVar(showHelp, "help", false, "Use this flag if you want help with this command") + showExport := showFlags.Bool("export", false, "Use this flag to prepend and \"export \" before every environment variable definition") availableCommands := fmt.Sprintf("%s,%s", pluginsCommand, showCommand) @@ -100,21 +101,27 @@ func main() { } providedVars[providerSpec] = vars } + + exportPrefix := "" + if *showExport { + exportPrefix = "export " + } + for providerSpec := range spec.providersFull { - fmt.Printf("%s - all variables:\n", providerSpec) + fmt.Printf("# Generated with %s - all variables:\n", providerSpec) for k, v := range providedVars[providerSpec] { - fmt.Printf("- %s=%s\n", k, v) + fmt.Printf("%s%s=%s\n", exportPrefix, k, v) } } for providerSpec, queriedVars := range spec.providerVars { - fmt.Printf("%s - specific variables:\n", providerSpec) + fmt.Printf("# Generated with %s - specific variables:\n", providerSpec) definedVars := providedVars[providerSpec] for k := range queriedVars { v, ok := definedVars[k] if !ok { - fmt.Printf("- UNDEFINED: %s\n", k) + fmt.Printf("# UNDEFINED: %s\n", k) } else { - fmt.Printf("- %s=%s\n", k, v) + fmt.Printf("%s%s=%s\n", exportPrefix, k, v) } } } diff --git a/go.mod b/go.mod index 460a655..f2e9111 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module github.com/bugout-dev/checkenv go 1.16 + +require ( + github.com/aws/aws-sdk-go-v2 v1.11.2 + github.com/aws/aws-sdk-go-v2/config v1.11.0 + github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5acab87 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs= +github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.11.0 h1:Czlld5zBB61A3/aoegA9/buZulwL9mHHfizh/Oq+Kqs= +github.com/aws/aws-sdk-go-v2/config v1.11.0/go.mod h1:VrQDJGFBM5yZe+IOeenNZ/DWoErdny+k2MHEIpwDsEY= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4 h1:2hvbUoHufns0lDIsaK8FVCMukT1WngtZPavN+W2FkSw= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4/go.mod h1:tTrhvBPHyPde4pdIPSba4Nv7RYr4wP9jxXEDa1bKn/8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 h1:KiN5TPOLrEjbGCvdTQR4t0U4T87vVwALZ5Bg3jpMqPY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2/go.mod h1:dF2F6tXEOgmW5X1ZFO/EPtWrcm7XkW07KNcJUGNtt4s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 h1:XJLnluKuUxQG255zPNe+04izXl7GSyUVafIsgfv9aw4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2/go.mod h1:SgKKNBIoDC/E1ZCDhhMW3yalWjwuLjMcpLzsM/QQnWo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 h1:EauRoYZVNPlidZSZJDscjJBQ22JhVF2+tdteatax2Ak= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2/go.mod h1:xT4XX6w5Sa3dhg50JrYyy3e4WPYo/+WjY/BXtqXVunU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 h1:IQup8Q6lorXeiA/rK72PeToWoWK8h7VAPgHNWdSrtgE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2/go.mod h1:VITe/MdW6EMXPb0o0txu/fsonXbMHUU2OC2Qp7ivU4o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 h1:CKdUNKmuilw/KNmO2Q53Av8u+ZyXMC2M9aX8Z+c/gzg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2/go.mod h1:FgR1tCsn8C6+Hf+N5qkfrE4IXvUL1RgW87sunJ+5J4I= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 h1:E/2WewR1wegBnthK8Yz+E87E8Mm4RJC/7R6vg6oAfl0= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1/go.mod h1:jqRk4h1lv2pV4G1DTYRj71JIMEoU/gEGvLU5O6ZnpLM= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 h1:2IDmvSb86KT44lSg1uU4ONpzgWLOuApRl6Tg54mZ6Dk= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2/go.mod h1:KnIpszaIdwI33tmc/W/GGXyn22c1USYxA/2KyvoeDY0= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 h1:QKR7wy5e650q70PFKMfGF9sTo0rZgUevSSJ4wxmyWXk= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=