From 0f6e1c6c22b1cf1d692e63ff971ffa9e8b700885 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 17 Dec 2021 20:29:16 +0000 Subject: [PATCH 1/9] Go version of parameters script --- scripts/sources/parameters/go.mod | 22 +++++ scripts/sources/parameters/go.sum | 41 ++++++++ scripts/sources/parameters/main.go | 149 +++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 scripts/sources/parameters/go.mod create mode 100644 scripts/sources/parameters/go.sum create mode 100644 scripts/sources/parameters/main.go diff --git a/scripts/sources/parameters/go.mod b/scripts/sources/parameters/go.mod new file mode 100644 index 0000000..44b4653 --- /dev/null +++ b/scripts/sources/parameters/go.mod @@ -0,0 +1,22 @@ +module github.com/bugout-dev/checkenv/scripts/sources/parameters + +go 1.17 + +require ( + github.com/aws/aws-sdk-go-v2/config v1.11.0 + github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect + github.com/aws/smithy-go v1.9.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/scripts/sources/parameters/go.sum b/scripts/sources/parameters/go.sum new file mode 100644 index 0000000..5acab87 --- /dev/null +++ b/scripts/sources/parameters/go.sum @@ -0,0 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs= +github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.11.0 h1:Czlld5zBB61A3/aoegA9/buZulwL9mHHfizh/Oq+Kqs= +github.com/aws/aws-sdk-go-v2/config v1.11.0/go.mod h1:VrQDJGFBM5yZe+IOeenNZ/DWoErdny+k2MHEIpwDsEY= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4 h1:2hvbUoHufns0lDIsaK8FVCMukT1WngtZPavN+W2FkSw= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4/go.mod h1:tTrhvBPHyPde4pdIPSba4Nv7RYr4wP9jxXEDa1bKn/8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 h1:KiN5TPOLrEjbGCvdTQR4t0U4T87vVwALZ5Bg3jpMqPY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2/go.mod h1:dF2F6tXEOgmW5X1ZFO/EPtWrcm7XkW07KNcJUGNtt4s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 h1:XJLnluKuUxQG255zPNe+04izXl7GSyUVafIsgfv9aw4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2/go.mod h1:SgKKNBIoDC/E1ZCDhhMW3yalWjwuLjMcpLzsM/QQnWo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 h1:EauRoYZVNPlidZSZJDscjJBQ22JhVF2+tdteatax2Ak= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2/go.mod h1:xT4XX6w5Sa3dhg50JrYyy3e4WPYo/+WjY/BXtqXVunU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 h1:IQup8Q6lorXeiA/rK72PeToWoWK8h7VAPgHNWdSrtgE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2/go.mod h1:VITe/MdW6EMXPb0o0txu/fsonXbMHUU2OC2Qp7ivU4o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 h1:CKdUNKmuilw/KNmO2Q53Av8u+ZyXMC2M9aX8Z+c/gzg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2/go.mod h1:FgR1tCsn8C6+Hf+N5qkfrE4IXvUL1RgW87sunJ+5J4I= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 h1:E/2WewR1wegBnthK8Yz+E87E8Mm4RJC/7R6vg6oAfl0= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1/go.mod h1:jqRk4h1lv2pV4G1DTYRj71JIMEoU/gEGvLU5O6ZnpLM= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 h1:2IDmvSb86KT44lSg1uU4ONpzgWLOuApRl6Tg54mZ6Dk= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2/go.mod h1:KnIpszaIdwI33tmc/W/GGXyn22c1USYxA/2KyvoeDY0= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 h1:QKR7wy5e650q70PFKMfGF9sTo0rZgUevSSJ4wxmyWXk= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/scripts/sources/parameters/main.go b/scripts/sources/parameters/main.go new file mode 100644 index 0000000..95f5297 --- /dev/null +++ b/scripts/sources/parameters/main.go @@ -0,0 +1,149 @@ +/* +Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go +*/ +package main + +import ( + "context" + "flag" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// SSMGetParametersAPI defines the interface for the GetParameters function. +// We use this interface to test the function using a mocked service. +type SSMGetParametersAPI interface { + GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.GetParametersOutput, error) +} + +type SSMDescribeParametersAPI interface { + DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.DescribeParametersOutput, error) +} + +// FindParameters retrieves an AWS Systems Manager string parameter +// Inputs: +// c is the context of the method call, which includes the AWS Region +// api is the interface that defines the method call +// input defines the input arguments to the service call. +// Output: +// If success, a GetParametersOutput object containing the result of the service call and nil +// Otherwise, nil and an error from the call to GetParameter +func FindParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { + return api.GetParameters(c, input) +} + +func FindParameterKeys(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { + return api.DescribeParameters(c, input) +} + +// Split list of reports on nested lists +func generateChunks(flatSlice []string, chunkSize int) [][]string { + if len(flatSlice) == 0 { + return nil + } + + chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) + + for i, v := range flatSlice { + if i%chunkSize == 0 { + chunks = append(chunks, make([]string, 0, chunkSize)) + } + chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) + } + + return chunks +} + +type Parameter struct { + Name string + Value string +} + +func main() { + var maxResults int + var productTag string + flag.IntVar(&maxResults, "max", 3, "The maximum number of items to return for call to AWS") + flag.StringVar(&productTag, "product", "", "Product tag") + flag.Parse() + + if productTag == "" { + log.Fatalln("Please specify the tag of product") + } + + // Load the Shared AWS Configuration (~/.aws/config) + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + log.Fatal(err) + } + client := ssm.NewFromConfig(cfg) + + // Set parameter filters + filterKey := "tag:Product" + parameterFilters := []types.ParameterStringFilter{ + { + Key: &filterKey, + Values: []string{productTag}, + }, + } + describeInput := &ssm.DescribeParametersInput{ + MaxResults: int32(maxResults), + ParameterFilters: parameterFilters, + } + + var parameterKeys []string + + n := 0 + for { + // Fetch list of parameter keys + results, err := FindParameterKeys(context.TODO(), client, describeInput) + if err != nil { + log.Fatal(err) + } + for _, p := range results.Parameters { + parameterKeys = append(parameterKeys, *p.Name) + } + + // If there are no more parameters break + if *&results.NextToken == nil { + break + } + describeInput.NextToken = *&results.NextToken + + n++ + if n >= 10 { + log.Fatal("To many iterations over DescribeParameters loop") + } + } + + var parameters []Parameter + + // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) + // and fetch values for required parameters + parameterKeyChunks := generateChunks(parameterKeys, 10) + for _, chunk := range parameterKeyChunks { + getInput := &ssm.GetParametersInput{ + Names: chunk, + } + results, err := FindParameters(context.TODO(), client, getInput) + if err != nil { + log.Fatal(err) + } + + for _, p := range results.Parameters { + parameters = append(parameters, Parameter{Name: *p.Name, Value: *p.Value}) + } + } + fmt.Println(parameters) +} From 7c9c92831909cf301e42e885605423850e7460a6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 19 Dec 2021 22:09:14 +0000 Subject: [PATCH 2/9] Added write to file, export prefix and tests --- scripts/sources/parameters/cmd/aws_ssm.go | 58 ++++++ .../sources/parameters/cmd/aws_ssm_test.go | 130 ++++++++++++ scripts/sources/parameters/cmd/data.go | 17 ++ scripts/sources/parameters/cmd/parameters.go | 186 ++++++++++++++++++ .../sources/parameters/cmd/parameters_test.go | 28 +++ scripts/sources/parameters/data.json | 44 +++++ scripts/sources/parameters/main.go | 144 +------------- 7 files changed, 465 insertions(+), 142 deletions(-) create mode 100644 scripts/sources/parameters/cmd/aws_ssm.go create mode 100644 scripts/sources/parameters/cmd/aws_ssm_test.go create mode 100644 scripts/sources/parameters/cmd/data.go create mode 100644 scripts/sources/parameters/cmd/parameters.go create mode 100644 scripts/sources/parameters/cmd/parameters_test.go create mode 100644 scripts/sources/parameters/data.json diff --git a/scripts/sources/parameters/cmd/aws_ssm.go b/scripts/sources/parameters/cmd/aws_ssm.go new file mode 100644 index 0000000..034c5ee --- /dev/null +++ b/scripts/sources/parameters/cmd/aws_ssm.go @@ -0,0 +1,58 @@ +/* +Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go +*/ +package parameters + +import ( + "context" + "log" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" +) + +// SSMGetParametersAPI and SSMDescribeParametersAPI defines the interface +// for the GetParameters and DescribeParameters function. +// We use this interface to test the function using a mocked service +type SSMGetParametersAPI interface { + GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.GetParametersOutput, error) +} + +type SSMDescribeParametersAPI interface { + DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.DescribeParametersOutput, error) +} + +// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter +// Inputs: +// c: is the context of the method call, which includes the AWS Region +// api: is the interface that defines the method call +// input: defines the input arguments to the service call +// Output: +// If success, a GetParametersOutput object containing the result of the service call and nil +// Otherwise, nil and an error from the call to GetParameters +func ExecGetParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { + return api.GetParameters(c, input) +} + +func ExecDescribeParameters(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { + return api.DescribeParameters(c, input) +} + +// Load the Shared AWS Configuration (~/.aws/config) +func InitAWSClient(ctx context.Context) *ssm.Client { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + log.Fatalln("Failed loading AWS Configuration", err) + } + client := ssm.NewFromConfig(cfg) + + return client +} diff --git a/scripts/sources/parameters/cmd/aws_ssm_test.go b/scripts/sources/parameters/cmd/aws_ssm_test.go new file mode 100644 index 0000000..c9ea2bd --- /dev/null +++ b/scripts/sources/parameters/cmd/aws_ssm_test.go @@ -0,0 +1,130 @@ +package parameters + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// Fill fake output data +type SSMGetParametersImpl struct{} +type SSMDescribeParametersImpl struct{} + +func (dt SSMGetParametersImpl) GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), +) (*ssm.GetParametersOutput, error) { + parameters := []types.Parameter{} + for _, d := range globalData { + parameters = append(parameters, types.Parameter{ + Name: aws.String(d.Name), + Value: aws.String(d.Value), + }) + } + output := &ssm.GetParametersOutput{ + Parameters: parameters, + } + + return output, nil +} +func (dt SSMDescribeParametersImpl) DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), +) (*ssm.DescribeParametersOutput, error) { + + // TODO(kompotkot): How to test filters? + parameters := []types.ParameterMetadata{ + {Name: aws.String("/test/dev/t1")}, + {Name: aws.String("/test/dev/t2")}, + } + output := &ssm.DescribeParametersOutput{ + Parameters: parameters, + } + + return output, nil +} + +type DataTags struct { + Product string `json:"Product"` +} + +type Data struct { + Name string `json:"Name"` + Value string `json:"Value"` + Tags []DataTags `json:"Tags"` +} + +var globalData []Data + +var globalParameterKeys []string + +func populateData(t *testing.T) error { + content, err := ioutil.ReadFile("../data.json") + if err != nil { + return err + } + + contentStr := string(content) + err = json.Unmarshal([]byte(contentStr), &globalData) + if err != nil { + return nil + } + + return nil +} + +func TestDescribeParameters(t *testing.T) { + err := populateData(t) + if err != nil { + t.Fatal("Failed to populate data") + } + + api := &SSMDescribeParametersImpl{} + + flags := Flags{ProductTag: "test"} + + // Test DescribeParameters + parameterKeys := FetchKeysOfParameters( + context.Background(), + *api, + flags, + ) + if len(parameterKeys) != 2 { + // TODO(kompotkot): Extract length of parameters from data.json + t.Logf("Length of parameter keys should be 2, but got %d", len(parameterKeys)) + t.Fail() + } + + for _, p := range parameterKeys { + globalParameterKeys = append(globalParameterKeys, p) + } +} + +func TestGetParameters(t *testing.T) { + parameterKeyChunks := GenerateChunks(globalParameterKeys, 10) + + api := &SSMGetParametersImpl{} + + flags := Flags{Export: false} + + parameters := FetchParameters( + context.Background(), + *api, + parameterKeyChunks, + flags, + ) + if len(parameters) != 2 { + // TODO(kompotkot): Extract length of parameters from data.json + t.Logf("Length of parameters should be 2, but got %d", len(parameters)) + t.Fail() + } + fmt.Println(parameters) +} diff --git a/scripts/sources/parameters/cmd/data.go b/scripts/sources/parameters/cmd/data.go new file mode 100644 index 0000000..338868a --- /dev/null +++ b/scripts/sources/parameters/cmd/data.go @@ -0,0 +1,17 @@ +package parameters + +// Parameter structure for storing final result from AWS SSM +type Parameter struct { + Name string + Value string + Export string +} + +// Contains command-line flags defined by user +type Flags struct { + Export bool + MaxResults int + Outfile string + ProductTag string + Update bool +} diff --git a/scripts/sources/parameters/cmd/parameters.go b/scripts/sources/parameters/cmd/parameters.go new file mode 100644 index 0000000..2556fc6 --- /dev/null +++ b/scripts/sources/parameters/cmd/parameters.go @@ -0,0 +1,186 @@ +package parameters + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// Fetch values for parameters +// Inputs: +// chunks: list of lists with parameter key values +func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]string, flags Flags) []Parameter { + var parameters []Parameter + + for _, chunk := range chunks { + getInput := &ssm.GetParametersInput{ + Names: chunk, + } + results, err := ExecGetParameters(ctx, api, getInput) + if err != nil { + log.Fatal(err) + } + + for _, p := range results.Parameters { + parameter := Parameter{ + Name: *p.Name, Value: *p.Value, + } + if flags.Export { + parameter.Export = "export " + } + parameters = append(parameters, parameter) + } + } + log.Println("Retrieved values for parameters") + + return parameters +} + +// Fetch list of parameter keys from AWS with defined filters +func FetchKeysOfParameters( + ctx context.Context, + api SSMDescribeParametersAPI, + flags Flags, +) []string { + var parameters []string + + // Set parameter filters + filterKey := "tag:Product" + parameterFilters := []types.ParameterStringFilter{ + { + Key: &filterKey, + Values: []string{flags.ProductTag}, + }, + } + describeInput := &ssm.DescribeParametersInput{ + MaxResults: int32(flags.MaxResults), + ParameterFilters: parameterFilters, + } + n := 0 + for { + // Fetch list of parameter keys + results, err := ExecDescribeParameters(ctx, api, describeInput) + if err != nil { + log.Fatal(err) + } + for _, p := range results.Parameters { + parameters = append(parameters, *p.Name) + } + + // If there are no more parameters break + if results.NextToken == nil { + break + } + describeInput.NextToken = *&results.NextToken + + n++ + if n >= 50 { + log.Fatal("To many iterations over DescribeParameters loop") + } + } + log.Printf("Retrieved %d parameters", len(parameters)) + + return parameters +} + +// Split list of reports on nested lists +func GenerateChunks(flatSlice []string, chunkSize int) [][]string { + if len(flatSlice) == 0 { + return [][]string{} + } + + chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) + + for i, v := range flatSlice { + if i%chunkSize == 0 { + chunks = append(chunks, make([]string, 0, chunkSize)) + } + chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) + } + + return chunks +} + +// WriteToFile generate or update existing file and +// flash to it environment variables +func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) { + flag := os.O_TRUNC | os.O_CREATE | os.O_WRONLY + if update { + flag = os.O_APPEND | os.O_CREATE | os.O_WRONLY + } + + f, err := os.OpenFile( + outfile, + flag, + 0644, + ) + if err != nil { + log.Fatalf("Unable to open file %s, error: %s", outfile, err) + } + defer f.Close() + + parametersStr := "" + for _, p := range parameters { + parametersStr += fmt.Sprintf("%s%s=%s\n", p.Export, p.Name, p.Value) + } + if _, err := f.WriteString(parametersStr); err != nil { + log.Fatalf("Unable to write to file %s, error: %s", outfile, err) + } +} + +// HandleSignals process Ctrl+C and all script interruptions +func HandleSignals(cancel context.CancelFunc) { + sigCh := make(chan os.Signal) + signal.Notify(sigCh, os.Interrupt) + for { + sig := <-sigCh + switch sig { + case os.Interrupt: + cancel() + return + } + } +} + +func Extract() { + var flags Flags + flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter") + flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS") + flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved") + flag.StringVar(&flags.ProductTag, "product", "", "Product tag") + flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)") + flag.Parse() + + if flags.ProductTag == "" { + log.Fatalln("Please specify the tag of product") + } + + ctx, cancel := context.WithCancel(context.Background()) + go HandleSignals(cancel) + + client := InitAWSClient(ctx) + + parameterKeys := FetchKeysOfParameters(ctx, client, flags) + + // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) + // and fetch values for required parameters + parameterKeyChunks := GenerateChunks(parameterKeys, 10) + if len(parameterKeyChunks) == 0 { + log.Fatalln("Nothing to generate, empty slice provided") + } + parameters := FetchParameters(ctx, client, parameterKeyChunks, flags) + + if flags.Outfile != "" { + WriteToFile(parameters, flags.Outfile, flags.Update, flags.Export) + } else { + for _, p := range parameters { + fmt.Printf("%s%s=%s\n", p.Export, p.Name, p.Value) + } + } +} diff --git a/scripts/sources/parameters/cmd/parameters_test.go b/scripts/sources/parameters/cmd/parameters_test.go new file mode 100644 index 0000000..7366da7 --- /dev/null +++ b/scripts/sources/parameters/cmd/parameters_test.go @@ -0,0 +1,28 @@ +package parameters + +import ( + "reflect" + "testing" +) + +func TestGenerateChunks(t *testing.T) { + var cases = []struct { + flatSlice []string + chunkSIze int + expected [][]string + }{ + {[]string{}, 1, [][]string{}}, + {[]string{}, 2, [][]string{}}, + {[]string{"val-1", "val-2"}, 2, [][]string{{"val-1", "val-2"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 1, [][]string{{"val-1"}, {"val-2"}, {"val-3"}, {"val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 2, [][]string{{"val-1", "val-2"}, {"val-3", "val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5", "val-6"}, 3, [][]string{{"val-1", "val-2", "val-3"}, {"val-4", "val-5", "val-6"}}}, + } + for _, c := range cases { + chunks := GenerateChunks(c.flatSlice, c.chunkSIze) + if !reflect.DeepEqual(chunks, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, chunks) + t.Fail() + } + } +} diff --git a/scripts/sources/parameters/data.json b/scripts/sources/parameters/data.json new file mode 100644 index 0000000..f413197 --- /dev/null +++ b/scripts/sources/parameters/data.json @@ -0,0 +1,44 @@ +[ + { + "Name": "/wrong/dev/y1", + "Value": "w1", + "Tags": [ + { + "Product": "wrong" + } + ] + }, + { + "Name": "/test/dev/t1", + "Value": "q1", + "Tags": [ + { + "Product": "test" + } + ] + }, + { + "Name": "/test/dev/t2", + "Value": "q2", + "Tags": [ + { + "Product": "test", + "Application": "dev" + } + ] + }, + { + "Name": "/test/dev/t3", + "Value": "q3", + "Tags": [] + }, + { + "Name": "/test/dev/t4", + "Value": "q4", + "Tags": [ + { + "Product": "wrong" + } + ] + } +] \ No newline at end of file diff --git a/scripts/sources/parameters/main.go b/scripts/sources/parameters/main.go index 95f5297..1130ac2 100644 --- a/scripts/sources/parameters/main.go +++ b/scripts/sources/parameters/main.go @@ -1,149 +1,9 @@ -/* -Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go -*/ package main import ( - "context" - "flag" - "fmt" - "log" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/aws/aws-sdk-go-v2/service/ssm/types" + "github.com/bugout-dev/checkenv/scripts/sources/parameters/cmd" ) -// SSMGetParametersAPI defines the interface for the GetParameters function. -// We use this interface to test the function using a mocked service. -type SSMGetParametersAPI interface { - GetParameters( - ctx context.Context, - params *ssm.GetParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.GetParametersOutput, error) -} - -type SSMDescribeParametersAPI interface { - DescribeParameters( - ctx context.Context, - params *ssm.DescribeParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.DescribeParametersOutput, error) -} - -// FindParameters retrieves an AWS Systems Manager string parameter -// Inputs: -// c is the context of the method call, which includes the AWS Region -// api is the interface that defines the method call -// input defines the input arguments to the service call. -// Output: -// If success, a GetParametersOutput object containing the result of the service call and nil -// Otherwise, nil and an error from the call to GetParameter -func FindParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { - return api.GetParameters(c, input) -} - -func FindParameterKeys(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { - return api.DescribeParameters(c, input) -} - -// Split list of reports on nested lists -func generateChunks(flatSlice []string, chunkSize int) [][]string { - if len(flatSlice) == 0 { - return nil - } - - chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) - - for i, v := range flatSlice { - if i%chunkSize == 0 { - chunks = append(chunks, make([]string, 0, chunkSize)) - } - chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) - } - - return chunks -} - -type Parameter struct { - Name string - Value string -} - func main() { - var maxResults int - var productTag string - flag.IntVar(&maxResults, "max", 3, "The maximum number of items to return for call to AWS") - flag.StringVar(&productTag, "product", "", "Product tag") - flag.Parse() - - if productTag == "" { - log.Fatalln("Please specify the tag of product") - } - - // Load the Shared AWS Configuration (~/.aws/config) - cfg, err := config.LoadDefaultConfig(context.TODO()) - if err != nil { - log.Fatal(err) - } - client := ssm.NewFromConfig(cfg) - - // Set parameter filters - filterKey := "tag:Product" - parameterFilters := []types.ParameterStringFilter{ - { - Key: &filterKey, - Values: []string{productTag}, - }, - } - describeInput := &ssm.DescribeParametersInput{ - MaxResults: int32(maxResults), - ParameterFilters: parameterFilters, - } - - var parameterKeys []string - - n := 0 - for { - // Fetch list of parameter keys - results, err := FindParameterKeys(context.TODO(), client, describeInput) - if err != nil { - log.Fatal(err) - } - for _, p := range results.Parameters { - parameterKeys = append(parameterKeys, *p.Name) - } - - // If there are no more parameters break - if *&results.NextToken == nil { - break - } - describeInput.NextToken = *&results.NextToken - - n++ - if n >= 10 { - log.Fatal("To many iterations over DescribeParameters loop") - } - } - - var parameters []Parameter - - // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) - // and fetch values for required parameters - parameterKeyChunks := generateChunks(parameterKeys, 10) - for _, chunk := range parameterKeyChunks { - getInput := &ssm.GetParametersInput{ - Names: chunk, - } - results, err := FindParameters(context.TODO(), client, getInput) - if err != nil { - log.Fatal(err) - } - - for _, p := range results.Parameters { - parameters = append(parameters, Parameter{Name: *p.Name, Value: *p.Value}) - } - } - fmt.Println(parameters) + parameters.Extract() } From bb0c048acc784169e35bc78867928180a5cf9f9d Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 20 Dec 2021 15:56:11 +0000 Subject: [PATCH 3/9] Added possibility to specify few tags for filter --- .../sources/parameters/cmd/aws_ssm_test.go | 2 +- scripts/sources/parameters/cmd/data.go | 8 +++- scripts/sources/parameters/cmd/parameters.go | 43 +++++++++++++++---- .../sources/parameters/cmd/parameters_test.go | 22 ++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/scripts/sources/parameters/cmd/aws_ssm_test.go b/scripts/sources/parameters/cmd/aws_ssm_test.go index c9ea2bd..389ef39 100644 --- a/scripts/sources/parameters/cmd/aws_ssm_test.go +++ b/scripts/sources/parameters/cmd/aws_ssm_test.go @@ -89,7 +89,7 @@ func TestDescribeParameters(t *testing.T) { api := &SSMDescribeParametersImpl{} - flags := Flags{ProductTag: "test"} + flags := Flags{FilterTags: []FilterTag{{Name: "Product", Value: "test"}}} // Test DescribeParameters parameterKeys := FetchKeysOfParameters( diff --git a/scripts/sources/parameters/cmd/data.go b/scripts/sources/parameters/cmd/data.go index 338868a..8d119ab 100644 --- a/scripts/sources/parameters/cmd/data.go +++ b/scripts/sources/parameters/cmd/data.go @@ -7,11 +7,17 @@ type Parameter struct { Export string } +// Tags for filter defined by user +type FilterTag struct { + Name string + Value string +} + // Contains command-line flags defined by user type Flags struct { Export bool MaxResults int Outfile string - ProductTag string + FilterTags []FilterTag Update bool } diff --git a/scripts/sources/parameters/cmd/parameters.go b/scripts/sources/parameters/cmd/parameters.go index 2556fc6..1a7a900 100644 --- a/scripts/sources/parameters/cmd/parameters.go +++ b/scripts/sources/parameters/cmd/parameters.go @@ -7,6 +7,7 @@ import ( "log" "os" "os/signal" + "strings" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" @@ -51,12 +52,13 @@ func FetchKeysOfParameters( var parameters []string // Set parameter filters - filterKey := "tag:Product" - parameterFilters := []types.ParameterStringFilter{ - { + parameterFilters := []types.ParameterStringFilter{} + for _, ft := range flags.FilterTags { + filterKey := fmt.Sprintf("tag:%s", ft.Name) + parameterFilters = append(parameterFilters, types.ParameterStringFilter{ Key: &filterKey, - Values: []string{flags.ProductTag}, - }, + Values: []string{ft.Value}, + }) } describeInput := &ssm.DescribeParametersInput{ MaxResults: int32(flags.MaxResults), @@ -107,6 +109,26 @@ func GenerateChunks(flatSlice []string, chunkSize int) [][]string { return chunks } +// ParseFilterTags convert string from user input to key value structure +func ParseFilterTags(filterTagsStr string) []FilterTag { + var filterTags []FilterTag + + filterTagsSlice := strings.Split(filterTagsStr, ",") + for _, t := range filterTagsSlice { + tagNameValue := strings.Split(t, ":") + if len(tagNameValue) != 2 || len(tagNameValue[0]) == 0 || len(tagNameValue[1]) == 0 { + log.Printf("Unable to parse tag name and value: %s", t) + continue + } + filterTags = append(filterTags, FilterTag{ + Name: tagNameValue[0], + Value: tagNameValue[1], + }) + } + + return filterTags +} + // WriteToFile generate or update existing file and // flash to it environment variables func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) { @@ -150,17 +172,22 @@ func HandleSignals(cancel context.CancelFunc) { func Extract() { var flags Flags + var filterTagsStr string flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter") flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS") flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved") - flag.StringVar(&flags.ProductTag, "product", "", "Product tag") + flag.StringVar(&filterTagsStr, "tags", "", "Product tags for filter separated by comma in format 'tagName1:tagValue1,tagName2:tagValue2'") flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)") flag.Parse() - if flags.ProductTag == "" { - log.Fatalln("Please specify the tag of product") + if filterTagsStr == "" { + log.Fatalln("Please specify the tags for filter, at least Product tag") } + // Convert string of tags for filter to key:value structure + filterTags := ParseFilterTags(filterTagsStr) + flags.FilterTags = filterTags + ctx, cancel := context.WithCancel(context.Background()) go HandleSignals(cancel) diff --git a/scripts/sources/parameters/cmd/parameters_test.go b/scripts/sources/parameters/cmd/parameters_test.go index 7366da7..bbd00ce 100644 --- a/scripts/sources/parameters/cmd/parameters_test.go +++ b/scripts/sources/parameters/cmd/parameters_test.go @@ -26,3 +26,25 @@ func TestGenerateChunks(t *testing.T) { } } } + +func TestFilterTags(t *testing.T) { + var emptyFilterTags []FilterTag + var cases = []struct { + filterTagsStr string + expected []FilterTag + }{ + {"Product", emptyFilterTags}, + {"Product:", emptyFilterTags}, + {":test", emptyFilterTags}, + {":", emptyFilterTags}, + {"Product:test", []FilterTag{{Name: "Product", Value: "test"}}}, + {"Product:test,Node:true", []FilterTag{{Name: "Product", Value: "test"}, {Name: "Node", Value: "true"}}}, + } + for _, c := range cases { + filterTags := ParseFilterTags(c.filterTagsStr) + if !reflect.DeepEqual(filterTags, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, filterTags) + t.Fatal() + } + } +} From c5d434f3dc727ff6fb49409941b0d8d9fd895afe Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 20 Dec 2021 10:03:41 -0800 Subject: [PATCH 4/9] Started reorganization to integration aws_ssm with checkenv --- aws_ssm/aws_ssm.go | 72 +++++++++++++ aws_ssm/aws_ssm_test.go | 130 ++++++++++++++++++++++ aws_ssm/data.go | 23 ++++ aws_ssm/parameters.go | 213 +++++++++++++++++++++++++++++++++++++ aws_ssm/parameters_test.go | 50 +++++++++ go.mod | 6 ++ go.sum | 41 +++++++ 7 files changed, 535 insertions(+) create mode 100644 aws_ssm/aws_ssm.go create mode 100644 aws_ssm/aws_ssm_test.go create mode 100644 aws_ssm/data.go create mode 100644 aws_ssm/parameters.go create mode 100644 aws_ssm/parameters_test.go create mode 100644 go.sum diff --git a/aws_ssm/aws_ssm.go b/aws_ssm/aws_ssm.go new file mode 100644 index 0000000..3cdf63d --- /dev/null +++ b/aws_ssm/aws_ssm.go @@ -0,0 +1,72 @@ +/* +Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go +*/ +package aws_ssm + +import ( + "context" + "log" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" +) + +// SSMGetParametersAPI and SSMDescribeParametersAPI defines the interface +// for the GetParameters and DescribeParameters function. +// We use this interface to test the function using a mocked service +type SSMGetParametersAPI interface { + GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.GetParametersOutput, error) +} + +type SSMDescribeParametersAPI interface { + DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.DescribeParametersOutput, error) +} + +type AWSSystemsManagerParameterStore interface { + GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.GetParametersOutput, error) + + DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), + ) (*ssm.DescribeParametersOutput, error) +} + +// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter +// Inputs: +// c: is the context of the method call, which includes the AWS Region +// api: is the interface that defines the method call +// input: defines the input arguments to the service call +// Output: +// If success, a GetParametersOutput object containing the result of the service call and nil +// Otherwise, nil and an error from the call to GetParameters +func ExecGetParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { + return api.GetParameters(c, input) +} + +func ExecDescribeParameters(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { + return api.DescribeParameters(c, input) +} + +// Load the Shared AWS Configuration (~/.aws/config) +func InitAWSClient(ctx context.Context) *ssm.Client { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + log.Fatalln("Failed loading AWS Configuration", err) + } + client := ssm.NewFromConfig(cfg) + + return client +} diff --git a/aws_ssm/aws_ssm_test.go b/aws_ssm/aws_ssm_test.go new file mode 100644 index 0000000..3ff00f3 --- /dev/null +++ b/aws_ssm/aws_ssm_test.go @@ -0,0 +1,130 @@ +package aws_ssm + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// Fill fake output data +type SSMGetParametersImpl struct{} +type SSMDescribeParametersImpl struct{} + +func (dt SSMGetParametersImpl) GetParameters( + ctx context.Context, + params *ssm.GetParametersInput, + optFns ...func(*ssm.Options), +) (*ssm.GetParametersOutput, error) { + parameters := []types.Parameter{} + for _, d := range globalData { + parameters = append(parameters, types.Parameter{ + Name: aws.String(d.Name), + Value: aws.String(d.Value), + }) + } + output := &ssm.GetParametersOutput{ + Parameters: parameters, + } + + return output, nil +} +func (dt SSMDescribeParametersImpl) DescribeParameters( + ctx context.Context, + params *ssm.DescribeParametersInput, + optFns ...func(*ssm.Options), +) (*ssm.DescribeParametersOutput, error) { + + // TODO(kompotkot): How to test filters? + parameters := []types.ParameterMetadata{ + {Name: aws.String("/test/dev/t1")}, + {Name: aws.String("/test/dev/t2")}, + } + output := &ssm.DescribeParametersOutput{ + Parameters: parameters, + } + + return output, nil +} + +type DataTags struct { + Product string `json:"Product"` +} + +type Data struct { + Name string `json:"Name"` + Value string `json:"Value"` + Tags []DataTags `json:"Tags"` +} + +var globalData []Data + +var globalParameterKeys []string + +func populateData(t *testing.T) error { + content, err := ioutil.ReadFile("../data.json") + if err != nil { + return err + } + + contentStr := string(content) + err = json.Unmarshal([]byte(contentStr), &globalData) + if err != nil { + return nil + } + + return nil +} + +func TestDescribeParameters(t *testing.T) { + err := populateData(t) + if err != nil { + t.Fatal("Failed to populate data") + } + + api := &SSMDescribeParametersImpl{} + + flags := Flags{FilterTags: []FilterTag{{Name: "Product", Value: "test"}}} + + // Test DescribeParameters + parameterKeys := FetchKeysOfParameters( + context.Background(), + *api, + flags, + ) + if len(parameterKeys) != 2 { + // TODO(kompotkot): Extract length of parameters from data.json + t.Logf("Length of parameter keys should be 2, but got %d", len(parameterKeys)) + t.Fail() + } + + for _, p := range parameterKeys { + globalParameterKeys = append(globalParameterKeys, p) + } +} + +func TestGetParameters(t *testing.T) { + parameterKeyChunks := GenerateChunks(globalParameterKeys, 10) + + api := &SSMGetParametersImpl{} + + flags := Flags{Export: false} + + parameters := FetchParameters( + context.Background(), + *api, + parameterKeyChunks, + flags, + ) + if len(parameters) != 2 { + // TODO(kompotkot): Extract length of parameters from data.json + t.Logf("Length of parameters should be 2, but got %d", len(parameters)) + t.Fail() + } + fmt.Println(parameters) +} diff --git a/aws_ssm/data.go b/aws_ssm/data.go new file mode 100644 index 0000000..9dfbe93 --- /dev/null +++ b/aws_ssm/data.go @@ -0,0 +1,23 @@ +package aws_ssm + +// Parameter structure for storing final result from AWS SSM +type Parameter struct { + Name string + Value string + Export string +} + +// Tags for filter defined by user +type FilterTag struct { + Name string + Value string +} + +// Contains command-line flags defined by user +type Flags struct { + Export bool + MaxResults int + Outfile string + FilterTags []FilterTag + Update bool +} diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go new file mode 100644 index 0000000..5f32701 --- /dev/null +++ b/aws_ssm/parameters.go @@ -0,0 +1,213 @@ +package aws_ssm + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// Fetch values for parameters +// Inputs: +// chunks: list of lists with parameter key values +func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]string, flags Flags) []Parameter { + var parameters []Parameter + + for _, chunk := range chunks { + getInput := &ssm.GetParametersInput{ + Names: chunk, + } + results, err := ExecGetParameters(ctx, api, getInput) + if err != nil { + log.Fatal(err) + } + + for _, p := range results.Parameters { + parameter := Parameter{ + Name: *p.Name, Value: *p.Value, + } + if flags.Export { + parameter.Export = "export " + } + parameters = append(parameters, parameter) + } + } + log.Println("Retrieved values for parameters") + + return parameters +} + +// Fetch list of parameter keys from AWS with defined filters +func FetchKeysOfParameters( + ctx context.Context, + api SSMDescribeParametersAPI, + flags Flags, +) []string { + var parameters []string + + // Set parameter filters + parameterFilters := []types.ParameterStringFilter{} + for _, ft := range flags.FilterTags { + filterKey := fmt.Sprintf("tag:%s", ft.Name) + parameterFilters = append(parameterFilters, types.ParameterStringFilter{ + Key: &filterKey, + Values: []string{ft.Value}, + }) + } + describeInput := &ssm.DescribeParametersInput{ + MaxResults: int32(flags.MaxResults), + ParameterFilters: parameterFilters, + } + n := 0 + for { + // Fetch list of parameter keys + results, err := ExecDescribeParameters(ctx, api, describeInput) + if err != nil { + log.Fatal(err) + } + for _, p := range results.Parameters { + parameters = append(parameters, *p.Name) + } + + // If there are no more parameters break + if results.NextToken == nil { + break + } + describeInput.NextToken = *&results.NextToken + + n++ + if n >= 50 { + log.Fatal("To many iterations over DescribeParameters loop") + } + } + log.Printf("Retrieved %d parameters", len(parameters)) + + return parameters +} + +// Split list of reports on nested lists +func GenerateChunks(flatSlice []string, chunkSize int) [][]string { + if len(flatSlice) == 0 { + return [][]string{} + } + + chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) + + for i, v := range flatSlice { + if i%chunkSize == 0 { + chunks = append(chunks, make([]string, 0, chunkSize)) + } + chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) + } + + return chunks +} + +// ParseFilterTags convert string from user input to key value structure +func ParseFilterTags(filterTagsStr string) []FilterTag { + var filterTags []FilterTag + + filterTagsSlice := strings.Split(filterTagsStr, ",") + for _, t := range filterTagsSlice { + tagNameValue := strings.Split(t, ":") + if len(tagNameValue) != 2 || len(tagNameValue[0]) == 0 || len(tagNameValue[1]) == 0 { + log.Printf("Unable to parse tag name and value: %s", t) + continue + } + filterTags = append(filterTags, FilterTag{ + Name: tagNameValue[0], + Value: tagNameValue[1], + }) + } + + return filterTags +} + +// WriteToFile generate or update existing file and +// flash to it environment variables +func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) { + flag := os.O_TRUNC | os.O_CREATE | os.O_WRONLY + if update { + flag = os.O_APPEND | os.O_CREATE | os.O_WRONLY + } + + f, err := os.OpenFile( + outfile, + flag, + 0644, + ) + if err != nil { + log.Fatalf("Unable to open file %s, error: %s", outfile, err) + } + defer f.Close() + + parametersStr := "" + for _, p := range parameters { + parametersStr += fmt.Sprintf("%s%s=%s\n", p.Export, p.Name, p.Value) + } + if _, err := f.WriteString(parametersStr); err != nil { + log.Fatalf("Unable to write to file %s, error: %s", outfile, err) + } +} + +// HandleSignals process Ctrl+C and all script interruptions +func HandleSignals(cancel context.CancelFunc) { + sigCh := make(chan os.Signal) + signal.Notify(sigCh, os.Interrupt) + for { + sig := <-sigCh + switch sig { + case os.Interrupt: + cancel() + return + } + } +} + +func Extract() { + var flags Flags + var filterTagsStr string + flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter") + flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS") + flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved") + flag.StringVar(&filterTagsStr, "tags", "", "Product tags for filter separated by comma in format 'tagName1:tagValue1,tagName2:tagValue2'") + flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)") + flag.Parse() + + if filterTagsStr == "" { + log.Fatalln("Please specify the tags for filter, at least Product tag") + } + + // Convert string of tags for filter to key:value structure + filterTags := ParseFilterTags(filterTagsStr) + flags.FilterTags = filterTags + + ctx, cancel := context.WithCancel(context.Background()) + go HandleSignals(cancel) + + client := InitAWSClient(ctx) + + parameterKeys := FetchKeysOfParameters(ctx, client, flags) + + // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) + // and fetch values for required parameters + parameterKeyChunks := GenerateChunks(parameterKeys, 10) + if len(parameterKeyChunks) == 0 { + log.Fatalln("Nothing to generate, empty slice provided") + } + parameters := FetchParameters(ctx, client, parameterKeyChunks, flags) + + if flags.Outfile != "" { + WriteToFile(parameters, flags.Outfile, flags.Update, flags.Export) + } else { + for _, p := range parameters { + fmt.Printf("%s%s=%s\n", p.Export, p.Name, p.Value) + } + } +} diff --git a/aws_ssm/parameters_test.go b/aws_ssm/parameters_test.go new file mode 100644 index 0000000..5a0d15a --- /dev/null +++ b/aws_ssm/parameters_test.go @@ -0,0 +1,50 @@ +package aws_ssm + +import ( + "reflect" + "testing" +) + +func TestGenerateChunks(t *testing.T) { + var cases = []struct { + flatSlice []string + chunkSIze int + expected [][]string + }{ + {[]string{}, 1, [][]string{}}, + {[]string{}, 2, [][]string{}}, + {[]string{"val-1", "val-2"}, 2, [][]string{{"val-1", "val-2"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 1, [][]string{{"val-1"}, {"val-2"}, {"val-3"}, {"val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 2, [][]string{{"val-1", "val-2"}, {"val-3", "val-4"}, {"val-5"}}}, + {[]string{"val-1", "val-2", "val-3", "val-4", "val-5", "val-6"}, 3, [][]string{{"val-1", "val-2", "val-3"}, {"val-4", "val-5", "val-6"}}}, + } + for _, c := range cases { + chunks := GenerateChunks(c.flatSlice, c.chunkSIze) + if !reflect.DeepEqual(chunks, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, chunks) + t.Fail() + } + } +} + +func TestFilterTags(t *testing.T) { + var emptyFilterTags []FilterTag + var cases = []struct { + filterTagsStr string + expected []FilterTag + }{ + {"Product", emptyFilterTags}, + {"Product:", emptyFilterTags}, + {":test", emptyFilterTags}, + {":", emptyFilterTags}, + {"Product:test", []FilterTag{{Name: "Product", Value: "test"}}}, + {"Product:test,Node:true", []FilterTag{{Name: "Product", Value: "test"}, {Name: "Node", Value: "true"}}}, + } + for _, c := range cases { + filterTags := ParseFilterTags(c.filterTagsStr) + if !reflect.DeepEqual(filterTags, c.expected) { + t.Logf("Value should be %s, but got %s", c.expected, filterTags) + t.Fatal() + } + } +} diff --git a/go.mod b/go.mod index 460a655..f2e9111 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module github.com/bugout-dev/checkenv go 1.16 + +require ( + github.com/aws/aws-sdk-go-v2 v1.11.2 + github.com/aws/aws-sdk-go-v2/config v1.11.0 + github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5acab87 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs= +github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.11.0 h1:Czlld5zBB61A3/aoegA9/buZulwL9mHHfizh/Oq+Kqs= +github.com/aws/aws-sdk-go-v2/config v1.11.0/go.mod h1:VrQDJGFBM5yZe+IOeenNZ/DWoErdny+k2MHEIpwDsEY= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4 h1:2hvbUoHufns0lDIsaK8FVCMukT1WngtZPavN+W2FkSw= +github.com/aws/aws-sdk-go-v2/credentials v1.6.4/go.mod h1:tTrhvBPHyPde4pdIPSba4Nv7RYr4wP9jxXEDa1bKn/8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 h1:KiN5TPOLrEjbGCvdTQR4t0U4T87vVwALZ5Bg3jpMqPY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2/go.mod h1:dF2F6tXEOgmW5X1ZFO/EPtWrcm7XkW07KNcJUGNtt4s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 h1:XJLnluKuUxQG255zPNe+04izXl7GSyUVafIsgfv9aw4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2/go.mod h1:SgKKNBIoDC/E1ZCDhhMW3yalWjwuLjMcpLzsM/QQnWo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 h1:EauRoYZVNPlidZSZJDscjJBQ22JhVF2+tdteatax2Ak= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2/go.mod h1:xT4XX6w5Sa3dhg50JrYyy3e4WPYo/+WjY/BXtqXVunU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 h1:IQup8Q6lorXeiA/rK72PeToWoWK8h7VAPgHNWdSrtgE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2/go.mod h1:VITe/MdW6EMXPb0o0txu/fsonXbMHUU2OC2Qp7ivU4o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 h1:CKdUNKmuilw/KNmO2Q53Av8u+ZyXMC2M9aX8Z+c/gzg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2/go.mod h1:FgR1tCsn8C6+Hf+N5qkfrE4IXvUL1RgW87sunJ+5J4I= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 h1:E/2WewR1wegBnthK8Yz+E87E8Mm4RJC/7R6vg6oAfl0= +github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1/go.mod h1:jqRk4h1lv2pV4G1DTYRj71JIMEoU/gEGvLU5O6ZnpLM= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 h1:2IDmvSb86KT44lSg1uU4ONpzgWLOuApRl6Tg54mZ6Dk= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.2/go.mod h1:KnIpszaIdwI33tmc/W/GGXyn22c1USYxA/2KyvoeDY0= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 h1:QKR7wy5e650q70PFKMfGF9sTo0rZgUevSSJ4wxmyWXk= +github.com/aws/aws-sdk-go-v2/service/sts v1.11.1/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From d9825ef91027b8a9151b0065433bc458b8ae9bab Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 20 Dec 2021 10:38:15 -0800 Subject: [PATCH 5/9] Integrated "aws_ssm" plugin for checkenv --- aws_ssm.go | 37 +++++++++++++++++++++++++++++++++++++ aws_ssm/aws_ssm.go | 23 ++--------------------- aws_ssm/aws_ssm_test.go | 11 +++++------ aws_ssm/parameters.go | 4 ++-- 4 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 aws_ssm.go diff --git a/aws_ssm.go b/aws_ssm.go new file mode 100644 index 0000000..20eb686 --- /dev/null +++ b/aws_ssm.go @@ -0,0 +1,37 @@ +// checkenv plugin that provides the environment variables defined in the checkenv process. + +package main + +import ( + "context" + + "github.com/bugout-dev/checkenv/aws_ssm" +) + +func AWSSystemsManagerParameterStoreProvider(filter string) (map[string]string, error) { + environment := make(map[string]string) + + AWSSystemsManagerFlags := aws_ssm.Flags{ + Export: false, + MaxResults: 10, + Outfile: "", + Update: false, + } + AWSSystemsManagerFlags.FilterTags = aws_ssm.ParseFilterTags(filter) + + ctx := context.Background() + api := aws_ssm.InitAWSClient(ctx) + keys := aws_ssm.FetchKeysOfParameters(ctx, api, AWSSystemsManagerFlags) + keyChunks := aws_ssm.GenerateChunks(keys, 10) + parameters := aws_ssm.FetchParameters(ctx, api, keyChunks, AWSSystemsManagerFlags) + for _, parameter := range parameters { + environment[parameter.Name] = parameter.Value + } + + return environment, nil +} + +func init() { + helpString := "Provides environment variables defined in AWS Systems Manager Parameter Store." + RegisterPlugin("aws_ssm", helpString, noop, AWSSystemsManagerParameterStoreProvider) +} diff --git a/aws_ssm/aws_ssm.go b/aws_ssm/aws_ssm.go index 3cdf63d..2016a4c 100644 --- a/aws_ssm/aws_ssm.go +++ b/aws_ssm/aws_ssm.go @@ -11,25 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ssm" ) -// SSMGetParametersAPI and SSMDescribeParametersAPI defines the interface -// for the GetParameters and DescribeParameters function. -// We use this interface to test the function using a mocked service -type SSMGetParametersAPI interface { - GetParameters( - ctx context.Context, - params *ssm.GetParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.GetParametersOutput, error) -} - -type SSMDescribeParametersAPI interface { - DescribeParameters( - ctx context.Context, - params *ssm.DescribeParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.DescribeParametersOutput, error) -} - type AWSSystemsManagerParameterStore interface { GetParameters( ctx context.Context, @@ -52,11 +33,11 @@ type AWSSystemsManagerParameterStore interface { // Output: // If success, a GetParametersOutput object containing the result of the service call and nil // Otherwise, nil and an error from the call to GetParameters -func ExecGetParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { +func ExecGetParameters(c context.Context, api AWSSystemsManagerParameterStore, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { return api.GetParameters(c, input) } -func ExecDescribeParameters(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { +func ExecDescribeParameters(c context.Context, api AWSSystemsManagerParameterStore, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { return api.DescribeParameters(c, input) } diff --git a/aws_ssm/aws_ssm_test.go b/aws_ssm/aws_ssm_test.go index 3ff00f3..0b53aef 100644 --- a/aws_ssm/aws_ssm_test.go +++ b/aws_ssm/aws_ssm_test.go @@ -13,10 +13,9 @@ import ( ) // Fill fake output data -type SSMGetParametersImpl struct{} -type SSMDescribeParametersImpl struct{} +type MockAWSSystemManagerParameterStore struct{} -func (dt SSMGetParametersImpl) GetParameters( +func (dt MockAWSSystemManagerParameterStore) GetParameters( ctx context.Context, params *ssm.GetParametersInput, optFns ...func(*ssm.Options), @@ -34,7 +33,7 @@ func (dt SSMGetParametersImpl) GetParameters( return output, nil } -func (dt SSMDescribeParametersImpl) DescribeParameters( +func (dt MockAWSSystemManagerParameterStore) DescribeParameters( ctx context.Context, params *ssm.DescribeParametersInput, optFns ...func(*ssm.Options), @@ -87,7 +86,7 @@ func TestDescribeParameters(t *testing.T) { t.Fatal("Failed to populate data") } - api := &SSMDescribeParametersImpl{} + api := &MockAWSSystemManagerParameterStore{} flags := Flags{FilterTags: []FilterTag{{Name: "Product", Value: "test"}}} @@ -111,7 +110,7 @@ func TestDescribeParameters(t *testing.T) { func TestGetParameters(t *testing.T) { parameterKeyChunks := GenerateChunks(globalParameterKeys, 10) - api := &SSMGetParametersImpl{} + api := &MockAWSSystemManagerParameterStore{} flags := Flags{Export: false} diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go index 5f32701..08e417d 100644 --- a/aws_ssm/parameters.go +++ b/aws_ssm/parameters.go @@ -16,7 +16,7 @@ import ( // Fetch values for parameters // Inputs: // chunks: list of lists with parameter key values -func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]string, flags Flags) []Parameter { +func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStore, chunks [][]string, flags Flags) []Parameter { var parameters []Parameter for _, chunk := range chunks { @@ -46,7 +46,7 @@ func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]st // Fetch list of parameter keys from AWS with defined filters func FetchKeysOfParameters( ctx context.Context, - api SSMDescribeParametersAPI, + api AWSSystemsManagerParameterStore, flags Flags, ) []string { var parameters []string From 0eacab1da09f5dcaf64972e7350ce42221d1e497 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 20 Dec 2021 10:48:59 -0800 Subject: [PATCH 6/9] Working aws_ssm plugin --- aws_ssm/parameters.go | 60 +------------------------------------------ checkenv.go | 17 ++++++++---- 2 files changed, 13 insertions(+), 64 deletions(-) diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go index 08e417d..7cee74a 100644 --- a/aws_ssm/parameters.go +++ b/aws_ssm/parameters.go @@ -2,11 +2,9 @@ package aws_ssm import ( "context" - "flag" "fmt" "log" "os" - "os/signal" "strings" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -79,7 +77,7 @@ func FetchKeysOfParameters( if results.NextToken == nil { break } - describeInput.NextToken = *&results.NextToken + describeInput.NextToken = results.NextToken n++ if n >= 50 { @@ -155,59 +153,3 @@ func WriteToFile(parameters []Parameter, outfile string, update bool, export boo log.Fatalf("Unable to write to file %s, error: %s", outfile, err) } } - -// HandleSignals process Ctrl+C and all script interruptions -func HandleSignals(cancel context.CancelFunc) { - sigCh := make(chan os.Signal) - signal.Notify(sigCh, os.Interrupt) - for { - sig := <-sigCh - switch sig { - case os.Interrupt: - cancel() - return - } - } -} - -func Extract() { - var flags Flags - var filterTagsStr string - flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter") - flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS") - flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved") - flag.StringVar(&filterTagsStr, "tags", "", "Product tags for filter separated by comma in format 'tagName1:tagValue1,tagName2:tagValue2'") - flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)") - flag.Parse() - - if filterTagsStr == "" { - log.Fatalln("Please specify the tags for filter, at least Product tag") - } - - // Convert string of tags for filter to key:value structure - filterTags := ParseFilterTags(filterTagsStr) - flags.FilterTags = filterTags - - ctx, cancel := context.WithCancel(context.Background()) - go HandleSignals(cancel) - - client := InitAWSClient(ctx) - - parameterKeys := FetchKeysOfParameters(ctx, client, flags) - - // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) - // and fetch values for required parameters - parameterKeyChunks := GenerateChunks(parameterKeys, 10) - if len(parameterKeyChunks) == 0 { - log.Fatalln("Nothing to generate, empty slice provided") - } - parameters := FetchParameters(ctx, client, parameterKeyChunks, flags) - - if flags.Outfile != "" { - WriteToFile(parameters, flags.Outfile, flags.Update, flags.Export) - } else { - for _, p := range parameters { - fmt.Printf("%s%s=%s\n", p.Export, p.Name, p.Value) - } - } -} diff --git a/checkenv.go b/checkenv.go index f159463..62b723a 100644 --- a/checkenv.go +++ b/checkenv.go @@ -64,6 +64,7 @@ func main() { showFlags := flag.NewFlagSet("show", flag.ExitOnError) showHelp := showFlags.Bool("h", false, "Use this flag if you want help with this command") showFlags.BoolVar(showHelp, "help", false, "Use this flag if you want help with this command") + showExport := showFlags.Bool("export", false, "Use this flag to prepend and \"export \" before every environment variable definition") availableCommands := fmt.Sprintf("%s,%s", pluginsCommand, showCommand) @@ -100,21 +101,27 @@ func main() { } providedVars[providerSpec] = vars } + + exportPrefix := "" + if *showExport { + exportPrefix = "export " + } + for providerSpec := range spec.providersFull { - fmt.Printf("%s - all variables:\n", providerSpec) + fmt.Printf("# %s - all variables:\n", providerSpec) for k, v := range providedVars[providerSpec] { - fmt.Printf("- %s=%s\n", k, v) + fmt.Printf("%s%s=%s\n", exportPrefix, k, v) } } for providerSpec, queriedVars := range spec.providerVars { - fmt.Printf("%s - specific variables:\n", providerSpec) + fmt.Printf("# %s - specific variables:\n", providerSpec) definedVars := providedVars[providerSpec] for k := range queriedVars { v, ok := definedVars[k] if !ok { - fmt.Printf("- UNDEFINED: %s\n", k) + fmt.Printf("# UNDEFINED: %s\n", k) } else { - fmt.Printf("- %s=%s\n", k, v) + fmt.Printf("%s%s=%s\n", exportPrefix, k, v) } } } From f032be3ea51014be18c1307a2c43b947a980af39 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 20 Dec 2021 10:51:48 -0800 Subject: [PATCH 7/9] Removed new aws_ssm parameters CLI in favor of checkenv integration --- aws_ssm/aws_ssm_test.go | 2 +- .../sources/parameters => aws_ssm}/data.json | 0 scripts/sources/parameters/cmd/aws_ssm.go | 58 ----- .../sources/parameters/cmd/aws_ssm_test.go | 130 ----------- scripts/sources/parameters/cmd/data.go | 23 -- scripts/sources/parameters/cmd/parameters.go | 213 ------------------ .../sources/parameters/cmd/parameters_test.go | 50 ---- scripts/sources/parameters/go.mod | 22 -- scripts/sources/parameters/go.sum | 41 ---- scripts/sources/parameters/main.go | 9 - 10 files changed, 1 insertion(+), 547 deletions(-) rename {scripts/sources/parameters => aws_ssm}/data.json (100%) delete mode 100644 scripts/sources/parameters/cmd/aws_ssm.go delete mode 100644 scripts/sources/parameters/cmd/aws_ssm_test.go delete mode 100644 scripts/sources/parameters/cmd/data.go delete mode 100644 scripts/sources/parameters/cmd/parameters.go delete mode 100644 scripts/sources/parameters/cmd/parameters_test.go delete mode 100644 scripts/sources/parameters/go.mod delete mode 100644 scripts/sources/parameters/go.sum delete mode 100644 scripts/sources/parameters/main.go diff --git a/aws_ssm/aws_ssm_test.go b/aws_ssm/aws_ssm_test.go index 0b53aef..619349c 100644 --- a/aws_ssm/aws_ssm_test.go +++ b/aws_ssm/aws_ssm_test.go @@ -66,7 +66,7 @@ var globalData []Data var globalParameterKeys []string func populateData(t *testing.T) error { - content, err := ioutil.ReadFile("../data.json") + content, err := ioutil.ReadFile("./data.json") if err != nil { return err } diff --git a/scripts/sources/parameters/data.json b/aws_ssm/data.json similarity index 100% rename from scripts/sources/parameters/data.json rename to aws_ssm/data.json diff --git a/scripts/sources/parameters/cmd/aws_ssm.go b/scripts/sources/parameters/cmd/aws_ssm.go deleted file mode 100644 index 034c5ee..0000000 --- a/scripts/sources/parameters/cmd/aws_ssm.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go -*/ -package parameters - -import ( - "context" - "log" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ssm" -) - -// SSMGetParametersAPI and SSMDescribeParametersAPI defines the interface -// for the GetParameters and DescribeParameters function. -// We use this interface to test the function using a mocked service -type SSMGetParametersAPI interface { - GetParameters( - ctx context.Context, - params *ssm.GetParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.GetParametersOutput, error) -} - -type SSMDescribeParametersAPI interface { - DescribeParameters( - ctx context.Context, - params *ssm.DescribeParametersInput, - optFns ...func(*ssm.Options), - ) (*ssm.DescribeParametersOutput, error) -} - -// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter -// Inputs: -// c: is the context of the method call, which includes the AWS Region -// api: is the interface that defines the method call -// input: defines the input arguments to the service call -// Output: -// If success, a GetParametersOutput object containing the result of the service call and nil -// Otherwise, nil and an error from the call to GetParameters -func ExecGetParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { - return api.GetParameters(c, input) -} - -func ExecDescribeParameters(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { - return api.DescribeParameters(c, input) -} - -// Load the Shared AWS Configuration (~/.aws/config) -func InitAWSClient(ctx context.Context) *ssm.Client { - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - log.Fatalln("Failed loading AWS Configuration", err) - } - client := ssm.NewFromConfig(cfg) - - return client -} diff --git a/scripts/sources/parameters/cmd/aws_ssm_test.go b/scripts/sources/parameters/cmd/aws_ssm_test.go deleted file mode 100644 index 389ef39..0000000 --- a/scripts/sources/parameters/cmd/aws_ssm_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package parameters - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/aws/aws-sdk-go-v2/service/ssm/types" -) - -// Fill fake output data -type SSMGetParametersImpl struct{} -type SSMDescribeParametersImpl struct{} - -func (dt SSMGetParametersImpl) GetParameters( - ctx context.Context, - params *ssm.GetParametersInput, - optFns ...func(*ssm.Options), -) (*ssm.GetParametersOutput, error) { - parameters := []types.Parameter{} - for _, d := range globalData { - parameters = append(parameters, types.Parameter{ - Name: aws.String(d.Name), - Value: aws.String(d.Value), - }) - } - output := &ssm.GetParametersOutput{ - Parameters: parameters, - } - - return output, nil -} -func (dt SSMDescribeParametersImpl) DescribeParameters( - ctx context.Context, - params *ssm.DescribeParametersInput, - optFns ...func(*ssm.Options), -) (*ssm.DescribeParametersOutput, error) { - - // TODO(kompotkot): How to test filters? - parameters := []types.ParameterMetadata{ - {Name: aws.String("/test/dev/t1")}, - {Name: aws.String("/test/dev/t2")}, - } - output := &ssm.DescribeParametersOutput{ - Parameters: parameters, - } - - return output, nil -} - -type DataTags struct { - Product string `json:"Product"` -} - -type Data struct { - Name string `json:"Name"` - Value string `json:"Value"` - Tags []DataTags `json:"Tags"` -} - -var globalData []Data - -var globalParameterKeys []string - -func populateData(t *testing.T) error { - content, err := ioutil.ReadFile("../data.json") - if err != nil { - return err - } - - contentStr := string(content) - err = json.Unmarshal([]byte(contentStr), &globalData) - if err != nil { - return nil - } - - return nil -} - -func TestDescribeParameters(t *testing.T) { - err := populateData(t) - if err != nil { - t.Fatal("Failed to populate data") - } - - api := &SSMDescribeParametersImpl{} - - flags := Flags{FilterTags: []FilterTag{{Name: "Product", Value: "test"}}} - - // Test DescribeParameters - parameterKeys := FetchKeysOfParameters( - context.Background(), - *api, - flags, - ) - if len(parameterKeys) != 2 { - // TODO(kompotkot): Extract length of parameters from data.json - t.Logf("Length of parameter keys should be 2, but got %d", len(parameterKeys)) - t.Fail() - } - - for _, p := range parameterKeys { - globalParameterKeys = append(globalParameterKeys, p) - } -} - -func TestGetParameters(t *testing.T) { - parameterKeyChunks := GenerateChunks(globalParameterKeys, 10) - - api := &SSMGetParametersImpl{} - - flags := Flags{Export: false} - - parameters := FetchParameters( - context.Background(), - *api, - parameterKeyChunks, - flags, - ) - if len(parameters) != 2 { - // TODO(kompotkot): Extract length of parameters from data.json - t.Logf("Length of parameters should be 2, but got %d", len(parameters)) - t.Fail() - } - fmt.Println(parameters) -} diff --git a/scripts/sources/parameters/cmd/data.go b/scripts/sources/parameters/cmd/data.go deleted file mode 100644 index 8d119ab..0000000 --- a/scripts/sources/parameters/cmd/data.go +++ /dev/null @@ -1,23 +0,0 @@ -package parameters - -// Parameter structure for storing final result from AWS SSM -type Parameter struct { - Name string - Value string - Export string -} - -// Tags for filter defined by user -type FilterTag struct { - Name string - Value string -} - -// Contains command-line flags defined by user -type Flags struct { - Export bool - MaxResults int - Outfile string - FilterTags []FilterTag - Update bool -} diff --git a/scripts/sources/parameters/cmd/parameters.go b/scripts/sources/parameters/cmd/parameters.go deleted file mode 100644 index 1a7a900..0000000 --- a/scripts/sources/parameters/cmd/parameters.go +++ /dev/null @@ -1,213 +0,0 @@ -package parameters - -import ( - "context" - "flag" - "fmt" - "log" - "os" - "os/signal" - "strings" - - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/aws/aws-sdk-go-v2/service/ssm/types" -) - -// Fetch values for parameters -// Inputs: -// chunks: list of lists with parameter key values -func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]string, flags Flags) []Parameter { - var parameters []Parameter - - for _, chunk := range chunks { - getInput := &ssm.GetParametersInput{ - Names: chunk, - } - results, err := ExecGetParameters(ctx, api, getInput) - if err != nil { - log.Fatal(err) - } - - for _, p := range results.Parameters { - parameter := Parameter{ - Name: *p.Name, Value: *p.Value, - } - if flags.Export { - parameter.Export = "export " - } - parameters = append(parameters, parameter) - } - } - log.Println("Retrieved values for parameters") - - return parameters -} - -// Fetch list of parameter keys from AWS with defined filters -func FetchKeysOfParameters( - ctx context.Context, - api SSMDescribeParametersAPI, - flags Flags, -) []string { - var parameters []string - - // Set parameter filters - parameterFilters := []types.ParameterStringFilter{} - for _, ft := range flags.FilterTags { - filterKey := fmt.Sprintf("tag:%s", ft.Name) - parameterFilters = append(parameterFilters, types.ParameterStringFilter{ - Key: &filterKey, - Values: []string{ft.Value}, - }) - } - describeInput := &ssm.DescribeParametersInput{ - MaxResults: int32(flags.MaxResults), - ParameterFilters: parameterFilters, - } - n := 0 - for { - // Fetch list of parameter keys - results, err := ExecDescribeParameters(ctx, api, describeInput) - if err != nil { - log.Fatal(err) - } - for _, p := range results.Parameters { - parameters = append(parameters, *p.Name) - } - - // If there are no more parameters break - if results.NextToken == nil { - break - } - describeInput.NextToken = *&results.NextToken - - n++ - if n >= 50 { - log.Fatal("To many iterations over DescribeParameters loop") - } - } - log.Printf("Retrieved %d parameters", len(parameters)) - - return parameters -} - -// Split list of reports on nested lists -func GenerateChunks(flatSlice []string, chunkSize int) [][]string { - if len(flatSlice) == 0 { - return [][]string{} - } - - chunks := make([][]string, 0, len(flatSlice)/chunkSize+1) - - for i, v := range flatSlice { - if i%chunkSize == 0 { - chunks = append(chunks, make([]string, 0, chunkSize)) - } - chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v) - } - - return chunks -} - -// ParseFilterTags convert string from user input to key value structure -func ParseFilterTags(filterTagsStr string) []FilterTag { - var filterTags []FilterTag - - filterTagsSlice := strings.Split(filterTagsStr, ",") - for _, t := range filterTagsSlice { - tagNameValue := strings.Split(t, ":") - if len(tagNameValue) != 2 || len(tagNameValue[0]) == 0 || len(tagNameValue[1]) == 0 { - log.Printf("Unable to parse tag name and value: %s", t) - continue - } - filterTags = append(filterTags, FilterTag{ - Name: tagNameValue[0], - Value: tagNameValue[1], - }) - } - - return filterTags -} - -// WriteToFile generate or update existing file and -// flash to it environment variables -func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) { - flag := os.O_TRUNC | os.O_CREATE | os.O_WRONLY - if update { - flag = os.O_APPEND | os.O_CREATE | os.O_WRONLY - } - - f, err := os.OpenFile( - outfile, - flag, - 0644, - ) - if err != nil { - log.Fatalf("Unable to open file %s, error: %s", outfile, err) - } - defer f.Close() - - parametersStr := "" - for _, p := range parameters { - parametersStr += fmt.Sprintf("%s%s=%s\n", p.Export, p.Name, p.Value) - } - if _, err := f.WriteString(parametersStr); err != nil { - log.Fatalf("Unable to write to file %s, error: %s", outfile, err) - } -} - -// HandleSignals process Ctrl+C and all script interruptions -func HandleSignals(cancel context.CancelFunc) { - sigCh := make(chan os.Signal) - signal.Notify(sigCh, os.Interrupt) - for { - sig := <-sigCh - switch sig { - case os.Interrupt: - cancel() - return - } - } -} - -func Extract() { - var flags Flags - var filterTagsStr string - flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter") - flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS") - flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved") - flag.StringVar(&filterTagsStr, "tags", "", "Product tags for filter separated by comma in format 'tagName1:tagValue1,tagName2:tagValue2'") - flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)") - flag.Parse() - - if filterTagsStr == "" { - log.Fatalln("Please specify the tags for filter, at least Product tag") - } - - // Convert string of tags for filter to key:value structure - filterTags := ParseFilterTags(filterTagsStr) - flags.FilterTags = filterTags - - ctx, cancel := context.WithCancel(context.Background()) - go HandleSignals(cancel) - - client := InitAWSClient(ctx) - - parameterKeys := FetchKeysOfParameters(ctx, client, flags) - - // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) - // and fetch values for required parameters - parameterKeyChunks := GenerateChunks(parameterKeys, 10) - if len(parameterKeyChunks) == 0 { - log.Fatalln("Nothing to generate, empty slice provided") - } - parameters := FetchParameters(ctx, client, parameterKeyChunks, flags) - - if flags.Outfile != "" { - WriteToFile(parameters, flags.Outfile, flags.Update, flags.Export) - } else { - for _, p := range parameters { - fmt.Printf("%s%s=%s\n", p.Export, p.Name, p.Value) - } - } -} diff --git a/scripts/sources/parameters/cmd/parameters_test.go b/scripts/sources/parameters/cmd/parameters_test.go deleted file mode 100644 index bbd00ce..0000000 --- a/scripts/sources/parameters/cmd/parameters_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package parameters - -import ( - "reflect" - "testing" -) - -func TestGenerateChunks(t *testing.T) { - var cases = []struct { - flatSlice []string - chunkSIze int - expected [][]string - }{ - {[]string{}, 1, [][]string{}}, - {[]string{}, 2, [][]string{}}, - {[]string{"val-1", "val-2"}, 2, [][]string{{"val-1", "val-2"}}}, - {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 1, [][]string{{"val-1"}, {"val-2"}, {"val-3"}, {"val-4"}, {"val-5"}}}, - {[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 2, [][]string{{"val-1", "val-2"}, {"val-3", "val-4"}, {"val-5"}}}, - {[]string{"val-1", "val-2", "val-3", "val-4", "val-5", "val-6"}, 3, [][]string{{"val-1", "val-2", "val-3"}, {"val-4", "val-5", "val-6"}}}, - } - for _, c := range cases { - chunks := GenerateChunks(c.flatSlice, c.chunkSIze) - if !reflect.DeepEqual(chunks, c.expected) { - t.Logf("Value should be %s, but got %s", c.expected, chunks) - t.Fail() - } - } -} - -func TestFilterTags(t *testing.T) { - var emptyFilterTags []FilterTag - var cases = []struct { - filterTagsStr string - expected []FilterTag - }{ - {"Product", emptyFilterTags}, - {"Product:", emptyFilterTags}, - {":test", emptyFilterTags}, - {":", emptyFilterTags}, - {"Product:test", []FilterTag{{Name: "Product", Value: "test"}}}, - {"Product:test,Node:true", []FilterTag{{Name: "Product", Value: "test"}, {Name: "Node", Value: "true"}}}, - } - for _, c := range cases { - filterTags := ParseFilterTags(c.filterTagsStr) - if !reflect.DeepEqual(filterTags, c.expected) { - t.Logf("Value should be %s, but got %s", c.expected, filterTags) - t.Fatal() - } - } -} diff --git a/scripts/sources/parameters/go.mod b/scripts/sources/parameters/go.mod deleted file mode 100644 index 44b4653..0000000 --- a/scripts/sources/parameters/go.mod +++ /dev/null @@ -1,22 +0,0 @@ -module github.com/bugout-dev/checkenv/scripts/sources/parameters - -go 1.17 - -require ( - github.com/aws/aws-sdk-go-v2/config v1.11.0 - github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 -) - -require ( - github.com/aws/aws-sdk-go-v2 v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect - github.com/aws/smithy-go v1.9.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect -) diff --git a/scripts/sources/parameters/go.sum b/scripts/sources/parameters/go.sum deleted file mode 100644 index 5acab87..0000000 --- a/scripts/sources/parameters/go.sum +++ /dev/null @@ -1,41 +0,0 @@ -github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs= -github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= -github.com/aws/aws-sdk-go-v2/config v1.11.0 h1:Czlld5zBB61A3/aoegA9/buZulwL9mHHfizh/Oq+Kqs= -github.com/aws/aws-sdk-go-v2/config v1.11.0/go.mod h1:VrQDJGFBM5yZe+IOeenNZ/DWoErdny+k2MHEIpwDsEY= -github.com/aws/aws-sdk-go-v2/credentials v1.6.4 h1:2hvbUoHufns0lDIsaK8FVCMukT1WngtZPavN+W2FkSw= -github.com/aws/aws-sdk-go-v2/credentials v1.6.4/go.mod h1:tTrhvBPHyPde4pdIPSba4Nv7RYr4wP9jxXEDa1bKn/8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 h1:KiN5TPOLrEjbGCvdTQR4t0U4T87vVwALZ5Bg3jpMqPY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2/go.mod h1:dF2F6tXEOgmW5X1ZFO/EPtWrcm7XkW07KNcJUGNtt4s= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 h1:XJLnluKuUxQG255zPNe+04izXl7GSyUVafIsgfv9aw4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2/go.mod h1:SgKKNBIoDC/E1ZCDhhMW3yalWjwuLjMcpLzsM/QQnWo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 h1:EauRoYZVNPlidZSZJDscjJBQ22JhVF2+tdteatax2Ak= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2/go.mod h1:xT4XX6w5Sa3dhg50JrYyy3e4WPYo/+WjY/BXtqXVunU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 h1:IQup8Q6lorXeiA/rK72PeToWoWK8h7VAPgHNWdSrtgE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2/go.mod h1:VITe/MdW6EMXPb0o0txu/fsonXbMHUU2OC2Qp7ivU4o= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 h1:CKdUNKmuilw/KNmO2Q53Av8u+ZyXMC2M9aX8Z+c/gzg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2/go.mod h1:FgR1tCsn8C6+Hf+N5qkfrE4IXvUL1RgW87sunJ+5J4I= -github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1 h1:E/2WewR1wegBnthK8Yz+E87E8Mm4RJC/7R6vg6oAfl0= -github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1/go.mod h1:jqRk4h1lv2pV4G1DTYRj71JIMEoU/gEGvLU5O6ZnpLM= -github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 h1:2IDmvSb86KT44lSg1uU4ONpzgWLOuApRl6Tg54mZ6Dk= -github.com/aws/aws-sdk-go-v2/service/sso v1.6.2/go.mod h1:KnIpszaIdwI33tmc/W/GGXyn22c1USYxA/2KyvoeDY0= -github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 h1:QKR7wy5e650q70PFKMfGF9sTo0rZgUevSSJ4wxmyWXk= -github.com/aws/aws-sdk-go-v2/service/sts v1.11.1/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= -github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= -github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/scripts/sources/parameters/main.go b/scripts/sources/parameters/main.go deleted file mode 100644 index 1130ac2..0000000 --- a/scripts/sources/parameters/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/bugout-dev/checkenv/scripts/sources/parameters/cmd" -) - -func main() { - parameters.Extract() -} From c6419e138c2cf240b8606341a26ea0c3dc7e8306 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 20 Dec 2021 20:03:15 +0000 Subject: [PATCH 8/9] Removed Flags struct, extended with comments and aws_ssm example in README --- README.md | 26 +++++- aws_ssm.go | 21 +++-- aws_ssm/aws_ssm.go | 9 +- aws_ssm/aws_ssm_test.go | 129 -------------------------- aws_ssm/data.go | 14 +-- aws_ssm/{data.json => data_test.json} | 0 aws_ssm/parameters.go | 41 +------- checkenv.go | 4 +- 8 files changed, 50 insertions(+), 194 deletions(-) delete mode 100644 aws_ssm/aws_ssm_test.go rename aws_ssm/{data.json => data_test.json} (100%) diff --git a/README.md b/README.md index 72f3f5c..d7cb130 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,11 @@ except they explicitly `export` the environment variables. We source these envir run the development versions of our applications. It can sometimes be difficult to understand: -1. Whether all the environment variables we *expect* to be defined in production actually have been. + +1. Whether all the environment variables we _expect_ to be defined in production actually have been. 2. What the particular value of a production environment actually is. 3. What the differences are between our expectations and the actual environment variables in a running -application process. + application process. We are building and maintaining `checkenv` to make it easier for us to diagnose and fix issues with application configuration via environment variables. We stand in solidarity with anyone else who @@ -40,3 +41,24 @@ binary which supports your needs. There is currently no need to support runtime plugins. Since doing so would make this program a lot more complicated, we have decided to forego runtime plugin functionality for now. + +## Usage + +```bash +./checkenv plugins +``` + +Available plugins: + +- env - Provides the environment variables defined in the checkenv process. +- file - Provides the environment variables defined in the env file with the given path. +- proc - Provides the environment variables set for the process with the given pid. +- aws_ssm - Provides environment variables defined in AWS Systems Manager Parameter Store. + +### aws_ssm plugin + +In order to fetch parameters with tags `Product` = `test` and `Node` = `true` with `export ` prefix execute following command + +```bash +./checkenv show -export aws_ssm+Product:test,Node:true +``` diff --git a/aws_ssm.go b/aws_ssm.go index 20eb686..9c913b6 100644 --- a/aws_ssm.go +++ b/aws_ssm.go @@ -1,4 +1,4 @@ -// checkenv plugin that provides the environment variables defined in the checkenv process. +// checkenv plugin that provides environment variables defined in AWS System Manager Parameter Store. package main @@ -11,19 +11,20 @@ import ( func AWSSystemsManagerParameterStoreProvider(filter string) (map[string]string, error) { environment := make(map[string]string) - AWSSystemsManagerFlags := aws_ssm.Flags{ - Export: false, - MaxResults: 10, - Outfile: "", - Update: false, - } - AWSSystemsManagerFlags.FilterTags = aws_ssm.ParseFilterTags(filter) + // Convert string of tags for filter to key:value structure + filterTags := aws_ssm.ParseFilterTags(filter) ctx := context.Background() + api := aws_ssm.InitAWSClient(ctx) - keys := aws_ssm.FetchKeysOfParameters(ctx, api, AWSSystemsManagerFlags) + + keys := aws_ssm.FetchKeysOfParameters(ctx, api, filterTags) + + // Split slice of parameter keys to chunks by 10 (max len allowed by AWS) + // and fetch values for required parameters keyChunks := aws_ssm.GenerateChunks(keys, 10) - parameters := aws_ssm.FetchParameters(ctx, api, keyChunks, AWSSystemsManagerFlags) + parameters := aws_ssm.FetchParameters(ctx, api, keyChunks) + for _, parameter := range parameters { environment[parameter.Name] = parameter.Value } diff --git a/aws_ssm/aws_ssm.go b/aws_ssm/aws_ssm.go index 2016a4c..14e5566 100644 --- a/aws_ssm/aws_ssm.go +++ b/aws_ssm/aws_ssm.go @@ -11,7 +11,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ssm" ) -type AWSSystemsManagerParameterStore interface { +// AWSSystemsManagerParameterStoreAPI defines the interface +// for the GetParameters and DescribeParameters function. +// We use this interface to test the function using a mocked service +type AWSSystemsManagerParameterStoreAPI interface { GetParameters( ctx context.Context, params *ssm.GetParametersInput, @@ -33,11 +36,11 @@ type AWSSystemsManagerParameterStore interface { // Output: // If success, a GetParametersOutput object containing the result of the service call and nil // Otherwise, nil and an error from the call to GetParameters -func ExecGetParameters(c context.Context, api AWSSystemsManagerParameterStore, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { +func ExecGetParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) { return api.GetParameters(c, input) } -func ExecDescribeParameters(c context.Context, api AWSSystemsManagerParameterStore, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { +func ExecDescribeParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) { return api.DescribeParameters(c, input) } diff --git a/aws_ssm/aws_ssm_test.go b/aws_ssm/aws_ssm_test.go deleted file mode 100644 index 619349c..0000000 --- a/aws_ssm/aws_ssm_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package aws_ssm - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/aws/aws-sdk-go-v2/service/ssm/types" -) - -// Fill fake output data -type MockAWSSystemManagerParameterStore struct{} - -func (dt MockAWSSystemManagerParameterStore) GetParameters( - ctx context.Context, - params *ssm.GetParametersInput, - optFns ...func(*ssm.Options), -) (*ssm.GetParametersOutput, error) { - parameters := []types.Parameter{} - for _, d := range globalData { - parameters = append(parameters, types.Parameter{ - Name: aws.String(d.Name), - Value: aws.String(d.Value), - }) - } - output := &ssm.GetParametersOutput{ - Parameters: parameters, - } - - return output, nil -} -func (dt MockAWSSystemManagerParameterStore) DescribeParameters( - ctx context.Context, - params *ssm.DescribeParametersInput, - optFns ...func(*ssm.Options), -) (*ssm.DescribeParametersOutput, error) { - - // TODO(kompotkot): How to test filters? - parameters := []types.ParameterMetadata{ - {Name: aws.String("/test/dev/t1")}, - {Name: aws.String("/test/dev/t2")}, - } - output := &ssm.DescribeParametersOutput{ - Parameters: parameters, - } - - return output, nil -} - -type DataTags struct { - Product string `json:"Product"` -} - -type Data struct { - Name string `json:"Name"` - Value string `json:"Value"` - Tags []DataTags `json:"Tags"` -} - -var globalData []Data - -var globalParameterKeys []string - -func populateData(t *testing.T) error { - content, err := ioutil.ReadFile("./data.json") - if err != nil { - return err - } - - contentStr := string(content) - err = json.Unmarshal([]byte(contentStr), &globalData) - if err != nil { - return nil - } - - return nil -} - -func TestDescribeParameters(t *testing.T) { - err := populateData(t) - if err != nil { - t.Fatal("Failed to populate data") - } - - api := &MockAWSSystemManagerParameterStore{} - - flags := Flags{FilterTags: []FilterTag{{Name: "Product", Value: "test"}}} - - // Test DescribeParameters - parameterKeys := FetchKeysOfParameters( - context.Background(), - *api, - flags, - ) - if len(parameterKeys) != 2 { - // TODO(kompotkot): Extract length of parameters from data.json - t.Logf("Length of parameter keys should be 2, but got %d", len(parameterKeys)) - t.Fail() - } - - for _, p := range parameterKeys { - globalParameterKeys = append(globalParameterKeys, p) - } -} - -func TestGetParameters(t *testing.T) { - parameterKeyChunks := GenerateChunks(globalParameterKeys, 10) - - api := &MockAWSSystemManagerParameterStore{} - - flags := Flags{Export: false} - - parameters := FetchParameters( - context.Background(), - *api, - parameterKeyChunks, - flags, - ) - if len(parameters) != 2 { - // TODO(kompotkot): Extract length of parameters from data.json - t.Logf("Length of parameters should be 2, but got %d", len(parameters)) - t.Fail() - } - fmt.Println(parameters) -} diff --git a/aws_ssm/data.go b/aws_ssm/data.go index 9dfbe93..646efe5 100644 --- a/aws_ssm/data.go +++ b/aws_ssm/data.go @@ -2,9 +2,8 @@ package aws_ssm // Parameter structure for storing final result from AWS SSM type Parameter struct { - Name string - Value string - Export string + Name string + Value string } // Tags for filter defined by user @@ -12,12 +11,3 @@ type FilterTag struct { Name string Value string } - -// Contains command-line flags defined by user -type Flags struct { - Export bool - MaxResults int - Outfile string - FilterTags []FilterTag - Update bool -} diff --git a/aws_ssm/data.json b/aws_ssm/data_test.json similarity index 100% rename from aws_ssm/data.json rename to aws_ssm/data_test.json diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go index 7cee74a..6a5f931 100644 --- a/aws_ssm/parameters.go +++ b/aws_ssm/parameters.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "os" "strings" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -14,7 +13,7 @@ import ( // Fetch values for parameters // Inputs: // chunks: list of lists with parameter key values -func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStore, chunks [][]string, flags Flags) []Parameter { +func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStoreAPI, chunks [][]string) []Parameter { var parameters []Parameter for _, chunk := range chunks { @@ -30,9 +29,6 @@ func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStore, c parameter := Parameter{ Name: *p.Name, Value: *p.Value, } - if flags.Export { - parameter.Export = "export " - } parameters = append(parameters, parameter) } } @@ -44,14 +40,14 @@ func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStore, c // Fetch list of parameter keys from AWS with defined filters func FetchKeysOfParameters( ctx context.Context, - api AWSSystemsManagerParameterStore, - flags Flags, + api AWSSystemsManagerParameterStoreAPI, + filterTags []FilterTag, ) []string { var parameters []string // Set parameter filters parameterFilters := []types.ParameterStringFilter{} - for _, ft := range flags.FilterTags { + for _, ft := range filterTags { filterKey := fmt.Sprintf("tag:%s", ft.Name) parameterFilters = append(parameterFilters, types.ParameterStringFilter{ Key: &filterKey, @@ -59,7 +55,7 @@ func FetchKeysOfParameters( }) } describeInput := &ssm.DescribeParametersInput{ - MaxResults: int32(flags.MaxResults), + MaxResults: 10, ParameterFilters: parameterFilters, } n := 0 @@ -126,30 +122,3 @@ func ParseFilterTags(filterTagsStr string) []FilterTag { return filterTags } - -// WriteToFile generate or update existing file and -// flash to it environment variables -func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) { - flag := os.O_TRUNC | os.O_CREATE | os.O_WRONLY - if update { - flag = os.O_APPEND | os.O_CREATE | os.O_WRONLY - } - - f, err := os.OpenFile( - outfile, - flag, - 0644, - ) - if err != nil { - log.Fatalf("Unable to open file %s, error: %s", outfile, err) - } - defer f.Close() - - parametersStr := "" - for _, p := range parameters { - parametersStr += fmt.Sprintf("%s%s=%s\n", p.Export, p.Name, p.Value) - } - if _, err := f.WriteString(parametersStr); err != nil { - log.Fatalf("Unable to write to file %s, error: %s", outfile, err) - } -} diff --git a/checkenv.go b/checkenv.go index 62b723a..d66f0d2 100644 --- a/checkenv.go +++ b/checkenv.go @@ -108,13 +108,13 @@ func main() { } for providerSpec := range spec.providersFull { - fmt.Printf("# %s - all variables:\n", providerSpec) + fmt.Printf("# Generated with %s - all variables:\n", providerSpec) for k, v := range providedVars[providerSpec] { fmt.Printf("%s%s=%s\n", exportPrefix, k, v) } } for providerSpec, queriedVars := range spec.providerVars { - fmt.Printf("# %s - specific variables:\n", providerSpec) + fmt.Printf("# Generated with %s - specific variables:\n", providerSpec) definedVars := providedVars[providerSpec] for k := range queriedVars { v, ok := definedVars[k] From f24785f886947a608cbaaebae5d9da5290700761 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 20 Dec 2021 20:50:03 +0000 Subject: [PATCH 9/9] CHECKENV_AWS_FETCH_LOOP_LIMIT to prevent fetching AWS too many params --- aws_ssm/parameters.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/aws_ssm/parameters.go b/aws_ssm/parameters.go index 6a5f931..dad09c8 100644 --- a/aws_ssm/parameters.go +++ b/aws_ssm/parameters.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log" + "os" + "strconv" "strings" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -58,6 +60,21 @@ func FetchKeysOfParameters( MaxResults: 10, ParameterFilters: parameterFilters, } + + // CHECKENV_AWS_FETCH_LOOP_LIMIT by default set to 10, + // it is allows to load 100 parameters from AWS and it is + // a limiter to prevent loading too many parameters without + // control during passing erroneous filters + var err error + var fetchLoopLimit int + fetchLoopLimitStr := os.Getenv("CHECKENV_AWS_FETCH_LOOP_LIMIT") + if fetchLoopLimitStr != "" { + fetchLoopLimit, err = strconv.Atoi(fetchLoopLimitStr) + } + if fetchLoopLimitStr == "" || err != nil { + fetchLoopLimit = 10 + } + n := 0 for { // Fetch list of parameter keys @@ -76,7 +93,7 @@ func FetchKeysOfParameters( describeInput.NextToken = results.NextToken n++ - if n >= 50 { + if n >= fetchLoopLimit { log.Fatal("To many iterations over DescribeParameters loop") } }