From 23cfffa46f41f3327b3b254c2bc1e425c27a5c87 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 10 Jul 2023 16:26:16 +0200 Subject: [PATCH 01/16] Embed git version into tracing info --- .github/workflows/release.yml | 3 +++ .github/workflows/tests.yml | 7 +++++++ .gitignore | 1 + tracing/main.go | 10 ++++++---- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3cf709cd..3574176b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,9 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 + - name: Go Generate + run: go generate ./... + - name: Run GoReleaser (publish) uses: goreleaser/goreleaser-action@v4 if: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17dc24b1..97a8c1d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,6 +16,9 @@ jobs: run: | go get -v -t -d ./... + - name: Go Generate + run: go generate ./... + - name: Go Vet run: go vet ./... @@ -32,6 +35,10 @@ jobs: with: go-version: '1.20' cache: false + + - name: Go Generate + run: go generate ./... + - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/.gitignore b/.gitignore index 6f2dc77f..2db3b32d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ go.work dist/ +tracing/commit.txt diff --git a/tracing/main.go b/tracing/main.go index 83969e71..346a375d 100644 --- a/tracing/main.go +++ b/tracing/main.go @@ -2,6 +2,7 @@ package tracing import ( "context" + _ "embed" "fmt" "os" "runtime/debug" @@ -23,10 +24,11 @@ import ( "go.opentelemetry.io/otel/trace" ) -const ( - instrumentationName = "github.com/overmindtech/ovm-cli" - instrumentationVersion = "0.0.1" -) +//go:generate sh -c "echo -n $(git describe --long) > commit.txt" +//go:embed commit.txt +var instrumentationVersion string + +const instrumentationName = "github.com/overmindtech/ovm-cli" var ( tracer = otel.GetTracerProvider().Tracer( From 19ac06c6ae6922bc70fa2ac6a639f4a910f90c17 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 10 Jul 2023 16:37:01 +0200 Subject: [PATCH 02/16] Add aws-source symlink/checkout to support generating tf transforms from it --- .github/actions/go_init/action.yml | 35 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 12 ++-------- .github/workflows/tests.yml | 19 ++++------------ sources/aws-source | 1 + 4 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 .github/actions/go_init/action.yml create mode 120000 sources/aws-source diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml new file mode 100644 index 00000000..5b6d85b9 --- /dev/null +++ b/.github/actions/go_init/action.yml @@ -0,0 +1,35 @@ +name: Go Init +description: Initializes go and runs go generate + +runs: + using: "composite" + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v4 + + - name: Reset sources + shell: bash + run: rm -Rf sources/ + + - name: Checkout + uses: actions/checkout@v3 + with: + repository: overmindtech/aws-source + path: sources/aws-source + + - name: Go Generate + shell: bash + run: | + go generate ./... + if [ -z "$(git status --porcelain)" ]; then + echo "No pending changes from `go generate`" + else + echo "Pending changes from `go generate` found, please run 'go generate ./...' and commit the changes" + git status + exit 1 + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3574176b..77c8265a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,16 +12,8 @@ jobs: contents: write packages: write steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v4 - - - name: Go Generate - run: go generate ./... + - name: Go Init + uses: ./.github/actions/go_init/action.yml - name: Run GoReleaser (publish) uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 97a8c1d3..3a3cf765 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,18 +7,13 @@ jobs: env: CGO_ENABLED: 0 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: 1.x + - name: Go Init + uses: ./.github/actions/go_init/action.yml - name: Get dependencies run: | go get -v -t -d ./... - - name: Go Generate - run: go generate ./... - - name: Go Vet run: go vet ./... @@ -30,14 +25,8 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '1.20' - cache: false - - - name: Go Generate - run: go generate ./... + - name: Go Init + uses: ./.github/actions/go_init/action.yml - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/sources/aws-source b/sources/aws-source new file mode 120000 index 00000000..1f473996 --- /dev/null +++ b/sources/aws-source @@ -0,0 +1 @@ +../../aws-source/ \ No newline at end of file From 82cd9d51ca63448c37f93d63670e9e7bd53032e3 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 10:40:03 +0200 Subject: [PATCH 03/16] First prototype of tfplan parsing and evaluating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All input data still manually assembled for testing purposes * add `go generate` script `extractmaps.go` as a placeholder for transforming the docs-data from aws-source into TfMapData. The data in extractmaps is manually copied over from aws-source for now. * add example resource from our dogfood environment plan to be extracted from `terraform show -json` * map the example resource into an `sdp.Query` and send that off as changing item to the rest of the processing Working run: ``` vscode ➜ /workspace/ovm-cli (main) $ go run main.go change-from-tfplan --frontend https://frontend-knkxto8fa.preview.overmind-demo.com/ ERRO[0000] Error reading config file err="Config File \"config\" Not Found in \"[]\"" INFO[0000] set log level from config fields.level=trace INFO[0000] otlptracehttp client configured itself: &{traces {api.honeycomb.io false map[x-honeycomb-team:Quu6uCifruYihvi1Lzj3MC] 0 10000000000 /v1/traces } {{api.honeycomb.io false map[x-honeycomb-team:Quu6uCifruYihvi1Lzj3MC] 0 10000000000 /v1/traces } {true 5000000000 30000000000 60000000000} 0 [] } 0xc9f440 0xc0005540f0 0xc0000483c0 {0 {0 0}}} DEBU[0000] Connecting to overmind API: https://api.df.overmind-demo.com/api/gateway [auth redacted] INFO[0017] created a new change change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 url="https://api.df.overmind-demo.com/api/gateway" INFO[0017] resolving items from terraform plan change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 url="https://api.df.overmind-demo.com/api/gateway" DEBU[0017] query status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 query=99d81090-03dd-4f6d-8783-d3360c68a446 status=STARTED url="https://api.df.overmind-demo.com/api/gateway" DEBU[0017] query status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 query=0588a10a-9cc1-41d0-abb5-02767172578f status=STARTED url="https://api.df.overmind-demo.com/api/gateway" INFO[0017] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=944651592624.eu-west-2.elbv2-load-balancer.ingress url="https://api.df.overmind-demo.com/api/gateway" INFO[0018] still waiting for responders change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 post_processing_complete=false queriesSent=true responders=2 summary="working:1 complete:1 responders:2" url="https://api.df.overmind-demo.com/api/gateway" DEBU[0018] query status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 query=99d81090-03dd-4f6d-8783-d3360c68a446 status=FINISHED url="https://api.df.overmind-demo.com/api/gateway" INFO[0019] still waiting for responders change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 post_processing_complete=false queriesSent=true responders=2 summary="working:1 complete:1 responders:2" url="https://api.df.overmind-demo.com/api/gateway" INFO[0019] still waiting for responders change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 post_processing_complete=false queriesSent=true responders=2 summary="working:1 complete:1 responders:2" url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AWSSSOServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.job-function/ViewOnlyAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.ReadOnlyAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.AdministratorAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AWSConfigServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AWSOrganizationsServiceTrustPolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AmazonSSMServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AWSTrustedAdvisorServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/AWSSupportServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.service-role/AWSLambdaBasicExecutionRole url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.aws-service-role/CloudTrailServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.AWSOrganizationsFullAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.PowerUserAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=aws.iam-policy.service-role/AWS_ConfigRole url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AWSSSOServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.job-function/ViewOnlyAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.AdministratorAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.ReadOnlyAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AWSOrganizationsServiceTrustPolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.AWSOrganizationsFullAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AWSSupportServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AmazonSSMServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AWSTrustedAdvisorServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.service-role/AWSLambdaBasicExecutionRole url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/CloudTrailServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.aws-service-role/AWSConfigServiceRolePolicy url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.PowerUserAccess url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] new item change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 item=337346983342.iam-policy.service-role/AWS_ConfigRole url="https://api.df.overmind-demo.com/api/gateway" DEBU[0023] query status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 query=0588a10a-9cc1-41d0-abb5-02767172578f status=FINISHED url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] all responders and queries done allDone=true change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 post_processing_complete=true queriesSent=true responders=2 summary="complete:2 responders:2" url="https://api.df.overmind-demo.com/api/gateway" INFO[0023] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING" url="https://api.df.overmind-demo.com/api/gateway" INFO[0024] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:1" url="https://api.df.overmind-demo.com/api/gateway" INFO[0025] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:10 numEdges:11" url="https://api.df.overmind-demo.com/api/gateway" INFO[0025] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:12 numEdges:13" url="https://api.df.overmind-demo.com/api/gateway" INFO[0026] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:14 numEdges:13" url="https://api.df.overmind-demo.com/api/gateway" INFO[0027] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:16 numEdges:13" url="https://api.df.overmind-demo.com/api/gateway" INFO[0028] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:17 numEdges:13" url="https://api.df.overmind-demo.com/api/gateway" INFO[0029] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:19 numEdges:13" url="https://api.df.overmind-demo.com/api/gateway" INFO[0030] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:33 numEdges:26" url="https://api.df.overmind-demo.com/api/gateway" INFO[0030] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:39 numEdges:35" url="https://api.df.overmind-demo.com/api/gateway" INFO[0030] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:44 numEdges:43" url="https://api.df.overmind-demo.com/api/gateway" INFO[0031] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:49 numEdges:47" url="https://api.df.overmind-demo.com/api/gateway" INFO[0031] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DISCOVERING numItems:51 numEdges:48" url="https://api.df.overmind-demo.com/api/gateway" INFO[0032] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_FINDING_APPS numItems:52 numEdges:55" url="https://api.df.overmind-demo.com/api/gateway" INFO[0032] status update change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 fields.msg="state:STATE_DONE numItems:52 numEdges:55" url="https://api.df.overmind-demo.com/api/gateway" INFO[0032] change ready change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 change-url="https://frontend-knkxto8fa.preview.overmind-demo.com//changes/b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2" url="https://api.df.overmind-demo.com/api/gateway" INFO[0033] Websocket closing change=b0209f6c-5b46-4ce5-8fae-b9d4d3c612a2 code=StatusNormalClosure reason= url="https://api.df.overmind-demo.com/api/gateway" TRAC[0033] tracing has shut down vscode ➜ /workspace/ovm-cli (main) $ ``` --- cmd/changefromtfplan.go | 55 ++++++++++++++++++++++- cmd/datamaps/awssource.go | 32 ++++++++++++++ cmd/datamaps/types.go | 12 +++++ extractmaps.go | 93 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 cmd/datamaps/awssource.go create mode 100644 cmd/datamaps/types.go create mode 100644 extractmaps.go diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index fd796e54..a8546cae 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -11,6 +11,7 @@ import ( "github.com/bufbuild/connect-go" "github.com/google/uuid" + "github.com/overmindtech/ovm-cli/cmd/datamaps" "github.com/overmindtech/ovm-cli/tracing" "github.com/overmindtech/sdp-go" log "github.com/sirupsen/logrus" @@ -73,13 +74,63 @@ var ( } ) -func changingItemQueriesFromTfplan() []*sdp.Query { +type TfData struct { + Address string + Type string + Values map[string]string +} + +func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) []*sdp.Query { + // read results from `terraform show -json ${tfplan file}` + exampleResults := map[string]TfData{ + "aws_iam_policy.aws_source_assume_customer": { + Address: "aws_iam_policy.aws_source_assume_customer", + Type: "aws_iam_policy", + Values: map[string]string{ + "arn": "arn:aws:iam::944651592624:policy/AwsSourceAssumeCustomerRole", + "description": "Allows the aws-source pod to assume the provided customer roles", + "id": "arn:aws:iam::944651592624:policy/AwsSourceAssumeCustomerRole", + "name": "AwsSourceAssumeCustomerRole", + // ... + }, + }, + } + var changing_items []*sdp.Query if viper.GetBool("test-affecting") { changing_items = []*sdp.Query{affecting_resource} } else { changing_items = []*sdp.Query{safe_resource} } + + // for all managed resources: + for _, r := range exampleResults { + mapData, ok := datamaps.AwssourceData[r.Type] + if !ok { + log.WithContext(ctx).WithFields(lf).WithField("terraform-address", r.Address).Warn("skipping unmapped resource") + break + } + + queryStr, ok := r.Values[mapData.QueryField] + if !ok { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", r.Address). + WithField("terraform-query-field", mapData.QueryField).Warn("skipping resource without query field") + break + } + + u := uuid.New() + changing_items = append(changing_items, &sdp.Query{ + Type: mapData.Type, + Method: mapData.Method, + Query: queryStr, + Scope: mapData.Scope, + RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, + UUID: u[:], + }) + } + return changing_items } @@ -133,7 +184,7 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { log.WithContext(ctx).WithFields(lf).Info("created a new change") log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") - queries := changingItemQueriesFromTfplan() + queries := changingItemQueriesFromTfplan(ctx, lf) options := &websocket.DialOptions{ HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), diff --git a/cmd/datamaps/awssource.go b/cmd/datamaps/awssource.go new file mode 100644 index 00000000..ec48ccdc --- /dev/null +++ b/cmd/datamaps/awssource.go @@ -0,0 +1,32 @@ +// Code generated by "extractmaps aws-source"; DO NOT EDIT + +package datamaps + +import "github.com/overmindtech/sdp-go" + +var AwssourceData = map[string]TfMapData{ + "aws_autoscaling_group": { + Type: "autoscaling-auto-scaling-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + "aws_iam_policy": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + "aws_iam_role_policy_attachment": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + "aws_iam_user_policy_attachment": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, +} diff --git a/cmd/datamaps/types.go b/cmd/datamaps/types.go new file mode 100644 index 00000000..675fcbf8 --- /dev/null +++ b/cmd/datamaps/types.go @@ -0,0 +1,12 @@ +package datamaps + +import "github.com/overmindtech/sdp-go" + +//go:generate go run ../../extractmaps.go aws-source + +type TfMapData struct { + Type string + Method sdp.QueryMethod + QueryField string + Scope string +} diff --git a/extractmaps.go b/extractmaps.go new file mode 100644 index 00000000..7c1a1670 --- /dev/null +++ b/extractmaps.go @@ -0,0 +1,93 @@ +//go:build ignore + +package main + +import ( + "fmt" + "html/template" + "os" + "strings" +) + +type Args struct { + Source string + SourceMunged string +} + +func main() { + fmt.Printf("Running %s go on %s\n", os.Args[0], os.Getenv("GOFILE")) + + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + fmt.Printf(" cwd = %s\n", cwd) + fmt.Printf(" os.Args = %#v\n", os.Args) + + for _, ev := range []string{"GOARCH", "GOOS", "GOFILE", "GOLINE", "GOPACKAGE", "DOLLAR"} { + fmt.Println(" ", ev, "=", os.Getenv(ev)) + } + + if len(os.Args) < 2 { + panic("Missing argument, aborting") + } + + args := Args{ + Source: os.Args[1], + SourceMunged: strings.ReplaceAll(os.Args[1], "-", ""), + } + + funcMap := template.FuncMap{ + "Title": strings.Title, + } + + template := template.New("simple").Funcs(funcMap) + template, err = template.Parse(`// Code generated by "extractmaps {{.Source}}"; DO NOT EDIT + +package datamaps + +import "github.com/overmindtech/sdp-go" + +var {{.SourceMunged | Title }}Data = map[string]TfMapData{ + "aws_autoscaling_group": { + Type: "autoscaling-auto-scaling-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + "aws_iam_policy": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + "aws_iam_role_policy_attachment": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + "aws_iam_user_policy_attachment": { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, +} +`) + if err != nil { + panic(err) + } + + f, err := os.Create(fmt.Sprintf("%v.go", strings.ToLower(args.SourceMunged))) + if err != nil { + panic(err) + } + defer f.Close() + + fmt.Printf("Generating handler for %v\n", args) + err = template.Execute(f, args) + if err != nil { + panic(err) + } +} From 8a1822a128a6ffa06757e00a97fcc830e918ee6e Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 15:11:26 +0200 Subject: [PATCH 04/16] Generate aws-source mappings Since some sources can have overlapping queries, this adds another layer of array into the data structure. --- cmd/changefromtfplan.go | 38 +- cmd/datamaps/awssource.go | 738 +++++++++++++++++++++++++++++++++++++- extractmaps.go | 97 +++-- 3 files changed, 812 insertions(+), 61 deletions(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index a8546cae..cc8f2194 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -105,30 +105,32 @@ func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) []*sdp.Qu // for all managed resources: for _, r := range exampleResults { - mapData, ok := datamaps.AwssourceData[r.Type] + mappings, ok := datamaps.AwssourceData[r.Type] if !ok { log.WithContext(ctx).WithFields(lf).WithField("terraform-address", r.Address).Warn("skipping unmapped resource") break } - queryStr, ok := r.Values[mapData.QueryField] - if !ok { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", r.Address). - WithField("terraform-query-field", mapData.QueryField).Warn("skipping resource without query field") - break - } + for _, mapData := range mappings { + queryStr, ok := r.Values[mapData.QueryField] + if !ok { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", r.Address). + WithField("terraform-query-field", mapData.QueryField).Warn("skipping resource without query field") + break + } - u := uuid.New() - changing_items = append(changing_items, &sdp.Query{ - Type: mapData.Type, - Method: mapData.Method, - Query: queryStr, - Scope: mapData.Scope, - RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, - UUID: u[:], - }) + u := uuid.New() + changing_items = append(changing_items, &sdp.Query{ + Type: mapData.Type, + Method: mapData.Method, + Query: queryStr, + Scope: mapData.Scope, + RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, + UUID: u[:], + }) + } } return changing_items diff --git a/cmd/datamaps/awssource.go b/cmd/datamaps/awssource.go index ec48ccdc..bde94178 100644 --- a/cmd/datamaps/awssource.go +++ b/cmd/datamaps/awssource.go @@ -4,29 +4,733 @@ package datamaps import "github.com/overmindtech/sdp-go" -var AwssourceData = map[string]TfMapData{ +var AwssourceData = map[string][]TfMapData{ + "aws_alb_listener": { + { + Type: "elbv2-listener", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_alb_listener_rule": { + { + Type: "elbv2-rule", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_alb_target_group": { + { + Type: "elbv2-target-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ami": { + { + Type: "ec2-image", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, "aws_autoscaling_group": { - Type: "autoscaling-auto-scaling-group", - Method: sdp.QueryMethod_GET, - QueryField: "name", - Scope: "*", + { + Type: "autoscaling-auto-scaling-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_cloudwatch_metric_alarm": { + { + Type: "cloudwatch-alarm", + Method: sdp.QueryMethod_GET, + QueryField: "alarm_name", + Scope: "*", + }, + }, + "aws_db_instance": { + { + Type: "rds-db-instance", + Method: sdp.QueryMethod_GET, + QueryField: "identifier", + Scope: "*", + }, + }, + "aws_db_instance_role_association": { + { + Type: "rds-db-instance", + Method: sdp.QueryMethod_GET, + QueryField: "db_instance_identifier", + Scope: "*", + }, + }, + "aws_db_option_group": { + { + Type: "rds-option-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_db_parameter_group": { + { + Type: "rds-db-parameter-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_db_subnet_group": { + { + Type: "rds-db-subnet-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_dynamodb_table": { + { + Type: "dynamodb-table", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ebs_volume": { + { + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_ecs_capacity_provider": { + { + Type: "ecs-capacity-provider", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ecs_cluster": { + { + Type: "ecs-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ecs_service": { + { + Type: "ecs-service", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_ecs_task_definition": { + { + Type: "ecs-task-definition", + Method: sdp.QueryMethod_GET, + QueryField: "revision", + Scope: "*", + }, + }, + "aws_efs_access_point": { + { + Type: "efs-access-point", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_backup_policy": { + { + Type: "efs-backup-policy", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_file_system": { + { + Type: "efs-file-system", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_mount_target": { + { + Type: "efs-mount-target", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_replication_configuration": { + { + Type: "efs-replication-configuration", + Method: sdp.QueryMethod_GET, + QueryField: "source_file_system_id", + Scope: "*", + }, + }, + "aws_eks_addon": { + { + Type: "eks-addon", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_eks_cluster": { + { + Type: "eks-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_eks_fargate_profile": { + { + Type: "eks-fargate-profile", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_eks_node_group": { + { + Type: "eks-nodegroup", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_elb": { + { + Type: "elb-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_elb_attachment": { + { + Type: "elb-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "elb", + Scope: "*", + }, + }, + "aws_iam_group": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_iam_group_membership": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "group", + Scope: "*", + }, }, "aws_iam_policy": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "arn", - Scope: "*", + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_iam_role": { + { + Type: "iam-role", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, }, "aws_iam_role_policy_attachment": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "policy_arn", - Scope: "*", + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + { + Type: "iam-role", + Method: sdp.QueryMethod_GET, + QueryField: "role", + Scope: "*", + }, + }, + "aws_iam_user": { + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_iam_user_group_membership": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "group", + Scope: "*", + }, + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, + }, + "aws_iam_user_policy": { + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, }, "aws_iam_user_policy_attachment": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "policy_arn", - Scope: "*", + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, + }, + "aws_instance": { + { + Type: "ec2-instance", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_internet_gateway": { + { + Type: "ec2-internet-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_key_pair": { + { + Type: "ec2-key-pair", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_lambda_function": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_lambda_function_event_invoke_config": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_function_url": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_invocation": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_layer_version": { + { + Type: "lambda-layer-version", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_launch_template": { + { + Type: "ec2-launch-template", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_lb": { + { + Type: "elbv2-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_lb_listener": { + { + Type: "elbv2-listener", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_lb_listener_rule": { + { + Type: "elbv2-rule", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_lb_target_group": { + { + Type: "elbv2-target-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_nat_gateway": { + { + Type: "ec2-nat-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_network_acl": { + { + Type: "ec2-network-acl", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_network_interface": { + { + Type: "ec2-network-interface", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_placement_group": { + { + Type: "ec2-placement-group", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_rds_cluster": { + { + Type: "rds-db-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "cluster_identifier", + Scope: "*", + }, + }, + "aws_rds_cluster_parameter_group": { + { + Type: "rds-db-cluster-parameter-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_route53_health_check": { + { + Type: "route53-health-check", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route53_hosted_zone_dnssec": { + { + Type: "route53-hosted-zone", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route53_record": { + { + Type: "route53-resource-record-set", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_route_table": { + { + Type: "ec2-route-table", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route_table_association": { + { + Type: "ec2-route-table", + Method: sdp.QueryMethod_GET, + QueryField: "route_table_id", + Scope: "*", + }, + { + Type: "ec2-subnet", + Method: sdp.QueryMethod_GET, + QueryField: "subnet_id", + Scope: "*", + }, + }, + "aws_s3_bucket": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_s3_bucket_acl": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_analytics_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_cors_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_intelligent_tiering_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_inventory": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_lifecycle_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_logging": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_metric": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_notification": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_object": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_object_lock_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_ownership_controls": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_policy": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_public_access_block": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_replication_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_request_payment_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_server_side_encryption_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_versioning": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_website_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_object": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_object_copy": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_security_group": { + { + Type: "ec2-security-group", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_security_group_rule": { + { + Type: "ec2-security-group-rule", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_subnet": { + { + Type: "ec2-subnet", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_vpc": { + { + Type: "ec2-vpc", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "egress_only_internet_gateway": { + { + Type: "ec2-egress-only-internet-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, }, } diff --git a/extractmaps.go b/extractmaps.go index 7c1a1670..526a7a79 100644 --- a/extractmaps.go +++ b/extractmaps.go @@ -3,6 +3,7 @@ package main import ( + "encoding/json" "fmt" "html/template" "os" @@ -12,6 +13,7 @@ import ( type Args struct { Source string SourceMunged string + Data map[string][]map[string]string } func main() { @@ -35,6 +37,7 @@ func main() { args := Args{ Source: os.Args[1], SourceMunged: strings.ReplaceAll(os.Args[1], "-", ""), + Data: dataFromFiles(fmt.Sprintf("../../sources/%v/docs-data", os.Args[1])), } funcMap := template.FuncMap{ @@ -48,31 +51,16 @@ package datamaps import "github.com/overmindtech/sdp-go" -var {{.SourceMunged | Title }}Data = map[string]TfMapData{ - "aws_autoscaling_group": { - Type: "autoscaling-auto-scaling-group", - Method: sdp.QueryMethod_GET, - QueryField: "name", - Scope: "*", - }, - "aws_iam_policy": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "arn", - Scope: "*", - }, - "aws_iam_role_policy_attachment": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "policy_arn", - Scope: "*", - }, - "aws_iam_user_policy_attachment": { - Type: "iam-policy", - Method: sdp.QueryMethod_LIST, - QueryField: "policy_arn", - Scope: "*", - }, +var {{.SourceMunged | Title }}Data = map[string][]TfMapData{ +{{- range $key, $mappings := .Data}} + "{{$key}}": { {{- range $mappings}} + { + Type: "{{index . "type"}}", + Method: sdp.QueryMethod_{{index . "method"}}, + QueryField: "{{index . "query-field"}}", + Scope: "{{index . "scope"}}", + },{{end}} + },{{end}} } `) if err != nil { @@ -85,9 +73,66 @@ var {{.SourceMunged | Title }}Data = map[string]TfMapData{ } defer f.Close() - fmt.Printf("Generating handler for %v\n", args) + fmt.Printf("Generating handler for %v\n", args.Source) err = template.Execute(f, args) if err != nil { panic(err) } } + +func dataFromFiles(path string) map[string][]map[string]string { + result := map[string][]map[string]string{} + entries, err := os.ReadDir(path) + if err != nil { + panic(err) + } + + for _, e := range entries { + if e.IsDir() { + continue + } + + if !strings.HasSuffix(e.Name(), ".json") { + continue + } + + fmt.Println(e.Name()) + contents, err := os.ReadFile(fmt.Sprintf("%v/%v", path, e.Name())) + if err != nil { + panic(err) + } + + var parsed map[string]any + err = json.Unmarshal(contents, &parsed) + if err != nil { + panic(err) + } + + queries, ok := parsed["terraformQuery"] + if !ok { + // skip if we don't have terraform query data + continue + } + for _, qAny := range queries.([]interface{}) { + q := qAny.(string) + data := map[string]string{ + "type": parsed["type"].(string), + } + + qSplit := strings.SplitN(q, ".", 2) + data["query-type"] = qSplit[0] + data["query-field"] = qSplit[1] + + data["scope"] = parsed["terraformScope"].(string) + switch parsed["terraformMethod"].(string) { + case "GET": + data["method"] = "GET" + case "SEARCH": + data["method"] = "LIST" + } + + result[data["query-type"]] = append(result[data["query-type"]], data) + } + } + return result +} From e65ff58852e63cefaba66fad5d6fc463c6fd68da Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:12:44 +0200 Subject: [PATCH 05/16] Mapping tf resources to SDP queries --- cmd/changefromtfplan.go | 86 +++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index cc8f2194..d907e6ec 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -77,34 +78,46 @@ var ( type TfData struct { Address string Type string - Values map[string]string + Values map[string]any } -func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) []*sdp.Query { +func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) ([]*sdp.Query, error) { // read results from `terraform show -json ${tfplan file}` - exampleResults := map[string]TfData{ - "aws_iam_policy.aws_source_assume_customer": { - Address: "aws_iam_policy.aws_source_assume_customer", - Type: "aws_iam_policy", - Values: map[string]string{ - "arn": "arn:aws:iam::944651592624:policy/AwsSourceAssumeCustomerRole", - "description": "Allows the aws-source pod to assume the provided customer roles", - "id": "arn:aws:iam::944651592624:policy/AwsSourceAssumeCustomerRole", - "name": "AwsSourceAssumeCustomerRole", - // ... - }, - }, + contents, err := os.ReadFile(viper.GetString("tfplan-json")) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", viper.GetString("tfplan-json"), err) } - var changing_items []*sdp.Query - if viper.GetBool("test-affecting") { - changing_items = []*sdp.Query{affecting_resource} - } else { - changing_items = []*sdp.Query{safe_resource} + changing_items_tf := map[string]TfData{} + + var parsed map[string]any + json.Unmarshal(contents, &parsed) + root_module := parsed["planned_values"].(map[string]any)["root_module"].(map[string]any) + resourceValues := map[string]map[string]any{} + resourceValuesFromModule(root_module, &resourceValues) + + resource_changes := parsed["resource_changes"].([]any) + for _, changed_item_data := range resource_changes { + changed_item_map := changed_item_data.(map[string]any) + change := changed_item_map["change"].(map[string]any) + actions := change["actions"].([]any) + if len(actions) == 0 || actions[0] == "no-op" { + // skip resources with no changes + continue + } + + changed_item := TfData{ + Address: changed_item_map["address"].(string), + Type: changed_item_map["type"].(string), + Values: resourceValues[changed_item_map["address"].(string)], + } + + changing_items_tf[changed_item.Address] = changed_item } + var changing_items []*sdp.Query // for all managed resources: - for _, r := range exampleResults { + for _, r := range changing_items_tf { mappings, ok := datamaps.AwssourceData[r.Type] if !ok { log.WithContext(ctx).WithFields(lf).WithField("terraform-address", r.Address).Warn("skipping unmapped resource") @@ -125,7 +138,7 @@ func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) []*sdp.Qu changing_items = append(changing_items, &sdp.Query{ Type: mapData.Type, Method: mapData.Method, - Query: queryStr, + Query: queryStr.(string), Scope: mapData.Scope, RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, UUID: u[:], @@ -133,7 +146,25 @@ func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) []*sdp.Qu } } - return changing_items + return changing_items, nil +} + +func resourceValuesFromModule(module map[string]any, result *map[string]map[string]any) { + resource_data, ok := module["resources"] + if ok { + for _, r := range resource_data.([]any) { + resource := r.(map[string]any) + (*result)[resource["address"].(string)] = resource["values"].(map[string]any) + } + } + + child_modules_data, ok := module["child_modules"] + if ok { + for _, cm := range child_modules_data.([]any) { + child_module := cm.(map[string]any) + resourceValuesFromModule(child_module, result) + } + } } func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { @@ -186,12 +217,17 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { log.WithContext(ctx).WithFields(lf).Info("created a new change") log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") - queries := changingItemQueriesFromTfplan(ctx, lf) + queries, err := changingItemQueriesFromTfplan(ctx, lf) + if err != nil { + log.WithContext(ctx).WithError(err).WithFields(lf).Error("failed to read terraform plan") + return 1 + } options := &websocket.DialOptions{ HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), } + log.WithContext(ctx).WithFields(lf).WithField("item_count", len(queries)).Info("identifying items") c, _, err := websocket.Dial(ctx, viper.GetString("url"), options) if err != nil { log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to connect to overmind API") @@ -345,6 +381,7 @@ responses: } } + log.WithContext(ctx).WithFields(lf).Info("updating changing items on the change record") resultStream, err := client.UpdateChangingItems(ctx, &connect.Request[sdp.UpdateChangingItemsRequest]{ Msg: &sdp.UpdateChangingItemsRequest{ ChangeUUID: createResponse.Msg.Change.Metadata.UUID, @@ -423,8 +460,7 @@ func init() { changeFromTfplanCmd.PersistentFlags().String("changes-url", "https://api.prod.overmind.tech", "The changes service API endpoint") changeFromTfplanCmd.PersistentFlags().String("frontend", "https://app.overmind.tech", "The frontend base URL") - changeFromTfplanCmd.PersistentFlags().String("terraform", "terraform", "The binary to use for calling terraform. Will be looked up in the system PATH.") - changeFromTfplanCmd.PersistentFlags().String("tfplan", "./tfplan", "Parse changing items from this terraform plan file.") + changeFromTfplanCmd.PersistentFlags().String("tfplan-json", "./tfplan.json", "Parse changing items from this terraform plan JSON file. Generate this using `terraform show -json PLAN_FILE`") changeFromTfplanCmd.PersistentFlags().String("title", "", "Short title for this change.") changeFromTfplanCmd.PersistentFlags().String("description", "", "Quick description of the change.") From 4fc5c94345d0def422bb35426e1d5b5c864b641c Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:30:34 +0200 Subject: [PATCH 06/16] Avoid infinite loop when there's no item mapped --- cmd/changefromtfplan.go | 294 +++++++++++++++++++++------------------- 1 file changed, 151 insertions(+), 143 deletions(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index d907e6ec..bd6c2173 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -196,6 +196,13 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") + queries, err := changingItemQueriesFromTfplan(ctx, lf) + if err != nil { + log.WithContext(ctx).WithError(err).WithFields(lf).Error("failed to read terraform plan") + return 1 + } + client := AuthenticatedChangesClient(ctx) createResponse, err := client.CreateChange(ctx, &connect.Request[sdp.CreateChangeRequest]{ Msg: &sdp.CreateChangeRequest{ @@ -216,172 +223,173 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { lf["change"] = createResponse.Msg.Change.Metadata.GetUUIDParsed() log.WithContext(ctx).WithFields(lf).Info("created a new change") - log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") - queries, err := changingItemQueriesFromTfplan(ctx, lf) - if err != nil { - log.WithContext(ctx).WithError(err).WithFields(lf).Error("failed to read terraform plan") - return 1 - } - - options := &websocket.DialOptions{ - HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), - } + receivedItems := []*sdp.Reference{} - log.WithContext(ctx).WithFields(lf).WithField("item_count", len(queries)).Info("identifying items") - c, _, err := websocket.Dial(ctx, viper.GetString("url"), options) - if err != nil { - log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to connect to overmind API") - return 1 - } - defer c.Close(websocket.StatusGoingAway, "") - - // the default, 32kB is too small for cert bundles and rds-db-cluster-parameter-groups - c.SetReadLimit(2 * 1024 * 1024) - - queriesSentChan := make(chan struct{}) - go func() { - for _, q := range queries { - req := sdp.GatewayRequest{ - MinStatusInterval: minStatusInterval, - RequestType: &sdp.GatewayRequest_Query{ - Query: q, - }, - } - err = wspb.Write(ctx, c, &req) - if err != nil { - log.WithContext(ctx).WithFields(lf).WithError(err).WithField("req", &req).Error("Failed to send request") - continue - } + if len(queries) > 0 { + options := &websocket.DialOptions{ + HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), } - queriesSentChan <- struct{}{} - }() - - responses := make(chan *sdp.GatewayResponse) - // Start a goroutine that reads responses - go func() { - for { - res := new(sdp.GatewayResponse) - - err = wspb.Read(ctx, c, res) - - if err != nil { - var e websocket.CloseError - if errors.As(err, &e) { - log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ - "code": e.Code.String(), - "reason": e.Reason, - }).Info("Websocket closing") - return + log.WithContext(ctx).WithFields(lf).WithField("item_count", len(queries)).Info("identifying items") + c, _, err := websocket.Dial(ctx, viper.GetString("url"), options) + if err != nil { + log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to connect to overmind API") + return 1 + } + defer c.Close(websocket.StatusGoingAway, "") + + // the default, 32kB is too small for cert bundles and rds-db-cluster-parameter-groups + c.SetReadLimit(2 * 1024 * 1024) + + queriesSentChan := make(chan struct{}) + go func() { + for _, q := range queries { + req := sdp.GatewayRequest{ + MinStatusInterval: minStatusInterval, + RequestType: &sdp.GatewayRequest_Query{ + Query: q, + }, + } + err = wspb.Write(ctx, c, &req) + if err != nil { + log.WithContext(ctx).WithFields(lf).WithError(err).WithField("req", &req).Error("Failed to send request") + continue } - log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to read response") - return } + queriesSentChan <- struct{}{} + }() + + responses := make(chan *sdp.GatewayResponse) + + // Start a goroutine that reads responses + go func() { + for { + res := new(sdp.GatewayResponse) + + err = wspb.Read(ctx, c, res) + + if err != nil { + var e websocket.CloseError + if errors.As(err, &e) { + log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ + "code": e.Code.String(), + "reason": e.Reason, + }).Info("Websocket closing") + return + } + log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to read response") + return + } - responses <- res - } - }() - - activeQueries := make(map[uuid.UUID]bool) - queriesSent := false - - receivedItems := []*sdp.Reference{} - - // Read the responses -responses: - for { - select { - case <-queriesSentChan: - queriesSent = true - - case <-signals: - log.WithContext(ctx).WithFields(lf).Info("Received interrupt, exiting") - return 1 - - case <-ctx.Done(): - log.WithContext(ctx).WithFields(lf).Info("Context cancelled, exiting") - return 1 + responses <- res + } + }() - case resp := <-responses: - switch resp.ResponseType.(type) { + activeQueries := make(map[uuid.UUID]bool) + queriesSent := false - case *sdp.GatewayResponse_Status: - status := resp.GetStatus() - statusFields := log.Fields{ - "summary": status.Summary, - "responders": status.Summary.Responders, - "queriesSent": queriesSent, - "post_processing_complete": status.PostProcessingComplete, - } + // Read the responses + responses: + for { + select { + case <-queriesSentChan: + queriesSent = true + + case <-signals: + log.WithContext(ctx).WithFields(lf).Info("Received interrupt, exiting") + return 1 + + case <-ctx.Done(): + log.WithContext(ctx).WithFields(lf).Info("Context cancelled, exiting") + return 1 + + case resp := <-responses: + switch resp.ResponseType.(type) { + + case *sdp.GatewayResponse_Status: + status := resp.GetStatus() + statusFields := log.Fields{ + "summary": status.Summary, + "responders": status.Summary.Responders, + "queriesSent": queriesSent, + "post_processing_complete": status.PostProcessingComplete, + } - if status.Done() { - // fall through from all "final" query states, check if there's still queries in progress; - // only break from the loop if all queries have already been sent - // TODO: see above, still needs DefaultStartTimeout implemented to account for slow sources - allDone := allDone(ctx, activeQueries, lf) - statusFields["allDone"] = allDone - if allDone && queriesSent { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders and queries done") - break responses + if status.Done() { + // fall through from all "final" query states, check if there's still queries in progress; + // only break from the loop if all queries have already been sent + // TODO: see above, still needs DefaultStartTimeout implemented to account for slow sources + allDone := allDone(ctx, activeQueries, lf) + statusFields["allDone"] = allDone + if allDone && queriesSent { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders and queries done") + break responses + } else { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders done, with unfinished queries") + } } else { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders done, with unfinished queries") + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("still waiting for responders") } - } else { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("still waiting for responders") - } - case *sdp.GatewayResponse_QueryStatus: - status := resp.GetQueryStatus() - statusFields := log.Fields{ - "status": status.Status.String(), - } - queryUuid := status.GetUUIDParsed() - if queryUuid == nil { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("Received QueryStatus with nil UUID") - continue responses - } - statusFields["query"] = queryUuid - - switch status.Status { - case sdp.QueryStatus_STARTED: - activeQueries[*queryUuid] = true - case sdp.QueryStatus_FINISHED: - activeQueries[*queryUuid] = false - case sdp.QueryStatus_ERRORED: - activeQueries[*queryUuid] = false - case sdp.QueryStatus_CANCELLED: - activeQueries[*queryUuid] = false - default: - statusFields["unexpected_status"] = true - } + case *sdp.GatewayResponse_QueryStatus: + status := resp.GetQueryStatus() + statusFields := log.Fields{ + "status": status.Status.String(), + } + queryUuid := status.GetUUIDParsed() + if queryUuid == nil { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("Received QueryStatus with nil UUID") + continue responses + } + statusFields["query"] = queryUuid + + switch status.Status { + case sdp.QueryStatus_STARTED: + activeQueries[*queryUuid] = true + case sdp.QueryStatus_FINISHED: + activeQueries[*queryUuid] = false + case sdp.QueryStatus_ERRORED: + activeQueries[*queryUuid] = false + case sdp.QueryStatus_CANCELLED: + activeQueries[*queryUuid] = false + default: + statusFields["unexpected_status"] = true + } - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("query status update") + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("query status update") - case *sdp.GatewayResponse_NewItem: - item := resp.GetNewItem() - log.WithContext(ctx).WithFields(lf).WithField("item", item.GloballyUniqueName()).Infof("new item") + case *sdp.GatewayResponse_NewItem: + item := resp.GetNewItem() + log.WithContext(ctx).WithFields(lf).WithField("item", item.GloballyUniqueName()).Infof("new item") - receivedItems = append(receivedItems, item.Reference()) + receivedItems = append(receivedItems, item.Reference()) - case *sdp.GatewayResponse_NewEdge: - log.WithContext(ctx).WithFields(lf).Debug("ignored edge") + case *sdp.GatewayResponse_NewEdge: + log.WithContext(ctx).WithFields(lf).Debug("ignored edge") - case *sdp.GatewayResponse_QueryError: - err := resp.GetQueryError() - log.WithContext(ctx).WithFields(lf).WithError(err).Errorf("Error from %v(%v)", err.ResponderName, err.SourceName) + case *sdp.GatewayResponse_QueryError: + err := resp.GetQueryError() + log.WithContext(ctx).WithFields(lf).WithError(err).Errorf("Error from %v(%v)", err.ResponderName, err.SourceName) - case *sdp.GatewayResponse_Error: - err := resp.GetError() - log.WithContext(ctx).WithFields(lf).WithField(log.ErrorKey, err).Errorf("generic error") + case *sdp.GatewayResponse_Error: + err := resp.GetError() + log.WithContext(ctx).WithFields(lf).WithField(log.ErrorKey, err).Errorf("generic error") - default: - j := protojson.Format(resp) - log.WithContext(ctx).WithFields(lf).Infof("Unknown %T Response:\n%v", resp.ResponseType, j) + default: + j := protojson.Format(resp) + log.WithContext(ctx).WithFields(lf).Infof("Unknown %T Response:\n%v", resp.ResponseType, j) + } } } + } else { + log.WithContext(ctx).WithFields(lf).Info("no item queries mapped, skipping changing items") } - log.WithContext(ctx).WithFields(lf).Info("updating changing items on the change record") + if len(receivedItems) > 0 { + log.WithContext(ctx).WithFields(lf).WithField("received_items", len(receivedItems)).Info("updating changing items on the change record") + } else { + log.WithContext(ctx).WithFields(lf).WithField("received_items", len(receivedItems)).Info("updating change record with no items") + } resultStream, err := client.UpdateChangingItems(ctx, &connect.Request[sdp.UpdateChangingItemsRequest]{ Msg: &sdp.UpdateChangingItemsRequest{ ChangeUUID: createResponse.Msg.Change.Metadata.UUID, From 36646c3381f953086abece9f6c410dd726cf7d69 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:30:48 +0200 Subject: [PATCH 07/16] Add debug logging for which items are getting mapped --- cmd/changefromtfplan.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index bd6c2173..2ef7b1ae 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -105,6 +105,10 @@ func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) ([]*sdp.Q // skip resources with no changes continue } + log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ + "actions": actions, + "address": changed_item_map["address"], + }).Debugf("mapping item") changed_item := TfData{ Address: changed_item_map["address"].(string), From b75079bfdcd5e8ef60b9d03a1db0c48701cfbe11 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:34:39 +0200 Subject: [PATCH 08/16] Remove test flag --- cmd/changefromtfplan.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index 2ef7b1ae..5c66e73d 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -481,5 +481,4 @@ func init() { // changeFromTfplanCmd.PersistentFlags().String("cc-emails", "", "A comma-separated list of emails to keep updated with the status of this change.") changeFromTfplanCmd.PersistentFlags().String("timeout", "1m", "How long to wait for responses") - changeFromTfplanCmd.PersistentFlags().Bool("test-affecting", true, "Choose from the hardcoded test data whether to use a resource that is affecting the test app or not.") } From 97baf61daa5258370359539f8266eb95b3c4b2e8 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:44:16 +0200 Subject: [PATCH 09/16] Fix CI --- .github/actions/go_init/action.yml | 5 ----- .github/workflows/release.yml | 5 +++++ .github/workflows/tests.yml | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml index 5b6d85b9..7f807876 100644 --- a/.github/actions/go_init/action.yml +++ b/.github/actions/go_init/action.yml @@ -4,11 +4,6 @@ description: Initializes go and runs go generate runs: using: "composite" steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up Go uses: actions/setup-go@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77c8265a..16009705 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,11 @@ jobs: contents: write packages: write steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Go Init uses: ./.github/actions/go_init/action.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a3cf765..c1e7a7ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,9 @@ jobs: env: CGO_ENABLED: 0 steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Go Init uses: ./.github/actions/go_init/action.yml @@ -25,6 +28,9 @@ jobs: name: lint runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Go Init uses: ./.github/actions/go_init/action.yml From 6f3bf2f44ad9004d228c7a5d8225f23eec5357ac Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:47:04 +0200 Subject: [PATCH 10/16] Fix CI even more --- .github/workflows/release.yml | 2 +- .github/workflows/tests.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16009705..27903d45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: fetch-depth: 0 - name: Go Init - uses: ./.github/actions/go_init/action.yml + uses: ./.github/actions/go_init - name: Run GoReleaser (publish) uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c1e7a7ac..1f9429ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v3 - name: Go Init - uses: ./.github/actions/go_init/action.yml + uses: ./.github/actions/go_init - name: Get dependencies run: | @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v3 - name: Go Init - uses: ./.github/actions/go_init/action.yml + uses: ./.github/actions/go_init - name: golangci-lint uses: golangci/golangci-lint-action@v3 From 52faf1605eafad3111b53828c1e4e8b2027c680a Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:49:12 +0200 Subject: [PATCH 11/16] Ignore intentional changes --- .github/actions/go_init/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml index 7f807876..b3d97310 100644 --- a/.github/actions/go_init/action.yml +++ b/.github/actions/go_init/action.yml @@ -21,7 +21,7 @@ runs: shell: bash run: | go generate ./... - if [ -z "$(git status --porcelain)" ]; then + if [ -z "$(git status --porcelain | grep -v 'typechange: sources/')" ]; then echo "No pending changes from `go generate`" else echo "Pending changes from `go generate` found, please run 'go generate ./...' and commit the changes" From edb5a9bb91832a6a9a0f565c720394c7a49c8f7c Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:51:02 +0200 Subject: [PATCH 12/16] Everything needs the git history for `go generate` --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f9429ff..f7a4b6ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Go Init uses: ./.github/actions/go_init @@ -30,6 +32,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Go Init uses: ./.github/actions/go_init From 877b207938760f3bb0b360c3399ffbb1196d52df Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:52:28 +0200 Subject: [PATCH 13/16] Fix shell quoting --- .github/actions/go_init/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml index b3d97310..2a4e8958 100644 --- a/.github/actions/go_init/action.yml +++ b/.github/actions/go_init/action.yml @@ -22,9 +22,9 @@ runs: run: | go generate ./... if [ -z "$(git status --porcelain | grep -v 'typechange: sources/')" ]; then - echo "No pending changes from `go generate`" + echo "No pending changes from 'go generate'" else - echo "Pending changes from `go generate` found, please run 'go generate ./...' and commit the changes" + echo "Pending changes from 'go generate' found, please run 'go generate ./...' and commit the changes" git status exit 1 fi From 5352d39bb34df6c09750e94687ea75e93762a673 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:56:36 +0200 Subject: [PATCH 14/16] Respect --porcelain format --- .github/actions/go_init/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml index 2a4e8958..c9cf550e 100644 --- a/.github/actions/go_init/action.yml +++ b/.github/actions/go_init/action.yml @@ -21,7 +21,7 @@ runs: shell: bash run: | go generate ./... - if [ -z "$(git status --porcelain | grep -v 'typechange: sources/')" ]; then + if [ -z "$(git status --porcelain | grep -v 'T sources/')" ]; then echo "No pending changes from 'go generate'" else echo "Pending changes from 'go generate' found, please run 'go generate ./...' and commit the changes" From e53b507ebd905cdc6765c3c69cfb46c9f616758f Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:58:38 +0200 Subject: [PATCH 15/16] Remove unused test data --- cmd/changefromtfplan.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index 5c66e73d..dd31174d 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -48,33 +48,6 @@ var changeFromTfplanCmd = &cobra.Command{ }, } -// test data -var ( - affecting_uuid uuid.UUID = uuid.New() - affecting_resource *sdp.Query = &sdp.Query{ - Type: "elbv2-load-balancer", - Method: sdp.QueryMethod_GET, - Query: "ingress", - RecursionBehaviour: &sdp.Query_RecursionBehaviour{ - LinkDepth: 0, - }, - Scope: "944651592624.eu-west-2", - UUID: affecting_uuid[:], - } - - safe_uuid uuid.UUID = uuid.New() - safe_resource *sdp.Query = &sdp.Query{ - Type: "ec2-security-group", - Method: sdp.QueryMethod_GET, - Query: "sg-09533c300cd1a41c1", - RecursionBehaviour: &sdp.Query_RecursionBehaviour{ - LinkDepth: 0, - }, - Scope: "944651592624.eu-west-2", - UUID: safe_uuid[:], - } -) - type TfData struct { Address string Type string From 037da8c2c6aae46b5e09174fcdf5800743935291 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 11 Jul 2023 16:58:54 +0200 Subject: [PATCH 16/16] Check for JSON parse errors --- cmd/changefromtfplan.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index dd31174d..13b9e0b1 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -64,7 +64,11 @@ func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) ([]*sdp.Q changing_items_tf := map[string]TfData{} var parsed map[string]any - json.Unmarshal(contents, &parsed) + err = json.Unmarshal(contents, &parsed) + if err != nil { + return nil, fmt.Errorf("failed to parse %v: %w", viper.GetString("tfplan-json"), err) + } + root_module := parsed["planned_values"].(map[string]any)["root_module"].(map[string]any) resourceValues := map[string]map[string]any{} resourceValuesFromModule(root_module, &resourceValues)