diff --git a/.github/actions/go_init/action.yml b/.github/actions/go_init/action.yml new file mode 100644 index 00000000..c9cf550e --- /dev/null +++ b/.github/actions/go_init/action.yml @@ -0,0 +1,30 @@ +name: Go Init +description: Initializes go and runs go generate + +runs: + using: "composite" + steps: + - name: Set up Go + uses: actions/setup-go@v4 + + - name: Reset sources + shell: bash + run: rm -Rf sources/ + + - name: Checkout + uses: actions/checkout@v3 + with: + repository: overmindtech/aws-source + path: sources/aws-source + + - name: Go Generate + shell: bash + run: | + go generate ./... + if [ -z "$(git status --porcelain | grep -v 'T sources/')" ]; then + echo "No pending changes from 'go generate'" + else + echo "Pending changes from 'go generate' found, please run 'go generate ./...' and commit the changes" + git status + exit 1 + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3cf709cd..27903d45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,8 +17,8 @@ jobs: with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 + - name: Go Init + uses: ./.github/actions/go_init - name: Run GoReleaser (publish) uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17dc24b1..f7a4b6ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,10 +7,13 @@ jobs: env: CGO_ENABLED: 0 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - name: Checkout + uses: actions/checkout@v3 with: - go-version: 1.x + fetch-depth: 0 + + - name: Go Init + uses: ./.github/actions/go_init - name: Get dependencies run: | @@ -27,11 +30,14 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - name: Checkout + uses: actions/checkout@v3 with: - go-version: '1.20' - cache: false + fetch-depth: 0 + + - name: Go Init + uses: ./.github/actions/go_init + - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/.gitignore b/.gitignore index 6f2dc77f..2db3b32d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ go.work dist/ +tracing/commit.txt diff --git a/cmd/changefromtfplan.go b/cmd/changefromtfplan.go index fd796e54..13b9e0b1 100644 --- a/cmd/changefromtfplan.go +++ b/cmd/changefromtfplan.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -11,6 +12,7 @@ import ( "github.com/bufbuild/connect-go" "github.com/google/uuid" + "github.com/overmindtech/ovm-cli/cmd/datamaps" "github.com/overmindtech/ovm-cli/tracing" "github.com/overmindtech/sdp-go" log "github.com/sirupsen/logrus" @@ -46,41 +48,104 @@ var changeFromTfplanCmd = &cobra.Command{ }, } -// test data -var ( - affecting_uuid uuid.UUID = uuid.New() - affecting_resource *sdp.Query = &sdp.Query{ - Type: "elbv2-load-balancer", - Method: sdp.QueryMethod_GET, - Query: "ingress", - RecursionBehaviour: &sdp.Query_RecursionBehaviour{ - LinkDepth: 0, - }, - Scope: "944651592624.eu-west-2", - UUID: affecting_uuid[:], +type TfData struct { + Address string + Type string + Values map[string]any +} + +func changingItemQueriesFromTfplan(ctx context.Context, lf log.Fields) ([]*sdp.Query, error) { + // read results from `terraform show -json ${tfplan file}` + contents, err := os.ReadFile(viper.GetString("tfplan-json")) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", viper.GetString("tfplan-json"), err) } - safe_uuid uuid.UUID = uuid.New() - safe_resource *sdp.Query = &sdp.Query{ - Type: "ec2-security-group", - Method: sdp.QueryMethod_GET, - Query: "sg-09533c300cd1a41c1", - RecursionBehaviour: &sdp.Query_RecursionBehaviour{ - LinkDepth: 0, - }, - Scope: "944651592624.eu-west-2", - UUID: safe_uuid[:], + changing_items_tf := map[string]TfData{} + + var parsed map[string]any + err = json.Unmarshal(contents, &parsed) + if err != nil { + return nil, fmt.Errorf("failed to parse %v: %w", viper.GetString("tfplan-json"), err) + } + + root_module := parsed["planned_values"].(map[string]any)["root_module"].(map[string]any) + resourceValues := map[string]map[string]any{} + resourceValuesFromModule(root_module, &resourceValues) + + resource_changes := parsed["resource_changes"].([]any) + for _, changed_item_data := range resource_changes { + changed_item_map := changed_item_data.(map[string]any) + change := changed_item_map["change"].(map[string]any) + actions := change["actions"].([]any) + if len(actions) == 0 || actions[0] == "no-op" { + // skip resources with no changes + continue + } + log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ + "actions": actions, + "address": changed_item_map["address"], + }).Debugf("mapping item") + + changed_item := TfData{ + Address: changed_item_map["address"].(string), + Type: changed_item_map["type"].(string), + Values: resourceValues[changed_item_map["address"].(string)], + } + + changing_items_tf[changed_item.Address] = changed_item } -) -func changingItemQueriesFromTfplan() []*sdp.Query { var changing_items []*sdp.Query - if viper.GetBool("test-affecting") { - changing_items = []*sdp.Query{affecting_resource} - } else { - changing_items = []*sdp.Query{safe_resource} + // for all managed resources: + for _, r := range changing_items_tf { + mappings, ok := datamaps.AwssourceData[r.Type] + if !ok { + log.WithContext(ctx).WithFields(lf).WithField("terraform-address", r.Address).Warn("skipping unmapped resource") + break + } + + for _, mapData := range mappings { + queryStr, ok := r.Values[mapData.QueryField] + if !ok { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", r.Address). + WithField("terraform-query-field", mapData.QueryField).Warn("skipping resource without query field") + break + } + + u := uuid.New() + changing_items = append(changing_items, &sdp.Query{ + Type: mapData.Type, + Method: mapData.Method, + Query: queryStr.(string), + Scope: mapData.Scope, + RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, + UUID: u[:], + }) + } + } + + return changing_items, nil +} + +func resourceValuesFromModule(module map[string]any, result *map[string]map[string]any) { + resource_data, ok := module["resources"] + if ok { + for _, r := range resource_data.([]any) { + resource := r.(map[string]any) + (*result)[resource["address"].(string)] = resource["values"].(map[string]any) + } + } + + child_modules_data, ok := module["child_modules"] + if ok { + for _, cm := range child_modules_data.([]any) { + child_module := cm.(map[string]any) + resourceValuesFromModule(child_module, result) + } } - return changing_items } func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { @@ -112,6 +177,13 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") + queries, err := changingItemQueriesFromTfplan(ctx, lf) + if err != nil { + log.WithContext(ctx).WithError(err).WithFields(lf).Error("failed to read terraform plan") + return 1 + } + client := AuthenticatedChangesClient(ctx) createResponse, err := client.CreateChange(ctx, &connect.Request[sdp.CreateChangeRequest]{ Msg: &sdp.CreateChangeRequest{ @@ -132,166 +204,173 @@ func ChangeFromTfplan(signals chan os.Signal, ready chan bool) int { lf["change"] = createResponse.Msg.Change.Metadata.GetUUIDParsed() log.WithContext(ctx).WithFields(lf).Info("created a new change") - log.WithContext(ctx).WithFields(lf).Info("resolving items from terraform plan") - queries := changingItemQueriesFromTfplan() - - options := &websocket.DialOptions{ - HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), - } + receivedItems := []*sdp.Reference{} - c, _, err := websocket.Dial(ctx, viper.GetString("url"), options) - if err != nil { - log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to connect to overmind API") - return 1 - } - defer c.Close(websocket.StatusGoingAway, "") - - // the default, 32kB is too small for cert bundles and rds-db-cluster-parameter-groups - c.SetReadLimit(2 * 1024 * 1024) - - queriesSentChan := make(chan struct{}) - go func() { - for _, q := range queries { - req := sdp.GatewayRequest{ - MinStatusInterval: minStatusInterval, - RequestType: &sdp.GatewayRequest_Query{ - Query: q, - }, - } - err = wspb.Write(ctx, c, &req) - if err != nil { - log.WithContext(ctx).WithFields(lf).WithError(err).WithField("req", &req).Error("Failed to send request") - continue - } + if len(queries) > 0 { + options := &websocket.DialOptions{ + HTTPClient: NewAuthenticatedClient(ctx, otelhttp.DefaultClient), } - queriesSentChan <- struct{}{} - }() - - responses := make(chan *sdp.GatewayResponse) - - // Start a goroutine that reads responses - go func() { - for { - res := new(sdp.GatewayResponse) - err = wspb.Read(ctx, c, res) - - if err != nil { - var e websocket.CloseError - if errors.As(err, &e) { - log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ - "code": e.Code.String(), - "reason": e.Reason, - }).Info("Websocket closing") - return + log.WithContext(ctx).WithFields(lf).WithField("item_count", len(queries)).Info("identifying items") + c, _, err := websocket.Dial(ctx, viper.GetString("url"), options) + if err != nil { + log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to connect to overmind API") + return 1 + } + defer c.Close(websocket.StatusGoingAway, "") + + // the default, 32kB is too small for cert bundles and rds-db-cluster-parameter-groups + c.SetReadLimit(2 * 1024 * 1024) + + queriesSentChan := make(chan struct{}) + go func() { + for _, q := range queries { + req := sdp.GatewayRequest{ + MinStatusInterval: minStatusInterval, + RequestType: &sdp.GatewayRequest_Query{ + Query: q, + }, + } + err = wspb.Write(ctx, c, &req) + if err != nil { + log.WithContext(ctx).WithFields(lf).WithError(err).WithField("req", &req).Error("Failed to send request") + continue } - log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to read response") - return } + queriesSentChan <- struct{}{} + }() + + responses := make(chan *sdp.GatewayResponse) + + // Start a goroutine that reads responses + go func() { + for { + res := new(sdp.GatewayResponse) + + err = wspb.Read(ctx, c, res) + + if err != nil { + var e websocket.CloseError + if errors.As(err, &e) { + log.WithContext(ctx).WithFields(lf).WithFields(log.Fields{ + "code": e.Code.String(), + "reason": e.Reason, + }).Info("Websocket closing") + return + } + log.WithContext(ctx).WithFields(lf).WithError(err).Error("Failed to read response") + return + } - responses <- res - } - }() - - activeQueries := make(map[uuid.UUID]bool) - queriesSent := false - - receivedItems := []*sdp.Reference{} - - // Read the responses -responses: - for { - select { - case <-queriesSentChan: - queriesSent = true - - case <-signals: - log.WithContext(ctx).WithFields(lf).Info("Received interrupt, exiting") - return 1 - - case <-ctx.Done(): - log.WithContext(ctx).WithFields(lf).Info("Context cancelled, exiting") - return 1 + responses <- res + } + }() - case resp := <-responses: - switch resp.ResponseType.(type) { + activeQueries := make(map[uuid.UUID]bool) + queriesSent := false - case *sdp.GatewayResponse_Status: - status := resp.GetStatus() - statusFields := log.Fields{ - "summary": status.Summary, - "responders": status.Summary.Responders, - "queriesSent": queriesSent, - "post_processing_complete": status.PostProcessingComplete, - } + // Read the responses + responses: + for { + select { + case <-queriesSentChan: + queriesSent = true + + case <-signals: + log.WithContext(ctx).WithFields(lf).Info("Received interrupt, exiting") + return 1 + + case <-ctx.Done(): + log.WithContext(ctx).WithFields(lf).Info("Context cancelled, exiting") + return 1 + + case resp := <-responses: + switch resp.ResponseType.(type) { + + case *sdp.GatewayResponse_Status: + status := resp.GetStatus() + statusFields := log.Fields{ + "summary": status.Summary, + "responders": status.Summary.Responders, + "queriesSent": queriesSent, + "post_processing_complete": status.PostProcessingComplete, + } - if status.Done() { - // fall through from all "final" query states, check if there's still queries in progress; - // only break from the loop if all queries have already been sent - // TODO: see above, still needs DefaultStartTimeout implemented to account for slow sources - allDone := allDone(ctx, activeQueries, lf) - statusFields["allDone"] = allDone - if allDone && queriesSent { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders and queries done") - break responses + if status.Done() { + // fall through from all "final" query states, check if there's still queries in progress; + // only break from the loop if all queries have already been sent + // TODO: see above, still needs DefaultStartTimeout implemented to account for slow sources + allDone := allDone(ctx, activeQueries, lf) + statusFields["allDone"] = allDone + if allDone && queriesSent { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders and queries done") + break responses + } else { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders done, with unfinished queries") + } } else { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("all responders done, with unfinished queries") + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("still waiting for responders") } - } else { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Info("still waiting for responders") - } - case *sdp.GatewayResponse_QueryStatus: - status := resp.GetQueryStatus() - statusFields := log.Fields{ - "status": status.Status.String(), - } - queryUuid := status.GetUUIDParsed() - if queryUuid == nil { - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("Received QueryStatus with nil UUID") - continue responses - } - statusFields["query"] = queryUuid - - switch status.Status { - case sdp.QueryStatus_STARTED: - activeQueries[*queryUuid] = true - case sdp.QueryStatus_FINISHED: - activeQueries[*queryUuid] = false - case sdp.QueryStatus_ERRORED: - activeQueries[*queryUuid] = false - case sdp.QueryStatus_CANCELLED: - activeQueries[*queryUuid] = false - default: - statusFields["unexpected_status"] = true - } + case *sdp.GatewayResponse_QueryStatus: + status := resp.GetQueryStatus() + statusFields := log.Fields{ + "status": status.Status.String(), + } + queryUuid := status.GetUUIDParsed() + if queryUuid == nil { + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("Received QueryStatus with nil UUID") + continue responses + } + statusFields["query"] = queryUuid + + switch status.Status { + case sdp.QueryStatus_STARTED: + activeQueries[*queryUuid] = true + case sdp.QueryStatus_FINISHED: + activeQueries[*queryUuid] = false + case sdp.QueryStatus_ERRORED: + activeQueries[*queryUuid] = false + case sdp.QueryStatus_CANCELLED: + activeQueries[*queryUuid] = false + default: + statusFields["unexpected_status"] = true + } - log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("query status update") + log.WithContext(ctx).WithFields(lf).WithFields(statusFields).Debugf("query status update") - case *sdp.GatewayResponse_NewItem: - item := resp.GetNewItem() - log.WithContext(ctx).WithFields(lf).WithField("item", item.GloballyUniqueName()).Infof("new item") + case *sdp.GatewayResponse_NewItem: + item := resp.GetNewItem() + log.WithContext(ctx).WithFields(lf).WithField("item", item.GloballyUniqueName()).Infof("new item") - receivedItems = append(receivedItems, item.Reference()) + receivedItems = append(receivedItems, item.Reference()) - case *sdp.GatewayResponse_NewEdge: - log.WithContext(ctx).WithFields(lf).Debug("ignored edge") + case *sdp.GatewayResponse_NewEdge: + log.WithContext(ctx).WithFields(lf).Debug("ignored edge") - case *sdp.GatewayResponse_QueryError: - err := resp.GetQueryError() - log.WithContext(ctx).WithFields(lf).WithError(err).Errorf("Error from %v(%v)", err.ResponderName, err.SourceName) + case *sdp.GatewayResponse_QueryError: + err := resp.GetQueryError() + log.WithContext(ctx).WithFields(lf).WithError(err).Errorf("Error from %v(%v)", err.ResponderName, err.SourceName) - case *sdp.GatewayResponse_Error: - err := resp.GetError() - log.WithContext(ctx).WithFields(lf).WithField(log.ErrorKey, err).Errorf("generic error") + case *sdp.GatewayResponse_Error: + err := resp.GetError() + log.WithContext(ctx).WithFields(lf).WithField(log.ErrorKey, err).Errorf("generic error") - default: - j := protojson.Format(resp) - log.WithContext(ctx).WithFields(lf).Infof("Unknown %T Response:\n%v", resp.ResponseType, j) + default: + j := protojson.Format(resp) + log.WithContext(ctx).WithFields(lf).Infof("Unknown %T Response:\n%v", resp.ResponseType, j) + } } } + } else { + log.WithContext(ctx).WithFields(lf).Info("no item queries mapped, skipping changing items") } + if len(receivedItems) > 0 { + log.WithContext(ctx).WithFields(lf).WithField("received_items", len(receivedItems)).Info("updating changing items on the change record") + } else { + log.WithContext(ctx).WithFields(lf).WithField("received_items", len(receivedItems)).Info("updating change record with no items") + } resultStream, err := client.UpdateChangingItems(ctx, &connect.Request[sdp.UpdateChangingItemsRequest]{ Msg: &sdp.UpdateChangingItemsRequest{ ChangeUUID: createResponse.Msg.Change.Metadata.UUID, @@ -370,8 +449,7 @@ func init() { changeFromTfplanCmd.PersistentFlags().String("changes-url", "https://api.prod.overmind.tech", "The changes service API endpoint") changeFromTfplanCmd.PersistentFlags().String("frontend", "https://app.overmind.tech", "The frontend base URL") - changeFromTfplanCmd.PersistentFlags().String("terraform", "terraform", "The binary to use for calling terraform. Will be looked up in the system PATH.") - changeFromTfplanCmd.PersistentFlags().String("tfplan", "./tfplan", "Parse changing items from this terraform plan file.") + changeFromTfplanCmd.PersistentFlags().String("tfplan-json", "./tfplan.json", "Parse changing items from this terraform plan JSON file. Generate this using `terraform show -json PLAN_FILE`") changeFromTfplanCmd.PersistentFlags().String("title", "", "Short title for this change.") changeFromTfplanCmd.PersistentFlags().String("description", "", "Quick description of the change.") @@ -380,5 +458,4 @@ func init() { // changeFromTfplanCmd.PersistentFlags().String("cc-emails", "", "A comma-separated list of emails to keep updated with the status of this change.") changeFromTfplanCmd.PersistentFlags().String("timeout", "1m", "How long to wait for responses") - changeFromTfplanCmd.PersistentFlags().Bool("test-affecting", true, "Choose from the hardcoded test data whether to use a resource that is affecting the test app or not.") } diff --git a/cmd/datamaps/awssource.go b/cmd/datamaps/awssource.go new file mode 100644 index 00000000..bde94178 --- /dev/null +++ b/cmd/datamaps/awssource.go @@ -0,0 +1,736 @@ +// Code generated by "extractmaps aws-source"; DO NOT EDIT + +package datamaps + +import "github.com/overmindtech/sdp-go" + +var AwssourceData = map[string][]TfMapData{ + "aws_alb_listener": { + { + Type: "elbv2-listener", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_alb_listener_rule": { + { + Type: "elbv2-rule", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_alb_target_group": { + { + Type: "elbv2-target-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ami": { + { + Type: "ec2-image", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_autoscaling_group": { + { + Type: "autoscaling-auto-scaling-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_cloudwatch_metric_alarm": { + { + Type: "cloudwatch-alarm", + Method: sdp.QueryMethod_GET, + QueryField: "alarm_name", + Scope: "*", + }, + }, + "aws_db_instance": { + { + Type: "rds-db-instance", + Method: sdp.QueryMethod_GET, + QueryField: "identifier", + Scope: "*", + }, + }, + "aws_db_instance_role_association": { + { + Type: "rds-db-instance", + Method: sdp.QueryMethod_GET, + QueryField: "db_instance_identifier", + Scope: "*", + }, + }, + "aws_db_option_group": { + { + Type: "rds-option-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_db_parameter_group": { + { + Type: "rds-db-parameter-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_db_subnet_group": { + { + Type: "rds-db-subnet-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_dynamodb_table": { + { + Type: "dynamodb-table", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ebs_volume": { + { + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_ecs_capacity_provider": { + { + Type: "ecs-capacity-provider", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ecs_cluster": { + { + Type: "ecs-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_ecs_service": { + { + Type: "ecs-service", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_ecs_task_definition": { + { + Type: "ecs-task-definition", + Method: sdp.QueryMethod_GET, + QueryField: "revision", + Scope: "*", + }, + }, + "aws_efs_access_point": { + { + Type: "efs-access-point", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_backup_policy": { + { + Type: "efs-backup-policy", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_file_system": { + { + Type: "efs-file-system", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_mount_target": { + { + Type: "efs-mount-target", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_efs_replication_configuration": { + { + Type: "efs-replication-configuration", + Method: sdp.QueryMethod_GET, + QueryField: "source_file_system_id", + Scope: "*", + }, + }, + "aws_eks_addon": { + { + Type: "eks-addon", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_eks_cluster": { + { + Type: "eks-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_eks_fargate_profile": { + { + Type: "eks-fargate-profile", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_eks_node_group": { + { + Type: "eks-nodegroup", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_elb": { + { + Type: "elb-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_elb_attachment": { + { + Type: "elb-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "elb", + Scope: "*", + }, + }, + "aws_iam_group": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_iam_group_membership": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "group", + Scope: "*", + }, + }, + "aws_iam_policy": { + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_iam_role": { + { + Type: "iam-role", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_iam_role_policy_attachment": { + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + { + Type: "iam-role", + Method: sdp.QueryMethod_GET, + QueryField: "role", + Scope: "*", + }, + }, + "aws_iam_user": { + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_iam_user_group_membership": { + { + Type: "iam-group", + Method: sdp.QueryMethod_GET, + QueryField: "group", + Scope: "*", + }, + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, + }, + "aws_iam_user_policy": { + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, + }, + "aws_iam_user_policy_attachment": { + { + Type: "iam-policy", + Method: sdp.QueryMethod_LIST, + QueryField: "policy_arn", + Scope: "*", + }, + { + Type: "iam-user", + Method: sdp.QueryMethod_GET, + QueryField: "user", + Scope: "*", + }, + }, + "aws_instance": { + { + Type: "ec2-instance", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_internet_gateway": { + { + Type: "ec2-internet-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_key_pair": { + { + Type: "ec2-key-pair", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_lambda_function": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_lambda_function_event_invoke_config": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_function_url": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_invocation": { + { + Type: "lambda-function", + Method: sdp.QueryMethod_GET, + QueryField: "function_name", + Scope: "*", + }, + }, + "aws_lambda_layer_version": { + { + Type: "lambda-layer-version", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_launch_template": { + { + Type: "ec2-launch-template", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_lb": { + { + Type: "elbv2-load-balancer", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_lb_listener": { + { + Type: "elbv2-listener", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_lb_listener_rule": { + { + Type: "elbv2-rule", + Method: sdp.QueryMethod_LIST, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_lb_target_group": { + { + Type: "elbv2-target-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_nat_gateway": { + { + Type: "ec2-nat-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_network_acl": { + { + Type: "ec2-network-acl", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_network_interface": { + { + Type: "ec2-network-interface", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_placement_group": { + { + Type: "ec2-placement-group", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_rds_cluster": { + { + Type: "rds-db-cluster", + Method: sdp.QueryMethod_GET, + QueryField: "cluster_identifier", + Scope: "*", + }, + }, + "aws_rds_cluster_parameter_group": { + { + Type: "rds-db-cluster-parameter-group", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_route53_health_check": { + { + Type: "route53-health-check", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route53_hosted_zone_dnssec": { + { + Type: "route53-hosted-zone", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route53_record": { + { + Type: "route53-resource-record-set", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_route_table": { + { + Type: "ec2-route-table", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_route_table_association": { + { + Type: "ec2-route-table", + Method: sdp.QueryMethod_GET, + QueryField: "route_table_id", + Scope: "*", + }, + { + Type: "ec2-subnet", + Method: sdp.QueryMethod_GET, + QueryField: "subnet_id", + Scope: "*", + }, + }, + "aws_s3_bucket": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_s3_bucket_acl": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_analytics_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_cors_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_intelligent_tiering_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_inventory": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_lifecycle_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_logging": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_metric": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_notification": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_object": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_object_lock_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_ownership_controls": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_policy": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_public_access_block": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_replication_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_request_payment_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_server_side_encryption_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_versioning": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_bucket_website_configuration": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_object": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_s3_object_copy": { + { + Type: "s3-bucket", + Method: sdp.QueryMethod_GET, + QueryField: "bucket", + Scope: "*", + }, + }, + "aws_security_group": { + { + Type: "ec2-security-group", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_security_group_rule": { + { + Type: "ec2-security-group-rule", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_subnet": { + { + Type: "ec2-subnet", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_vpc": { + { + Type: "ec2-vpc", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "egress_only_internet_gateway": { + { + Type: "ec2-egress-only-internet-gateway", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, +} diff --git a/cmd/datamaps/types.go b/cmd/datamaps/types.go new file mode 100644 index 00000000..675fcbf8 --- /dev/null +++ b/cmd/datamaps/types.go @@ -0,0 +1,12 @@ +package datamaps + +import "github.com/overmindtech/sdp-go" + +//go:generate go run ../../extractmaps.go aws-source + +type TfMapData struct { + Type string + Method sdp.QueryMethod + QueryField string + Scope string +} diff --git a/extractmaps.go b/extractmaps.go new file mode 100644 index 00000000..526a7a79 --- /dev/null +++ b/extractmaps.go @@ -0,0 +1,138 @@ +//go:build ignore + +package main + +import ( + "encoding/json" + "fmt" + "html/template" + "os" + "strings" +) + +type Args struct { + Source string + SourceMunged string + Data map[string][]map[string]string +} + +func main() { + fmt.Printf("Running %s go on %s\n", os.Args[0], os.Getenv("GOFILE")) + + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + fmt.Printf(" cwd = %s\n", cwd) + fmt.Printf(" os.Args = %#v\n", os.Args) + + for _, ev := range []string{"GOARCH", "GOOS", "GOFILE", "GOLINE", "GOPACKAGE", "DOLLAR"} { + fmt.Println(" ", ev, "=", os.Getenv(ev)) + } + + if len(os.Args) < 2 { + panic("Missing argument, aborting") + } + + args := Args{ + Source: os.Args[1], + SourceMunged: strings.ReplaceAll(os.Args[1], "-", ""), + Data: dataFromFiles(fmt.Sprintf("../../sources/%v/docs-data", os.Args[1])), + } + + funcMap := template.FuncMap{ + "Title": strings.Title, + } + + template := template.New("simple").Funcs(funcMap) + template, err = template.Parse(`// Code generated by "extractmaps {{.Source}}"; DO NOT EDIT + +package datamaps + +import "github.com/overmindtech/sdp-go" + +var {{.SourceMunged | Title }}Data = map[string][]TfMapData{ +{{- range $key, $mappings := .Data}} + "{{$key}}": { {{- range $mappings}} + { + Type: "{{index . "type"}}", + Method: sdp.QueryMethod_{{index . "method"}}, + QueryField: "{{index . "query-field"}}", + Scope: "{{index . "scope"}}", + },{{end}} + },{{end}} +} +`) + if err != nil { + panic(err) + } + + f, err := os.Create(fmt.Sprintf("%v.go", strings.ToLower(args.SourceMunged))) + if err != nil { + panic(err) + } + defer f.Close() + + fmt.Printf("Generating handler for %v\n", args.Source) + err = template.Execute(f, args) + if err != nil { + panic(err) + } +} + +func dataFromFiles(path string) map[string][]map[string]string { + result := map[string][]map[string]string{} + entries, err := os.ReadDir(path) + if err != nil { + panic(err) + } + + for _, e := range entries { + if e.IsDir() { + continue + } + + if !strings.HasSuffix(e.Name(), ".json") { + continue + } + + fmt.Println(e.Name()) + contents, err := os.ReadFile(fmt.Sprintf("%v/%v", path, e.Name())) + if err != nil { + panic(err) + } + + var parsed map[string]any + err = json.Unmarshal(contents, &parsed) + if err != nil { + panic(err) + } + + queries, ok := parsed["terraformQuery"] + if !ok { + // skip if we don't have terraform query data + continue + } + for _, qAny := range queries.([]interface{}) { + q := qAny.(string) + data := map[string]string{ + "type": parsed["type"].(string), + } + + qSplit := strings.SplitN(q, ".", 2) + data["query-type"] = qSplit[0] + data["query-field"] = qSplit[1] + + data["scope"] = parsed["terraformScope"].(string) + switch parsed["terraformMethod"].(string) { + case "GET": + data["method"] = "GET" + case "SEARCH": + data["method"] = "LIST" + } + + result[data["query-type"]] = append(result[data["query-type"]], data) + } + } + return result +} diff --git a/sources/aws-source b/sources/aws-source new file mode 120000 index 00000000..1f473996 --- /dev/null +++ b/sources/aws-source @@ -0,0 +1 @@ +../../aws-source/ \ No newline at end of file diff --git a/tracing/main.go b/tracing/main.go index 83969e71..346a375d 100644 --- a/tracing/main.go +++ b/tracing/main.go @@ -2,6 +2,7 @@ package tracing import ( "context" + _ "embed" "fmt" "os" "runtime/debug" @@ -23,10 +24,11 @@ import ( "go.opentelemetry.io/otel/trace" ) -const ( - instrumentationName = "github.com/overmindtech/ovm-cli" - instrumentationVersion = "0.0.1" -) +//go:generate sh -c "echo -n $(git describe --long) > commit.txt" +//go:embed commit.txt +var instrumentationVersion string + +const instrumentationName = "github.com/overmindtech/ovm-cli" var ( tracer = otel.GetTracerProvider().Tracer(