diff --git a/.air.toml b/.air.toml index 2525b9fd..db38ebe9 100644 --- a/.air.toml +++ b/.air.toml @@ -16,7 +16,7 @@ exclude_regex = [ "_templ\\.go", "\\.sql\\.go", ".*models/(models|copyfrom|db).go", - "cmd/datamaps", + "datamaps", ] exclude_unchanged = false follow_symlink = false diff --git a/cmd/changes_submit_plan.go b/cmd/changes_submit_plan.go index 9fa8fb2c..5c2a8814 100644 --- a/cmd/changes_submit_plan.go +++ b/cmd/changes_submit_plan.go @@ -2,23 +2,20 @@ package cmd import ( "context" - "encoding/json" "fmt" "os" "os/exec" "os/user" - "slices" "strings" "time" "connectrpc.com/connect" - "github.com/getsentry/sentry-go" "github.com/google/uuid" + "github.com/overmindtech/cli/datamaps" "github.com/overmindtech/sdp-go" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "google.golang.org/protobuf/types/known/structpb" ) // submitPlanCmd represents the submit-plan command @@ -47,274 +44,6 @@ type TfData struct { Values map[string]any } -// maskAllData masks every entry in attributes as redacted -func maskAllData(attributes map[string]any) map[string]any { - for k, v := range attributes { - if mv, ok := v.(map[string]any); ok { - attributes[k] = maskAllData(mv) - } else { - attributes[k] = "(sensitive value)" - } - } - return attributes -} - -// maskSensitiveData masks every entry in attributes that is set to true in sensitive. returns the redacted attributes -func maskSensitiveData(attributes, sensitive any) any { - if sensitive == true { - return "(sensitive value)" - } else if sensitiveMap, ok := sensitive.(map[string]any); ok { - if attributesMap, ok := attributes.(map[string]any); ok { - result := map[string]any{} - for k, v := range attributesMap { - result[k] = maskSensitiveData(v, sensitiveMap[k]) - } - return result - } else { - return "(sensitive value) (type mismatch)" - } - } else if sensitiveArr, ok := sensitive.([]any); ok { - if attributesArr, ok := attributes.([]any); ok { - if len(sensitiveArr) != len(attributesArr) { - return "(sensitive value) (len mismatch)" - } - result := make([]any, len(attributesArr)) - for i, v := range attributesArr { - result[i] = maskSensitiveData(v, sensitiveArr[i]) - } - return result - } else { - return "(sensitive value) (type mismatch)" - } - } - return attributes -} - -func itemAttributesFromResourceChangeData(attributesMsg, sensitiveMsg json.RawMessage) (*sdp.ItemAttributes, error) { - var attributes map[string]any - err := json.Unmarshal(attributesMsg, &attributes) - if err != nil { - return nil, fmt.Errorf("failed to parse attributes: %w", err) - } - - // sensitiveMsg can be a bool or a map[string]any - var isSensitive bool - err = json.Unmarshal(sensitiveMsg, &isSensitive) - if err == nil && isSensitive { - attributes = maskAllData(attributes) - } else if err != nil { - // only try parsing as map if parsing as bool failed - var sensitive map[string]any - err = json.Unmarshal(sensitiveMsg, &sensitive) - if err != nil { - return nil, fmt.Errorf("failed to parse sensitive: %w", err) - } - attributes = maskSensitiveData(attributes, sensitive).(map[string]any) - } - - return sdp.ToAttributesSorted(attributes) -} - -// Converts a ResourceChange form a terraform plan to an ItemDiff in SDP format. -// These items will use the scope `terraform_plan` since we haven't mapped them -// to an actual item in the infrastructure yet -func itemDiffFromResourceChange(resourceChange ResourceChange) (*sdp.ItemDiff, error) { - status := sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UNSPECIFIED - - if slices.Equal(resourceChange.Change.Actions, []string{"no-op"}) || slices.Equal(resourceChange.Change.Actions, []string{"read"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UNCHANGED - } else if slices.Equal(resourceChange.Change.Actions, []string{"create"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_CREATED - } else if slices.Equal(resourceChange.Change.Actions, []string{"update"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UPDATED - } else if slices.Equal(resourceChange.Change.Actions, []string{"delete", "create"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_REPLACED - } else if slices.Equal(resourceChange.Change.Actions, []string{"create", "delete"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_REPLACED - } else if slices.Equal(resourceChange.Change.Actions, []string{"delete"}) { - status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_DELETED - } - - beforeAttributes, err := itemAttributesFromResourceChangeData(resourceChange.Change.Before, resourceChange.Change.BeforeSensitive) - if err != nil { - return nil, fmt.Errorf("failed to parse before attributes: %w", err) - } - afterAttributes, err := itemAttributesFromResourceChangeData(resourceChange.Change.After, resourceChange.Change.AfterSensitive) - if err != nil { - return nil, fmt.Errorf("failed to parse after attributes: %w", err) - } - - err = handleKnownAfterApply(beforeAttributes, afterAttributes, resourceChange.Change.AfterUnknown) - if err != nil { - return nil, fmt.Errorf("failed to remove known after apply fields: %w", err) - } - - result := &sdp.ItemDiff{ - // Item: filled in by item mapping in UpdatePlannedChanges - Status: status, - } - - // shorten the address by removing the type prefix if and only if it is the - // first part. Longer terraform addresses created in modules will not be - // shortened to avoid confusion. - trimmedAddress, _ := strings.CutPrefix(resourceChange.Address, fmt.Sprintf("%v.", resourceChange.Type)) - - if beforeAttributes != nil { - result.Before = &sdp.Item{ - Type: resourceChange.Type, - UniqueAttribute: "terraform_name", - Attributes: beforeAttributes, - Scope: "terraform_plan", - } - - err = result.GetBefore().GetAttributes().Set("terraform_name", trimmedAddress) - if err != nil { - // since Address is a string, this should never happen - sentry.CaptureException(fmt.Errorf("failed to set terraform_name '%v' on before attributes: %w", trimmedAddress, err)) - } - - err = result.GetBefore().GetAttributes().Set("terraform_address", resourceChange.Address) - if err != nil { - // since Address is a string, this should never happen - sentry.CaptureException(fmt.Errorf("failed to set terraform_address of type %T (%v) on before attributes: %w", resourceChange.Address, resourceChange.Address, err)) - } - } - - if afterAttributes != nil { - result.After = &sdp.Item{ - Type: resourceChange.Type, - UniqueAttribute: "terraform_name", - Attributes: afterAttributes, - Scope: "terraform_plan", - } - - err = result.GetAfter().GetAttributes().Set("terraform_name", trimmedAddress) - if err != nil { - // since Address is a string, this should never happen - sentry.CaptureException(fmt.Errorf("failed to set terraform_name '%v' on after attributes: %w", trimmedAddress, err)) - } - - err = result.GetAfter().GetAttributes().Set("terraform_address", resourceChange.Address) - if err != nil { - // since Address is a string, this should never happen - sentry.CaptureException(fmt.Errorf("failed to set terraform_address of type %T (%v) on after attributes: %w", resourceChange.Address, resourceChange.Address, err)) - } - } - - return result, nil -} - -// Finds fields from the `before` and `after` attributes that are known after -// apply and replaces the "after" value with the string "(known after apply)" -func handleKnownAfterApply(before, after *sdp.ItemAttributes, afterUnknown json.RawMessage) error { - var afterUnknownInterface interface{} - err := json.Unmarshal(afterUnknown, &afterUnknownInterface) - if err != nil { - return fmt.Errorf("could not unmarshal `after_unknown` from plan: %w", err) - } - - // Convert the parent struct to a value so that we can treat them all the - // same when we recurse - beforeValue := structpb.Value{ - Kind: &structpb.Value_StructValue{ - StructValue: before.GetAttrStruct(), - }, - } - - afterValue := structpb.Value{ - Kind: &structpb.Value_StructValue{ - StructValue: after.GetAttrStruct(), - }, - } - - err = insertKnownAfterApply(&beforeValue, &afterValue, afterUnknownInterface) - - if err != nil { - return fmt.Errorf("failed to remove known after apply fields: %w", err) - } - - return nil -} - -const KnownAfterApply = `(known after apply)` - -// Inserts the text "(known after apply)" in place of null values in the planned -// "after" values for fields that are known after apply. By default these are -// `null` which produces a bad diff, so we replace them with (known after apply) -// to more accurately mirror what Terraform does in the CLI -func insertKnownAfterApply(before, after *structpb.Value, afterUnknown interface{}) error { - switch afterUnknown.(type) { - case map[string]interface{}: - for k, v := range afterUnknown.(map[string]interface{}) { - if v == true { - if afterFields := after.GetStructValue().GetFields(); afterFields != nil { - // Insert this in the after fields even if it doesn't exist. - // This is because sometimes you will get a plan that only - // has a before value for a know after apply field, so we - // want to still make sure it shows up - afterFields[k] = &structpb.Value{ - Kind: &structpb.Value_StringValue{ - StringValue: KnownAfterApply, - }, - } - } - } else if v == false { - // Do nothing - continue - } else { - // Recurse into the nested fields - err := insertKnownAfterApply(before.GetStructValue().GetFields()[k], after.GetStructValue().GetFields()[k], v) - if err != nil { - return err - } - } - } - case []interface{}: - for i, v := range afterUnknown.([]interface{}) { - if v == true { - // If this value in a slice is true, set the corresponding value - // in after to (know after apply) - if after.GetListValue() != nil && len(after.GetListValue().GetValues()) > i { - after.GetListValue().Values[i] = &structpb.Value{ - Kind: &structpb.Value_StringValue{ - StringValue: KnownAfterApply, - }, - } - } - } else if v == false { - // Do nothing - continue - } else { - // Make sure that the before and after both actually have a - // valid list item at this position, if they don't we can just - // pass `nil` to the `removeUnknownFields` function and it'll - // handle it - beforeListValues := before.GetListValue().GetValues() - afterListValues := after.GetListValue().GetValues() - var nestedBeforeValue *structpb.Value - var nestedAfterValue *structpb.Value - - if len(beforeListValues) > i { - nestedBeforeValue = beforeListValues[i] - } - - if len(afterListValues) > i { - nestedAfterValue = afterListValues[i] - } - - err := insertKnownAfterApply(nestedBeforeValue, nestedAfterValue, v) - if err != nil { - return err - } - } - } - default: - return nil - } - - return nil -} - type plannedChangeGroups struct { supported map[string][]*sdp.MappedItemDiff unsupported map[string][]*sdp.MappedItemDiff @@ -367,35 +96,6 @@ func (g *plannedChangeGroups) Add(typ string, item *sdp.MappedItemDiff) { groups[typ] = append(list, item) } -// Checks if the supplied JSON bytes are a state file. It's a common mistake to -// pass a state file to Overmind rather than a plan file since the commands to -// create them are similar -func isStateFile(bytes []byte) bool { - fields := make(map[string]interface{}) - - err := json.Unmarshal(bytes, &fields) - - if err != nil { - return false - } - - if _, exists := fields["values"]; exists { - return true - } - - return false -} - -// Returns the name of the provider from the config key. If the resource isn't -// in a module, the ProviderConfigKey will be something like "kubernetes", -// however if it's in a module it's be something like -// "module.something:kubernetes". In both scenarios we want to return -// "kubernetes" -func extractProviderNameFromConfigKey(providerConfigKey string) string { - sections := strings.Split(providerConfigKey, ":") - return sections[len(sections)-1] -} - func changeTitle(arg string) string { if arg != "" { // easy, return the user's choice @@ -460,7 +160,7 @@ func SubmitPlan(cmd *cobra.Command, args []string) error { lf := log.Fields{} for _, f := range args { lf["file"] = f - result, err := MappedItemDiffsFromPlanFile(ctx, f, lf) + result, err := datamaps.MappedItemDiffsFromPlanFile(ctx, f, lf) if err != nil { return loggedError{ err: err, diff --git a/cmd/changes_submit_plan_test.go b/cmd/changes_submit_plan_test.go deleted file mode 100644 index ec6b1749..00000000 --- a/cmd/changes_submit_plan_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "testing" - - "github.com/overmindtech/sdp-go" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" -) - -func TestWithStateFile(t *testing.T) { - _, err := MappedItemDiffsFromPlanFile(context.Background(), "testdata/state.json", logrus.Fields{}) - - if err == nil { - t.Error("Expected error when running with state file, got none") - } -} - -// note that these tests need to allocate the input map for every test to avoid -// false positives from maskSensitiveData mutating the data -func TestMaskSensitiveData(t *testing.T) { - t.Parallel() - - t.Run("empty", func(t *testing.T) { - t.Parallel() - got := maskSensitiveData(map[string]any{}, map[string]any{}) - require.Equal(t, map[string]any{}, got) - }) - - t.Run("easy", func(t *testing.T) { - t.Parallel() - require.Equal(t, - map[string]any{ - "foo": "bar", - }, - maskSensitiveData( - map[string]any{ - "foo": "bar", - }, - map[string]any{})) - - require.Equal(t, - map[string]any{ - "foo": "(sensitive value)", - }, - maskSensitiveData( - map[string]any{ - "foo": "bar", - }, - map[string]any{"foo": true})) - - }) - - t.Run("deep", func(t *testing.T) { - t.Parallel() - require.Equal(t, - map[string]any{ - "foo": map[string]any{"key": "bar"}, - }, - maskSensitiveData( - map[string]any{ - "foo": map[string]any{"key": "bar"}, - }, - map[string]any{})) - - require.Equal(t, - map[string]any{ - "foo": "(sensitive value)", - }, - maskSensitiveData( - map[string]any{ - "foo": map[string]any{"key": "bar"}, - }, - map[string]any{"foo": true})) - - require.Equal(t, - map[string]any{ - "foo": map[string]any{"key": "(sensitive value)"}, - }, - maskSensitiveData( - map[string]any{ - "foo": map[string]any{"key": "bar"}, - }, - map[string]any{"foo": map[string]any{"key": true}})) - - }) - - t.Run("arrays", func(t *testing.T) { - t.Parallel() - require.Equal(t, - map[string]any{ - "foo": []any{"one", "two"}, - }, - maskSensitiveData( - map[string]any{ - "foo": []any{"one", "two"}, - }, - map[string]any{})) - - require.Equal(t, - map[string]any{ - "foo": "(sensitive value)", - }, - maskSensitiveData( - map[string]any{ - "foo": []any{"one", "two"}, - }, - map[string]any{"foo": true})) - - require.Equal(t, - map[string]any{ - "foo": []any{"one", "(sensitive value)"}, - }, - maskSensitiveData( - map[string]any{ - "foo": []any{"one", "two"}, - }, - map[string]any{"foo": []any{false, true}})) - - }) -} - -func TestExtractProviderNameFromConfigKey(t *testing.T) { - tests := []struct { - ConfigKey string - Expected string - }{ - { - ConfigKey: "kubernetes", - Expected: "kubernetes", - }, - { - ConfigKey: "module.core:kubernetes", - Expected: "kubernetes", - }, - } - - for _, test := range tests { - t.Run(test.ConfigKey, func(t *testing.T) { - actual := extractProviderNameFromConfigKey(test.ConfigKey) - if actual != test.Expected { - t.Errorf("Expected %v, got %v", test.Expected, actual) - } - }) - } -} - -func TestHandleKnownAfterApply(t *testing.T) { - before, err := sdp.ToAttributes(map[string]interface{}{ - "string_value": "foo", - "int_value": 42, - "bool_value": true, - "float_value": 3.14, - "data": "secret", // Known after apply but doesn't exist in the "after" map, this happens sometimes - "list_value": []interface{}{ - "foo", - "bar", - }, - "map_value": map[string]interface{}{ - "foo": "bar", - "bar": "baz", - }, - "map_value2": map[string]interface{}{ - "ding": map[string]interface{}{ - "foo": "bar", - }, - }, - "nested_list": []interface{}{ - []interface{}{}, - []interface{}{ - "foo", - "bar", - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - after, err := sdp.ToAttributes(map[string]interface{}{ - "string_value": "bar", // I want to see a diff here - "int_value": nil, // These are going to be known after apply - "bool_value": nil, // These are going to be known after apply - "float_value": 3.14, - "list_value": []interface{}{ - "foo", - "bar", - "baz", // So is this one - }, - "map_value": map[string]interface{}{ // This whole thing will be known after apply - "foo": "bar", - }, - "map_value2": map[string]interface{}{ - "ding": map[string]interface{}{ - "foo": nil, // This will be known after apply - }, - }, - "nested_list": []interface{}{ - []interface{}{ - "foo", - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - afterUnknown := json.RawMessage(`{ - "int_value": true, - "bool_value": true, - "float_value": false, - "data": true, - "list_value": [ - false, - false, - true - ], - "map_value": true, - "map_value2": { - "ding": { - "foo": true - } - }, - "nested_list": [ - [ - false, - true - ], - [ - false, - true - ] - ] - }`) - - err = handleKnownAfterApply(before, after, afterUnknown) - if err != nil { - t.Fatal(err) - } - - beforeJSON, err := json.MarshalIndent(before, "", " ") - if err != nil { - t.Fatal(err) - } - afterJSON, err := json.MarshalIndent(after, "", " ") - if err != nil { - t.Fatal(err) - } - - fmt.Println("BEFORE:") - fmt.Println(string(beforeJSON)) - fmt.Println("\n\nAFTER:") - fmt.Println(string(afterJSON)) - - if val, _ := after.Get("int_value"); val != KnownAfterApply { - t.Errorf("expected int_value to be %v, got %v", KnownAfterApply, val) - } - - if val, _ := after.Get("bool_value"); val != KnownAfterApply { - t.Errorf("expected bool_value to be %v, got %v", KnownAfterApply, val) - } - - i, err := after.Get("list_value") - if err != nil { - t.Error(err) - } - - if list, ok := i.([]interface{}); ok { - if list[2] != KnownAfterApply { - t.Errorf("expected third string_value to be %v, got %v", KnownAfterApply, list[2]) - } - } else { - t.Error("list_value is not a string slice") - } - - if val, _ := after.Get("data"); val != KnownAfterApply { - t.Errorf("expected data to be %v, got %v", KnownAfterApply, val) - } -} diff --git a/cmd/plan_mapper.go b/cmd/plan_mapper.go deleted file mode 100644 index 8844e2e4..00000000 --- a/cmd/plan_mapper.go +++ /dev/null @@ -1,296 +0,0 @@ -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "time" - - "github.com/google/uuid" - "github.com/overmindtech/cli/cmd/datamaps" - "github.com/overmindtech/sdp-go" - log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/types/known/timestamppb" -) - -type MapStatus int - -const ( - MapStatusSuccess MapStatus = iota - MapStatusNotEnoughInfo - MapStatusUnsupported -) - -type PlannedChangeMapResult struct { - // The name of the resource in the Terraform plan - TerraformName string - - // The status of the mapping - Status MapStatus - - // The message that should be printed next to the status e.g. "mapped" or - // "missing arn" - Message string - - *sdp.MappedItemDiff -} - -type PlanMappingResult struct { - Results []PlannedChangeMapResult - RemovedSecrets int -} - -func (r *PlanMappingResult) NumSuccess() int { - return r.numStatus(MapStatusSuccess) -} - -func (r *PlanMappingResult) NumNotEnoughInfo() int { - return r.numStatus(MapStatusNotEnoughInfo) -} - -func (r *PlanMappingResult) NumUnsupported() int { - return r.numStatus(MapStatusUnsupported) -} - -func (r *PlanMappingResult) NumTotal() int { - return len(r.Results) -} - -func (r *PlanMappingResult) GetItemDiffs() []*sdp.MappedItemDiff { - diffs := make([]*sdp.MappedItemDiff, 0) - - for _, result := range r.Results { - if result.MappedItemDiff != nil { - diffs = append(diffs, result.MappedItemDiff) - } - } - - return diffs -} - -func (r *PlanMappingResult) numStatus(status MapStatus) int { - count := 0 - for _, result := range r.Results { - if result.Status == status { - count++ - } - } - return count -} - -func MappedItemDiffsFromPlanFile(ctx context.Context, fileName string, lf log.Fields) (*PlanMappingResult, error) { - // read results from `terraform show -json ${tfplan file}` - planJSON, err := os.ReadFile(fileName) - if err != nil { - log.WithContext(ctx).WithError(err).WithFields(lf).Error("Failed to read terraform plan") - return nil, err - } - - return MappedItemDiffsFromPlan(ctx, planJSON, fileName, lf) -} - -// MappedItemDiffsFromPlan takes a plan JSON, file name, and log fields as input -// and returns the mapping results and an error. It parses the plan JSON, -// extracts resource changes, and creates mapped item differences for each -// resource change. It also generates mapping queries based on the resource type -// and current resource values. The function categorizes the mapped item -// differences into supported and unsupported changes. Finally, it logs the -// number of supported and unsupported changes and returns the mapped item -// differences. -func MappedItemDiffsFromPlan(ctx context.Context, planJson []byte, fileName string, lf log.Fields) (*PlanMappingResult, error) { - // Check that we haven't been passed a state file - if isStateFile(planJson) { - return nil, fmt.Errorf("'%v' appears to be a state file, not a plan file", fileName) - } - - var plan Plan - err := json.Unmarshal(planJson, &plan) - if err != nil { - return nil, fmt.Errorf("failed to parse '%v': %w", fileName, err) - } - - results := PlanMappingResult{ - Results: make([]PlannedChangeMapResult, 0), - RemovedSecrets: countSensitiveValuesInConfig(plan.Config.RootModule) + countSensitiveValuesInState(plan.PlannedValues.RootModule), - } - - // for all managed resources: - for _, resourceChange := range plan.ResourceChanges { - if len(resourceChange.Change.Actions) == 0 || resourceChange.Change.Actions[0] == "no-op" || resourceChange.Mode == "data" { - // skip resources with no changes and data updates - continue - } - - itemDiff, err := itemDiffFromResourceChange(resourceChange) - if err != nil { - return nil, fmt.Errorf("failed to create item diff for resource change: %w", err) - } - - // Load mappings for this type. These mappings tell us how to create an - // SDP query that will return this resource - awsMappings := datamaps.AwssourceData[resourceChange.Type] - k8sMappings := datamaps.K8ssourceData[resourceChange.Type] - mappings := append(awsMappings, k8sMappings...) - - if len(mappings) == 0 { - log.WithContext(ctx).WithFields(lf).WithField("terraform-address", resourceChange.Address).Debug("Skipping unmapped resource") - results.Results = append(results.Results, PlannedChangeMapResult{ - TerraformName: resourceChange.Address, - Status: MapStatusUnsupported, - Message: "unsupported", - MappedItemDiff: &sdp.MappedItemDiff{ - Item: itemDiff, - MappingQuery: nil, // unmapped item has no mapping query - }, - }) - continue - } - - for _, mapData := range mappings { - var currentResource *Resource - - // Look for the resource in the prior values first, since this is - // the *previous* state we're like to be able to find it in the - // actual infra - if plan.PriorState.Values != nil { - currentResource = plan.PriorState.Values.RootModule.DigResource(resourceChange.Address) - } - - // If we didn't find it, look in the planned values - if currentResource == nil { - currentResource = plan.PlannedValues.RootModule.DigResource(resourceChange.Address) - } - - if currentResource == nil { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", resourceChange.Address). - WithField("terraform-query-field", mapData.QueryField).Warn("Skipping resource without values") - continue - } - - query, ok := currentResource.AttributeValues.Dig(mapData.QueryField) - if !ok { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", resourceChange.Address). - WithField("terraform-query-field", mapData.QueryField).Debug("Missing mapping field, cannot create Overmind query") - results.Results = append(results.Results, PlannedChangeMapResult{ - TerraformName: resourceChange.Address, - Status: MapStatusNotEnoughInfo, - Message: fmt.Sprintf("missing %v", mapData.QueryField), - MappedItemDiff: &sdp.MappedItemDiff{ - Item: itemDiff, - MappingQuery: nil, // unmapped item has no mapping query - }, - }) - continue - } - - // Create the map that variables will pull data from - dataMap := make(map[string]any) - - // Populate resource values - dataMap["values"] = currentResource.AttributeValues - - if overmindMappingsOutput, ok := plan.PlannedValues.Outputs["overmind_mappings"]; ok { - configResource := plan.Config.RootModule.DigResource(resourceChange.Address) - - if configResource == nil { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", resourceChange.Address). - Debug("Skipping provider mapping for resource without config") - } else { - // Look up the provider config key in the mappings - mappings := make(map[string]map[string]string) - - err = json.Unmarshal(overmindMappingsOutput.Value, &mappings) - - if err != nil { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", resourceChange.Address). - WithError(err). - Error("Failed to parse overmind_mappings output") - } else { - // We need to split out the module section of the name - // here. If the resource isn't in a module, the - // ProviderConfigKey will be something like - // "kubernetes", however if it's in a module it's be - // something like "module.something:kubernetes" - providerName := extractProviderNameFromConfigKey(configResource.ProviderConfigKey) - currentProviderMappings, ok := mappings[providerName] - - if ok { - log.WithContext(ctx). - WithFields(lf). - WithField("terraform-address", resourceChange.Address). - WithField("provider-config-key", configResource.ProviderConfigKey). - Debug("Found provider mappings") - - // We have mappings for this provider, so set them - // in the `provider_mapping` value - dataMap["provider_mapping"] = currentProviderMappings - } - } - } - } - - // Interpolate variables in the scope - scope, err := InterpolateScope(mapData.Scope, dataMap) - - if err != nil { - log.WithContext(ctx).WithError(err).Debugf("Could not find scope mapping variables %v, adding them will result in better results. Error: ", mapData.Scope) - scope = "*" - } - - u := uuid.New() - newQuery := &sdp.Query{ - Type: mapData.Type, - Method: mapData.Method, - Query: fmt.Sprintf("%v", query), - Scope: scope, - RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, - UUID: u[:], - Deadline: timestamppb.New(time.Now().Add(60 * time.Second)), - } - - // cleanup item metadata from mapping query - if itemDiff.GetBefore() != nil { - itemDiff.Before.Type = newQuery.GetType() - if newQuery.GetScope() != "*" { - itemDiff.Before.Scope = newQuery.GetScope() - } - } - - // cleanup item metadata from mapping query - if itemDiff.GetAfter() != nil { - itemDiff.After.Type = newQuery.GetType() - if newQuery.GetScope() != "*" { - itemDiff.After.Scope = newQuery.GetScope() - } - } - - results.Results = append(results.Results, PlannedChangeMapResult{ - TerraformName: resourceChange.Address, - Status: MapStatusSuccess, - Message: "mapped", - MappedItemDiff: &sdp.MappedItemDiff{ - Item: itemDiff, - MappingQuery: newQuery, - }, - }) - - log.WithContext(ctx).WithFields(log.Fields{ - "scope": newQuery.GetScope(), - "type": newQuery.GetType(), - "query": newQuery.GetQuery(), - "method": newQuery.GetMethod().String(), - }).Debug("Mapped resource to query") - } - } - - return &results, nil -} diff --git a/cmd/plan_test.go b/cmd/plan_test.go deleted file mode 100644 index 433113a4..00000000 --- a/cmd/plan_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package cmd - -import "testing" - -func TestInterpolateScope(t *testing.T) { - t.Run("with no interpolation", func(t *testing.T) { - t.Parallel() - - result, err := InterpolateScope("foo", map[string]any{}) - - if err != nil { - t.Error(err) - } - - if result != "foo" { - t.Errorf("Expected result to be foo, got %s", result) - } - }) - - t.Run("with a single variable", func(t *testing.T) { - t.Parallel() - - result, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}", map[string]any{ - "outputs": map[string]any{ - "overmind_kubernetes_cluster_name": "foo", - }, - }) - - if err != nil { - t.Error(err) - } - - if result != "foo" { - t.Errorf("Expected result to be foo, got %s", result) - } - }) - - t.Run("with multiple variables", func(t *testing.T) { - t.Parallel() - - result, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}.${values.metadata.namespace}", map[string]any{ - "outputs": map[string]any{ - "overmind_kubernetes_cluster_name": "foo", - }, - "values": map[string]any{ - "metadata": map[string]any{ - "namespace": "bar", - }, - }, - }) - - if err != nil { - t.Error(err) - } - - if result != "foo.bar" { - t.Errorf("Expected result to be foo.bar, got %s", result) - } - }) - - t.Run("with a variable that doesn't exist", func(t *testing.T) { - t.Parallel() - - _, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}", map[string]any{}) - - if err == nil { - t.Error("Expected error, got nil") - } - }) -} diff --git a/cmd/tea_submitplan.go b/cmd/tea_submitplan.go index c448a7e3..3ee72fd7 100644 --- a/cmd/tea_submitplan.go +++ b/cmd/tea_submitplan.go @@ -15,6 +15,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/google/uuid" "github.com/muesli/reflow/wordwrap" + "github.com/overmindtech/cli/datamaps" "github.com/overmindtech/sdp-go" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -37,7 +38,7 @@ type submitPlanModel struct { removingSecretsTask taskModel resourceExtractionTask taskModel - planMappingResult PlanMappingResult + planMappingResult datamaps.PlanMappingResult uploadChangesTask taskModel @@ -118,7 +119,7 @@ func (m submitPlanModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // better idea on the table. cmds = append(cmds, tea.Sequence(func() tea.Msg { return msg.wrapped }, m.waitForSubmitPlanActivity)) - case *PlanMappingResult: + case *datamaps.PlanMappingResult: m.planMappingResult = *msg case submitPlanFinishedMsg: @@ -222,11 +223,11 @@ func (m submitPlanModel) View() string { for _, mapping := range m.planMappingResult.Results { var icon string switch mapping.Status { - case MapStatusSuccess: + case datamaps.MapStatusSuccess: icon = RenderOk() - case MapStatusNotEnoughInfo: + case datamaps.MapStatusNotEnoughInfo: icon = RenderUnknown() - case MapStatusUnsupported: + case datamaps.MapStatusUnsupported: icon = RenderErr() } bits = append(bits, fmt.Sprintf(" %v %v (%v)", icon, mapping.TerraformName, mapping.Message)) @@ -315,23 +316,23 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { m.processing <- submitPlanUpdateMsg{m.resourceExtractionTask.UpdateTitleMsg( "Extracting 13 changing resources: 4 supported 9 unsupported", )} - mappingResult := PlanMappingResult{ - Results: []PlannedChangeMapResult{ + mappingResult := datamaps.PlanMappingResult{ + Results: []datamaps.PlannedChangeMapResult{ { TerraformName: "kubernetes_deployment.nats_box", - Status: MapStatusSuccess, + Status: datamaps.MapStatusSuccess, Message: "mapped", MappedItemDiff: &sdp.MappedItemDiff{}, }, { TerraformName: "kubernetes_deployment.api_server", - Status: MapStatusNotEnoughInfo, + Status: datamaps.MapStatusNotEnoughInfo, Message: "missing arn", MappedItemDiff: &sdp.MappedItemDiff{}, }, { TerraformName: "aws_fake_resource", - Status: MapStatusUnsupported, + Status: datamaps.MapStatusUnsupported, Message: "unsupported", MappedItemDiff: &sdp.MappedItemDiff{}, }, @@ -529,7 +530,7 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { time.Sleep(200 * time.Millisecond) // give the UI a little time to update // Map the terraform changes to Overmind queries - mappingResponse, err := MappedItemDiffsFromPlan(ctx, planJson, m.planFile, log.Fields{}) + mappingResponse, err := datamaps.MappedItemDiffsFromPlan(ctx, planJson, m.planFile, log.Fields{}) if err != nil { m.processing <- submitPlanUpdateMsg{m.resourceExtractionTask.UpdateStatusMsg(taskStatusError)} m.processing <- submitPlanUpdateMsg{m.risksError("failed to parse terraform plan", err)} @@ -549,7 +550,7 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { ))} // Sort the supported and unsupported changes so that they display nicely - slices.SortFunc(mappingResponse.Results, func(a, b PlannedChangeMapResult) int { + slices.SortFunc(mappingResponse.Results, func(a, b datamaps.PlannedChangeMapResult) int { return int(a.Status) - int(b.Status) }) m.processing <- submitPlanUpdateMsg{mappingResponse} diff --git a/cmd/terraform_plan.go b/cmd/terraform_plan.go index 16b50dfe..b8e81691 100644 --- a/cmd/terraform_plan.go +++ b/cmd/terraform_plan.go @@ -3,7 +3,6 @@ package cmd import ( "context" "crypto/sha256" - "encoding/json" "fmt" "os" "strings" @@ -159,85 +158,6 @@ func getTicketLinkFromPlan(planFile string) (string, error) { return fmt.Sprintf("tfplan://{SHA256}%x", h.Sum(nil)), nil } -func countSensitiveValuesInConfig(m ConfigModule) int { - removedSecrets := 0 - for _, v := range m.Variables { - if v.Sensitive { - removedSecrets++ - } - } - for _, o := range m.Outputs { - if o.Sensitive { - removedSecrets++ - } - } - for _, c := range m.ModuleCalls { - removedSecrets += countSensitiveValuesInConfig(c.Module) - } - return removedSecrets -} - -func countSensitiveValuesInState(m Module) int { - removedSecrets := 0 - for _, r := range m.Resources { - removedSecrets += countSensitiveValuesInResource(r) - } - for _, c := range m.ChildModules { - removedSecrets += countSensitiveValuesInState(c) - } - return removedSecrets -} - -// follow itemAttributesFromResourceChangeData and maskSensitiveData -// implementation to count sensitive values -func countSensitiveValuesInResource(r Resource) int { - // sensitiveMsg can be a bool or a map[string]any - var isSensitive bool - err := json.Unmarshal(r.SensitiveValues, &isSensitive) - if err == nil && isSensitive { - return 1 // one very large secret - } else if err != nil { - // only try parsing as map if parsing as bool failed - var sensitive map[string]any - err = json.Unmarshal(r.SensitiveValues, &sensitive) - if err != nil { - return 0 - } - return countSensitiveAttributes(r.AttributeValues, sensitive) - } - return 0 -} - -func countSensitiveAttributes(attributes, sensitive any) int { - if sensitive == true { - return 1 - } else if sensitiveMap, ok := sensitive.(map[string]any); ok { - if attributesMap, ok := attributes.(map[string]any); ok { - result := 0 - for k, v := range attributesMap { - result += countSensitiveAttributes(v, sensitiveMap[k]) - } - return result - } else { - return 1 - } - } else if sensitiveArr, ok := sensitive.([]any); ok { - if attributesArr, ok := attributes.([]any); ok { - if len(sensitiveArr) != len(attributesArr) { - return 1 - } - result := 0 - for i, v := range attributesArr { - result += countSensitiveAttributes(v, sensitiveArr[i]) - } - return result - } else { - return 1 - } - } - return 0 -} - func addTerraformBaseFlags(cmd *cobra.Command) { cmd.PersistentFlags().Bool("reset-stored-config", false, "Set this to reset the sources config stored in Overmind and input fresh values.") cmd.PersistentFlags().String("aws-config", "", "The chosen AWS config method, best set through the initial wizard when running the CLI. Options: 'profile_input', 'aws_profile', 'defaults', 'managed'.") diff --git a/cmd/datamaps/awssource.go b/datamaps/awssource.go similarity index 100% rename from cmd/datamaps/awssource.go rename to datamaps/awssource.go diff --git a/cmd/datamaps/k8ssource.go b/datamaps/k8ssource.go similarity index 100% rename from cmd/datamaps/k8ssource.go rename to datamaps/k8ssource.go diff --git a/cmd/plan.go b/datamaps/plan.go similarity index 94% rename from cmd/plan.go rename to datamaps/plan.go index 5681f6a7..c5e00d13 100644 --- a/cmd/plan.go +++ b/datamaps/plan.go @@ -1,8 +1,7 @@ -package cmd +package datamaps import ( "encoding/json" - "fmt" "regexp" "strconv" "strings" @@ -130,44 +129,6 @@ func TerraformDig(srcMapPtr interface{}, path string) interface{} { var escapeRegex = regexp.MustCompile(`\${([\w\.\[\]]*)}`) -// InterpolateScope Will interpolate variables in the scope string. These -// variables can come from the following places: -// -// * `outputs` - These are the outputs from the plan -// * `values` - These are the values from the resource in question -// -// Interpolation is done using the Terraform interpolation syntax: -// https://www.terraform.io/docs/configuration/interpolation.html -func InterpolateScope(scope string, data map[string]any) (string, error) { - // Find all instances of ${} in the Scope - matches := escapeRegex.FindAllStringSubmatch(scope, -1) - - interpolated := scope - - for _, match := range matches { - // The first match is the entire string, the second match is the - // variable name - variableName := match[1] - - value := TerraformDig(&data, variableName) - - if value == nil { - return "", fmt.Errorf("variable '%v' not found", variableName) - } - - // Convert the value to a string - valueString, ok := value.(string) - - if !ok { - return "", fmt.Errorf("variable '%v' is not a string", variableName) - } - - interpolated = strings.Replace(interpolated, match[0], valueString, 1) - } - - return interpolated, nil -} - // Digs for a config resource in this module or its children func (m ConfigModule) DigResource(address string) *ConfigResource { addressSections := strings.Split(address, ".") diff --git a/datamaps/plan_mapper.go b/datamaps/plan_mapper.go new file mode 100644 index 00000000..6fcc4fd6 --- /dev/null +++ b/datamaps/plan_mapper.go @@ -0,0 +1,713 @@ +package datamaps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/getsentry/sentry-go" + "github.com/google/uuid" + "github.com/overmindtech/sdp-go" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type MapStatus int + +const ( + MapStatusSuccess MapStatus = iota + MapStatusNotEnoughInfo + MapStatusUnsupported +) + +const KnownAfterApply = `(known after apply)` + +type PlannedChangeMapResult struct { + // The name of the resource in the Terraform plan + TerraformName string + + // The status of the mapping + Status MapStatus + + // The message that should be printed next to the status e.g. "mapped" or + // "missing arn" + Message string + + *sdp.MappedItemDiff +} + +type PlanMappingResult struct { + Results []PlannedChangeMapResult + RemovedSecrets int +} + +func (r *PlanMappingResult) NumSuccess() int { + return r.numStatus(MapStatusSuccess) +} + +func (r *PlanMappingResult) NumNotEnoughInfo() int { + return r.numStatus(MapStatusNotEnoughInfo) +} + +func (r *PlanMappingResult) NumUnsupported() int { + return r.numStatus(MapStatusUnsupported) +} + +func (r *PlanMappingResult) NumTotal() int { + return len(r.Results) +} + +func (r *PlanMappingResult) GetItemDiffs() []*sdp.MappedItemDiff { + diffs := make([]*sdp.MappedItemDiff, 0) + + for _, result := range r.Results { + if result.MappedItemDiff != nil { + diffs = append(diffs, result.MappedItemDiff) + } + } + + return diffs +} + +func (r *PlanMappingResult) numStatus(status MapStatus) int { + count := 0 + for _, result := range r.Results { + if result.Status == status { + count++ + } + } + return count +} + +func MappedItemDiffsFromPlanFile(ctx context.Context, fileName string, lf log.Fields) (*PlanMappingResult, error) { + // read results from `terraform show -json ${tfplan file}` + planJSON, err := os.ReadFile(fileName) + if err != nil { + log.WithContext(ctx).WithError(err).WithFields(lf).Error("Failed to read terraform plan") + return nil, err + } + + return MappedItemDiffsFromPlan(ctx, planJSON, fileName, lf) +} + +// MappedItemDiffsFromPlan takes a plan JSON, file name, and log fields as input +// and returns the mapping results and an error. It parses the plan JSON, +// extracts resource changes, and creates mapped item differences for each +// resource change. It also generates mapping queries based on the resource type +// and current resource values. The function categorizes the mapped item +// differences into supported and unsupported changes. Finally, it logs the +// number of supported and unsupported changes and returns the mapped item +// differences. +func MappedItemDiffsFromPlan(ctx context.Context, planJson []byte, fileName string, lf log.Fields) (*PlanMappingResult, error) { + // Check that we haven't been passed a state file + if isStateFile(planJson) { + return nil, fmt.Errorf("'%v' appears to be a state file, not a plan file", fileName) + } + + var plan Plan + err := json.Unmarshal(planJson, &plan) + if err != nil { + return nil, fmt.Errorf("failed to parse '%v': %w", fileName, err) + } + + results := PlanMappingResult{ + Results: make([]PlannedChangeMapResult, 0), + RemovedSecrets: countSensitiveValuesInConfig(plan.Config.RootModule) + countSensitiveValuesInState(plan.PlannedValues.RootModule), + } + + // for all managed resources: + for _, resourceChange := range plan.ResourceChanges { + if len(resourceChange.Change.Actions) == 0 || resourceChange.Change.Actions[0] == "no-op" || resourceChange.Mode == "data" { + // skip resources with no changes and data updates + continue + } + + itemDiff, err := itemDiffFromResourceChange(resourceChange) + if err != nil { + return nil, fmt.Errorf("failed to create item diff for resource change: %w", err) + } + + // Load mappings for this type. These mappings tell us how to create an + // SDP query that will return this resource + awsMappings := AwssourceData[resourceChange.Type] + k8sMappings := K8ssourceData[resourceChange.Type] + mappings := append(awsMappings, k8sMappings...) + + if len(mappings) == 0 { + log.WithContext(ctx).WithFields(lf).WithField("terraform-address", resourceChange.Address).Debug("Skipping unmapped resource") + results.Results = append(results.Results, PlannedChangeMapResult{ + TerraformName: resourceChange.Address, + Status: MapStatusUnsupported, + Message: "unsupported", + MappedItemDiff: &sdp.MappedItemDiff{ + Item: itemDiff, + MappingQuery: nil, // unmapped item has no mapping query + }, + }) + continue + } + + for _, mapData := range mappings { + var currentResource *Resource + + // Look for the resource in the prior values first, since this is + // the *previous* state we're like to be able to find it in the + // actual infra + if plan.PriorState.Values != nil { + currentResource = plan.PriorState.Values.RootModule.DigResource(resourceChange.Address) + } + + // If we didn't find it, look in the planned values + if currentResource == nil { + currentResource = plan.PlannedValues.RootModule.DigResource(resourceChange.Address) + } + + if currentResource == nil { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", resourceChange.Address). + WithField("terraform-query-field", mapData.QueryField).Warn("Skipping resource without values") + continue + } + + query, ok := currentResource.AttributeValues.Dig(mapData.QueryField) + if !ok { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", resourceChange.Address). + WithField("terraform-query-field", mapData.QueryField).Debug("Missing mapping field, cannot create Overmind query") + results.Results = append(results.Results, PlannedChangeMapResult{ + TerraformName: resourceChange.Address, + Status: MapStatusNotEnoughInfo, + Message: fmt.Sprintf("missing %v", mapData.QueryField), + MappedItemDiff: &sdp.MappedItemDiff{ + Item: itemDiff, + MappingQuery: nil, // unmapped item has no mapping query + }, + }) + continue + } + + // Create the map that variables will pull data from + dataMap := make(map[string]any) + + // Populate resource values + dataMap["values"] = currentResource.AttributeValues + + if overmindMappingsOutput, ok := plan.PlannedValues.Outputs["overmind_mappings"]; ok { + configResource := plan.Config.RootModule.DigResource(resourceChange.Address) + + if configResource == nil { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", resourceChange.Address). + Debug("Skipping provider mapping for resource without config") + } else { + // Look up the provider config key in the mappings + mappings := make(map[string]map[string]string) + + err = json.Unmarshal(overmindMappingsOutput.Value, &mappings) + + if err != nil { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", resourceChange.Address). + WithError(err). + Error("Failed to parse overmind_mappings output") + } else { + // We need to split out the module section of the name + // here. If the resource isn't in a module, the + // ProviderConfigKey will be something like + // "kubernetes", however if it's in a module it's be + // something like "module.something:kubernetes" + providerName := extractProviderNameFromConfigKey(configResource.ProviderConfigKey) + currentProviderMappings, ok := mappings[providerName] + + if ok { + log.WithContext(ctx). + WithFields(lf). + WithField("terraform-address", resourceChange.Address). + WithField("provider-config-key", configResource.ProviderConfigKey). + Debug("Found provider mappings") + + // We have mappings for this provider, so set them + // in the `provider_mapping` value + dataMap["provider_mapping"] = currentProviderMappings + } + } + } + } + + // Interpolate variables in the scope + scope, err := InterpolateScope(mapData.Scope, dataMap) + + if err != nil { + log.WithContext(ctx).WithError(err).Debugf("Could not find scope mapping variables %v, adding them will result in better results. Error: ", mapData.Scope) + scope = "*" + } + + u := uuid.New() + newQuery := &sdp.Query{ + Type: mapData.Type, + Method: mapData.Method, + Query: fmt.Sprintf("%v", query), + Scope: scope, + RecursionBehaviour: &sdp.Query_RecursionBehaviour{}, + UUID: u[:], + Deadline: timestamppb.New(time.Now().Add(60 * time.Second)), + } + + // cleanup item metadata from mapping query + if itemDiff.GetBefore() != nil { + itemDiff.Before.Type = newQuery.GetType() + if newQuery.GetScope() != "*" { + itemDiff.Before.Scope = newQuery.GetScope() + } + } + + // cleanup item metadata from mapping query + if itemDiff.GetAfter() != nil { + itemDiff.After.Type = newQuery.GetType() + if newQuery.GetScope() != "*" { + itemDiff.After.Scope = newQuery.GetScope() + } + } + + results.Results = append(results.Results, PlannedChangeMapResult{ + TerraformName: resourceChange.Address, + Status: MapStatusSuccess, + Message: "mapped", + MappedItemDiff: &sdp.MappedItemDiff{ + Item: itemDiff, + MappingQuery: newQuery, + }, + }) + + log.WithContext(ctx).WithFields(log.Fields{ + "scope": newQuery.GetScope(), + "type": newQuery.GetType(), + "query": newQuery.GetQuery(), + "method": newQuery.GetMethod().String(), + }).Debug("Mapped resource to query") + } + } + + return &results, nil +} + +// Checks if the supplied JSON bytes are a state file. It's a common mistake to +// pass a state file to Overmind rather than a plan file since the commands to +// create them are similar +func isStateFile(bytes []byte) bool { + fields := make(map[string]interface{}) + + err := json.Unmarshal(bytes, &fields) + + if err != nil { + return false + } + + if _, exists := fields["values"]; exists { + return true + } + + return false +} + +// Returns the name of the provider from the config key. If the resource isn't +// in a module, the ProviderConfigKey will be something like "kubernetes", +// however if it's in a module it's be something like +// "module.something:kubernetes". In both scenarios we want to return +// "kubernetes" +func extractProviderNameFromConfigKey(providerConfigKey string) string { + sections := strings.Split(providerConfigKey, ":") + return sections[len(sections)-1] +} + +// InterpolateScope Will interpolate variables in the scope string. These +// variables can come from the following places: +// +// * `outputs` - These are the outputs from the plan +// * `values` - These are the values from the resource in question +// +// Interpolation is done using the Terraform interpolation syntax: +// https://www.terraform.io/docs/configuration/interpolation.html +func InterpolateScope(scope string, data map[string]any) (string, error) { + // Find all instances of ${} in the Scope + matches := escapeRegex.FindAllStringSubmatch(scope, -1) + + interpolated := scope + + for _, match := range matches { + // The first match is the entire string, the second match is the + // variable name + variableName := match[1] + + value := TerraformDig(&data, variableName) + + if value == nil { + return "", fmt.Errorf("variable '%v' not found", variableName) + } + + // Convert the value to a string + valueString, ok := value.(string) + + if !ok { + return "", fmt.Errorf("variable '%v' is not a string", variableName) + } + + interpolated = strings.Replace(interpolated, match[0], valueString, 1) + } + + return interpolated, nil +} + +func countSensitiveValuesInConfig(m ConfigModule) int { + removedSecrets := 0 + for _, v := range m.Variables { + if v.Sensitive { + removedSecrets++ + } + } + for _, o := range m.Outputs { + if o.Sensitive { + removedSecrets++ + } + } + for _, c := range m.ModuleCalls { + removedSecrets += countSensitiveValuesInConfig(c.Module) + } + return removedSecrets +} + +func countSensitiveValuesInState(m Module) int { + removedSecrets := 0 + for _, r := range m.Resources { + removedSecrets += countSensitiveValuesInResource(r) + } + for _, c := range m.ChildModules { + removedSecrets += countSensitiveValuesInState(c) + } + return removedSecrets +} + +// follow itemAttributesFromResourceChangeData and maskSensitiveData +// implementation to count sensitive values +func countSensitiveValuesInResource(r Resource) int { + // sensitiveMsg can be a bool or a map[string]any + var isSensitive bool + err := json.Unmarshal(r.SensitiveValues, &isSensitive) + if err == nil && isSensitive { + return 1 // one very large secret + } else if err != nil { + // only try parsing as map if parsing as bool failed + var sensitive map[string]any + err = json.Unmarshal(r.SensitiveValues, &sensitive) + if err != nil { + return 0 + } + return countSensitiveAttributes(r.AttributeValues, sensitive) + } + return 0 +} + +func countSensitiveAttributes(attributes, sensitive any) int { + if sensitive == true { + return 1 + } else if sensitiveMap, ok := sensitive.(map[string]any); ok { + if attributesMap, ok := attributes.(map[string]any); ok { + result := 0 + for k, v := range attributesMap { + result += countSensitiveAttributes(v, sensitiveMap[k]) + } + return result + } else { + return 1 + } + } else if sensitiveArr, ok := sensitive.([]any); ok { + if attributesArr, ok := attributes.([]any); ok { + if len(sensitiveArr) != len(attributesArr) { + return 1 + } + result := 0 + for i, v := range attributesArr { + result += countSensitiveAttributes(v, sensitiveArr[i]) + } + return result + } else { + return 1 + } + } + return 0 +} + +// Converts a ResourceChange form a terraform plan to an ItemDiff in SDP format. +// These items will use the scope `terraform_plan` since we haven't mapped them +// to an actual item in the infrastructure yet +func itemDiffFromResourceChange(resourceChange ResourceChange) (*sdp.ItemDiff, error) { + status := sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UNSPECIFIED + + if slices.Equal(resourceChange.Change.Actions, []string{"no-op"}) || slices.Equal(resourceChange.Change.Actions, []string{"read"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UNCHANGED + } else if slices.Equal(resourceChange.Change.Actions, []string{"create"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_CREATED + } else if slices.Equal(resourceChange.Change.Actions, []string{"update"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_UPDATED + } else if slices.Equal(resourceChange.Change.Actions, []string{"delete", "create"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_REPLACED + } else if slices.Equal(resourceChange.Change.Actions, []string{"create", "delete"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_REPLACED + } else if slices.Equal(resourceChange.Change.Actions, []string{"delete"}) { + status = sdp.ItemDiffStatus_ITEM_DIFF_STATUS_DELETED + } + + beforeAttributes, err := itemAttributesFromResourceChangeData(resourceChange.Change.Before, resourceChange.Change.BeforeSensitive) + if err != nil { + return nil, fmt.Errorf("failed to parse before attributes: %w", err) + } + afterAttributes, err := itemAttributesFromResourceChangeData(resourceChange.Change.After, resourceChange.Change.AfterSensitive) + if err != nil { + return nil, fmt.Errorf("failed to parse after attributes: %w", err) + } + + err = handleKnownAfterApply(beforeAttributes, afterAttributes, resourceChange.Change.AfterUnknown) + if err != nil { + return nil, fmt.Errorf("failed to remove known after apply fields: %w", err) + } + + result := &sdp.ItemDiff{ + // Item: filled in by item mapping in UpdatePlannedChanges + Status: status, + } + + // shorten the address by removing the type prefix if and only if it is the + // first part. Longer terraform addresses created in modules will not be + // shortened to avoid confusion. + trimmedAddress, _ := strings.CutPrefix(resourceChange.Address, fmt.Sprintf("%v.", resourceChange.Type)) + + if beforeAttributes != nil { + result.Before = &sdp.Item{ + Type: resourceChange.Type, + UniqueAttribute: "terraform_name", + Attributes: beforeAttributes, + Scope: "terraform_plan", + } + + err = result.GetBefore().GetAttributes().Set("terraform_name", trimmedAddress) + if err != nil { + // since Address is a string, this should never happen + sentry.CaptureException(fmt.Errorf("failed to set terraform_name '%v' on before attributes: %w", trimmedAddress, err)) + } + + err = result.GetBefore().GetAttributes().Set("terraform_address", resourceChange.Address) + if err != nil { + // since Address is a string, this should never happen + sentry.CaptureException(fmt.Errorf("failed to set terraform_address of type %T (%v) on before attributes: %w", resourceChange.Address, resourceChange.Address, err)) + } + } + + if afterAttributes != nil { + result.After = &sdp.Item{ + Type: resourceChange.Type, + UniqueAttribute: "terraform_name", + Attributes: afterAttributes, + Scope: "terraform_plan", + } + + err = result.GetAfter().GetAttributes().Set("terraform_name", trimmedAddress) + if err != nil { + // since Address is a string, this should never happen + sentry.CaptureException(fmt.Errorf("failed to set terraform_name '%v' on after attributes: %w", trimmedAddress, err)) + } + + err = result.GetAfter().GetAttributes().Set("terraform_address", resourceChange.Address) + if err != nil { + // since Address is a string, this should never happen + sentry.CaptureException(fmt.Errorf("failed to set terraform_address of type %T (%v) on after attributes: %w", resourceChange.Address, resourceChange.Address, err)) + } + } + + return result, nil +} + +func itemAttributesFromResourceChangeData(attributesMsg, sensitiveMsg json.RawMessage) (*sdp.ItemAttributes, error) { + var attributes map[string]any + err := json.Unmarshal(attributesMsg, &attributes) + if err != nil { + return nil, fmt.Errorf("failed to parse attributes: %w", err) + } + + // sensitiveMsg can be a bool or a map[string]any + var isSensitive bool + err = json.Unmarshal(sensitiveMsg, &isSensitive) + if err == nil && isSensitive { + attributes = maskAllData(attributes) + } else if err != nil { + // only try parsing as map if parsing as bool failed + var sensitive map[string]any + err = json.Unmarshal(sensitiveMsg, &sensitive) + if err != nil { + return nil, fmt.Errorf("failed to parse sensitive: %w", err) + } + attributes = maskSensitiveData(attributes, sensitive).(map[string]any) + } + + return sdp.ToAttributesSorted(attributes) +} + +// maskAllData masks every entry in attributes as redacted +func maskAllData(attributes map[string]any) map[string]any { + for k, v := range attributes { + if mv, ok := v.(map[string]any); ok { + attributes[k] = maskAllData(mv) + } else { + attributes[k] = "(sensitive value)" + } + } + return attributes +} + +// maskSensitiveData masks every entry in attributes that is set to true in sensitive. returns the redacted attributes +func maskSensitiveData(attributes, sensitive any) any { + if sensitive == true { + return "(sensitive value)" + } else if sensitiveMap, ok := sensitive.(map[string]any); ok { + if attributesMap, ok := attributes.(map[string]any); ok { + result := map[string]any{} + for k, v := range attributesMap { + result[k] = maskSensitiveData(v, sensitiveMap[k]) + } + return result + } else { + return "(sensitive value) (type mismatch)" + } + } else if sensitiveArr, ok := sensitive.([]any); ok { + if attributesArr, ok := attributes.([]any); ok { + if len(sensitiveArr) != len(attributesArr) { + return "(sensitive value) (len mismatch)" + } + result := make([]any, len(attributesArr)) + for i, v := range attributesArr { + result[i] = maskSensitiveData(v, sensitiveArr[i]) + } + return result + } else { + return "(sensitive value) (type mismatch)" + } + } + return attributes +} + +// Finds fields from the `before` and `after` attributes that are known after +// apply and replaces the "after" value with the string "(known after apply)" +func handleKnownAfterApply(before, after *sdp.ItemAttributes, afterUnknown json.RawMessage) error { + var afterUnknownInterface interface{} + err := json.Unmarshal(afterUnknown, &afterUnknownInterface) + if err != nil { + return fmt.Errorf("could not unmarshal `after_unknown` from plan: %w", err) + } + + // Convert the parent struct to a value so that we can treat them all the + // same when we recurse + beforeValue := structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: before.GetAttrStruct(), + }, + } + + afterValue := structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: after.GetAttrStruct(), + }, + } + + err = insertKnownAfterApply(&beforeValue, &afterValue, afterUnknownInterface) + + if err != nil { + return fmt.Errorf("failed to remove known after apply fields: %w", err) + } + + return nil +} + +// Inserts the text "(known after apply)" in place of null values in the planned +// "after" values for fields that are known after apply. By default these are +// `null` which produces a bad diff, so we replace them with (known after apply) +// to more accurately mirror what Terraform does in the CLI +func insertKnownAfterApply(before, after *structpb.Value, afterUnknown interface{}) error { + switch afterUnknown.(type) { + case map[string]interface{}: + for k, v := range afterUnknown.(map[string]interface{}) { + if v == true { + if afterFields := after.GetStructValue().GetFields(); afterFields != nil { + // Insert this in the after fields even if it doesn't exist. + // This is because sometimes you will get a plan that only + // has a before value for a know after apply field, so we + // want to still make sure it shows up + afterFields[k] = &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: KnownAfterApply, + }, + } + } + } else if v == false { + // Do nothing + continue + } else { + // Recurse into the nested fields + err := insertKnownAfterApply(before.GetStructValue().GetFields()[k], after.GetStructValue().GetFields()[k], v) + if err != nil { + return err + } + } + } + case []interface{}: + for i, v := range afterUnknown.([]interface{}) { + if v == true { + // If this value in a slice is true, set the corresponding value + // in after to (know after apply) + if after.GetListValue() != nil && len(after.GetListValue().GetValues()) > i { + after.GetListValue().Values[i] = &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: KnownAfterApply, + }, + } + } + } else if v == false { + // Do nothing + continue + } else { + // Make sure that the before and after both actually have a + // valid list item at this position, if they don't we can just + // pass `nil` to the `removeUnknownFields` function and it'll + // handle it + beforeListValues := before.GetListValue().GetValues() + afterListValues := after.GetListValue().GetValues() + var nestedBeforeValue *structpb.Value + var nestedAfterValue *structpb.Value + + if len(beforeListValues) > i { + nestedBeforeValue = beforeListValues[i] + } + + if len(afterListValues) > i { + nestedAfterValue = afterListValues[i] + } + + err := insertKnownAfterApply(nestedBeforeValue, nestedAfterValue, v) + if err != nil { + return err + } + } + } + default: + return nil + } + + return nil +} diff --git a/cmd/plan_mapper_test.go b/datamaps/plan_mapper_test.go similarity index 55% rename from cmd/plan_mapper_test.go rename to datamaps/plan_mapper_test.go index 9653eb99..8ded5c9e 100644 --- a/cmd/plan_mapper_test.go +++ b/datamaps/plan_mapper_test.go @@ -1,13 +1,49 @@ -package cmd +package datamaps import ( "context" + "encoding/json" + "fmt" "testing" "github.com/overmindtech/sdp-go" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" ) +func TestWithStateFile(t *testing.T) { + _, err := MappedItemDiffsFromPlanFile(context.Background(), "testdata/state.json", logrus.Fields{}) + + if err == nil { + t.Error("Expected error when running with state file, got none") + } +} + +func TestExtractProviderNameFromConfigKey(t *testing.T) { + tests := []struct { + ConfigKey string + Expected string + }{ + { + ConfigKey: "kubernetes", + Expected: "kubernetes", + }, + { + ConfigKey: "module.core:kubernetes", + Expected: "kubernetes", + }, + } + + for _, test := range tests { + t.Run(test.ConfigKey, func(t *testing.T) { + actual := extractProviderNameFromConfigKey(test.ConfigKey) + if actual != test.Expected { + t.Errorf("Expected %v, got %v", test.Expected, actual) + } + }) + } +} + func TestMappedItemDiffsFromPlan(t *testing.T) { results, err := MappedItemDiffsFromPlanFile(context.Background(), "testdata/plan.json", logrus.Fields{}) if err != nil { @@ -201,3 +237,307 @@ func TestPlanMappingResultNumFuncs(t *testing.T) { t.Errorf("Expected 1 unsupported, got %v", result.NumUnsupported()) } } + +func TestInterpolateScope(t *testing.T) { + t.Run("with no interpolation", func(t *testing.T) { + t.Parallel() + + result, err := InterpolateScope("foo", map[string]any{}) + + if err != nil { + t.Error(err) + } + + if result != "foo" { + t.Errorf("Expected result to be foo, got %s", result) + } + }) + + t.Run("with a single variable", func(t *testing.T) { + t.Parallel() + + result, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}", map[string]any{ + "outputs": map[string]any{ + "overmind_kubernetes_cluster_name": "foo", + }, + }) + + if err != nil { + t.Error(err) + } + + if result != "foo" { + t.Errorf("Expected result to be foo, got %s", result) + } + }) + + t.Run("with multiple variables", func(t *testing.T) { + t.Parallel() + + result, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}.${values.metadata.namespace}", map[string]any{ + "outputs": map[string]any{ + "overmind_kubernetes_cluster_name": "foo", + }, + "values": map[string]any{ + "metadata": map[string]any{ + "namespace": "bar", + }, + }, + }) + + if err != nil { + t.Error(err) + } + + if result != "foo.bar" { + t.Errorf("Expected result to be foo.bar, got %s", result) + } + }) + + t.Run("with a variable that doesn't exist", func(t *testing.T) { + t.Parallel() + + _, err := InterpolateScope("${outputs.overmind_kubernetes_cluster_name}", map[string]any{}) + + if err == nil { + t.Error("Expected error, got nil") + } + }) +} + +// note that these tests need to allocate the input map for every test to avoid +// false positives from maskSensitiveData mutating the data +func TestMaskSensitiveData(t *testing.T) { + t.Parallel() + + t.Run("empty", func(t *testing.T) { + t.Parallel() + got := maskSensitiveData(map[string]any{}, map[string]any{}) + require.Equal(t, map[string]any{}, got) + }) + + t.Run("easy", func(t *testing.T) { + t.Parallel() + require.Equal(t, + map[string]any{ + "foo": "bar", + }, + maskSensitiveData( + map[string]any{ + "foo": "bar", + }, + map[string]any{})) + + require.Equal(t, + map[string]any{ + "foo": "(sensitive value)", + }, + maskSensitiveData( + map[string]any{ + "foo": "bar", + }, + map[string]any{"foo": true})) + + }) + + t.Run("deep", func(t *testing.T) { + t.Parallel() + require.Equal(t, + map[string]any{ + "foo": map[string]any{"key": "bar"}, + }, + maskSensitiveData( + map[string]any{ + "foo": map[string]any{"key": "bar"}, + }, + map[string]any{})) + + require.Equal(t, + map[string]any{ + "foo": "(sensitive value)", + }, + maskSensitiveData( + map[string]any{ + "foo": map[string]any{"key": "bar"}, + }, + map[string]any{"foo": true})) + + require.Equal(t, + map[string]any{ + "foo": map[string]any{"key": "(sensitive value)"}, + }, + maskSensitiveData( + map[string]any{ + "foo": map[string]any{"key": "bar"}, + }, + map[string]any{"foo": map[string]any{"key": true}})) + + }) + + t.Run("arrays", func(t *testing.T) { + t.Parallel() + require.Equal(t, + map[string]any{ + "foo": []any{"one", "two"}, + }, + maskSensitiveData( + map[string]any{ + "foo": []any{"one", "two"}, + }, + map[string]any{})) + + require.Equal(t, + map[string]any{ + "foo": "(sensitive value)", + }, + maskSensitiveData( + map[string]any{ + "foo": []any{"one", "two"}, + }, + map[string]any{"foo": true})) + + require.Equal(t, + map[string]any{ + "foo": []any{"one", "(sensitive value)"}, + }, + maskSensitiveData( + map[string]any{ + "foo": []any{"one", "two"}, + }, + map[string]any{"foo": []any{false, true}})) + + }) +} + +func TestHandleKnownAfterApply(t *testing.T) { + before, err := sdp.ToAttributes(map[string]interface{}{ + "string_value": "foo", + "int_value": 42, + "bool_value": true, + "float_value": 3.14, + "data": "secret", // Known after apply but doesn't exist in the "after" map, this happens sometimes + "list_value": []interface{}{ + "foo", + "bar", + }, + "map_value": map[string]interface{}{ + "foo": "bar", + "bar": "baz", + }, + "map_value2": map[string]interface{}{ + "ding": map[string]interface{}{ + "foo": "bar", + }, + }, + "nested_list": []interface{}{ + []interface{}{}, + []interface{}{ + "foo", + "bar", + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + after, err := sdp.ToAttributes(map[string]interface{}{ + "string_value": "bar", // I want to see a diff here + "int_value": nil, // These are going to be known after apply + "bool_value": nil, // These are going to be known after apply + "float_value": 3.14, + "list_value": []interface{}{ + "foo", + "bar", + "baz", // So is this one + }, + "map_value": map[string]interface{}{ // This whole thing will be known after apply + "foo": "bar", + }, + "map_value2": map[string]interface{}{ + "ding": map[string]interface{}{ + "foo": nil, // This will be known after apply + }, + }, + "nested_list": []interface{}{ + []interface{}{ + "foo", + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + afterUnknown := json.RawMessage(`{ + "int_value": true, + "bool_value": true, + "float_value": false, + "data": true, + "list_value": [ + false, + false, + true + ], + "map_value": true, + "map_value2": { + "ding": { + "foo": true + } + }, + "nested_list": [ + [ + false, + true + ], + [ + false, + true + ] + ] + }`) + + err = handleKnownAfterApply(before, after, afterUnknown) + if err != nil { + t.Fatal(err) + } + + beforeJSON, err := json.MarshalIndent(before, "", " ") + if err != nil { + t.Fatal(err) + } + afterJSON, err := json.MarshalIndent(after, "", " ") + if err != nil { + t.Fatal(err) + } + + fmt.Println("BEFORE:") + fmt.Println(string(beforeJSON)) + fmt.Println("\n\nAFTER:") + fmt.Println(string(afterJSON)) + + if val, _ := after.Get("int_value"); val != KnownAfterApply { + t.Errorf("expected int_value to be %v, got %v", KnownAfterApply, val) + } + + if val, _ := after.Get("bool_value"); val != KnownAfterApply { + t.Errorf("expected bool_value to be %v, got %v", KnownAfterApply, val) + } + + i, err := after.Get("list_value") + if err != nil { + t.Error(err) + } + + if list, ok := i.([]interface{}); ok { + if list[2] != KnownAfterApply { + t.Errorf("expected third string_value to be %v, got %v", KnownAfterApply, list[2]) + } + } else { + t.Error("list_value is not a string slice") + } + + if val, _ := after.Get("data"); val != KnownAfterApply { + t.Errorf("expected data to be %v, got %v", KnownAfterApply, val) + } +} diff --git a/cmd/testdata/plan.json b/datamaps/testdata/plan.json similarity index 100% rename from cmd/testdata/plan.json rename to datamaps/testdata/plan.json diff --git a/cmd/testdata/state.json b/datamaps/testdata/state.json similarity index 100% rename from cmd/testdata/state.json rename to datamaps/testdata/state.json diff --git a/cmd/datamaps/types.go b/datamaps/types.go similarity index 82% rename from cmd/datamaps/types.go rename to datamaps/types.go index cc024aec..46184599 100644 --- a/cmd/datamaps/types.go +++ b/datamaps/types.go @@ -4,8 +4,8 @@ import ( "github.com/overmindtech/sdp-go" ) -//go:generate go run ../../extractmaps.go aws-source -//go:generate go run ../../extractmaps.go k8s-source +//go:generate go run ../extractmaps.go aws-source +//go:generate go run ../extractmaps.go k8s-source type TfMapData struct { // The overmind type name diff --git a/extractmaps.go b/extractmaps.go index 8f9686c3..e134d588 100644 --- a/extractmaps.go +++ b/extractmaps.go @@ -37,7 +37,7 @@ func main() { args := Args{ Source: os.Args[1], SourceMunged: strings.ReplaceAll(os.Args[1], "-", ""), - Data: dataFromFiles(fmt.Sprintf("../../sources/%v/docs-data", os.Args[1])), + Data: dataFromFiles(fmt.Sprintf("../sources/%v/docs-data", os.Args[1])), } funcMap := template.FuncMap{