Skip to content

Improve capabilities of planned data redaction #118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 10, 2023
Merged

Improve capabilities of planned data redaction #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions cmd/submitplan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ type TfData struct {
}

// maskAllData masks every entry in attributes as redacted
func maskAllData(attributes map[string]interface{}) map[string]interface{} {
func maskAllData(attributes map[string]any) map[string]any {
for k, v := range attributes {
if mv, ok := v.(map[string]interface{}); ok {
if mv, ok := v.(map[string]any); ok {
attributes[k] = maskAllData(mv)
} else {
attributes[k] = "REDACTED"
Expand All @@ -93,35 +93,50 @@ func maskAllData(attributes map[string]interface{}) map[string]interface{} {
}

// maskSensitiveData masks every entry in attributes that is set to true in sensitive. returns the redacted attributes
func maskSensitiveData(attributes, sensitive map[string]interface{}) map[string]interface{} {
func maskSensitiveData(attributes, sensitive map[string]any) map[string]any {
for k, s := range sensitive {
log.Debugf("checking %v", k)
if mv, ok := s.(map[string]interface{}); ok {
if sub, ok := attributes[k].(map[string]interface{}); ok {
log.Debugf("checking %v: %v", k, s)
if mv, ok := s.(map[string]any); ok {
if sub, ok := attributes[k].(map[string]any); ok {
attributes[k] = maskSensitiveData(sub, mv)
}
} else if arr, ok := s.([]any); ok {
if sub, ok := attributes[k].([]any); ok {
if len(arr) != len(sub) {
attributes[k] = "REDACTED (len mismatch)"
continue
}
for i, v := range arr {
if v == true {
sub[i] = "REDACTED"
}
}
attributes[k] = sub
}
} else {
attributes[k] = "REDACTED"
if _, ok := attributes[k]; ok {
attributes[k] = "REDACTED"
}
}
}
return attributes
}

func itemAttributesFromResourceChangeData(attributesMsg, sensitiveMsg json.RawMessage) (*sdp.ItemAttributes, error) {
var attributes map[string]interface{}
var attributes map[string]any
err := json.Unmarshal(attributesMsg, &attributes)
if err != nil {
return nil, fmt.Errorf("failed to parse attributes: %w", err)
}

// sensitiveMsg can be a bool or a map[string]interface{}
// sensitiveMsg can be a bool or a map[string]any
var isSensitive bool
err = json.Unmarshal(sensitiveMsg, &isSensitive)
if err == nil && isSensitive {
attributes = maskAllData(attributes)
} else if err != nil {
// only try parsing as map if parsing as bool failed
var sensitive map[string]interface{}
var sensitive map[string]any
err = json.Unmarshal(sensitiveMsg, &sensitive)
if err != nil {
return nil, fmt.Errorf("failed to parse sensitive: %w", err)
Expand Down Expand Up @@ -164,7 +179,7 @@ func itemDiffFromResourceChange(resourceChange ResourceChange) (*sdp.ItemDiff, e
}

if beforeAttributes != nil {
result.Before = &sdp.Item{ // DODO: check for empty before/after attributes and set to nil
result.Before = &sdp.Item{
Type: resourceChange.Type,
UniqueAttribute: "terraform_address",
Attributes: beforeAttributes,
Expand Down Expand Up @@ -326,7 +341,7 @@ func mappedItemDiffsFromPlan(ctx context.Context, fileName string, lf log.Fields
}

// Create the map that variables will pull data from
dataMap := make(map[string]interface{})
dataMap := make(map[string]any)

// Populate resource values
dataMap["values"] = currentResource.AttributeValues
Expand Down
105 changes: 105 additions & 0 deletions cmd/submitplan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/overmindtech/sdp-go"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)

func TestMappedItemDiffsFromPlan(t *testing.T) {
Expand Down Expand Up @@ -136,3 +137,107 @@ func TestMappedItemDiffsFromPlan(t *testing.T) {
t.Errorf("Expected aws_iam_policy query query to be 'arn:aws:iam::123456789012:policy/test-alb-ingress', got '%v'", aws_iam_policy.MappingQuery.Query)
}
}

// note that these tests need to allocate the input map for every test as
// maskSensitiveData mutates its inputs
func TestMaskSensitiveData(t *testing.T) {
t.Parallel()

t.Run("empty", func(t *testing.T) {
t.Parallel()
got := maskSensitiveData(map[string]any{}, map[string]any{})
require.Equal(t, got, map[string]any{})
})

t.Run("easy", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": "bar",
},
maskSensitiveData(
map[string]any{
"foo": "bar",
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": "bar",
},
map[string]any{"foo": true}))

})

t.Run("deep", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": map[string]any{"key": "bar"},
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{"foo": true}))

require.Equal(t,
map[string]any{
"foo": map[string]any{"key": "REDACTED"},
},
maskSensitiveData(
map[string]any{
"foo": map[string]any{"key": "bar"},
},
map[string]any{"foo": map[string]any{"key": true}}))

})

t.Run("arrays", func(t *testing.T) {
t.Parallel()
require.Equal(t,
map[string]any{
"foo": []any{"one", "two"},
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{}))

require.Equal(t,
map[string]any{
"foo": "REDACTED",
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{"foo": true}))

require.Equal(t,
map[string]any{
"foo": []any{"one", "REDACTED"},
},
maskSensitiveData(
map[string]any{
"foo": []any{"one", "two"},
},
map[string]any{"foo": []any{false, true}}))

})
}
5 changes: 4 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ require (
github.com/mattn/go-isatty v0.0.20
github.com/overmindtech/sdp-go v0.57.0
github.com/sirupsen/logrus v1.9.3
github.com/sourcegraph/conc v0.3.0
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.17.0
github.com/stretchr/testify v1.8.4
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31
github.com/uptrace/opentelemetry-go-extra/otellogrus v0.2.3
github.com/xiam/dig v0.0.0-20191116195832-893b5fb5093b
Expand All @@ -32,6 +32,7 @@ require (
github.com/auth0/go-jwt-middleware/v2 v2.1.0 // indirect
github.com/aws/aws-sdk-go v1.45.16 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand All @@ -50,9 +51,11 @@ require (
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand Down
15 changes: 0 additions & 15 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -192,8 +191,6 @@ github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/overmindtech/sdp-go v0.56.0 h1:1VFF6kulsNIVVctBClZbeZxOsHKRQDPzfZPVmwcWGSo=
github.com/overmindtech/sdp-go v0.56.0/go.mod h1:WZ/CsRATgtF0KZpgrlJdzLieBO3Gk9mk5XcdrSVOkbQ=
github.com/overmindtech/sdp-go v0.57.0 h1:hUqBij/gmc11TaQF4OOKj15kZH6SZJsObQgeCah0vRc=
github.com/overmindtech/sdp-go v0.57.0/go.mod h1:G01shsRPrtJBMQb2HizIG1BIst7jG8Q6BZ+G8/9YPHM=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
Expand Down Expand Up @@ -227,8 +224,6 @@ github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY=
github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
Expand Down Expand Up @@ -302,8 +297,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -376,8 +369,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand All @@ -389,8 +380,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0=
golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -445,8 +434,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand All @@ -461,8 +448,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down