From e65a14642431f7bae648804dd9d19fd49a668242 Mon Sep 17 00:00:00 2001 From: Leandro Carneiro Date: Tue, 2 Aug 2022 13:59:58 -0300 Subject: [PATCH 1/2] feat: mask environment variables from outputs --- cmd/common.go | 8 +- cmd/common_konnect.go | 8 +- cmd/diff.go | 13 +- cmd/konnect_diff.go | 11 +- cmd/konnect_sync.go | 4 +- cmd/reset.go | 13 +- cmd/sync.go | 11 +- diff/diff.go | 47 ++++- diff/diff_helpers.go | 35 ++++ diff/diff_helpers_test.go | 33 ++++ tests/integration/diff_test.go | 167 +++++++++++++++++- tests/integration/test_utils.go | 18 +- .../diff/001-not-existing-workspace/kong.yaml | 12 +- .../001-not-existing-workspace/kong3x.yaml | 12 +- 14 files changed, 344 insertions(+), 48 deletions(-) diff --git a/cmd/common.go b/cmd/common.go index ab9160601..f5fdf61ac 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -81,7 +81,7 @@ func getWorkspaceName(workspaceFlag string, targetContent *file.Content) string } func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, - delay int, workspace string, + delay int, workspace string, noMaskValues bool, ) error { // read target file targetContent, err := file.GetContentFromFiles(filenames) @@ -225,7 +225,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, return err } - totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient) + totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient, noMaskValues) if err != nil { return err } @@ -273,7 +273,7 @@ func fetchCurrentState(ctx context.Context, client *kong.Client, dumpConfig dump } func performDiff(ctx context.Context, currentState, targetState *state.KongState, - dry bool, parallelism int, delay int, client *kong.Client, + dry bool, parallelism int, delay int, client *kong.Client, noMaskValues bool, ) (int, error) { s, err := diff.NewSyncer(diff.SyncerOpts{ CurrentState: currentState, @@ -285,7 +285,7 @@ func performDiff(ctx context.Context, currentState, targetState *state.KongState return 0, err } - stats, errs := s.Solve(ctx, parallelism, dry) + stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go index bc9c5e032..5d6259e93 100644 --- a/cmd/common_konnect.go +++ b/cmd/common_konnect.go @@ -123,7 +123,7 @@ func GetKongClientForKonnectMode( }) } -func resetKonnectV2(ctx context.Context) error { +func resetKonnectV2(ctx context.Context, resetNoMaskDeckEnvVarsValue bool) error { client, err := GetKongClientForKonnectMode(ctx, konnectConfig) if err != nil { return err @@ -136,7 +136,7 @@ func resetKonnectV2(ctx context.Context) error { if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client, resetNoMaskDeckEnvVarsValue) if err != nil { return err } @@ -170,7 +170,7 @@ func dumpKonnectV2(ctx context.Context) error { } func syncKonnect(ctx context.Context, - filenames []string, dry bool, parallelism int, + filenames []string, dry bool, parallelism int, noMaskValues bool, ) error { httpClient := utils.HTTPClient() @@ -249,7 +249,7 @@ func syncKonnect(ctx context.Context, return err } - stats, errs := s.Solve(ctx, parallelism, dry) + stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/diff.go b/cmd/diff.go index d6055b68c..02d490b23 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -7,10 +7,11 @@ import ( ) var ( - diffCmdKongStateFile []string - diffCmdParallelism int - diffCmdNonZeroExitCode bool - diffWorkspace string + diffCmdKongStateFile []string + diffCmdParallelism int + diffCmdNoMaskDeckEnvVarsValue bool + diffCmdNonZeroExitCode bool + diffWorkspace string ) // newDiffCmd represents the diff command @@ -27,7 +28,7 @@ that will be created, updated, or deleted. Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), diffCmdKongStateFile, true, - diffCmdParallelism, 0, diffWorkspace) + diffCmdParallelism, 0, diffWorkspace, diffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(diffCmdKongStateFile) == 0 { @@ -51,6 +52,8 @@ that will be created, updated, or deleted. "any plugins associated with consumers") diffCmd.Flags().IntVar(&diffCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + diffCmd.Flags().BoolVar(&diffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") diffCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are diffed.\n"+ diff --git a/cmd/konnect_diff.go b/cmd/konnect_diff.go index a3d1c52eb..656f37c67 100644 --- a/cmd/konnect_diff.go +++ b/cmd/konnect_diff.go @@ -7,9 +7,10 @@ import ( ) var ( - konnectDiffCmdKongStateFile []string - konnectDiffCmdParallelism int - konnectDiffCmdNonZeroExitCode bool + konnectDiffCmdKongStateFile []string + konnectDiffCmdParallelism int + konnectDiffCmdNoMaskDeckEnvVarsValue bool + konnectDiffCmdNonZeroExitCode bool ) // newKonnectDiffCmd represents the 'deck konnect diff' command. @@ -32,7 +33,7 @@ func newKonnectDiffCmd() *cobra.Command { konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, true, - konnectDiffCmdParallelism) + konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -47,6 +48,8 @@ func newKonnectDiffCmd() *cobra.Command { "with consumers.") konnectDiffCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNonZeroExitCode, "non-zero-exit-code", false, "return exit code 2 if there is a diff present,\n"+ "exit code 0 if no diff is found,\n"+ diff --git a/cmd/konnect_sync.go b/cmd/konnect_sync.go index 200bd6b3d..11590f39b 100644 --- a/cmd/konnect_sync.go +++ b/cmd/konnect_sync.go @@ -24,7 +24,7 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, false, - konnectDiffCmdParallelism) + konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -39,6 +39,8 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, "with consumers.") konnectSyncCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") + konnectSyncCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") addSilenceEventsFlag(konnectSyncCmd.Flags()) return konnectSyncCmd } diff --git a/cmd/reset.go b/cmd/reset.go index 755eed088..a0690e0d7 100644 --- a/cmd/reset.go +++ b/cmd/reset.go @@ -9,9 +9,10 @@ import ( ) var ( - resetCmdForce bool - resetWorkspace string - resetAllWorkspaces bool + resetCmdForce bool + resetWorkspace string + resetAllWorkspaces bool + resetNoMaskDeckEnvVarsValue bool ) // newResetCmd represents the reset command @@ -47,7 +48,7 @@ By default, this command will ask for confirmation.`, mode := getMode(nil) if mode == modeKonnect { _ = sendAnalytics("reset", "", mode) - return resetKonnectV2(ctx) + return resetKonnectV2(ctx, resetNoMaskDeckEnvVarsValue) } rootClient, err := utils.GetKongClient(rootConfig) @@ -99,7 +100,7 @@ By default, this command will ask for confirmation.`, if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient, resetNoMaskDeckEnvVarsValue) if err != nil { return err } @@ -118,6 +119,8 @@ By default, this command will ask for confirmation.`, "(Kong Enterprise only).") resetCmd.Flags().BoolVar(&resetAllWorkspaces, "all-workspaces", false, "reset configuration of all workspaces (Kong Enterprise only).") + resetCmd.Flags().BoolVar(&resetNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") resetCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are deleted.\n"+ diff --git a/cmd/sync.go b/cmd/sync.go index 9c91bc40c..4da34c292 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -7,9 +7,10 @@ import ( ) var ( - syncCmdParallelism int - syncCmdDBUpdateDelay int - syncWorkspace string + syncCmdParallelism int + syncCmdDBUpdateDelay int + syncCmdNoMaskDeckEnvVarsValue bool + syncWorkspace string ) // newSyncCmd represents the sync command @@ -24,7 +25,7 @@ to get Kong's state in sync with the input state.`, Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), syncCmdKongStateFile, false, - syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace) + syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace, syncCmdNoMaskDeckEnvVarsValue) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(syncCmdKongStateFile) == 0 { @@ -48,6 +49,8 @@ to get Kong's state in sync with the input state.`, "any plugins associated with consumers.") syncCmd.Flags().IntVar(&syncCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") + syncCmd.Flags().BoolVar(&syncCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + false, "do not mask DECK_ environment variable values at diff output.") syncCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, "only entities matching tags specified via this flag are synced.\n"+ diff --git a/diff/diff.go b/diff/diff.go index 1c8c19381..4f7818f2f 100644 --- a/diff/diff.go +++ b/diff/diff.go @@ -322,8 +322,34 @@ type Stats struct { DeleteOps *utils.AtomicInt32Counter } +// Generete Diff output for 'sync' and 'diff' commands +func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, error) { + var diffString string + var err error + if oldObj, ok := e.OldObj.(*state.Document); ok { + if !isDelete { + diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) + } else { + diffString, err = getDocumentDiff(e.Obj.(*state.Document), oldObj) + } + } else { + if !isDelete { + diffString, err = getDiff(e.OldObj, e.Obj) + } else { + diffString, err = getDiff(e.Obj, e.OldObj) + } + } + if err != nil { + return "", err + } + if !noMaskValues { + diffString = maskEnvVarValue(diffString) + } + return diffString, err +} + // Solve generates a diff and walks the graph. -func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, []error) { +func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, noMaskValues bool) (Stats, []error) { stats := Stats{ CreateOps: &utils.AtomicInt32Counter{}, UpdateOps: &utils.AtomicInt32Counter{}, @@ -347,20 +373,23 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, c := e.Obj.(state.ConsoleString) switch e.Op { case crud.Create: - cprint.CreatePrintln("creating", e.Kind, c.Console()) - case crud.Update: - var diffString string - if oldObj, ok := e.OldObj.(*state.Document); ok { - diffString, err = getDocumentDiff(oldObj, e.Obj.(*state.Document)) - } else { - diffString, err = getDiff(e.OldObj, e.Obj) + diffString, err := generateDiffString(e, false, noMaskValues) + if err != nil { + return nil, err } + cprint.CreatePrintln("creating", e.Kind, c.Console(), diffString) + case crud.Update: + diffString, err := generateDiffString(e, false, noMaskValues) if err != nil { return nil, err } cprint.UpdatePrintln("updating", e.Kind, c.Console(), diffString) case crud.Delete: - cprint.DeletePrintln("deleting", e.Kind, c.Console()) + diffString, err := generateDiffString(e, true, noMaskValues) + if err != nil { + return nil, err + } + cprint.DeletePrintln("deleting", e.Kind, c.Console(), diffString) default: panic("unknown operation " + e.Op.String()) } diff --git a/diff/diff_helpers.go b/diff/diff_helpers.go index e2eeb71ec..6bbd0c768 100644 --- a/diff/diff_helpers.go +++ b/diff/diff_helpers.go @@ -3,6 +3,9 @@ package diff import ( "encoding/json" "fmt" + "os" + "sort" + "strings" "github.com/Kong/gojsondiff" "github.com/Kong/gojsondiff/formatter" @@ -83,3 +86,35 @@ func getDiff(a, b interface{}) (string, error) { diffString, err := formatter.Format(d) return diffString, err } + +type EnvVar struct { + Key string + Value string +} + +func parseDeckEnvVars() []EnvVar { + const envVarPrefix = "DECK_" + var parsedEnvVars []EnvVar + + for _, envVarStr := range os.Environ() { + envPair := strings.SplitN(envVarStr, "=", 2) + if strings.HasPrefix(envPair[0], envVarPrefix) { + envVar := EnvVar{} + envVar.Key = envPair[0] + envVar.Value = envPair[1] + parsedEnvVars = append(parsedEnvVars, envVar) + } + } + + sort.Slice(parsedEnvVars, func(i, j int) bool { + return len(parsedEnvVars[i].Value) > len(parsedEnvVars[j].Value) + }) + return parsedEnvVars +} + +func maskEnvVarValue(diffString string) string { + for _, envVar := range parseDeckEnvVars() { + diffString = strings.Replace(diffString, envVar.Value, "[masked]", -1) + } + return diffString +} diff --git a/diff/diff_helpers_test.go b/diff/diff_helpers_test.go index 8f96e5446..453214e2a 100644 --- a/diff/diff_helpers_test.go +++ b/diff/diff_helpers_test.go @@ -1,6 +1,7 @@ package diff import ( + "os" "testing" "github.com/kong/deck/konnect" @@ -145,3 +146,35 @@ bar }) } } + +func Test_MaskEnvVarsValues(t *testing.T) { + tests := []struct { + name string + args string + want string + envVars map[string]string + }{ + { + name: "JSON", + envVars: map[string]string{ + "DECK_BAR": "barbar", + "DECK_BAZ": "bazbaz", + }, + args: `{"foo":"foo","bar":"barbar","baz":"bazbaz"}`, + want: `{"foo":"foo","bar":"[masked]","baz":"[masked]"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + if got := maskEnvVarValue(tt.args); got != tt.want { + t.Errorf("maskEnvVarValue() = %v\nwant %v", got, tt.want) + } + }) + } +} diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index 4aca76c82..4af042af1 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -3,59 +3,214 @@ package integration import ( + "os" "testing" "github.com/kong/deck/utils" "github.com/stretchr/testify/assert" ) +var ( + expectedOutputMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "[masked]" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "tags": [ ++ "[masked] is an external host. I like [masked]!", ++ "foo:foo", ++ "baz:[masked]", ++ "another:[masked]", ++ "bar:[masked]" ++ ] ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` + + expectedOutputUnMasked = `creating workspace test +creating service svc1 { ++ "connect_timeout": 60000 ++ "host": "mockbin.org" ++ "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d" ++ "name": "svc1" ++ "protocol": "http" ++ "read_timeout": 60000 ++ "tags": [ ++ "mockbin.org is an external host. I like mockbin.org!", ++ "foo:foo", ++ "baz:bazbaz", ++ "another:bazbaz", ++ "bar:barbar" ++ ] ++ "write_timeout": 60000 + } + +creating plugin rate-limiting (global) { ++ "config": { ++ "minute": 123 ++ } ++ "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e" ++ "name": "rate-limiting" + } + +Summary: + Created: 2 + Updated: 0 + Deleted: 0 +` + + diffEnvVars = map[string]string{ + "DECK_SVC1_HOSTNAME": "mockbin.org", + "DECK_BARR": "barbar", + "DECK_BAZZ": "bazbaz", // used more than once + "DECK_FUB": "fubfub", // unused + "DECK_FOO": "foo_test", // unused, partial match + } +) + // test scope: // - 1.x // - 2.x -func Test_Diff_Workspace_OlderThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_OlderThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: diffEnvVars, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } runWhen(t, "kong", "<3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_OlderThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", + envVars: diffEnvVars, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + runWhen(t, "kong", "<3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } // test scope: // - 3.x -func Test_Diff_Workspace_NewerThan3x(t *testing.T) { +func Test_Diff_Workspace_UnMasked_NewerThan3x(t *testing.T) { + tests := []struct { + name string + stateFile string + expectedState utils.KongRawState + envVars map[string]string + }{ + { + name: "diff with not existent workspace doesn't error out", + stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: diffEnvVars, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } + runWhen(t, "kong", ">=3.0.0") + teardown := setup(t) + defer teardown(t) + + out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputUnMasked) + }) + } +} +func Test_Diff_Workspace_Masked_NewerThan3x(t *testing.T) { tests := []struct { name string stateFile string expectedState utils.KongRawState + envVars map[string]string }{ { name: "diff with not existent workspace doesn't error out", stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", + envVars: diffEnvVars, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + os.Setenv(k, v) + defer func(k string) { + os.Unsetenv(k) + }(k) + } runWhen(t, "kong", ">=3.0.0") teardown := setup(t) defer teardown(t) - err := diff(tc.stateFile) - assert.Nil(t, err) + out, err := diff(tc.stateFile) + assert.NoError(t, err) + assert.Equal(t, out, expectedOutputMasked) }) } } diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index f0417417a..b03298987 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -3,9 +3,11 @@ package integration import ( "context" + "io/ioutil" "os" "testing" + "github.com/fatih/color" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/kong/deck/cmd" @@ -173,12 +175,24 @@ func sync(kongFile string, opts ...string) error { return deckCmd.ExecuteContext(context.Background()) } -func diff(kongFile string, opts ...string) error { +func diff(kongFile string, opts ...string) (string, error) { deckCmd := cmd.NewRootCmd() args := []string{"diff", "-s", kongFile} if len(opts) > 0 { args = append(args, opts...) } deckCmd.SetArgs(args) - return deckCmd.ExecuteContext(context.Background()) + + // overwrite default standard output + r, w, _ := os.Pipe() + color.Output = w + + // execute decK command + cmdErr := deckCmd.ExecuteContext(context.Background()) + + // read command output + w.Close() + out, _ := ioutil.ReadAll(r) + + return string(out), cmdErr } diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml index 33b3a9cdf..01b37490e 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong.yaml @@ -1,8 +1,16 @@ _workspace: test services: -- name: svc1 - host: mockbin.org + - name: svc1 + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} + tags: + - ${{ env "DECK_SVC1_HOSTNAME" }} is an external host. I like mockbin.org! + - foo:foo + - baz:${{ env "DECK_BAZZ" }} + - another:${{ env "DECK_BAZZ" }} + - bar:${{ env "DECK_BARR" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123 diff --git a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml index 328e12366..84c9692d0 100644 --- a/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml +++ b/tests/integration/testdata/diff/001-not-existing-workspace/kong3x.yaml @@ -1,9 +1,17 @@ _format_version: "3.0" _workspace: test services: -- name: svc1 - host: mockbin.org + - name: svc1 + id: 9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d + host: ${{ env "DECK_SVC1_HOSTNAME" }} + tags: + - ${{ env "DECK_SVC1_HOSTNAME" }} is an external host. I like mockbin.org! + - foo:foo + - baz:${{ env "DECK_BAZZ" }} + - another:${{ env "DECK_BAZZ" }} + - bar:${{ env "DECK_BARR" }} plugins: - name: rate-limiting + id: a1368a28-cb5c-4eee-86d8-03a6bdf94b5e config: minute: 123 From 599ad93cd016e9a7e32a5aaf0ebc7635e96f778d Mon Sep 17 00:00:00 2001 From: Leandro Carneiro Date: Fri, 14 Oct 2022 10:26:12 -0300 Subject: [PATCH 2/2] refact(mask env vars): noMaskValues as a global var --- cmd/common.go | 14 ++++++++------ cmd/common_konnect.go | 9 +++++---- cmd/diff.go | 13 ++++++------- cmd/konnect_diff.go | 11 +++++------ cmd/konnect_sync.go | 4 ++-- cmd/reset.go | 13 ++++++------- cmd/sync.go | 11 +++++------ diff/diff.go | 14 ++++++++++---- 8 files changed, 47 insertions(+), 42 deletions(-) diff --git a/cmd/common.go b/cmd/common.go index f5fdf61ac..d0f5ece26 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -27,8 +27,9 @@ const ( ) var ( - dumpConfig dump.Config - assumeYes bool + dumpConfig dump.Config + assumeYes bool + noMaskValues bool ) type mode int @@ -81,7 +82,7 @@ func getWorkspaceName(workspaceFlag string, targetContent *file.Content) string } func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, - delay int, workspace string, noMaskValues bool, + delay int, workspace string, ) error { // read target file targetContent, err := file.GetContentFromFiles(filenames) @@ -225,7 +226,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, return err } - totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient, noMaskValues) + totalOps, err := performDiff(ctx, currentState, targetState, dry, parallelism, delay, kongClient) if err != nil { return err } @@ -273,19 +274,20 @@ func fetchCurrentState(ctx context.Context, client *kong.Client, dumpConfig dump } func performDiff(ctx context.Context, currentState, targetState *state.KongState, - dry bool, parallelism int, delay int, client *kong.Client, noMaskValues bool, + dry bool, parallelism int, delay int, client *kong.Client, ) (int, error) { s, err := diff.NewSyncer(diff.SyncerOpts{ CurrentState: currentState, TargetState: targetState, KongClient: client, StageDelaySec: delay, + NoMaskValues: noMaskValues, }) if err != nil { return 0, err } - stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) + stats, errs := s.Solve(ctx, parallelism, dry) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go index 5d6259e93..9a5887213 100644 --- a/cmd/common_konnect.go +++ b/cmd/common_konnect.go @@ -123,7 +123,7 @@ func GetKongClientForKonnectMode( }) } -func resetKonnectV2(ctx context.Context, resetNoMaskDeckEnvVarsValue bool) error { +func resetKonnectV2(ctx context.Context) error { client, err := GetKongClientForKonnectMode(ctx, konnectConfig) if err != nil { return err @@ -136,7 +136,7 @@ func resetKonnectV2(ctx context.Context, resetNoMaskDeckEnvVarsValue bool) error if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client, resetNoMaskDeckEnvVarsValue) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, client) if err != nil { return err } @@ -170,7 +170,7 @@ func dumpKonnectV2(ctx context.Context) error { } func syncKonnect(ctx context.Context, - filenames []string, dry bool, parallelism int, noMaskValues bool, + filenames []string, dry bool, parallelism int, ) error { httpClient := utils.HTTPClient() @@ -244,12 +244,13 @@ func syncKonnect(ctx context.Context, TargetState: targetState, KongClient: kongClient, KonnectClient: konnectClient, + NoMaskValues: noMaskValues, }) if err != nil { return err } - stats, errs := s.Solve(ctx, parallelism, dry, noMaskValues) + stats, errs := s.Solve(ctx, parallelism, dry) // print stats before error to report completed operations printStats(stats) if errs != nil { diff --git a/cmd/diff.go b/cmd/diff.go index 02d490b23..8bb66ff92 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -7,11 +7,10 @@ import ( ) var ( - diffCmdKongStateFile []string - diffCmdParallelism int - diffCmdNoMaskDeckEnvVarsValue bool - diffCmdNonZeroExitCode bool - diffWorkspace string + diffCmdKongStateFile []string + diffCmdParallelism int + diffCmdNonZeroExitCode bool + diffWorkspace string ) // newDiffCmd represents the diff command @@ -28,7 +27,7 @@ that will be created, updated, or deleted. Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), diffCmdKongStateFile, true, - diffCmdParallelism, 0, diffWorkspace, diffCmdNoMaskDeckEnvVarsValue) + diffCmdParallelism, 0, diffWorkspace) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(diffCmdKongStateFile) == 0 { @@ -52,7 +51,7 @@ that will be created, updated, or deleted. "any plugins associated with consumers") diffCmd.Flags().IntVar(&diffCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") - diffCmd.Flags().BoolVar(&diffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + diffCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", false, "do not mask DECK_ environment variable values at diff output.") diffCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, diff --git a/cmd/konnect_diff.go b/cmd/konnect_diff.go index 656f37c67..b5cf55ade 100644 --- a/cmd/konnect_diff.go +++ b/cmd/konnect_diff.go @@ -7,10 +7,9 @@ import ( ) var ( - konnectDiffCmdKongStateFile []string - konnectDiffCmdParallelism int - konnectDiffCmdNoMaskDeckEnvVarsValue bool - konnectDiffCmdNonZeroExitCode bool + konnectDiffCmdKongStateFile []string + konnectDiffCmdParallelism int + konnectDiffCmdNonZeroExitCode bool ) // newKonnectDiffCmd represents the 'deck konnect diff' command. @@ -33,7 +32,7 @@ func newKonnectDiffCmd() *cobra.Command { konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, true, - konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) + konnectDiffCmdParallelism) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -48,7 +47,7 @@ func newKonnectDiffCmd() *cobra.Command { "with consumers.") konnectDiffCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") - konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + konnectDiffCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", false, "do not mask DECK_ environment variable values at diff output.") konnectDiffCmd.Flags().BoolVar(&konnectDiffCmdNonZeroExitCode, "non-zero-exit-code", false, "return exit code 2 if there is a diff present,\n"+ diff --git a/cmd/konnect_sync.go b/cmd/konnect_sync.go index 11590f39b..2bfd9d23c 100644 --- a/cmd/konnect_sync.go +++ b/cmd/konnect_sync.go @@ -24,7 +24,7 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, konnectConfig.Address = defaultLegacyKonnectURL } return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, false, - konnectDiffCmdParallelism, konnectDiffCmdNoMaskDeckEnvVarsValue) + konnectDiffCmdParallelism) }, PreRunE: func(cmd *cobra.Command, args []string) error { return preRunSilenceEventsFlag() @@ -39,7 +39,7 @@ to get Konnect's state in sync with the input state.` + konnectAlphaState, "with consumers.") konnectSyncCmd.Flags().IntVar(&konnectDiffCmdParallelism, "parallelism", 100, "Maximum number of concurrent operations.") - konnectSyncCmd.Flags().BoolVar(&konnectDiffCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + konnectSyncCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", false, "do not mask DECK_ environment variable values at diff output.") addSilenceEventsFlag(konnectSyncCmd.Flags()) return konnectSyncCmd diff --git a/cmd/reset.go b/cmd/reset.go index a0690e0d7..203a68ac8 100644 --- a/cmd/reset.go +++ b/cmd/reset.go @@ -9,10 +9,9 @@ import ( ) var ( - resetCmdForce bool - resetWorkspace string - resetAllWorkspaces bool - resetNoMaskDeckEnvVarsValue bool + resetCmdForce bool + resetWorkspace string + resetAllWorkspaces bool ) // newResetCmd represents the reset command @@ -48,7 +47,7 @@ By default, this command will ask for confirmation.`, mode := getMode(nil) if mode == modeKonnect { _ = sendAnalytics("reset", "", mode) - return resetKonnectV2(ctx, resetNoMaskDeckEnvVarsValue) + return resetKonnectV2(ctx) } rootClient, err := utils.GetKongClient(rootConfig) @@ -100,7 +99,7 @@ By default, this command will ask for confirmation.`, if err != nil { return err } - _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient, resetNoMaskDeckEnvVarsValue) + _, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient) if err != nil { return err } @@ -119,7 +118,7 @@ By default, this command will ask for confirmation.`, "(Kong Enterprise only).") resetCmd.Flags().BoolVar(&resetAllWorkspaces, "all-workspaces", false, "reset configuration of all workspaces (Kong Enterprise only).") - resetCmd.Flags().BoolVar(&resetNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + resetCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", false, "do not mask DECK_ environment variable values at diff output.") resetCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, diff --git a/cmd/sync.go b/cmd/sync.go index 4da34c292..c35895327 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -7,10 +7,9 @@ import ( ) var ( - syncCmdParallelism int - syncCmdDBUpdateDelay int - syncCmdNoMaskDeckEnvVarsValue bool - syncWorkspace string + syncCmdParallelism int + syncCmdDBUpdateDelay int + syncWorkspace string ) // newSyncCmd represents the sync command @@ -25,7 +24,7 @@ to get Kong's state in sync with the input state.`, Args: validateNoArgs, RunE: func(cmd *cobra.Command, args []string) error { return syncMain(cmd.Context(), syncCmdKongStateFile, false, - syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace, syncCmdNoMaskDeckEnvVarsValue) + syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace) }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(syncCmdKongStateFile) == 0 { @@ -49,7 +48,7 @@ to get Kong's state in sync with the input state.`, "any plugins associated with consumers.") syncCmd.Flags().IntVar(&syncCmdParallelism, "parallelism", 10, "Maximum number of concurrent operations.") - syncCmd.Flags().BoolVar(&syncCmdNoMaskDeckEnvVarsValue, "no-mask-deck-env-vars-value", + syncCmd.Flags().BoolVar(&noMaskValues, "no-mask-deck-env-vars-value", false, "do not mask DECK_ environment variable values at diff output.") syncCmd.Flags().StringSliceVar(&dumpConfig.SelectorTags, "select-tag", []string{}, diff --git a/diff/diff.go b/diff/diff.go index 4f7818f2f..a90224c63 100644 --- a/diff/diff.go +++ b/diff/diff.go @@ -55,6 +55,8 @@ type Syncer struct { konnectClient *konnect.Client entityDiffers map[types.EntityType]types.Differ + + noMaskValues bool } type SyncerOpts struct { @@ -66,6 +68,8 @@ type SyncerOpts struct { SilenceWarnings bool StageDelaySec int + + NoMaskValues bool } // NewSyncer constructs a Syncer. @@ -79,6 +83,8 @@ func NewSyncer(opts SyncerOpts) (*Syncer, error) { silenceWarnings: opts.SilenceWarnings, stageDelaySec: opts.StageDelaySec, + + noMaskValues: opts.NoMaskValues, } err := s.init() @@ -349,7 +355,7 @@ func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, } // Solve generates a diff and walks the graph. -func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, noMaskValues bool) (Stats, []error) { +func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool) (Stats, []error) { stats := Stats{ CreateOps: &utils.AtomicInt32Counter{}, UpdateOps: &utils.AtomicInt32Counter{}, @@ -373,19 +379,19 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, noMaskVa c := e.Obj.(state.ConsoleString) switch e.Op { case crud.Create: - diffString, err := generateDiffString(e, false, noMaskValues) + diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err } cprint.CreatePrintln("creating", e.Kind, c.Console(), diffString) case crud.Update: - diffString, err := generateDiffString(e, false, noMaskValues) + diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err } cprint.UpdatePrintln("updating", e.Kind, c.Console(), diffString) case crud.Delete: - diffString, err := generateDiffString(e, true, noMaskValues) + diffString, err := generateDiffString(e, true, sc.noMaskValues) if err != nil { return nil, err }