feat(convert): add new format

Kong · Apr 13, 2023 · 81625b6 · 81625b6
1 parent 3184491
commit 81625b6
Show file tree

Hide file tree

Showing 25 changed files with 602 additions and 77 deletions.
diff --git a/cmd/common.go b/cmd/common.go
@@ -84,7 +84,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	delay int, workspace string,
 ) error {
 	// read target file
-	targetContent, err := file.GetContentFromFiles(filenames)
+	targetContent, err := file.GetContentFromFiles(filenames, false)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go
@@ -183,7 +183,7 @@ func syncKonnect(ctx context.Context,
 	httpClient := utils.HTTPClient()
 
 	// read target file
-	targetContent, err := file.GetContentFromFiles(filenames)
+	targetContent, err := file.GetContentFromFiles(filenames, false)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/convert.go b/cmd/convert.go
@@ -11,11 +11,12 @@ import (
 )
 
 var (
-	convertCmdSourceFormat      string
-	convertCmdDestinationFormat string
-	convertCmdInputFile         string
-	convertCmdOutputFile        string
-	convertCmdAssumeYes         bool
+	convertCmdSourceFormat       string
+	convertCmdDestinationFormat  string
+	convertCmdInputFile          []string
+	convertCmdOutputFile         string
+	convertCmdAssumeYes          bool
+	convertCmdDisableMockEnvVars bool
 )
 
 // newConvertCmd represents the convert command
@@ -37,7 +38,7 @@ can be converted into a 'konnect' configuration file.`,
 				return err
 			}
 
-			if convertCmdInputFile != "" {
+			if len(convertCmdInputFile) != 0 {
 				if yes, err := utils.ConfirmFileOverwrite(
 					convertCmdOutputFile, "", convertCmdAssumeYes,
 				); err != nil {
@@ -46,7 +47,11 @@ can be converted into a 'konnect' configuration file.`,
 					return nil
 				}
 
-				err = convert.Convert(convertCmdInputFile, convertCmdOutputFile, sourceFormat, destinationFormat)
+				err = convert.Convert(convertCmdInputFile,
+					convertCmdOutputFile,
+					sourceFormat,
+					destinationFormat,
+					!convertCmdDisableMockEnvVars)
 				if err != nil {
 					return fmt.Errorf("converting file: %v", err)
 				}
@@ -60,7 +65,7 @@ can be converted into a 'konnect' configuration file.`,
 					return fmt.Errorf("getting files from directory: %w", err)
 				}
 				for _, filename := range files {
-					err = convert.Convert(filename, filename, sourceFormat, destinationFormat)
+					err = convert.Convert([]string{filename}, filename, sourceFormat, destinationFormat, !convertCmdDisableMockEnvVars)
 					if err != nil {
 						return fmt.Errorf("converting '%s' file: %v", filename, err)
 					}
@@ -75,18 +80,20 @@ can be converted into a 'konnect' configuration file.`,
 		},
 	}
 
-	sourceFormats := []convert.Format{convert.FormatKongGateway, convert.FormatKongGateway2x}
+	sourceFormats := []convert.Format{convert.FormatKongGateway, convert.FormatKongGateway2x, convert.FormatDistributed}
 	destinationFormats := []convert.Format{convert.FormatKonnect, convert.FormatKongGateway3x}
 	convertCmd.Flags().StringVar(&convertCmdSourceFormat, "from", "",
 		fmt.Sprintf("format of the source file, allowed formats: %v", sourceFormats))
 	convertCmd.Flags().StringVar(&convertCmdDestinationFormat, "to", "",
 		fmt.Sprintf("desired format of the output, allowed formats: %v", destinationFormats))
-	convertCmd.Flags().StringVar(&convertCmdInputFile, "input-file", "",
-		"configuration file to be converted. Use `-` to read from stdin.")
+	convertCmd.Flags().StringSliceVar(&convertCmdInputFile, "input-file", []string{},
+		"configuration files to be converted. Use `-` to read from stdin.")
 	convertCmd.Flags().StringVar(&convertCmdOutputFile, "output-file", "kong.yaml",
 		"file to write configuration to after conversion. Use `-` to write to stdout.")
 	convertCmd.Flags().BoolVar(&convertCmdAssumeYes, "yes",
 		false, "assume `yes` to prompts and run non-interactively.")
+	convertCmd.Flags().BoolVar(&convertCmdDisableMockEnvVars, "disable-mock-env",
+		false, "disables the mocking of environment variables.")
 	return convertCmd
 }
 

diff --git a/cmd/validate.go b/cmd/validate.go
@@ -43,7 +43,7 @@ this command unless --online flag is used.
 			_ = sendAnalytics("validate", "", mode)
 			// read target file
 			// this does json schema validation as well
-			targetContent, err := file.GetContentFromFiles(validateCmdKongStateFile)
+			targetContent, err := file.GetContentFromFiles(validateCmdKongStateFile, false)
 			if err != nil {
 				return err
 			}

diff --git a/convert/testdata/2/.gitignore → convert/.gitignore b/convert/testdata/2/.gitignore → convert/.gitignore
diff --git a/convert/convert.go b/convert/convert.go
@@ -1,18 +1,24 @@
 package convert
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
+	"github.com/blang/semver/v4"
 	"github.com/kong/deck/cprint"
+	"github.com/kong/deck/dump"
 	"github.com/kong/deck/file"
+	"github.com/kong/deck/state"
 	"github.com/kong/deck/utils"
 	"github.com/kong/go-kong/kong"
 )
 
 type Format string
 
 const (
+	// FormatDistributed represents the Deck configuration format.
+	FormatDistributed Format = "distributed"
 	// FormatKongGateway represents the Kong gateway format.
 	FormatKongGateway Format = "kong-gateway"
 	// FormatKonnect represents the Konnect format.
@@ -37,38 +43,49 @@ func ParseFormat(key string) (Format, error) {
 		return FormatKongGateway2x, nil
 	case FormatKongGateway3x:
 		return FormatKongGateway3x, nil
+	case FormatDistributed:
+		return FormatDistributed, nil
 	default:
 		return "", fmt.Errorf("invalid format: '%v'", key)
 	}
 }
 
-func Convert(inputFilename, outputFilename string, from, to Format) error {
+func Convert(inputFilenames []string, outputFilename string, from, to Format, mockEnvVars bool) error {
+	const outputFormat = file.YAML
 	var (
 		outputContent *file.Content
 		err           error
 	)
 
-	inputContent, err := file.GetContentFromFiles([]string{inputFilename})
+	inputContent, err := file.GetContentFromFiles(inputFilenames, mockEnvVars)
 	if err != nil {
 		return err
 	}
 
 	switch {
 	case from == FormatKongGateway && to == FormatKonnect:
-		outputContent, err = convertKongGatewayToKonnect(inputContent)
-		if err != nil {
-			return err
+		if len(inputFilenames) > 1 {
+			return fmt.Errorf("only one input file can be provided when converting from Kong to Konnect format")
 		}
+		outputContent, err = convertKongGatewayToKonnect(inputContent)
 	case from == FormatKongGateway2x && to == FormatKongGateway3x:
-		outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilename)
-		if err != nil {
-			return err
+		if len(inputFilenames) > 1 {
+			return fmt.Errorf("only one input file can be provided when converting from Kong 2.x to Kong 3.x format")
 		}
+		outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilenames[0])
+	case from == FormatDistributed && to == FormatKongGateway:
+	case from == FormatDistributed && to == FormatKongGateway2x:
+	case from == FormatDistributed && to == FormatKongGateway3x:
+		outputContent, err = convertDistributedToKong(inputContent, outputFilename, outputFormat, to)
 	default:
 		return fmt.Errorf("cannot convert from '%s' to '%s' format", from, to)
 	}
 
-	err = file.WriteContentToFile(outputContent, outputFilename, file.YAML)
+	if err != nil {
+		return err
+	}
+
+	err = file.WriteContentToFile(outputContent, outputFilename, outputFormat)
 	if err != nil {
 		return err
 	}
@@ -195,3 +212,43 @@ func removeServiceName(service *file.FService) *file.FService {
 	serviceCopy.ID = kong.String(utils.UUID())
 	return serviceCopy
 }
+
+// convertDistributedToKong is used to convert one or many distributed format
+// files to create one Kong Gateway declarative config. It also leverages some
+// deck features like the defaults/centralized plugin configurations.
+func convertDistributedToKong(
+	targetContent *file.Content,
+	outputFilename string,
+	format file.Format,
+	kongFormat Format,
+) (*file.Content, error) {
+	var version semver.Version
+
+	switch kongFormat { //nolint:exhaustive
+	case FormatKongGateway:
+	case FormatKongGateway3x:
+		version = semver.Version{Major: 3, Minor: 0}
+	case FormatKongGateway2x:
+		version = semver.Version{Major: 2, Minor: 8}
+	}
+
+	s, _ := state.NewKongState()
+	rawState, err := file.Get(context.Background(), targetContent, file.RenderConfig{
+		CurrentState: s,
+		KongVersion:  version,
+	}, dump.Config{}, nil)
+	if err != nil {
+		return nil, err
+	}
+	targetState, err := state.Get(rawState)
+	if err != nil {
+		return nil, err
+	}
+
+	// file.KongStateToContent calls file.WriteContentToFile
+	return file.KongStateToContent(targetState, file.WriteConfig{
+		Filename:    outputFilename,
+		FileFormat:  format,
+		KongVersion: version.String(),
+	})
+}
diff --git a/convert/convert_test.go b/convert/convert_test.go
@@ -158,9 +158,12 @@ func zeroOutID(sp file.FServicePackage) file.FServicePackage {
 func Test_Convert(t *testing.T) {
 	type args struct {
 		inputFilename          string
+		inputFilenames         []string
 		outputFilename         string
 		fromFormat             Format
 		toFormat               Format
+		disableMocks           bool
+		envVars                map[string]string
 		expectedOutputFilename string
 	}
 	tests := []struct {
@@ -237,21 +240,101 @@ func Test_Convert(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "converts from distributed to kong gateway (no deck specific fields)",
+			args: args{
+				inputFilename:          "testdata/5/input.yaml",
+				outputFilename:         "testdata/5/output.yaml",
+				expectedOutputFilename: "testdata/5/output-expected.yaml",
+				fromFormat:             FormatDistributed,
+				toFormat:               FormatKongGateway,
+			},
+			wantErr: false,
+		},
+		{
+			name: "converts from distributed to kong gateway with defaults",
+			args: args{
+				inputFilename:          "testdata/6/input.yaml",
+				outputFilename:         "testdata/6/output.yaml",
+				expectedOutputFilename: "testdata/6/output-expected.yaml",
+				fromFormat:             FormatDistributed,
+				toFormat:               FormatKongGateway,
+			},
+			wantErr: false,
+		},
+		{
+			name: "converts from distributed to kong gateway with multiple files",
+			args: args{
+				inputFilenames:         []string{"testdata/7/input-1.yaml", "testdata/7/input-2.yaml"},
+				outputFilename:         "testdata/7/output.yaml",
+				expectedOutputFilename: "testdata/7/output-expected.yaml",
+				fromFormat:             FormatDistributed,
+				toFormat:               FormatKongGateway,
+			},
+			wantErr: false,
+		},
+		{
+			name: "converts from distributed to kong gateway with env variables",
+			args: args{
+				inputFilenames:         []string{"testdata/8/input.yaml"},
+				outputFilename:         "testdata/8/output.yaml",
+				expectedOutputFilename: "testdata/8/output-expected.yaml",
+				fromFormat:             FormatDistributed,
+				toFormat:               FormatKongGateway,
+				disableMocks:           true,
+				envVars: map[string]string{
+					"DECK_MOCKBIN_HOST":    "mockbin.org",
+					"DECK_MOCKBIN_ENABLED": "true",
+					"DECK_WRITE_TIMEOUT":   "777",
+					"DECK_FOO_FLOAT":       "666",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "converts from distributed to kong gateway with env variables (mocked)",
+			args: args{
+				inputFilenames:         []string{"testdata/9/input.yaml"},
+				outputFilename:         "testdata/9/output.yaml",
+				expectedOutputFilename: "testdata/9/output-expected.yaml",
+				fromFormat:             FormatDistributed,
+				toFormat:               FormatKongGateway,
+				disableMocks:           false,
+			},
+			wantErr: false,
+		},
+		{
+			name: "errors from distributed to kong gateway with env variables not set",
+			args: args{
+				inputFilenames: []string{"testdata/9/input.yaml"},
+				fromFormat:     FormatDistributed,
+				toFormat:       FormatKongGateway,
+				disableMocks:   true,
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := Convert(tt.args.inputFilename, tt.args.outputFilename, tt.args.fromFormat,
-				tt.args.toFormat)
+			inputFiles := tt.args.inputFilenames
+			if tt.args.inputFilename != "" {
+				inputFiles = []string{tt.args.inputFilename}
+			}
+			for k, v := range tt.args.envVars {
+				t.Setenv(k, v)
+			}
+			err := Convert(inputFiles, tt.args.outputFilename, tt.args.fromFormat,
+				tt.args.toFormat, !tt.args.disableMocks)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Convert() error = %v, wantErr %v", err, tt.wantErr)
 			}
 
 			if err == nil {
-				got, err := file.GetContentFromFiles([]string{tt.args.outputFilename})
+				got, err := file.GetContentFromFiles([]string{tt.args.outputFilename}, !tt.args.disableMocks)
 				if err != nil {
 					t.Errorf("failed to read output file: %v", err)
 				}
-				want, err := file.GetContentFromFiles([]string{tt.args.expectedOutputFilename})
+				want, err := file.GetContentFromFiles([]string{tt.args.expectedOutputFilename}, !tt.args.disableMocks)
 				if err != nil {
 					t.Errorf("failed to read output file: %v", err)
 				}

diff --git a/convert/testdata/3/output.yaml b/convert/testdata/3/output.yaml
diff --git a/convert/testdata/4/output.yaml b/convert/testdata/4/output.yaml