fix: Refactored logic for viper and mapstructure workaround

internal/domains/config.go

@@ -115,10 +115,11 @@ type TransformerConfig struct {
 	// https://github.com/spf13/viper/issues/373
 	// Instead we have to use workaround and parse it manually
 	Params toolkit.Params `mapstructure:"-" yaml:"-" json:"-"` // yaml:"params" json:"params,omitempty"`
-	// TempParams - the https://github.com/spf13/viper/issues/373 workaround
-	// we decode the values into TempParams and then Unmarshal it manually and set to Params
-	// The related code is in internal/utils/config/viper_workaround.go
-	TempParams map[string]any `mapstructure:"-" yaml:"params,omitempty" json:"params,omitempty"`
+	// MetadataParams - encoded transformer parameters - uses only for storing into storage
+	// TODO: You need to get rid of it by creating a separate structure for storing metadata in
+	//   internal/db/postgres/storage/metadata_json.go
+	// this is used only due to https://github.com/spf13/viper/issues/373
+	MetadataParams map[string]any `mapstructure:"-" yaml:"params,omitempty" json:"params,omitempty"`
 }
 
 type Table struct {
@@ -129,3 +130,16 @@ type Table struct {
 	Transformers        []*TransformerConfig `mapstructure:"transformers" yaml:"transformers" json:"transformers,omitempty"`
 	ColumnsTypeOverride map[string]string    `mapstructure:"columns_type_override" yaml:"columns_type_override" json:"columns_type_override,omitempty"`
 }
+
+// DummyConfig - This is a dummy config to the viper workaround
+// It is used to parse the transformation parameters manually only avoiding parsing other pars of the config
+// The reason why is there https://github.com/GreenmaskIO/greenmask/discussions/85
+type DummyConfig struct {
+	Dump struct {
+		Transformation []struct {
+			Transformers []struct {
+				Params map[string]interface{} `yaml:"params" json:"params"`
+			} `yaml:"transformers" json:"transformers"`
+		} `yaml:"transformation" json:"transformation"`
+	} `yaml:"dump" json:"dump"`
+}

internal/utils/config/viper_workaround.go

@@ -32,8 +32,7 @@ import (
 // To overcome this problem we need use default yaml and json parsers avoiding vaiper or mapstructure usage.
 func ParseTransformerParamsManually(cfgFilePath string, cfg *domains.Config) error {
 	ext := path.Ext(cfgFilePath)
-	//cfgMap := make(map[string]any)
-	tmpCfg := domains.NewConfig()
+	tmpCfg := &domains.DummyConfig{}
 	f, err := os.Open(cfgFilePath)
 	if err != nil {
 		return err
@@ -55,14 +54,15 @@ func ParseTransformerParamsManually(cfgFilePath string, cfg *domains.Config) err
 	return setTransformerParams(tmpCfg, cfg)
 }
 
-// setTransformerParams - get the value from domains.TransformerConfig.TempParams, marshall this value and store into
+// setTransformerParams - get the value from domains.TransformerConfig.MetadataParams, marshall this value and store into
 // domains.TransformerConfig.Params
-func setTransformerParams(tmpCfg, cfg *domains.Config) (err error) {
+func setTransformerParams(tmpCfg *domains.DummyConfig, cfg *domains.Config) (err error) {
 	for tableIdx, tableObj := range tmpCfg.Dump.Transformation {
 		for transformationIdx, transformationObj := range tableObj.Transformers {
 			transformer := cfg.Dump.Transformation[tableIdx].Transformers[transformationIdx]
+			tmpTransformer := tmpCfg.Dump.Transformation[tableIdx].Transformers[transformationIdx]
 			paramsMap := make(map[string]toolkit.ParamsValue, len(transformationObj.Params))
-			for paramName, decodedValue := range transformer.TempParams {
+			for paramName, decodedValue := range tmpTransformer.Params {
 				var encodedVal toolkit.ParamsValue
 				switch v := decodedValue.(type) {
 				case string:
@@ -76,6 +76,7 @@ func setTransformerParams(tmpCfg, cfg *domains.Config) (err error) {
 				paramsMap[paramName] = encodedVal
 			}
 			transformer.Params = paramsMap
+			transformer.MetadataParams = tmpTransformer.Params
 		}
 	}
 	return nil

tests/integration/greenmask/backward_compatibility_test.go

@@ -16,71 +16,57 @@ package greenmask
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
+	"text/template"
 	"time"
 
 	"github.com/jackc/pgx/v5"
 	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/suite"
-
-	"github.com/greenmaskio/greenmask/internal/db/postgres/pgdump"
-	"github.com/greenmaskio/greenmask/internal/domains"
-	"github.com/greenmaskio/greenmask/internal/storages/builder"
-	"github.com/greenmaskio/greenmask/internal/storages/directory"
-	"github.com/greenmaskio/greenmask/pkg/toolkit"
 )
 
-var config = &domains.Config{
-	Common: domains.Common{
-		PgBinPath:     "/usr/local/opt/postgresql@16/bin",
-		TempDirectory: "/tmp",
-	},
-	Log: domains.LogConfig{
-		Level:  "debug",
-		Format: "text",
-	},
-	Storage: domains.StorageConfig{
-		Type: builder.DirectoryStorageType,
-		Directory: &directory.Config{
-			Path: "/tmp",
-		},
-	},
-	Dump: domains.Dump{
-		PgDumpOptions: pgdump.Options{
-			DbName: "host=localhost user=postgres password=example dbname=demo port=54316",
-			Jobs:   10,
-		},
-		Transformation: []*domains.Table{
-			{
-				Schema: "bookings",
-				Name:   "flights",
-				Transformers: []*domains.TransformerConfig{
-					{
-						Name: "RandomDate",
-						Params: map[string]toolkit.ParamsValue{
-							"min":    toolkit.ParamsValue("2023-01-01 00:00:00.0+03"),
-							"max":    toolkit.ParamsValue("2023-01-02 00:00:00.0+03"),
-							"column": toolkit.ParamsValue("scheduled_departure"),
-						},
-					},
-					{
-						Name: "RandomDate",
-						Params: map[string]toolkit.ParamsValue{
-							"min":    toolkit.ParamsValue("2023-02-02 01:00:00.0+03"),
-							"max":    toolkit.ParamsValue("2023-03-03 00:00:00.0+03"),
-							"column": toolkit.ParamsValue("scheduled_arrival"),
-						},
-					},
-				},
-			},
-		},
-	},
-}
+var configStr = template.Must(template.New("config").Parse(`
+common:
+  pg_bin_path: "{{ .pgBinPath }}"
+  tmp_dir: "{{ .tmpDir }}"
+
+log:
+  level: debug
+  format: json
+
+storage:
+  type: "directory"
+  directory:
+    path: "{{ .storageDir }}"
+
+dump:
+  pg_dump_options:
+    dbname: "{{ .uri }}"
+    jobs: 10
+    load-via-partition-root: true
+    schema: public
+
+  transformation:
+    - schema: "bookings"
+      name: "flights"
+      transformers:
+
+        - name: "RandomDate"
+          params:
+            "min": "2023-01-01 00:00:00.0+03"
+            "max": "2023-01-02 00:00:00.0+03"
+            "column": "scheduled_departure"
+
+        - name: "RandomDate"
+          params:
+            "min": "2023-02-02 01:00:00.0+03"
+            "max": "2023-03-03 00:00:00.0+03"
+            "column": "scheduled_arrival"
+`))
 
 type BackwardCompatibilitySuite struct {
 	suite.Suite
@@ -114,16 +100,18 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() {
 	err = os.Mkdir(suite.runtimeTmpDir, 0700)
 	suite.Require().NoError(err, "error creating tmp dir")
 
-	config.Common.TempDirectory = suite.tmpDir
-	config.Storage.Directory.Path = suite.storageDir
-	config.Dump.PgDumpOptions.DbName = uri
-	config.Common.PgBinPath = pgBinPath
-
-	suite.configFilePath = path.Join(suite.tmpDir, "config.json")
+	suite.configFilePath = path.Join(suite.tmpDir, "config.yaml")
 	confFile, err := os.Create(suite.configFilePath)
-	suite.Require().NoError(err, "error creating config.yml file")
+	suite.Require().NoError(err, "error creating config.yaml file")
 	defer confFile.Close()
-	err = json.NewEncoder(confFile).Encode(config)
+	err = configStr.Execute(
+		confFile,
+		map[string]string{
+			"pgBinPath":  pgBinPath,
+			"tmpDir":     suite.tmpDir,
+			"uri":        uri,
+			"storageDir": suite.storageDir,
+		})
 	suite.Require().NoError(err, "error encoding config into yaml")
 
 	suite.conn, err = pgx.Connect(context.Background(), uri)
@@ -134,15 +122,23 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() {
 	log.Info().Str("dbname", suite.restorationDbName).Msg("creating database")
 	_, err = suite.conn.Exec(context.Background(), fmt.Sprintf("create database %s", suite.restorationDbName))
 	suite.Require().NoError(err, "error creating database")
+
+	restoreDbConn, err := pgx.Connect(context.Background(), fmt.Sprintf("%s dbname=%s", uri, suite.restorationDbName))
+	suite.Require().NoError(err, "error connecting to restore db")
+	defer restoreDbConn.Close(context.Background())
+	_, err = restoreDbConn.Exec(context.Background(), "drop schema public;")
+	suite.Require().NoError(err, "error creating database")
 }
 
 func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
 	suite.Run("dumping data using greenmask", func() {
 		cmd := exec.Command(path.Join(greenmaskBinPath, "greenmask"),
 			"--config", suite.configFilePath, "dump",
 		)
+		log.Debug().Str("cmd", cmd.String()).Msg("running greenmask")
 		cmd.Stderr = os.Stderr
 		cmd.Stdout = os.Stdout
+		log.Info().Str("cmd", cmd.String()).Msg("greenmask stdout and stderr forwarding")
 
 		err := cmd.Run()
 		suite.Require().NoError(err, "error running greenmask")
@@ -158,6 +154,7 @@ func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
 		cmd := exec.Command(path.Join(pgBinPath, "pg_restore"),
 			"-l", path.Join(suite.storageDir, lastDump.Name()),
 		)
+		log.Info().Str("cmd", cmd.String()).Msg("running pg_restore list")
 		out, err := cmd.Output()
 		if len(out) > 0 {
 			log.Info().Msg("pg_restore stout forwarding")
@@ -187,6 +184,7 @@ func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
 			"-v",
 			path.Join(suite.storageDir, lastDump.Name()),
 		)
+		log.Info().Str("cmd", cmd.String()).Msg("running pg_restore")
 		cmd.Stderr = os.Stderr
 		cmd.Stdout = os.Stdout
 		log.Info().Str("cmd", cmd.String()).Msg("pg_restore stdout and stderr forwarding")

tests/integration/storages/s3_test.go

@@ -21,10 +21,10 @@ import (
 	"path"
 	"slices"
 
-	"github.com/greenmaskio/greenmask/internal/storages"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/greenmaskio/greenmask/internal/storages"
 	"github.com/greenmaskio/greenmask/internal/storages/s3"
 )
 
@@ -90,7 +90,7 @@ func (suite *S3StorageSuite) TestS3Ops() {
 		suite.Require().Len(dirs, 1)
 		suite.Require().Equal("test.txt", files[0])
 		s3Dir := dirs[0].(*s3.Storage)
-		suite.Require().Equal(path.Join(suite.cfg.Bucket, suite.cfg.Prefix, "testdb")+"/", s3Dir.GetCwd())
+		suite.Require().Equal(path.Join(suite.cfg.Prefix, "testdb")+"/", s3Dir.GetCwd())
 
 		nextDir := dirs[0]
 		files, dirs, err = nextDir.ListDir(context.Background())