Skip to content

Commit

Permalink
Merge pull request #86 from GreenmaskIO/fix/viper_workaround_for_para…
Browse files Browse the repository at this point in the history 
…ms_only

fix: Refactored logic for viper and mapstructure workaround
  • Loading branch information
@wwoytenko
wwoytenko committed Apr 29, 2024
2 parents fe1940d + 2cfd63e commit 759a2a3
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 70 deletions.
22 changes: 18 additions & 4 deletions internal/domains/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,11 @@ type TransformerConfig struct {
// https://github.com/spf13/viper/issues/373
// Instead we have to use workaround and parse it manually
Params toolkit.Params `mapstructure:"-" yaml:"-" json:"-"` // yaml:"params" json:"params,omitempty"`
// TempParams - the https://github.com/spf13/viper/issues/373 workaround
// we decode the values into TempParams and then Unmarshal it manually and set to Params
// The related code is in internal/utils/config/viper_workaround.go
TempParams map[string]any `mapstructure:"-" yaml:"params,omitempty" json:"params,omitempty"`
// MetadataParams - encoded transformer parameters - uses only for storing into storage
// TODO: You need to get rid of it by creating a separate structure for storing metadata in
// internal/db/postgres/storage/metadata_json.go
// this is used only due to https://github.com/spf13/viper/issues/373
MetadataParams map[string]any `mapstructure:"-" yaml:"params,omitempty" json:"params,omitempty"`
}

type Table struct {
Expand All @@ -129,3 +130,16 @@ type Table struct {
Transformers []*TransformerConfig `mapstructure:"transformers" yaml:"transformers" json:"transformers,omitempty"`
ColumnsTypeOverride map[string]string `mapstructure:"columns_type_override" yaml:"columns_type_override" json:"columns_type_override,omitempty"`
}

// DummyConfig - This is a dummy config to the viper workaround
// It is used to parse the transformation parameters manually only avoiding parsing other pars of the config
// The reason why is there https://github.com/GreenmaskIO/greenmask/discussions/85
type DummyConfig struct {
Dump struct {
Transformation []struct {
Transformers []struct {
Params map[string]interface{} `yaml:"params" json:"params"`
} `yaml:"transformers" json:"transformers"`
} `yaml:"transformation" json:"transformation"`
} `yaml:"dump" json:"dump"`
}
11 changes: 6 additions & 5 deletions internal/utils/config/viper_workaround.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ import (
// To overcome this problem we need use default yaml and json parsers avoiding vaiper or mapstructure usage.
func ParseTransformerParamsManually(cfgFilePath string, cfg *domains.Config) error {
ext := path.Ext(cfgFilePath)
//cfgMap := make(map[string]any)
tmpCfg := domains.NewConfig()
tmpCfg := &domains.DummyConfig{}
f, err := os.Open(cfgFilePath)
if err != nil {
return err
Expand All @@ -55,14 +54,15 @@ func ParseTransformerParamsManually(cfgFilePath string, cfg *domains.Config) err
return setTransformerParams(tmpCfg, cfg)
}

// setTransformerParams - get the value from domains.TransformerConfig.TempParams, marshall this value and store into
// setTransformerParams - get the value from domains.TransformerConfig.MetadataParams, marshall this value and store into
// domains.TransformerConfig.Params
func setTransformerParams(tmpCfg, cfg *domains.Config) (err error) {
func setTransformerParams(tmpCfg *domains.DummyConfig, cfg *domains.Config) (err error) {
for tableIdx, tableObj := range tmpCfg.Dump.Transformation {
for transformationIdx, transformationObj := range tableObj.Transformers {
transformer := cfg.Dump.Transformation[tableIdx].Transformers[transformationIdx]
tmpTransformer := tmpCfg.Dump.Transformation[tableIdx].Transformers[transformationIdx]
paramsMap := make(map[string]toolkit.ParamsValue, len(transformationObj.Params))
for paramName, decodedValue := range transformer.TempParams {
for paramName, decodedValue := range tmpTransformer.Params {
var encodedVal toolkit.ParamsValue
switch v := decodedValue.(type) {
case string:
Expand All @@ -76,6 +76,7 @@ func setTransformerParams(tmpCfg, cfg *domains.Config) (err error) {
paramsMap[paramName] = encodedVal
}
transformer.Params = paramsMap
transformer.MetadataParams = tmpTransformer.Params
}
}
return nil
Expand Down
120 changes: 59 additions & 61 deletions tests/integration/greenmask/backward_compatibility_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,71 +16,57 @@ package greenmask

import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path"
"text/template"
"time"

"github.com/jackc/pgx/v5"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/suite"

"github.com/greenmaskio/greenmask/internal/db/postgres/pgdump"
"github.com/greenmaskio/greenmask/internal/domains"
"github.com/greenmaskio/greenmask/internal/storages/builder"
"github.com/greenmaskio/greenmask/internal/storages/directory"
"github.com/greenmaskio/greenmask/pkg/toolkit"
)

var config = &domains.Config{
Common: domains.Common{
PgBinPath: "/usr/local/opt/postgresql@16/bin",
TempDirectory: "/tmp",
},
Log: domains.LogConfig{
Level: "debug",
Format: "text",
},
Storage: domains.StorageConfig{
Type: builder.DirectoryStorageType,
Directory: &directory.Config{
Path: "/tmp",
},
},
Dump: domains.Dump{
PgDumpOptions: pgdump.Options{
DbName: "host=localhost user=postgres password=example dbname=demo port=54316",
Jobs: 10,
},
Transformation: []*domains.Table{
{
Schema: "bookings",
Name: "flights",
Transformers: []*domains.TransformerConfig{
{
Name: "RandomDate",
Params: map[string]toolkit.ParamsValue{
"min": toolkit.ParamsValue("2023-01-01 00:00:00.0+03"),
"max": toolkit.ParamsValue("2023-01-02 00:00:00.0+03"),
"column": toolkit.ParamsValue("scheduled_departure"),
},
},
{
Name: "RandomDate",
Params: map[string]toolkit.ParamsValue{
"min": toolkit.ParamsValue("2023-02-02 01:00:00.0+03"),
"max": toolkit.ParamsValue("2023-03-03 00:00:00.0+03"),
"column": toolkit.ParamsValue("scheduled_arrival"),
},
},
},
},
},
},
}
var configStr = template.Must(template.New("config").Parse(`
common:
pg_bin_path: "{{ .pgBinPath }}"
tmp_dir: "{{ .tmpDir }}"
log:
level: debug
format: json
storage:
type: "directory"
directory:
path: "{{ .storageDir }}"
dump:
pg_dump_options:
dbname: "{{ .uri }}"
jobs: 10
load-via-partition-root: true
schema: public
transformation:
- schema: "bookings"
name: "flights"
transformers:
- name: "RandomDate"
params:
"min": "2023-01-01 00:00:00.0+03"
"max": "2023-01-02 00:00:00.0+03"
"column": "scheduled_departure"
- name: "RandomDate"
params:
"min": "2023-02-02 01:00:00.0+03"
"max": "2023-03-03 00:00:00.0+03"
"column": "scheduled_arrival"
`))

type BackwardCompatibilitySuite struct {
suite.Suite
Expand Down Expand Up @@ -114,16 +100,18 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() {
err = os.Mkdir(suite.runtimeTmpDir, 0700)
suite.Require().NoError(err, "error creating tmp dir")

config.Common.TempDirectory = suite.tmpDir
config.Storage.Directory.Path = suite.storageDir
config.Dump.PgDumpOptions.DbName = uri
config.Common.PgBinPath = pgBinPath

suite.configFilePath = path.Join(suite.tmpDir, "config.json")
suite.configFilePath = path.Join(suite.tmpDir, "config.yaml")
confFile, err := os.Create(suite.configFilePath)
suite.Require().NoError(err, "error creating config.yml file")
suite.Require().NoError(err, "error creating config.yaml file")
defer confFile.Close()
err = json.NewEncoder(confFile).Encode(config)
err = configStr.Execute(
confFile,
map[string]string{
"pgBinPath": pgBinPath,
"tmpDir": suite.tmpDir,
"uri": uri,
"storageDir": suite.storageDir,
})
suite.Require().NoError(err, "error encoding config into yaml")

suite.conn, err = pgx.Connect(context.Background(), uri)
Expand All @@ -134,15 +122,23 @@ func (suite *BackwardCompatibilitySuite) SetupSuite() {
log.Info().Str("dbname", suite.restorationDbName).Msg("creating database")
_, err = suite.conn.Exec(context.Background(), fmt.Sprintf("create database %s", suite.restorationDbName))
suite.Require().NoError(err, "error creating database")

restoreDbConn, err := pgx.Connect(context.Background(), fmt.Sprintf("%s dbname=%s", uri, suite.restorationDbName))
suite.Require().NoError(err, "error connecting to restore db")
defer restoreDbConn.Close(context.Background())
_, err = restoreDbConn.Exec(context.Background(), "drop schema public;")
suite.Require().NoError(err, "error creating database")
}

func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
suite.Run("dumping data using greenmask", func() {
cmd := exec.Command(path.Join(greenmaskBinPath, "greenmask"),
"--config", suite.configFilePath, "dump",
)
log.Debug().Str("cmd", cmd.String()).Msg("running greenmask")
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
log.Info().Str("cmd", cmd.String()).Msg("greenmask stdout and stderr forwarding")

err := cmd.Run()
suite.Require().NoError(err, "error running greenmask")
Expand All @@ -158,6 +154,7 @@ func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
cmd := exec.Command(path.Join(pgBinPath, "pg_restore"),
"-l", path.Join(suite.storageDir, lastDump.Name()),
)
log.Info().Str("cmd", cmd.String()).Msg("running pg_restore list")
out, err := cmd.Output()
if len(out) > 0 {
log.Info().Msg("pg_restore stout forwarding")
Expand Down Expand Up @@ -187,6 +184,7 @@ func (suite *BackwardCompatibilitySuite) TestGreenmaskCompatibility() {
"-v",
path.Join(suite.storageDir, lastDump.Name()),
)
log.Info().Str("cmd", cmd.String()).Msg("running pg_restore")
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
log.Info().Str("cmd", cmd.String()).Msg("pg_restore stdout and stderr forwarding")
Expand Down

0 comments on commit 759a2a3

Please sign in to comment.