diff --git a/_test/import-csv.yaml b/_test/import-csv.yaml index a534253..864d8fa 100644 --- a/_test/import-csv.yaml +++ b/_test/import-csv.yaml @@ -1,7 +1,40 @@ --- +# kind: source +# spec: +# name: test1 +# registry: local +# path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file +# version: v1.0.0 +# tables: +# ["*"] +# destinations: +# - sqlite +# spec: +# file: ./test.csv +# format: csv +# table: T1 +# separator: "," +# filter: _.color startsWith 'b' +# columns: +# - name: color +# type: string +# key: true +# unique: true +# notnull: true +# - name: value +# type: string +# unique: true +# notnull: true +# - name: optimized +# type: boolean +# notnull: true +# - name: count +# type: integer +# notnull: true +# --- kind: source spec: - name: local_csv + name: test2 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,20 +45,25 @@ spec: spec: file: ./test.csv format: csv - table: my_csv_data + table: T2 separator: "," + filter: _.color not startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true --- kind: destination spec: diff --git a/_test/import-json.yaml b/_test/import-json.yaml index a866c2c..eb2949b 100644 --- a/_test/import-json.yaml +++ b/_test/import-json.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_json + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,19 +12,56 @@ spec: spec: file: ./test.json format: json - table: my_json_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.json + format: json + table: T2 + filter: "_.color not startsWith 'b'" + columns: + - name: color + type: string + key: true + unique: true + notnull: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true + - name: count + type: integer + notnull: true --- kind: destination spec: diff --git a/_test/import-xlsx.yaml b/_test/import-xlsx.yaml index 7e44ba2..7197989 100644 --- a/_test/import-xlsx.yaml +++ b/_test/import-xlsx.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_xlsx + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,20 +12,56 @@ spec: spec: file: ./test.xlsx format: xlsx - table: my_xlsx_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean notnull: true # comment to get a null - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.xlsx + format: xlsx + table: T2 + filter: _.color not startsWith 'b' + columns: + - name: color + type: string + key: true + unique: true + notnull: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true # comment to get a null + - name: count + type: integer + #notnull: true --- kind: destination spec: diff --git a/_test/import-yaml.yaml b/_test/import-yaml.yaml index ea60bb1..00d12dd 100644 --- a/_test/import-yaml.yaml +++ b/_test/import-yaml.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_yaml + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,19 +12,56 @@ spec: spec: file: ./test.yaml format: yaml - table: my_yaml_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true + notnull: true unique: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.yaml + format: yaml + table: T2 + filter: _.color not startsWith 'b' + columns: + - name: color + type: string + key: true + notnull: true + unique: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true + - name: count + type: integer + notnull: true --- kind: destination spec: diff --git a/client/spec.go b/client/spec.go index 64fac31..3ceec29 100644 --- a/client/spec.go +++ b/client/spec.go @@ -1,6 +1,10 @@ package client -import "text/template" +import ( + "text/template" + + "github.com/antonmedv/expr/vm" +) type Column struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` @@ -10,13 +14,15 @@ type Column struct { Unique bool `json:"unique,omitempty" yaml:"unique,omitempty"` NotNull bool `json:"notnull,omitempty" yaml:"notnull,omitempty"` Transform *string `json:"transform,omitempty" yaml:"transform,omitempty"` - Template *template.Template `json:"_template" yaml:"-"` + Template *template.Template `json:"-" yaml:"-"` } type Spec struct { - File string `json:"file,omitempty" yaml:"file,omitempty"` - Format string `json:"format,omitempty" yaml:"format,omitempty"` - Table string `json:"table,omitempty" yaml:"table,omitempty"` - Columns []*Column `json:"columns,omitempty" yaml:"columns,omitempty"` - Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only - Sheets []string `json:"sheets,omitempty" yaml:"sheets,omitempty"` // XLSX only + File string `json:"file,omitempty" yaml:"file,omitempty"` + Format string `json:"format,omitempty" yaml:"format,omitempty"` + Table string `json:"table,omitempty" yaml:"table,omitempty"` + Filter *string `json:"filter,omitempty" yaml:"filter,omitempty"` + Evaluator *vm.Program `json:"-,omitempty" yaml:"-,omitempty"` + Columns []*Column `json:"columns,omitempty" yaml:"columns,omitempty"` + Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only + Sheets []string `json:"sheets,omitempty" yaml:"sheets,omitempty"` // XLSX only } diff --git a/go.mod b/go.mod index afad0ad..b2f573f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/Masterminds/sprig/v3 v3.2.3 + github.com/antonmedv/expr v1.12.5 github.com/cloudquery/plugin-sdk v1.45.0 github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c github.com/rs/zerolog v1.29.1 diff --git a/go.sum b/go.sum index 2bb654f..c3661d5 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E= +github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/resources/data.go b/resources/data.go index 9bc1b0c..3563d74 100644 --- a/resources/data.go +++ b/resources/data.go @@ -12,6 +12,7 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" + "github.com/antonmedv/expr" "github.com/cloudquery/plugin-sdk/schema" "github.com/dihedron/cq-plugin-utils/format" "github.com/dihedron/cq-plugin-utils/pointer" @@ -20,11 +21,17 @@ import ( "gopkg.in/yaml.v3" ) +type Env struct { + Row map[string]any `expr:"row"` +} + // GetTable uses data in the spec section of the client configuration to // dynamically build the information about the columns being imported. func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, error) { client := meta.(*client.Client) + row := map[string]any{} + columns := []schema.Column{} for _, c := range client.Specs.Columns { client.Logger.Debug().Str("name", c.Name).Msg("adding column") @@ -59,21 +66,41 @@ func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, erro case "string", "str", "s": client.Logger.Debug().Str("name", c.Name).Msg("column is of type string") column.Type = schema.TypeString + row[c.Name] = "" case "integer", "int", "i": client.Logger.Debug().Str("name", c.Name).Msg("column is of type int") column.Type = schema.TypeInt + row[c.Name] = 0 case "boolean", "bool", "b": client.Logger.Debug().Str("name", c.Name).Msg("column is of type bool") column.Type = schema.TypeBool + row[c.Name] = false default: client.Logger.Debug().Str("name", c.Name).Msg("column is of unmapped type, assuming string") column.Type = schema.TypeString + row[c.Name] = "" } columns = append(columns, column) + } + // now initialise the filter + if client.Specs.Filter != nil { + env := map[string]any{ + "_": row, + "string": func(v any) string { + return fmt.Sprintf("%v", v) + }, + } + if program, err := expr.Compile(*client.Specs.Filter, expr.Env(env), expr.AsBool()); err != nil { + client.Logger.Error().Err(err).Str("filter", *client.Specs.Filter).Msg("error compiling expression evaluator") + } else { + client.Logger.Debug().Str("filter", *client.Specs.Filter).Msg("expression evaluator successfully compiled") + client.Specs.Evaluator = program + } } client.Logger.Debug().Msg("returning table") + return []*schema.Table{ { Name: client.Specs.Table, @@ -204,9 +231,29 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou } for _, row := range rows { - client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("returning single row") - res <- row + accepted := true + if client.Specs.Evaluator != nil { + accepted = false + env := map[string]any{ + "_": row, + } + + if output, err := expr.Run(client.Specs.Evaluator, env); err != nil { + client.Logger.Error().Err(err).Msg("error running evaluator") + } else { + client.Logger.Debug().Any("output", output).Msg("received output") + accepted = output.(bool) + } + } + + if accepted { + client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("returning single row") + res <- row + } else { + client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("rejecting single row") + } } + return nil }