diff --git a/_test/import-csv.yaml b/_test/import-csv.yaml index a534253..864d8fa 100644 --- a/_test/import-csv.yaml +++ b/_test/import-csv.yaml @@ -1,7 +1,40 @@ --- +# kind: source +# spec: +# name: test1 +# registry: local +# path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file +# version: v1.0.0 +# tables: +# ["*"] +# destinations: +# - sqlite +# spec: +# file: ./test.csv +# format: csv +# table: T1 +# separator: "," +# filter: _.color startsWith 'b' +# columns: +# - name: color +# type: string +# key: true +# unique: true +# notnull: true +# - name: value +# type: string +# unique: true +# notnull: true +# - name: optimized +# type: boolean +# notnull: true +# - name: count +# type: integer +# notnull: true +# --- kind: source spec: - name: local_csv + name: test2 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,20 +45,25 @@ spec: spec: file: ./test.csv format: csv - table: my_csv_data + table: T2 separator: "," + filter: _.color not startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true --- kind: destination spec: diff --git a/_test/import-json.yaml b/_test/import-json.yaml index a866c2c..eb2949b 100644 --- a/_test/import-json.yaml +++ b/_test/import-json.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_json + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,19 +12,56 @@ spec: spec: file: ./test.json format: json - table: my_json_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.json + format: json + table: T2 + filter: "_.color not startsWith 'b'" + columns: + - name: color + type: string + key: true + unique: true + notnull: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true + - name: count + type: integer + notnull: true --- kind: destination spec: diff --git a/_test/import-xlsx.yaml b/_test/import-xlsx.yaml index 7e44ba2..7197989 100644 --- a/_test/import-xlsx.yaml +++ b/_test/import-xlsx.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_xlsx + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,20 +12,56 @@ spec: spec: file: ./test.xlsx format: xlsx - table: my_xlsx_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true unique: true + notnull: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean notnull: true # comment to get a null - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.xlsx + format: xlsx + table: T2 + filter: _.color not startsWith 'b' + columns: + - name: color + type: string + key: true + unique: true + notnull: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true # comment to get a null + - name: count + type: integer + #notnull: true --- kind: destination spec: diff --git a/_test/import-yaml.yaml b/_test/import-yaml.yaml index ea60bb1..00d12dd 100644 --- a/_test/import-yaml.yaml +++ b/_test/import-yaml.yaml @@ -1,7 +1,7 @@ --- kind: source spec: - name: local_yaml + name: test1 registry: local path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file version: v1.0.0 @@ -12,19 +12,56 @@ spec: spec: file: ./test.yaml format: yaml - table: my_yaml_data + table: T1 + filter: _.color startsWith 'b' columns: - name: color type: string key: true + notnull: true unique: true - name: value type: string unique: true + notnull: true - name: optimized type: boolean + notnull: true - name: count type: integer + notnull: true +--- +kind: source +spec: + name: test2 + registry: local + path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file + version: v1.0.0 + tables: + ["*"] + destinations: + - sqlite + spec: + file: ./test.yaml + format: yaml + table: T2 + filter: _.color not startsWith 'b' + columns: + - name: color + type: string + key: true + notnull: true + unique: true + - name: value + type: string + unique: true + notnull: true + - name: optimized + type: boolean + notnull: true + - name: count + type: integer + notnull: true --- kind: destination spec: diff --git a/client/client.go b/client/client.go index c097d78..72bd353 100644 --- a/client/client.go +++ b/client/client.go @@ -14,7 +14,6 @@ import ( type Client struct { Logger zerolog.Logger Specs *Spec - //Data []map[string]any } func (c *Client) ID() string { diff --git a/client/spec.go b/client/spec.go index a847a80..3ceec29 100644 --- a/client/spec.go +++ b/client/spec.go @@ -1,18 +1,28 @@ package client +import ( + "text/template" + + "github.com/antonmedv/expr/vm" +) + type Column struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Description *string `json:"description,omitempty" yaml:"description,omitempty"` - Type string `json:"type,omitempty" yaml:"type,omitempty"` - Key bool `json:"key,omitempty" yaml:"pk,omitempty"` - Unique bool `json:"unique,omitempty" yaml:"unique,omitempty"` - NotNull bool `json:"notnull,omitempty" yaml:"notnull,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Description *string `json:"description,omitempty" yaml:"description,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Key bool `json:"key,omitempty" yaml:"pk,omitempty"` + Unique bool `json:"unique,omitempty" yaml:"unique,omitempty"` + NotNull bool `json:"notnull,omitempty" yaml:"notnull,omitempty"` + Transform *string `json:"transform,omitempty" yaml:"transform,omitempty"` + Template *template.Template `json:"-" yaml:"-"` } type Spec struct { - File string `json:"file,omitempty" yaml:"file,omitempty"` - Format string `json:"format,omitempty" yaml:"format,omitempty"` - Table string `json:"table,omitempty" yaml:"table,omitempty"` - Columns []Column `json:"columns,omitempty" yaml:"columns,omitempty"` - Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only - Sheet *string `json:"sheet,omitempty" yaml:"sheet,omitempty"` // XLSX only + File string `json:"file,omitempty" yaml:"file,omitempty"` + Format string `json:"format,omitempty" yaml:"format,omitempty"` + Table string `json:"table,omitempty" yaml:"table,omitempty"` + Filter *string `json:"filter,omitempty" yaml:"filter,omitempty"` + Evaluator *vm.Program `json:"-,omitempty" yaml:"-,omitempty"` + Columns []*Column `json:"columns,omitempty" yaml:"columns,omitempty"` + Separator *string `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only + Sheets []string `json:"sheets,omitempty" yaml:"sheets,omitempty"` // XLSX only } diff --git a/go.mod b/go.mod index 1864eac..b2f573f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/dihedron/cq-source-file go 1.20 require ( + github.com/Masterminds/sprig/v3 v3.2.3 + github.com/antonmedv/expr v1.12.5 github.com/cloudquery/plugin-sdk v1.45.0 github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c github.com/rs/zerolog v1.29.1 @@ -11,6 +13,8 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.20.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect @@ -18,13 +22,18 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect + github.com/huandu/xstrings v1.3.3 // indirect + github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index d7fdc6b..c3661d5 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,14 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E= +github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -117,6 +125,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -129,7 +138,11 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -150,6 +163,10 @@ github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp9 github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -175,6 +192,9 @@ github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -184,6 +204,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -214,6 +235,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -281,6 +303,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= @@ -334,12 +357,14 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= @@ -350,6 +375,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -493,6 +519,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/resources/data.go b/resources/data.go index 129c5c2..3563d74 100644 --- a/resources/data.go +++ b/resources/data.go @@ -9,7 +9,10 @@ import ( "os" "reflect" "strings" + "text/template" + "github.com/Masterminds/sprig/v3" + "github.com/antonmedv/expr" "github.com/cloudquery/plugin-sdk/schema" "github.com/dihedron/cq-plugin-utils/format" "github.com/dihedron/cq-plugin-utils/pointer" @@ -18,14 +21,34 @@ import ( "gopkg.in/yaml.v3" ) +type Env struct { + Row map[string]any `expr:"row"` +} + // GetTable uses data in the spec section of the client configuration to // dynamically build the information about the columns being imported. func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, error) { client := meta.(*client.Client) + row := map[string]any{} + columns := []schema.Column{} for _, c := range client.Specs.Columns { client.Logger.Debug().Str("name", c.Name).Msg("adding column") + + // prepare the template for value transformation if there is a transform + if c.Transform != nil { + tpl, err := template.New(c.Name).Funcs(sprig.FuncMap()).Parse(*c.Transform) + if err != nil { + client.Logger.Error().Err(err).Str("column", c.Name).Str("transform", *c.Transform).Msg("error parsing column transform") + return nil, fmt.Errorf("error parsing transform for column %q: %w", c.Name, err) + } else { + c.Template = tpl + client.Logger.Debug().Str("template", format.ToJSON(tpl)).Str("transform", *c.Transform).Msg("template after having parsed transform") + } + client.Logger.Debug().Str("column", c.Name).Str("specs", format.ToJSON(client.Specs.Columns)).Msg("column metadata after having parsed transform") + } + if c.Description == nil { c.Description = pointer.To(fmt.Sprintf("The column mapping the %q field from the input data", c.Name)) } @@ -43,20 +66,41 @@ func GetTables(ctx context.Context, meta schema.ClientMeta) (schema.Tables, erro case "string", "str", "s": client.Logger.Debug().Str("name", c.Name).Msg("column is of type string") column.Type = schema.TypeString + row[c.Name] = "" case "integer", "int", "i": client.Logger.Debug().Str("name", c.Name).Msg("column is of type int") column.Type = schema.TypeInt + row[c.Name] = 0 case "boolean", "bool", "b": client.Logger.Debug().Str("name", c.Name).Msg("column is of type bool") column.Type = schema.TypeBool + row[c.Name] = false default: client.Logger.Debug().Str("name", c.Name).Msg("column is of unmapped type, assuming string") column.Type = schema.TypeString + row[c.Name] = "" } columns = append(columns, column) } + // now initialise the filter + if client.Specs.Filter != nil { + env := map[string]any{ + "_": row, + "string": func(v any) string { + return fmt.Sprintf("%v", v) + }, + } + if program, err := expr.Compile(*client.Specs.Filter, expr.Env(env), expr.AsBool()); err != nil { + client.Logger.Error().Err(err).Str("filter", *client.Specs.Filter).Msg("error compiling expression evaluator") + } else { + client.Logger.Debug().Str("filter", *client.Specs.Filter).Msg("expression evaluator successfully compiled") + client.Specs.Evaluator = program + } + } + client.Logger.Debug().Msg("returning table") + return []*schema.Table{ { Name: client.Specs.Table, @@ -147,50 +191,69 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou } }() // get all the rows in the requested (or the active) sheet - if client.Specs.Sheet == nil { + if len(client.Specs.Sheets) == 0 { // get the currently active sheet in the file - client.Specs.Sheet = pointer.To(xls.GetSheetName(xls.GetActiveSheetIndex())) - } - client.Logger.Debug().Str("sheet", *client.Specs.Sheet).Msg("getting data from sheet") - xlsrows, err := xls.GetRows(*client.Specs.Sheet) - if err != nil { - client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error getting rows") - return fmt.Errorf("error getting rows from input file %q: %w", client.Specs.File, err) + client.Specs.Sheets = []string{xls.GetSheetName(xls.GetActiveSheetIndex())} } + for _, sheet := range client.Specs.Sheets { + client.Logger.Debug().Str("sheet", sheet).Msg("getting data from sheet") + xlsrows, err := xls.GetRows(sheet) + if err != nil { + client.Logger.Error().Err(err).Str("file", client.Specs.File).Msg("error getting rows") + return fmt.Errorf("error getting rows from input file %q: %w", client.Specs.File, err) + } - var keys []string - first := true - for _, xlsrow := range xlsrows { - if first { - first = false - keys = xlsrow - } else { - values := xlsrow - row := map[string]any{} - for i := 0; i < len(keys); i++ { - if i < len(values) { - // XLSX rows can be sparse, in which case all TRAILING empty cells are removed - // from the returned slice; empty cells in the middle are still valid - row[keys[i]] = values[i] - } else { - row[keys[i]] = nil + var keys []string + first := true + for _, xlsrow := range xlsrows { + if first { + first = false + keys = xlsrow + } else { + values := xlsrow + row := map[string]any{} + for i := 0; i < len(keys); i++ { + if i < len(values) { + // XLSX rows can be sparse, in which case all TRAILING empty cells are removed + // from the returned slice; empty cells in the middle are still valid + row[keys[i]] = values[i] + } else { + row[keys[i]] = nil + } } + rows = append(rows, row) } - rows = append(rows, row) } } - - // TODO: add more formats default: client.Logger.Error().Str("format", client.Specs.Format).Msg("unsupported format") return fmt.Errorf("unsupported format: %q", client.Specs.Format) - } for _, row := range rows { - client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("returning single row") - res <- row + accepted := true + if client.Specs.Evaluator != nil { + accepted = false + env := map[string]any{ + "_": row, + } + + if output, err := expr.Run(client.Specs.Evaluator, env); err != nil { + client.Logger.Error().Err(err).Msg("error running evaluator") + } else { + client.Logger.Debug().Any("output", output).Msg("received output") + accepted = output.(bool) + } + } + + if accepted { + client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("returning single row") + res <- row + } else { + client.Logger.Debug().Str("row", format.ToJSON(row)).Msg("rejecting single row") + } } + return nil } @@ -199,9 +262,37 @@ func fetchData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resou func fetchColumn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { client := meta.(*client.Client) // client.Logger.Debug().Str("resource", format.ToJSON(resource)).Str("column", format.ToJSON(c)).Str("item type", fmt.Sprintf("%T", resource.Item)).Msg("fetching column...") - item := resource.Item.(map[string]any) - value := item[c.Name] + row := resource.Item.(map[string]any) + value := row[c.Name] client.Logger.Debug().Str("value", fmt.Sprintf("%v", value)).Str("type", fmt.Sprintf("%T", value)).Msg("checking value type") + + // now apply the transform if it is available + for _, spec := range client.Specs.Columns { + if spec.Name == c.Name && spec.Template != nil { + client.Logger.Debug().Msg("applying transform...") + var buffer bytes.Buffer + target := struct { + Name string + Value any + Type schema.ValueType + Row map[string]any + }{ + Name: c.Name, + Value: value, + Type: c.Type, + Row: row, + } + if err := spec.Template.Execute(&buffer, target); err != nil { + client.Logger.Error().Err(err).Any("value", value).Str("transform", *spec.Transform).Any("row", row).Msg("error applying transform") + return err + } + value = buffer.String() + break + } + } + + client.Logger.Debug().Any("value", value).Msg("after transform...") + if value == nil { client.Logger.Warn().Msg("value is nil") if c.CreationOptions.NotNull {