Pull into main

_test/import-csv.yaml

@@ -1,7 +1,40 @@
 ---
+# kind: source
+# spec:
+#   name: test1
+#   registry: local
+#   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+#   version: v1.0.0
+#   tables: 
+#     ["*"]
+#   destinations:
+#     - sqlite
+#   spec:
+#     file: ./test.csv
+#     format: csv
+#     table: T1
+#     separator: ","
+#     filter: _.color startsWith 'b'
+#     columns:
+#       - name: color
+#         type: string
+#         key: true
+#         unique: true
+#         notnull: true
+#       - name: value
+#         type: string
+#         unique: true
+#         notnull: true
+#       - name: optimized
+#         type: boolean
+#         notnull: true
+#       - name: count
+#         type: integer
+#         notnull: true
+# ---
 kind: source
 spec:
-  name: local_csv
+  name: test2
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,20 +45,25 @@ spec:
   spec:
     file: ./test.csv
     format: csv
-    table: my_csv_data
+    table: T2
     separator: ","
+    filter: _.color not startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
 ---
 kind: destination
 spec:

_test/import-json.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_json
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,19 +12,56 @@ spec:
   spec:
     file: ./test.json
     format: json
-    table: my_json_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.json
+    format: json
+    table: T2
+    filter: "_.color not startsWith 'b'"
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+        notnull: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true
+      - name: count
+        type: integer
+        notnull: true
 ---
 kind: destination
 spec:

_test/import-xlsx.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_xlsx
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,20 +12,56 @@ spec:
   spec:
     file: ./test.xlsx
     format: xlsx
-    table: my_xlsx_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
         notnull: true # comment to get a null
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.xlsx
+    format: xlsx
+    table: T2
+    filter: _.color not startsWith 'b'
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+        notnull: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true # comment to get a null
+      - name: count
+        type: integer
+        #notnull: true
 ---
 kind: destination
 spec:

_test/import-yaml.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_yaml
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,19 +12,56 @@ spec:
   spec:
     file: ./test.yaml
     format: yaml
-    table: my_yaml_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
+        notnull: true
         unique: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.yaml
+    format: yaml
+    table: T2
+    filter: _.color not startsWith 'b'
+    columns:
+      - name: color
+        type: string
+        key: true
+        notnull: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true
+      - name: count
+        type: integer
+        notnull: true
 ---
 kind: destination
 spec:

client/client.go

@@ -14,7 +14,6 @@ import (
 type Client struct {
 	Logger zerolog.Logger
 	Specs  *Spec
-	//Data   []map[string]any
 }
 
 func (c *Client) ID() string {

client/spec.go

@@ -1,18 +1,28 @@
 package client
 
+import (
+	"text/template"
+
+	"github.com/antonmedv/expr/vm"
+)
+
 type Column struct {
-	Name        string  `json:"name,omitempty" yaml:"name,omitempty"`
-	Description *string `json:"description,omitempty" yaml:"description,omitempty"`
-	Type        string  `json:"type,omitempty" yaml:"type,omitempty"`
-	Key         bool    `json:"key,omitempty" yaml:"pk,omitempty"`
-	Unique      bool    `json:"unique,omitempty" yaml:"unique,omitempty"`
-	NotNull     bool    `json:"notnull,omitempty" yaml:"notnull,omitempty"`
+	Name        string             `json:"name,omitempty" yaml:"name,omitempty"`
+	Description *string            `json:"description,omitempty" yaml:"description,omitempty"`
+	Type        string             `json:"type,omitempty" yaml:"type,omitempty"`
+	Key         bool               `json:"key,omitempty" yaml:"pk,omitempty"`
+	Unique      bool               `json:"unique,omitempty" yaml:"unique,omitempty"`
+	NotNull     bool               `json:"notnull,omitempty" yaml:"notnull,omitempty"`
+	Transform   *string            `json:"transform,omitempty" yaml:"transform,omitempty"`
+	Template    *template.Template `json:"-" yaml:"-"`
 }
 type Spec struct {
-	File      string   `json:"file,omitempty" yaml:"file,omitempty"`
-	Format    string   `json:"format,omitempty" yaml:"format,omitempty"`
-	Table     string   `json:"table,omitempty" yaml:"table,omitempty"`
-	Columns   []Column `json:"columns,omitempty" yaml:"columns,omitempty"`
-	Separator *string  `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
-	Sheet     *string  `json:"sheet,omitempty" yaml:"sheet,omitempty"`         // XLSX only
+	File      string      `json:"file,omitempty" yaml:"file,omitempty"`
+	Format    string      `json:"format,omitempty" yaml:"format,omitempty"`
+	Table     string      `json:"table,omitempty" yaml:"table,omitempty"`
+	Filter    *string     `json:"filter,omitempty" yaml:"filter,omitempty"`
+	Evaluator *vm.Program `json:"-,omitempty" yaml:"-,omitempty"`
+	Columns   []*Column   `json:"columns,omitempty" yaml:"columns,omitempty"`
+	Separator *string     `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
+	Sheets    []string    `json:"sheets,omitempty" yaml:"sheets,omitempty"`       // XLSX only
 }

go.mod

@@ -3,6 +3,8 @@ module github.com/dihedron/cq-source-file
 go 1.20
 
 require (
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/antonmedv/expr v1.12.5
 	github.com/cloudquery/plugin-sdk v1.45.0
 	github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c
 	github.com/rs/zerolog v1.29.1
@@ -11,20 +13,27 @@ require (
 )
 
 require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.20.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.0 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect