Add support for record filtering

_test/import-csv.yaml

@@ -1,7 +1,40 @@
 ---
+# kind: source
+# spec:
+#   name: test1
+#   registry: local
+#   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+#   version: v1.0.0
+#   tables: 
+#     ["*"]
+#   destinations:
+#     - sqlite
+#   spec:
+#     file: ./test.csv
+#     format: csv
+#     table: T1
+#     separator: ","
+#     filter: _.color startsWith 'b'
+#     columns:
+#       - name: color
+#         type: string
+#         key: true
+#         unique: true
+#         notnull: true
+#       - name: value
+#         type: string
+#         unique: true
+#         notnull: true
+#       - name: optimized
+#         type: boolean
+#         notnull: true
+#       - name: count
+#         type: integer
+#         notnull: true
+# ---
 kind: source
 spec:
-  name: local_csv
+  name: test2
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,20 +45,25 @@ spec:
   spec:
     file: ./test.csv
     format: csv
-    table: my_csv_data
+    table: T2
     separator: ","
+    filter: _.color not startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
 ---
 kind: destination
 spec:

_test/import-json.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_json
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,19 +12,56 @@ spec:
   spec:
     file: ./test.json
     format: json
-    table: my_json_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.json
+    format: json
+    table: T2
+    filter: "_.color not startsWith 'b'"
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+        notnull: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true
+      - name: count
+        type: integer
+        notnull: true
 ---
 kind: destination
 spec:

_test/import-xlsx.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_xlsx
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,20 +12,56 @@ spec:
   spec:
     file: ./test.xlsx
     format: xlsx
-    table: my_xlsx_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
         unique: true
+        notnull: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
         notnull: true # comment to get a null
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.xlsx
+    format: xlsx
+    table: T2
+    filter: _.color not startsWith 'b'
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+        notnull: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true # comment to get a null
+      - name: count
+        type: integer
+        #notnull: true
 ---
 kind: destination
 spec:

_test/import-yaml.yaml

@@ -1,7 +1,7 @@
 ---
 kind: source
 spec:
-  name: local_yaml
+  name: test1
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
@@ -12,19 +12,56 @@ spec:
   spec:
     file: ./test.yaml
     format: yaml
-    table: my_yaml_data
+    table: T1
+    filter: _.color startsWith 'b'
     columns:
       - name: color
         type: string
         key: true
+        notnull: true
         unique: true
       - name: value
         type: string
         unique: true
+        notnull: true
       - name: optimized
         type: boolean
+        notnull: true
       - name: count
         type: integer
+        notnull: true
+---
+kind: source
+spec:
+  name: test2
+  registry: local
+  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  version: v1.0.0
+  tables: 
+    ["*"]
+  destinations:
+    - sqlite
+  spec:
+    file: ./test.yaml
+    format: yaml
+    table: T2
+    filter: _.color not startsWith 'b'
+    columns:
+      - name: color
+        type: string
+        key: true
+        notnull: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+        notnull: true
+      - name: optimized
+        type: boolean
+        notnull: true
+      - name: count
+        type: integer
+        notnull: true
 ---
 kind: destination
 spec:

client/spec.go

@@ -1,6 +1,10 @@
 package client
 
-import "text/template"
+import (
+	"text/template"
+
+	"github.com/antonmedv/expr/vm"
+)
 
 type Column struct {
 	Name        string             `json:"name,omitempty" yaml:"name,omitempty"`
@@ -10,13 +14,15 @@ type Column struct {
 	Unique      bool               `json:"unique,omitempty" yaml:"unique,omitempty"`
 	NotNull     bool               `json:"notnull,omitempty" yaml:"notnull,omitempty"`
 	Transform   *string            `json:"transform,omitempty" yaml:"transform,omitempty"`
-	Template    *template.Template `json:"_template" yaml:"-"`
+	Template    *template.Template `json:"-" yaml:"-"`
 }
 type Spec struct {
-	File      string    `json:"file,omitempty" yaml:"file,omitempty"`
-	Format    string    `json:"format,omitempty" yaml:"format,omitempty"`
-	Table     string    `json:"table,omitempty" yaml:"table,omitempty"`
-	Columns   []*Column `json:"columns,omitempty" yaml:"columns,omitempty"`
-	Separator *string   `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
-	Sheets    []string  `json:"sheets,omitempty" yaml:"sheets,omitempty"`       // XLSX only
+	File      string      `json:"file,omitempty" yaml:"file,omitempty"`
+	Format    string      `json:"format,omitempty" yaml:"format,omitempty"`
+	Table     string      `json:"table,omitempty" yaml:"table,omitempty"`
+	Filter    *string     `json:"filter,omitempty" yaml:"filter,omitempty"`
+	Evaluator *vm.Program `json:"-,omitempty" yaml:"-,omitempty"`
+	Columns   []*Column   `json:"columns,omitempty" yaml:"columns,omitempty"`
+	Separator *string     `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
+	Sheets    []string    `json:"sheets,omitempty" yaml:"sheets,omitempty"`       // XLSX only
 }

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/antonmedv/expr v1.12.5
 	github.com/cloudquery/plugin-sdk v1.45.0
 	github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c
 	github.com/rs/zerolog v1.29.1

go.sum

@@ -39,6 +39,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E=
+github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
 github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=