Feature/schema support

_test/import-csv.yaml

@@ -14,10 +14,18 @@ spec:
     format: csv
     table: my_csv_data
     separator: ","
-    keys: [color]
-    types:
-      value: string
-      color: string
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+      - name: optimized
+        type: boolean
+      - name: count
+        type: integer
 ---
 kind: destination
 spec:

_test/import-json.yaml

@@ -13,10 +13,18 @@ spec:
     file: ./test.json
     format: json
     table: my_json_data
-    keys: [color]
-    types:
-      value: string
-      color: string
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+      - name: optimized
+        type: boolean
+      - name: count
+        type: integer
 ---
 kind: destination
 spec:

_test/import-xlsx.yaml

@@ -13,10 +13,19 @@ spec:
     file: ./test.xlsx
     format: xlsx
     table: my_xlsx_data
-    keys: [color]
-    types:
-      value: string
-      color: string
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+      - name: optimized
+        type: boolean
+        notnull: true # comment to get a null
+      - name: count
+        type: integer
 ---
 kind: destination
 spec:

_test/import-yaml.yaml

@@ -13,11 +13,18 @@ spec:
     file: ./test.yaml
     format: yaml
     table: my_yaml_data
-    keys:
-      - color
-    types:
-      value: string
-      color: string
+    columns:
+      - name: color
+        type: string
+        key: true
+        unique: true
+      - name: value
+        type: string
+        unique: true
+      - name: optimized
+        type: boolean
+      - name: count
+        type: integer
 ---
 kind: destination
 spec:

_test/test.csv

@@ -1,8 +1,10 @@
-color,value
-red,#f00
-green,#0f0
-blue,#00f
-cyan,#0ff
-magenta,#f0f
-yellow,#ff0
-black,#000
+color,value,optimized,count
+red,#f00,false,-123
+green,#0f0,true,345
+blue,#00f,false,-345
+cyan,#0ff,true,678
+magenta,#f0f,false,-678
+yellow,#ff0,true,901
+black,#000,false,-901
+grey,#aaa,false,
+white,#fff,,-222

_test/test.json

@@ -1,30 +1,44 @@
 [
 	{
 		"color": "red",
-		"value": "#f00"
+		"value": "#f00",
+		"optimized": true,
+		"count": 123
 	},
 	{
 		"color": "green",
-		"value": "#0f0"
+		"value": "#0f0",
+		"optimized": false,
+		"count": -123
 	},
 	{
 		"color": "blue",
-		"value": "#00f"
+		"value": "#00f",
+		"optimized": true,
+		"count": -321
 	},
 	{
 		"color": "cyan",
-		"value": "#0ff"
+		"value": "#0ff",
+		"optimized": false,
+		"count": 321
 	},
 	{
 		"color": "magenta",
-		"value": "#f0f"
+		"value": "#f0f",
+		"optimized": true,
+		"count": 456
 	},
 	{
 		"color": "yellow",
-		"value": "#ff0"
+		"value": "#ff0",
+		"optimized": false,
+		"count": -456
 	},
 	{
 		"color": "black",
-		"value": "#000"
+		"value": "#000",
+		"optimized": true,
+		"count": 789
 	}
 ]

_test/test.yaml

@@ -1,15 +1,29 @@
 ---
 - color: red
   value: "#f00"
+  optimized: true
+  count: 123
 - color: green
   value: "#0f0"
+  optimized: false
+  count: -123
 - color: blue
   value: "#00f"
+  optimized: true
+  count: -321
 - color: cyan
   value: "#0ff"
+  optimized: false
+  count: 321
 - color: magenta
   value: "#f0f"
+  optimized: true
+  count: 456
 - color: yellow
   value: "#ff0"
+  optimized: false
+  count: -456
 - color: black
-  value: "#000"
+  value: "#000"
+  optimized: true
+  count: 789

client/client.go

@@ -14,7 +14,7 @@ import (
 type Client struct {
 	Logger zerolog.Logger
 	Specs  *Spec
-	Data   []map[string]any
+	//Data   []map[string]any
 }
 
 func (c *Client) ID() string {

client/spec.go

@@ -1,11 +1,18 @@
 package client
 
+type Column struct {
+	Name        string  `json:"name,omitempty" yaml:"name,omitempty"`
+	Description *string `json:"description,omitempty" yaml:"description,omitempty"`
+	Type        string  `json:"type,omitempty" yaml:"type,omitempty"`
+	Key         bool    `json:"key,omitempty" yaml:"pk,omitempty"`
+	Unique      bool    `json:"unique,omitempty" yaml:"unique,omitempty"`
+	NotNull     bool    `json:"notnull,omitempty" yaml:"notnull,omitempty"`
+}
 type Spec struct {
-	File      string            `json:"file,omitempty" yaml:"file,omitempty"`
-	Format    string            `json:"format,omitempty" yaml:"format,omitempty"`
-	Table     string            `json:"table,omitempty" yaml:"table,omitempty"`
-	Keys      []string          `json:"keys,omitempty" yaml:"keys,omitempty"`
-	Types     map[string]string `json:"types,omitempty" yaml:"types,omitempty"`
-	Separator *string           `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
-	Sheet     *string           `json:"sheet,omitempty" yaml:"sheet,omitempty"`         // XLSX only
+	File      string   `json:"file,omitempty" yaml:"file,omitempty"`
+	Format    string   `json:"format,omitempty" yaml:"format,omitempty"`
+	Table     string   `json:"table,omitempty" yaml:"table,omitempty"`
+	Columns   []Column `json:"columns,omitempty" yaml:"columns,omitempty"`
+	Separator *string  `json:"separator,omitempty" yaml:"separator,omitempty"` // CSV only
+	Sheet     *string  `json:"sheet,omitempty" yaml:"sheet,omitempty"`         // XLSX only
 }