From 624bf3f69c188a7d8b002adb406315f89454667f Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Wed, 10 Aug 2022 22:24:59 +0300
Subject: [PATCH 01/25] feat: Remove connection spec

---
 specs/connection.go  |  6 ------
 specs/source.go      | 16 +---------------
 specs/spec.go        |  2 --
 specs/spec_reader.go | 29 ++++++++---------------------
 4 files changed, 9 insertions(+), 44 deletions(-)
 delete mode 100644 specs/connection.go

diff --git a/specs/connection.go b/specs/connection.go
deleted file mode 100644
index 5cd410ee1c..0000000000
--- a/specs/connection.go
+++ /dev/null
@@ -1,6 +0,0 @@
-package specs
-
-type ConnectionSpec struct {
-	Source      string `yaml:"source"`
-	Destination string `yaml:"destination"`
-}
diff --git a/specs/source.go b/specs/source.go
index bc1d618a5c..02be11d900 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -17,24 +17,10 @@ type SourceSpec struct {
 	MaxGoRoutines uint64   `json:"max_goroutines" yaml:"max_goroutines"`
 	Tables        []string `json:"tables" yaml:"tables"`
 	SkipTables    []string `json:"skip_tables" yaml:"skip_tables"`
+	Destinations  []string `json:"destinations" yaml:"destinations"`
 	// Spec          yaml.Node `json:"spec" yaml:"spec"`
 }
 
-func SetSourceSpecDefault(s *SourceSpec) {
-	if s.Registry == "" {
-		s.Registry = "github"
-	}
-	if s.Path == "" {
-		s.Path = s.Name
-	}
-	if s.Version == "" {
-		s.Version = "latest"
-	}
-	if !strings.Contains(s.Path, "/") {
-		s.Path = "cloudquery/" + s.Path
-	}
-}
-
 func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
 	type S SourceSpec
 	type T struct {
diff --git a/specs/spec.go b/specs/spec.go
index 17573fff41..aeff7c732e 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -27,8 +27,6 @@ func (s *Spec) UnmarshalYAML(n *yaml.Node) error {
 		s.Spec = new(SourceSpec)
 	case "destination":
 		s.Spec = new(DestinationSpec)
-	case "connection":
-		s.Spec = new(ConnectionSpec)
 	default:
 		return fmt.Errorf("unknown kind %s", s.Kind)
 	}
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index 4da475ced6..687e6c6760 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -13,14 +13,12 @@ import (
 type SpecReader struct {
 	sources      map[string]SourceSpec
 	destinations map[string]DestinationSpec
-	connections  map[string]ConnectionSpec
 }
 
 func NewSpecReader(directory string) (*SpecReader, error) {
 	reader := SpecReader{
 		sources:      make(map[string]SourceSpec),
 		destinations: make(map[string]DestinationSpec),
-		connections:  make(map[string]ConnectionSpec),
 	}
 	files, err := ioutil.ReadDir(directory)
 	if err != nil {
@@ -42,8 +40,6 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 				reader.sources[file.Name()] = *s.Spec.(*SourceSpec)
 			case "destination":
 				reader.destinations[file.Name()] = *s.Spec.(*DestinationSpec)
-			case "connection":
-				reader.connections[file.Name()] = *s.Spec.(*ConnectionSpec)
 			default:
 				return nil, fmt.Errorf("unknown kind %s", s.Kind)
 			}
@@ -52,6 +48,14 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 	return &reader, nil
 }
 
+func (r *SpecReader) GetSources() []SourceSpec {
+	sources := make([]SourceSpec, 0, len(r.sources))
+	for _, spec := range r.sources {
+		sources = append(sources, spec)
+	}
+	return sources
+}
+
 func (s *SpecReader) GetSourceByName(name string) SourceSpec {
 	for _, spec := range s.sources {
 		if spec.Name == name {
@@ -69,20 +73,3 @@ func (s *SpecReader) GetDestinatinoByName(name string) DestinationSpec {
 	}
 	return DestinationSpec{}
 }
-
-func (s *SpecReader) GetConnectionByName(name string) ConnectionSpec {
-	for _, spec := range s.connections {
-		if spec.Source == name {
-			return spec
-		}
-	}
-	return ConnectionSpec{}
-}
-
-func (s *SpecReader) Connections() []ConnectionSpec {
-	connections := make([]ConnectionSpec, 0, len(s.connections))
-	for _, spec := range s.connections {
-		connections = append(connections, spec)
-	}
-	return connections
-}

From 2c76bddd21a7b5f60711ce0aa7d7ab3623fd9fc7 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Thu, 11 Aug 2022 13:18:44 +0300
Subject: [PATCH 02/25] fix SourceSpec

---
 specs/source.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/specs/source.go b/specs/source.go
index 02be11d900..e15ac05d25 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -13,12 +13,12 @@ type SourceSpec struct {
 	// Path is the path in the registry
 	Path string `json:"path" yaml:"path"`
 	// Registry can be github,local,grpc. Might support things like https in the future.
-	Registry      string   `json:"registry" yaml:"registry"`
-	MaxGoRoutines uint64   `json:"max_goroutines" yaml:"max_goroutines"`
-	Tables        []string `json:"tables" yaml:"tables"`
-	SkipTables    []string `json:"skip_tables" yaml:"skip_tables"`
-	Destinations  []string `json:"destinations" yaml:"destinations"`
-	// Spec          yaml.Node `json:"spec" yaml:"spec"`
+	Registry      string    `json:"registry" yaml:"registry"`
+	MaxGoRoutines uint64    `json:"max_goroutines" yaml:"max_goroutines"`
+	Tables        []string  `json:"tables" yaml:"tables"`
+	SkipTables    []string  `json:"skip_tables" yaml:"skip_tables"`
+	Destinations  []string  `json:"destinations" yaml:"destinations"`
+	Spec          yaml.Node `json:"spec" yaml:"spec"`
 }
 
 func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {

From 4f35bf2446c96329eced76f9e6462afea1a262de Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sat, 13 Aug 2022 22:03:54 +0300
Subject: [PATCH 03/25] working on destination plugin

---
 clients/destination.go             |  82 +++--------
 internal/pb/destination.pb.go      | 224 ++++++++++++++---------------
 internal/pb/destination.proto      |  21 +--
 internal/pb/destination_grpc.pb.go | 160 +++++++++++----------
 internal/servers/destinations.go   |  15 +-
 internal/servers/source.go         |   7 +-
 plugins/destination.go             |  62 +++++++-
 plugins/source.go                  |  97 ++++++++++---
 plugins/source_testing.go          |   2 +-
 schema/meta.go                     |   5 +
 schema/resource.go                 |  63 ++++----
 schema/table.go                    |  62 ++++++--
 specs/destination.go               |  22 ++-
 specs/registry.go                  |  13 ++
 specs/source.go                    |   8 +-
 15 files changed, 504 insertions(+), 339 deletions(-)
 create mode 100644 specs/registry.go

diff --git a/clients/destination.go b/clients/destination.go
index 31bc46942a..ab78a39535 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -7,10 +7,7 @@ import (
 	"github.com/cloudquery/plugin-sdk/internal/pb"
 	"github.com/cloudquery/plugin-sdk/plugins"
 	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
-	"github.com/vmihailenco/msgpack/v5"
 	"google.golang.org/grpc"
-	"gopkg.in/yaml.v3"
 )
 
 type DestinationClient struct {
@@ -31,67 +28,34 @@ func NewLocalDestinationClient(p plugins.DestinationPlugin) *DestinationClient {
 	}
 }
 
-func (c *DestinationClient) Configure(ctx context.Context, s specs.DestinationSpec) error {
+func (c *DestinationClient) Write(ctx context.Context, resource *schema.Resource) error {
+	// var saveClient pb.Destination_SaveClient
+	// var err error
+	// if c.pbClient != nil {
+	// 	saveClient, err = c.pbClient.Write(ctx)
+	// 	if err != nil {
+	// 		return fmt.Errorf("failed to create save client: %w", err)
+	// 	}
+	// }
 	if c.localClient != nil {
-		return c.localClient.Configure(ctx, s)
-	}
-	b, err := yaml.Marshal(s)
-	if err != nil {
-		return fmt.Errorf("failed to marshal spec: %w", err)
-	}
-	if _, err := c.pbClient.Configure(ctx, &pb.Configure_Request{Config: b}); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error) {
-	if c.localClient != nil {
-		return c.localClient.GetExampleConfig(ctx), nil
-	}
-	res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{})
-	if err != nil {
-		return "", err
-	}
-	return res.Config, nil
-}
-
-func (c *DestinationClient) Save(ctx context.Context, msg *FetchResultMessage) error {
-	var saveClient pb.Destination_SaveClient
-	var err error
-	if c.pbClient != nil {
-		saveClient, err = c.pbClient.Save(ctx)
-		if err != nil {
-			return fmt.Errorf("failed to create save client: %w", err)
-		}
-	}
-	if c.localClient != nil {
-		var resource schema.Resource
-		if err := msgpack.Unmarshal(msg.Resource, &resource); err != nil {
-			return fmt.Errorf("failed to unmarshal resources: %w", err)
-		}
-		if err := c.localClient.Save(ctx, []*schema.Resource{&resource}); err != nil {
+		if err := c.localClient.Write(ctx, resource); err != nil {
 			return fmt.Errorf("failed to save resources: %w", err)
 		}
-	} else {
-		if err := saveClient.Send(&pb.Save_Request{Resources: msg.Resource}); err != nil {
-			return err
-		}
 	}
 
 	return nil
 }
 
-func (c *DestinationClient) CreateTables(ctx context.Context, tables []*schema.Table) error {
-	if c.localClient != nil {
-		return c.localClient.CreateTables(ctx, tables)
-	}
-	b, err := yaml.Marshal(tables)
-	if err != nil {
-		return fmt.Errorf("failed to marshal tables: %w", err)
-	}
-	if _, err := c.pbClient.CreateTables(ctx, &pb.CreateTables_Request{Tables: b}); err != nil {
-		return err
-	}
-	return nil
-}
+// func (c *DestinationClient) CreateTables(ctx context.Context, tables []*schema.Table) error {
+// 	if c.localClient != nil {
+// 		return c.localClient.CreateTables(ctx, tables)
+// 	}
+// 	b, err := yaml.Marshal(tables)
+// 	if err != nil {
+// 		return fmt.Errorf("failed to marshal tables: %w", err)
+// 	}
+// 	if _, err := c.pbClient.CreateTables(ctx, &pb.CreateTables_Request{Tables: b}); err != nil {
+// 		return err
+// 	}
+// 	return nil
+// }
diff --git a/internal/pb/destination.pb.go b/internal/pb/destination.pb.go
index 3ab55efca8..80fbbb0909 100644
--- a/internal/pb/destination.pb.go
+++ b/internal/pb/destination.pb.go
@@ -20,14 +20,14 @@ const (
 	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )
 
-type CreateTables struct {
+type Migrate struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *CreateTables) Reset() {
-	*x = CreateTables{}
+func (x *Migrate) Reset() {
+	*x = Migrate{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -35,13 +35,13 @@ func (x *CreateTables) Reset() {
 	}
 }
 
-func (x *CreateTables) String() string {
+func (x *Migrate) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*CreateTables) ProtoMessage() {}
+func (*Migrate) ProtoMessage() {}
 
-func (x *CreateTables) ProtoReflect() protoreflect.Message {
+func (x *Migrate) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -53,19 +53,19 @@ func (x *CreateTables) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use CreateTables.ProtoReflect.Descriptor instead.
-func (*CreateTables) Descriptor() ([]byte, []int) {
+// Deprecated: Use Migrate.ProtoReflect.Descriptor instead.
+func (*Migrate) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{0}
 }
 
-type Save struct {
+type Write struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *Save) Reset() {
-	*x = Save{}
+func (x *Write) Reset() {
+	*x = Write{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[1]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -73,13 +73,13 @@ func (x *Save) Reset() {
 	}
 }
 
-func (x *Save) String() string {
+func (x *Write) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Save) ProtoMessage() {}
+func (*Write) ProtoMessage() {}
 
-func (x *Save) ProtoReflect() protoreflect.Message {
+func (x *Write) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[1]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -91,22 +91,22 @@ func (x *Save) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Save.ProtoReflect.Descriptor instead.
-func (*Save) Descriptor() ([]byte, []int) {
+// Deprecated: Use Write.ProtoReflect.Descriptor instead.
+func (*Write) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{1}
 }
 
-type CreateTables_Request struct {
+type Migrate_Request struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// marshalled []*schema.Table
-	Tables []byte `protobuf:"bytes,1,opt,name=tables,proto3" json:"tables,omitempty"`
+	// marshalled sourcePlugin
+	SourcePlugin []byte `protobuf:"bytes,1,opt,name=source_plugin,json=sourcePlugin,proto3" json:"source_plugin,omitempty"`
 }
 
-func (x *CreateTables_Request) Reset() {
-	*x = CreateTables_Request{}
+func (x *Migrate_Request) Reset() {
+	*x = Migrate_Request{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[2]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -114,13 +114,13 @@ func (x *CreateTables_Request) Reset() {
 	}
 }
 
-func (x *CreateTables_Request) String() string {
+func (x *Migrate_Request) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*CreateTables_Request) ProtoMessage() {}
+func (*Migrate_Request) ProtoMessage() {}
 
-func (x *CreateTables_Request) ProtoReflect() protoreflect.Message {
+func (x *Migrate_Request) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[2]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -132,26 +132,26 @@ func (x *CreateTables_Request) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use CreateTables_Request.ProtoReflect.Descriptor instead.
-func (*CreateTables_Request) Descriptor() ([]byte, []int) {
+// Deprecated: Use Migrate_Request.ProtoReflect.Descriptor instead.
+func (*Migrate_Request) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{0, 0}
 }
 
-func (x *CreateTables_Request) GetTables() []byte {
+func (x *Migrate_Request) GetSourcePlugin() []byte {
 	if x != nil {
-		return x.Tables
+		return x.SourcePlugin
 	}
 	return nil
 }
 
-type CreateTables_Response struct {
+type Migrate_Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *CreateTables_Response) Reset() {
-	*x = CreateTables_Response{}
+func (x *Migrate_Response) Reset() {
+	*x = Migrate_Response{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[3]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -159,13 +159,13 @@ func (x *CreateTables_Response) Reset() {
 	}
 }
 
-func (x *CreateTables_Response) String() string {
+func (x *Migrate_Response) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*CreateTables_Response) ProtoMessage() {}
+func (*Migrate_Response) ProtoMessage() {}
 
-func (x *CreateTables_Response) ProtoReflect() protoreflect.Message {
+func (x *Migrate_Response) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[3]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -177,22 +177,22 @@ func (x *CreateTables_Response) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use CreateTables_Response.ProtoReflect.Descriptor instead.
-func (*CreateTables_Response) Descriptor() ([]byte, []int) {
+// Deprecated: Use Migrate_Response.ProtoReflect.Descriptor instead.
+func (*Migrate_Response) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{0, 1}
 }
 
-type Save_Request struct {
+type Write_Request struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// marshalled []*schema.Resources
-	Resources []byte `protobuf:"bytes,1,opt,name=resources,proto3" json:"resources,omitempty"`
+	// marshalled *schema.Resources
+	Resource []byte `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"`
 }
 
-func (x *Save_Request) Reset() {
-	*x = Save_Request{}
+func (x *Write_Request) Reset() {
+	*x = Write_Request{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[4]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -200,13 +200,13 @@ func (x *Save_Request) Reset() {
 	}
 }
 
-func (x *Save_Request) String() string {
+func (x *Write_Request) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Save_Request) ProtoMessage() {}
+func (*Write_Request) ProtoMessage() {}
 
-func (x *Save_Request) ProtoReflect() protoreflect.Message {
+func (x *Write_Request) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[4]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -218,29 +218,29 @@ func (x *Save_Request) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Save_Request.ProtoReflect.Descriptor instead.
-func (*Save_Request) Descriptor() ([]byte, []int) {
+// Deprecated: Use Write_Request.ProtoReflect.Descriptor instead.
+func (*Write_Request) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{1, 0}
 }
 
-func (x *Save_Request) GetResources() []byte {
+func (x *Write_Request) GetResource() []byte {
 	if x != nil {
-		return x.Resources
+		return x.Resource
 	}
 	return nil
 }
 
-type Save_Response struct {
+type Write_Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// marshalled goschemajson.Result
+	// error
 	Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
 }
 
-func (x *Save_Response) Reset() {
-	*x = Save_Response{}
+func (x *Write_Response) Reset() {
+	*x = Write_Response{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_destination_proto_msgTypes[5]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -248,13 +248,13 @@ func (x *Save_Response) Reset() {
 	}
 }
 
-func (x *Save_Response) String() string {
+func (x *Write_Response) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Save_Response) ProtoMessage() {}
+func (*Write_Response) ProtoMessage() {}
 
-func (x *Save_Response) ProtoReflect() protoreflect.Message {
+func (x *Write_Response) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_destination_proto_msgTypes[5]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -266,12 +266,12 @@ func (x *Save_Response) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Save_Response.ProtoReflect.Descriptor instead.
-func (*Save_Response) Descriptor() ([]byte, []int) {
+// Deprecated: Use Write_Response.ProtoReflect.Descriptor instead.
+func (*Write_Response) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{1, 1}
 }
 
-func (x *Save_Response) GetError() string {
+func (x *Write_Response) GetError() string {
 	if x != nil {
 		return x.Error
 	}
@@ -284,36 +284,36 @@ var file_internal_pb_destination_proto_rawDesc = []byte{
 	0x0a, 0x1d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65,
 	0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
 	0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
-	0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3d,
-	0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x21,
-	0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62,
-	0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65,
-	0x73, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x51, 0x0a,
-	0x04, 0x53, 0x61, 0x76, 0x65, 0x1a, 0x27, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0c, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x20,
-	0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72,
-	0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72,
-	0x32, 0xa6, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e,
-	0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
-	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65,
-	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47,
-	0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
-	0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x61, 0x76,
-	0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x2e, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53,
-	0x61, 0x76, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x49,
-	0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x1b,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62,
-	0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73,
-	0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x45,
+	0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x1a, 0x2e, 0x0a, 0x07, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70,
+	0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x1a, 0x25,
+	0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x9a, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74,
+	0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78,
+	0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40,
+	0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x3a, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72,
+	0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05,
+	0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72,
+	0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x28, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -330,26 +330,26 @@ func file_internal_pb_destination_proto_rawDescGZIP() []byte {
 
 var file_internal_pb_destination_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
 var file_internal_pb_destination_proto_goTypes = []interface{}{
-	(*CreateTables)(nil),              // 0: proto.CreateTables
-	(*Save)(nil),                      // 1: proto.Save
-	(*CreateTables_Request)(nil),      // 2: proto.CreateTables.Request
-	(*CreateTables_Response)(nil),     // 3: proto.CreateTables.Response
-	(*Save_Request)(nil),              // 4: proto.Save.Request
-	(*Save_Response)(nil),             // 5: proto.Save.Response
-	(*Configure_Request)(nil),         // 6: proto.Configure.Request
-	(*GetExampleConfig_Request)(nil),  // 7: proto.GetExampleConfig.Request
-	(*Configure_Response)(nil),        // 8: proto.Configure.Response
-	(*GetExampleConfig_Response)(nil), // 9: proto.GetExampleConfig.Response
+	(*Migrate)(nil),                   // 0: proto.Migrate
+	(*Write)(nil),                     // 1: proto.Write
+	(*Migrate_Request)(nil),           // 2: proto.Migrate.Request
+	(*Migrate_Response)(nil),          // 3: proto.Migrate.Response
+	(*Write_Request)(nil),             // 4: proto.Write.Request
+	(*Write_Response)(nil),            // 5: proto.Write.Response
+	(*GetExampleConfig_Request)(nil),  // 6: proto.GetExampleConfig.Request
+	(*Configure_Request)(nil),         // 7: proto.Configure.Request
+	(*GetExampleConfig_Response)(nil), // 8: proto.GetExampleConfig.Response
+	(*Configure_Response)(nil),        // 9: proto.Configure.Response
 }
 var file_internal_pb_destination_proto_depIdxs = []int32{
-	6, // 0: proto.Destination.Configure:input_type -> proto.Configure.Request
-	7, // 1: proto.Destination.GetExampleConfig:input_type -> proto.GetExampleConfig.Request
-	4, // 2: proto.Destination.Save:input_type -> proto.Save.Request
-	2, // 3: proto.Destination.CreateTables:input_type -> proto.CreateTables.Request
-	8, // 4: proto.Destination.Configure:output_type -> proto.Configure.Response
-	9, // 5: proto.Destination.GetExampleConfig:output_type -> proto.GetExampleConfig.Response
-	5, // 6: proto.Destination.Save:output_type -> proto.Save.Response
-	3, // 7: proto.Destination.CreateTables:output_type -> proto.CreateTables.Response
+	6, // 0: proto.Destination.GetExampleConfig:input_type -> proto.GetExampleConfig.Request
+	7, // 1: proto.Destination.Configure:input_type -> proto.Configure.Request
+	2, // 2: proto.Destination.Migrate:input_type -> proto.Migrate.Request
+	4, // 3: proto.Destination.Write:input_type -> proto.Write.Request
+	8, // 4: proto.Destination.GetExampleConfig:output_type -> proto.GetExampleConfig.Response
+	9, // 5: proto.Destination.Configure:output_type -> proto.Configure.Response
+	3, // 6: proto.Destination.Migrate:output_type -> proto.Migrate.Response
+	5, // 7: proto.Destination.Write:output_type -> proto.Write.Response
 	4, // [4:8] is the sub-list for method output_type
 	0, // [0:4] is the sub-list for method input_type
 	0, // [0:0] is the sub-list for extension type_name
@@ -365,7 +365,7 @@ func file_internal_pb_destination_proto_init() {
 	file_internal_pb_base_proto_init()
 	if !protoimpl.UnsafeEnabled {
 		file_internal_pb_destination_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*CreateTables); i {
+			switch v := v.(*Migrate); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -377,7 +377,7 @@ func file_internal_pb_destination_proto_init() {
 			}
 		}
 		file_internal_pb_destination_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Save); i {
+			switch v := v.(*Write); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -389,7 +389,7 @@ func file_internal_pb_destination_proto_init() {
 			}
 		}
 		file_internal_pb_destination_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*CreateTables_Request); i {
+			switch v := v.(*Migrate_Request); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -401,7 +401,7 @@ func file_internal_pb_destination_proto_init() {
 			}
 		}
 		file_internal_pb_destination_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*CreateTables_Response); i {
+			switch v := v.(*Migrate_Response); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -413,7 +413,7 @@ func file_internal_pb_destination_proto_init() {
 			}
 		}
 		file_internal_pb_destination_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Save_Request); i {
+			switch v := v.(*Write_Request); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -425,7 +425,7 @@ func file_internal_pb_destination_proto_init() {
 			}
 		}
 		file_internal_pb_destination_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Save_Response); i {
+			switch v := v.(*Write_Response); i {
 			case 0:
 				return &v.state
 			case 1:
diff --git a/internal/pb/destination.proto b/internal/pb/destination.proto
index 2393932811..f8fe68d1e7 100644
--- a/internal/pb/destination.proto
+++ b/internal/pb/destination.proto
@@ -4,32 +4,33 @@ option go_package = "/pb";
 import "internal/pb/base.proto";
 
 service Destination {
-  rpc Configure(Configure.Request) returns (Configure.Response);
   // Get an example configuration for the source plugin
   rpc GetExampleConfig(GetExampleConfig.Request) returns (GetExampleConfig.Response);
-  // Save resources
-  rpc Save(stream Save.Request) returns (Save.Response);
-  // Create tables
-  rpc CreateTables(CreateTables.Request) returns (CreateTables.Response);
+  // Configure the destination plugin with the given credentials and mode
+  rpc Configure(Configure.Request) returns (Configure.Response);
+  // Migrate tables to the given source plugin version
+  rpc Migrate(Migrate.Request) returns (Migrate.Response);
+  // Write resources
+  rpc Write(stream Write.Request) returns (Write.Response);
 }
 
 
-message CreateTables {
+message Migrate {
   message Request {
-    // marshalled []*schema.Table
-    bytes tables = 1;
+    // marshalled sourcePlugin
+    bytes source_plugin = 1;
   }
   message Response {
   }
 }
 
-message Save {
+message Write {
   message Request {
     // marshalled *schema.Resources
     bytes resource = 1;
   }
   message Response {
-    // marshalled goschemajson.Result
+    // error
     string error = 1;
   }
 }
\ No newline at end of file
diff --git a/internal/pb/destination_grpc.pb.go b/internal/pb/destination_grpc.pb.go
index ecac6dea33..7e2b94d44e 100644
--- a/internal/pb/destination_grpc.pb.go
+++ b/internal/pb/destination_grpc.pb.go
@@ -22,13 +22,14 @@ const _ = grpc.SupportPackageIsVersion7
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type DestinationClient interface {
-	Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error)
 	// Get an example configuration for the source plugin
 	GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error)
-	// Save resources
-	Save(ctx context.Context, opts ...grpc.CallOption) (Destination_SaveClient, error)
-	// Create tables
-	CreateTables(ctx context.Context, in *CreateTables_Request, opts ...grpc.CallOption) (*CreateTables_Response, error)
+	// Configure the destination plugin with the given credentials and mode
+	Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error)
+	// Migrate tables to the given source plugin version
+	Migrate(ctx context.Context, in *Migrate_Request, opts ...grpc.CallOption) (*Migrate_Response, error)
+	// Write resources
+	Write(ctx context.Context, opts ...grpc.CallOption) (Destination_WriteClient, error)
 }
 
 type destinationClient struct {
@@ -39,6 +40,15 @@ func NewDestinationClient(cc grpc.ClientConnInterface) DestinationClient {
 	return &destinationClient{cc}
 }
 
+func (c *destinationClient) GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) {
+	out := new(GetExampleConfig_Response)
+	err := c.cc.Invoke(ctx, "/proto.Destination/GetExampleConfig", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *destinationClient) Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) {
 	out := new(Configure_Response)
 	err := c.cc.Invoke(ctx, "/proto.Destination/Configure", in, out, opts...)
@@ -48,69 +58,61 @@ func (c *destinationClient) Configure(ctx context.Context, in *Configure_Request
 	return out, nil
 }
 
-func (c *destinationClient) GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) {
-	out := new(GetExampleConfig_Response)
-	err := c.cc.Invoke(ctx, "/proto.Destination/GetExampleConfig", in, out, opts...)
+func (c *destinationClient) Migrate(ctx context.Context, in *Migrate_Request, opts ...grpc.CallOption) (*Migrate_Response, error) {
+	out := new(Migrate_Response)
+	err := c.cc.Invoke(ctx, "/proto.Destination/Migrate", in, out, opts...)
 	if err != nil {
 		return nil, err
 	}
 	return out, nil
 }
 
-func (c *destinationClient) Save(ctx context.Context, opts ...grpc.CallOption) (Destination_SaveClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Destination_ServiceDesc.Streams[0], "/proto.Destination/Save", opts...)
+func (c *destinationClient) Write(ctx context.Context, opts ...grpc.CallOption) (Destination_WriteClient, error) {
+	stream, err := c.cc.NewStream(ctx, &Destination_ServiceDesc.Streams[0], "/proto.Destination/Write", opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &destinationSaveClient{stream}
+	x := &destinationWriteClient{stream}
 	return x, nil
 }
 
-type Destination_SaveClient interface {
-	Send(*Save_Request) error
-	CloseAndRecv() (*Save_Response, error)
+type Destination_WriteClient interface {
+	Send(*Write_Request) error
+	CloseAndRecv() (*Write_Response, error)
 	grpc.ClientStream
 }
 
-type destinationSaveClient struct {
+type destinationWriteClient struct {
 	grpc.ClientStream
 }
 
-func (x *destinationSaveClient) Send(m *Save_Request) error {
+func (x *destinationWriteClient) Send(m *Write_Request) error {
 	return x.ClientStream.SendMsg(m)
 }
 
-func (x *destinationSaveClient) CloseAndRecv() (*Save_Response, error) {
+func (x *destinationWriteClient) CloseAndRecv() (*Write_Response, error) {
 	if err := x.ClientStream.CloseSend(); err != nil {
 		return nil, err
 	}
-	m := new(Save_Response)
+	m := new(Write_Response)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
 	return m, nil
 }
 
-func (c *destinationClient) CreateTables(ctx context.Context, in *CreateTables_Request, opts ...grpc.CallOption) (*CreateTables_Response, error) {
-	out := new(CreateTables_Response)
-	err := c.cc.Invoke(ctx, "/proto.Destination/CreateTables", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 // DestinationServer is the server API for Destination service.
 // All implementations must embed UnimplementedDestinationServer
 // for forward compatibility
 type DestinationServer interface {
-	Configure(context.Context, *Configure_Request) (*Configure_Response, error)
 	// Get an example configuration for the source plugin
 	GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error)
-	// Save resources
-	Save(Destination_SaveServer) error
-	// Create tables
-	CreateTables(context.Context, *CreateTables_Request) (*CreateTables_Response, error)
+	// Configure the destination plugin with the given credentials and mode
+	Configure(context.Context, *Configure_Request) (*Configure_Response, error)
+	// Migrate tables to the given source plugin version
+	Migrate(context.Context, *Migrate_Request) (*Migrate_Response, error)
+	// Write resources
+	Write(Destination_WriteServer) error
 	mustEmbedUnimplementedDestinationServer()
 }
 
@@ -118,17 +120,17 @@ type DestinationServer interface {
 type UnimplementedDestinationServer struct {
 }
 
-func (UnimplementedDestinationServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
-}
 func (UnimplementedDestinationServer) GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetExampleConfig not implemented")
 }
-func (UnimplementedDestinationServer) Save(Destination_SaveServer) error {
-	return status.Errorf(codes.Unimplemented, "method Save not implemented")
+func (UnimplementedDestinationServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
+}
+func (UnimplementedDestinationServer) Migrate(context.Context, *Migrate_Request) (*Migrate_Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Migrate not implemented")
 }
-func (UnimplementedDestinationServer) CreateTables(context.Context, *CreateTables_Request) (*CreateTables_Response, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CreateTables not implemented")
+func (UnimplementedDestinationServer) Write(Destination_WriteServer) error {
+	return status.Errorf(codes.Unimplemented, "method Write not implemented")
 }
 func (UnimplementedDestinationServer) mustEmbedUnimplementedDestinationServer() {}
 
@@ -143,6 +145,24 @@ func RegisterDestinationServer(s grpc.ServiceRegistrar, srv DestinationServer) {
 	s.RegisterService(&Destination_ServiceDesc, srv)
 }
 
+func _Destination_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetExampleConfig_Request)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DestinationServer).GetExampleConfig(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/proto.Destination/GetExampleConfig",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DestinationServer).GetExampleConfig(ctx, req.(*GetExampleConfig_Request))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Destination_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Configure_Request)
 	if err := dec(in); err != nil {
@@ -161,68 +181,50 @@ func _Destination_Configure_Handler(srv interface{}, ctx context.Context, dec fu
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Destination_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetExampleConfig_Request)
+func _Destination_Migrate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Migrate_Request)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(DestinationServer).GetExampleConfig(ctx, in)
+		return srv.(DestinationServer).Migrate(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
 		Server:     srv,
-		FullMethod: "/proto.Destination/GetExampleConfig",
+		FullMethod: "/proto.Destination/Migrate",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(DestinationServer).GetExampleConfig(ctx, req.(*GetExampleConfig_Request))
+		return srv.(DestinationServer).Migrate(ctx, req.(*Migrate_Request))
 	}
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Destination_Save_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(DestinationServer).Save(&destinationSaveServer{stream})
+func _Destination_Write_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(DestinationServer).Write(&destinationWriteServer{stream})
 }
 
-type Destination_SaveServer interface {
-	SendAndClose(*Save_Response) error
-	Recv() (*Save_Request, error)
+type Destination_WriteServer interface {
+	SendAndClose(*Write_Response) error
+	Recv() (*Write_Request, error)
 	grpc.ServerStream
 }
 
-type destinationSaveServer struct {
+type destinationWriteServer struct {
 	grpc.ServerStream
 }
 
-func (x *destinationSaveServer) SendAndClose(m *Save_Response) error {
+func (x *destinationWriteServer) SendAndClose(m *Write_Response) error {
 	return x.ServerStream.SendMsg(m)
 }
 
-func (x *destinationSaveServer) Recv() (*Save_Request, error) {
-	m := new(Save_Request)
+func (x *destinationWriteServer) Recv() (*Write_Request, error) {
+	m := new(Write_Request)
 	if err := x.ServerStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
 	return m, nil
 }
 
-func _Destination_CreateTables_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(CreateTables_Request)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(DestinationServer).CreateTables(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/proto.Destination/CreateTables",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(DestinationServer).CreateTables(ctx, req.(*CreateTables_Request))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
 // Destination_ServiceDesc is the grpc.ServiceDesc for Destination service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -230,23 +232,23 @@ var Destination_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "proto.Destination",
 	HandlerType: (*DestinationServer)(nil),
 	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Configure",
-			Handler:    _Destination_Configure_Handler,
-		},
 		{
 			MethodName: "GetExampleConfig",
 			Handler:    _Destination_GetExampleConfig_Handler,
 		},
 		{
-			MethodName: "CreateTables",
-			Handler:    _Destination_CreateTables_Handler,
+			MethodName: "Configure",
+			Handler:    _Destination_Configure_Handler,
+		},
+		{
+			MethodName: "Migrate",
+			Handler:    _Destination_Migrate_Handler,
 		},
 	},
 	Streams: []grpc.StreamDesc{
 		{
-			StreamName:    "Save",
-			Handler:       _Destination_Save_Handler,
+			StreamName:    "Write",
+			Handler:       _Destination_Write_Handler,
 			ClientStreams: true,
 		},
 	},
diff --git a/internal/servers/destinations.go b/internal/servers/destinations.go
index cc96e81004..f93b7d6fa1 100644
--- a/internal/servers/destinations.go
+++ b/internal/servers/destinations.go
@@ -2,6 +2,7 @@ package servers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 
@@ -29,25 +30,25 @@ func (s *DestinationServer) Configure(ctx context.Context, req *pb.Configure_Req
 
 func (s *DestinationServer) GetExampleConfig(ctx context.Context, req *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
 	return &pb.GetExampleConfig_Response{
-		Config: s.Plugin.GetExampleConfig(ctx),
+		Config: s.Plugin.GetExampleConfig(),
 	}, nil
 }
 
-func (s *DestinationServer) Save(msg pb.Destination_SaveServer) error {
+func (s *DestinationServer) Write(msg pb.Destination_WriteServer) error {
 	for {
 		r, err := msg.Recv()
 		if err != nil {
 			if err == io.EOF {
 				return nil
 			}
-			return fmt.Errorf("Save: failed to receive msg: %w", err)
+			return fmt.Errorf("write: failed to receive msg: %w", err)
 		}
-		var resources []*schema.Resource
-		if err := yaml.Unmarshal(r.Resources, &resources); err != nil {
+		var resource *schema.Resource
+		if err := json.Unmarshal(r.Resource, &resource); err != nil {
 			return status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err)
 		}
-		if err := s.Plugin.Save(context.Background(), resources); err != nil {
-			return fmt.Errorf("Save: failed to save resources: %w", err)
+		if err := s.Plugin.Write(context.Background(), resource); err != nil {
+			return fmt.Errorf("write: failed to write resource: %w", err)
 		}
 	}
 }
diff --git a/internal/servers/source.go b/internal/servers/source.go
index ee9200fbd6..0a417e7712 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -39,7 +39,7 @@ func (s *SourceServer) Configure(ctx context.Context, req *pb.Configure_Request)
 	if err := yaml.Unmarshal(req.Config, &spec); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal config")
 	}
-	jsonschemaResult, err := s.Plugin.Init(ctx, spec)
+	jsonschemaResult, err := s.Plugin.Configure(ctx, spec)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure source")
 	}
@@ -63,7 +63,10 @@ func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer
 	}()
 
 	for resource := range resources {
-		b, err := msgpack.Marshal(resource)
+		b, err := msgpack.Marshal(schema.WireResource{
+			Data:      resource.Data,
+			TableName: resource.Table.Name,
+		})
 		if err != nil {
 			return errors.Wrap(err, "failed to marshal resource")
 		}
diff --git a/plugins/destination.go b/plugins/destination.go
index 87b9a2128d..ae8e199c9f 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -5,16 +5,64 @@ import (
 
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
-	"github.com/rs/zerolog"
 )
 
-type DestinationPluginOptions struct {
-	Logger zerolog.Logger
-}
+// type DestinationOption func(*DestinationPlugin)
+
+// type WriteFunc func(ctx context.Context, spec specs.DestinationSpec, tables []*schema.Table, resources <-chan *schema.Resource) error
+
+// type DestinationPlugin struct {
+// 	// Name of the plugin.
+// 	name string
+// 	// Version is the version of the plugin.
+// 	version string
+// 	// JsonSchema for specific source plugin spec
+// 	jsonSchema string
+// 	// ExampleConfig is the example configuration for this plugin
+// 	exampleConfig string
+// 	// Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead.
+// 	logger zerolog.Logger
+// 	// Write is a function that get a stream of resources and write them to the configured destination
+// 	// with the configured mode.
+// 	Write WriteFunc
+// }
 
 type DestinationPlugin interface {
 	Configure(ctx context.Context, spec specs.DestinationSpec) error
-	CreateTables(ctx context.Context, table []*schema.Table) error
-	Save(ctx context.Context, resources []*schema.Resource) error
-	GetExampleConfig(ctx context.Context) string
+	Migrate(ctx context.Context, plugin *SourcePlugin) error
+	Write(ctx context.Context, resources *schema.Resource) error
+	GetExampleConfig() string
 }
+
+// func WithExampleConfig(exampleConfig string) DestinationOption {
+// 	return func(p *DestinationPlugin) {
+// 		p.exampleConfig = exampleConfig
+// 	}
+// }
+
+// func WithJsonSchema(jsonSchema string) DestinationOption {
+// 	return func(p *DestinationPlugin) {
+// 		p.jsonSchema = jsonSchema
+// 	}
+// }
+
+// func WithLogger(logger zerolog.Logger) DestinationOption {
+// 	return func(p *DestinationPlugin) {
+// 		p.logger = logger
+// 	}
+// }
+
+// func NewDestinationClient(name string, version string, writeFunc WriteFunc, opts ...DestinationOption) *DestinationPlugin {
+// 	p := DestinationPlugin{
+// 		name:    name,
+// 		version: version,
+// 	}
+// 	for _, opt := range opts {
+// 		opt(&p)
+// 	}
+// 	return &p
+// }
+
+// func (p *DestinationPlugin) GetExampleConfig() string {
+// 	return p.exampleConfig
+// }
diff --git a/plugins/source.go b/plugins/source.go
index 707c413aaa..b215e6a4a1 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -30,6 +30,8 @@ const ExampleSourceConfig = `
 # skip_tables: []
 `
 
+type SourceConfigureFunc func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
+
 // SourcePlugin is the base structure required to pass to sdk.serve
 // We take a similar/declerative approach to API here similar to Cobra
 type SourcePlugin struct {
@@ -37,10 +39,12 @@ type SourcePlugin struct {
 	Name string
 	// Version of the plugin
 	Version string
+	// Classify error and return it's severity and type
+	ClassifyError schema.ClassifyErrorFunc
 	// Called upon configure call to validate and init configuration
-	Configure func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
+	configure SourceConfigureFunc
 	// Tables is all tables supported by this source plugin
-	Tables []*schema.Table
+	Tables schema.Tables
 	// JsonSchema for specific source plugin spec
 	JsonSchema string
 	// ExampleConfig is the example configuration for this plugin
@@ -51,9 +55,59 @@ type SourcePlugin struct {
 	// Internal fields set by configure
 	clientMeta schema.ClientMeta
 	spec       *specs.SourceSpec
+	m          sync.Mutex
+}
+
+type SourceOption func(*SourcePlugin)
+
+func WithSourceExampleConfig(exampleConfig string) SourceOption {
+	return func(p *SourcePlugin) {
+		p.ExampleConfig = exampleConfig
+	}
+}
+
+func WithSourceJsonSchema(jsonSchema string) SourceOption {
+	return func(p *SourcePlugin) {
+		p.JsonSchema = jsonSchema
+	}
+}
+
+func WithSourceLogger(logger zerolog.Logger) SourceOption {
+	return func(p *SourcePlugin) {
+		p.Logger = logger
+	}
+}
+
+func WithClassifyError(classifyError schema.ClassifyErrorFunc) SourceOption {
+	return func(p *SourcePlugin) {
+		p.ClassifyError = classifyError
+	}
+}
+
+func NewSourcePlugin(name string, version string, tables []*schema.Table, configure SourceConfigureFunc, opts ...SourceOption) *SourcePlugin {
+	p := SourcePlugin{
+		Name:      name,
+		Version:   version,
+		Tables:    tables,
+		configure: configure,
+	}
+	if configure == nil {
+		panic("configure function not defined for source plugin:" + name)
+	}
+	for _, opt := range opts {
+		opt(&p)
+	}
+	return &p
 }
 
-func (p *SourcePlugin) Init(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
+func (p *SourcePlugin) Configure(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
+	// we permit only one configure per source plugin for security reasons.
+	// in the grpc layer this will behave similarly and for every new configuration/creds the cli will have to create a new process.
+	p.m.Lock()
+	defer p.m.Unlock()
+	if p.spec != nil {
+		return nil, fmt.Errorf("source plugin %s already configured", p.Name)
+	}
 	res, err := specs.ValidateSpec(sourceSchema, spec)
 	if err != nil {
 		return nil, err
@@ -61,10 +115,14 @@ func (p *SourcePlugin) Init(ctx context.Context, spec specs.SourceSpec) (*gojson
 	if !res.Valid() {
 		return res, nil
 	}
-	if p.Configure == nil {
-		return nil, fmt.Errorf("configure function not defined")
+
+	// if resources ["*"] is requested we will fetch all resources
+	p.spec.Tables, err = p.interpolateAllResources(p.spec.Tables)
+	if err != nil {
+		return res, fmt.Errorf("failed to interpolate resources: %w", err)
 	}
-	p.clientMeta, err = p.Configure(ctx, p, spec)
+
+	p.clientMeta, err = p.configure(ctx, p, spec)
 	if err != nil {
 		return res, fmt.Errorf("failed to configure source plugin: %w", err)
 	}
@@ -75,12 +133,7 @@ func (p *SourcePlugin) Init(ctx context.Context, spec specs.SourceSpec) (*gojson
 // Fetch fetches data according to source configuration and
 func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) error {
 	if p.spec == nil {
-		return fmt.Errorf("source plugin not initialized")
-	}
-	// if resources ["*"] is requested we will fetch all resources
-	tables, err := p.interpolateAllResources(p.spec.Tables)
-	if err != nil {
-		return fmt.Errorf("failed to interpolate resources: %w", err)
+		return fmt.Errorf("source plugin not configured")
 	}
 
 	// limiter used to limit the amount of resources fetched concurrently
@@ -92,9 +145,12 @@ func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) e
 	goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
 
 	w := sync.WaitGroup{}
+	totalResources := 0
+	startTime := time.Now()
+	tableNames := p.Tables.TableNames()
 	for _, table := range p.Tables {
 		table := table
-		if funk.ContainsString(p.spec.SkipTables, table.Name) || !funk.ContainsString(tables, table.Name) {
+		if funk.ContainsString(p.spec.SkipTables, table.Name) || !funk.ContainsString(tableNames, table.Name) {
 			p.Logger.Info().Str("table", table.Name).Msg("skipping table")
 			continue
 		}
@@ -102,6 +158,10 @@ func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) e
 		if table.Multiplex != nil {
 			clients = table.Multiplex(p.clientMeta)
 		}
+		// because table can't import sourceplugin we need to set classifyError if it is not set by table
+		if table.ClassifyError == nil {
+			table.ClassifyError = p.ClassifyError
+		}
 		// we call this here because we dont know when the following goroutine will be called and we do want an order
 		// of table by table
 		totalClients := len(clients)
@@ -113,7 +173,8 @@ func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) e
 			defer goroutinesSem.Release(int64(totalClients) - newN)
 			wg := sync.WaitGroup{}
 			p.Logger.Info().Str("table", table.Name).Msg("fetch start")
-			startTime := time.Now()
+			tableStartTime := time.Now()
+			totalTableResources := 0
 			for i, client := range clients {
 				client := client
 				i := i
@@ -131,15 +192,17 @@ func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) e
 					if newN > 0 && i >= (totalClients-int(newN)) {
 						defer goroutinesSem.Release(1)
 					}
-					table.Resolve(ctx, client, nil, res)
+
+					totalTableResources += table.Resolve(ctx, client, nil, res)
 				}()
 			}
 			wg.Wait()
-			p.Logger.Info().Str("table", table.Name).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished")
+			totalResources += totalTableResources
+			p.Logger.Info().Str("table", table.Name).Int("total_resources", totalTableResources).TimeDiff("duration", time.Now(), tableStartTime).Msg("fetch table finished")
 		}()
 	}
 	w.Wait()
-
+	p.Logger.Info().Int("total_resources", totalResources).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished")
 	return nil
 }
 
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index b7e99590d2..9528a540d5 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -54,7 +54,7 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
 	if err := yaml.Unmarshal([]byte(tc.Config), &spec); err != nil {
 		t.Fatal("failed to unmarshal source spec:", err)
 	}
-	validationResult, err := tc.Plugin.Init(context.Background(), spec)
+	validationResult, err := tc.Plugin.Configure(context.Background(), spec)
 	if err != nil {
 		t.Fatal("failed to init plugin:", err)
 	}
diff --git a/schema/meta.go b/schema/meta.go
index f3c72cd979..f55e2dd44a 100644
--- a/schema/meta.go
+++ b/schema/meta.go
@@ -17,6 +17,11 @@ type Meta struct {
 
 const FetchIdMetaKey = "cq_fetch_id"
 
+var CqColumns = []Column{
+	{Name: "cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", CreationOptions: ColumnCreationOptions{Unique: true}},
+	{Name: "cq_fetch_time", Type: TypeTimestamp, Description: "Internal CQ row of when fetch was started (this will be the same for all rows in a single fetch)"},
+}
+
 var (
 	// cqMeta = Column{
 	// 	Name:        "cq_meta",
diff --git a/schema/resource.go b/schema/resource.go
index bbedb93ab2..6e5e40f292 100644
--- a/schema/resource.go
+++ b/schema/resource.go
@@ -4,35 +4,39 @@ import (
 	"fmt"
 
 	"github.com/google/uuid"
-	"github.com/thoas/go-funk"
+	// "github.com/segmentio/objconv/json"
 )
 
 type Resources []*Resource
 
+// this is sent back to the cli "over the wire" and we want to keep this as small as possible and have specific marshal/unmarshal
+// for this struct
+type WireResource struct {
+	Data      []interface{} `json:"data"`
+	TableName string        `json:"table_name"`
+}
+
 // Resource represents a row in it's associated table, it carries a reference to the original item, and automatically
 // generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods
 type Resource struct {
 	// Original resource item that wa from prior resolve
 	Item interface{}
 	// Set if this is an embedded table
-	Parent *Resource `msgpack:"parent"`
+	Parent *Resource
 	// internal fields
-	Table    *Table                 `msgpack:"table"`
-	Data     map[string]interface{} `msgpack:"data"`
-	cqId     uuid.UUID
-	metadata map[string]interface{}
-	CColumns []string `msgpack:"columns"`
+	Table *Table
+	// This is sorted result data by column name
+	Data []interface{}
+	cqId uuid.UUID
 }
 
 func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource {
 	return &Resource{
-		Item:     item,
-		Parent:   parent,
-		Table:    t,
-		Data:     make(map[string]interface{}),
-		cqId:     uuid.New(),
-		CColumns: t.Columns.Names(),
-		// metadata: metadata,
+		Item:   item,
+		Parent: parent,
+		Table:  t,
+		Data:   make([]interface{}, len(t.Columns)),
+		cqId:   uuid.New(),
 	}
 }
 
@@ -64,15 +68,20 @@ func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource {
 // }
 
 func (r *Resource) Get(key string) interface{} {
-	return r.Data[key]
+	i := r.Table.ColumnIndex(key)
+	if i == -1 {
+		return nil
+	}
+	return r.Data[i]
 }
 
 func (r *Resource) Set(key string, value interface{}) error {
-	columnExists := funk.ContainsString(r.CColumns, key)
-	if !columnExists {
+	i := r.Table.ColumnIndex(key)
+	if i == -1 {
 		return fmt.Errorf("column %s does not exist", key)
 	}
-	r.Data[key] = value
+
+	r.Data[i] = value
 	return nil
 }
 
@@ -81,7 +90,7 @@ func (r *Resource) Id() uuid.UUID {
 }
 
 func (r *Resource) Columns() []string {
-	return r.CColumns
+	return r.Table.Columns.Names()
 }
 
 // func (r *Resource) Values() ([]interface{}, error) {
@@ -130,13 +139,13 @@ func (r *Resource) TableName() string {
 	return r.Table.Name
 }
 
-func (r Resource) GetMeta(key string) (interface{}, bool) {
-	if r.metadata == nil {
-		return nil, false
-	}
-	v, ok := r.metadata[key]
-	return v, ok
-}
+// func (r Resource) GetMeta(key string) (interface{}, bool) {
+// 	if r.metadata == nil {
+// 		return nil, false
+// 	}
+// 	v, ok := r.metadata[key]
+// 	return v, ok
+// }
 
 // func (r Resource) getColumnByName(column string) *Column {
 // 	for _, c := range r.Table.Columns {
@@ -165,7 +174,7 @@ func (rr Resources) ColumnNames() []string {
 	if len(rr) == 0 {
 		return []string{}
 	}
-	return rr[0].CColumns
+	return rr[0].Table.Columns.Names()
 }
 
 // func hashUUID(objs interface{}) (uuid.UUID, error) {
diff --git a/schema/table.go b/schema/table.go
index 4f57914d77..2a97c6f0a1 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -3,6 +3,8 @@ package schema
 import (
 	"context"
 	"runtime/debug"
+	"sync"
+	"time"
 
 	"github.com/cloudquery/plugin-sdk/helpers"
 	"github.com/iancoleman/strcase"
@@ -23,6 +25,19 @@ type IgnoreErrorFunc func(err error) bool
 
 type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error
 
+// Classify error and return it's severity and type
+type ClassifyErrorFunc func(err error) (bool, string)
+
+type Tables []*Table
+
+func (tt Tables) TableNames() []string {
+	ret := []string{}
+	for _, t := range tt {
+		ret = append(ret, t.TableNames()...)
+	}
+	return ret
+}
+
 type Table struct {
 	// Name of table
 	Name string
@@ -34,8 +49,8 @@ type Table struct {
 	Relations []*Table
 	// Resolver is the main entry point to fetching table data and
 	Resolver TableResolver `msgpack:"-"`
-	// Ignore errors checks if returned error from table resolver should be ignored.
-	IgnoreError IgnoreErrorFunc `msgpack:"-"`
+	// ClassifyError is a function that classifies error and returns it's severity and type
+	ClassifyError ClassifyErrorFunc `msgpack:"-"`
 	// Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them
 	Multiplex func(meta ClientMeta) []ClientMeta `msgpack:"-"`
 	// Post resource resolver is called after all columns have been resolved, and before resource is inserted to database.
@@ -55,6 +70,8 @@ type Table struct {
 
 	// Serial is used to force a signature change, which forces new table creation and cascading removal of old table and relations
 	Serial string
+
+	columnsMap map[string]int
 }
 
 // TableCreationOptions allow modifying how table is created such as defining primary keys, indices, foreign keys and constraints.
@@ -72,6 +89,22 @@ func (t Table) Column(name string) *Column {
 	return nil
 }
 
+func (t Table) ColumnIndex(name string) int {
+	var once sync.Once
+	once.Do(func() {
+		if t.columnsMap == nil {
+			t.columnsMap = make(map[string]int)
+			for i, c := range t.Columns {
+				t.columnsMap[c.Name] = i
+			}
+		}
+	})
+	if index, ok := t.columnsMap[name]; ok {
+		return index
+	}
+	return -1
+}
+
 // func (tco TableCreationOptions) signature() string {
 // 	return strings.Join(tco.PrimaryKeys, ";")
 // }
@@ -85,29 +118,38 @@ func (t Table) TableNames() []string {
 }
 
 // Call the table resolver with with all of it's relation for every reolved resource
-func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, resolvedResources chan<- *Resource) {
+func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, resolvedResources chan<- *Resource) int {
 	res := make(chan interface{})
+	startTime := time.Now()
 	go func() {
 		defer func() {
 			if r := recover(); r != nil {
 				stack := string(debug.Stack())
-				meta.Logger().Error().Str("table_name", t.Name).Str("stack", stack).Msg("table resolver finished with panic")
+				meta.Logger().Error().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Str("stack", stack).Msg("table resolver finished with panic")
 			}
 			close(res)
 		}()
 		meta.Logger().Debug().Str("table_name", t.Name).Msg("table resolver started")
 		if err := t.Resolver(ctx, meta, parent, res); err != nil {
-			meta.Logger().Error().Str("table_name", t.Name).Err(err).Msg("table resolver finished with error")
+			if t.ClassifyError != nil {
+				if classify, errType := t.ClassifyError(err); classify {
+					meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Str("error_type", errType).Msg("table resolver finished with error")
+					return
+				}
+			}
+			meta.Logger().Error().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Err(err).Msg("table resolver finished with error")
+			return
 		}
-		meta.Logger().Debug().Str("table_name", t.Name).Msg("table resolver finished successfully")
+		meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Msg("table resolver finished successfully")
 	}()
-
+	totalResources := 0
 	// each result is an array of interface{}
 	for elem := range res {
 		objects := helpers.InterfaceSlice(elem)
 		if len(objects) == 0 {
 			continue
 		}
+		totalResources += len(objects)
 		for i := range objects {
 			resource := NewResourceData(&t, parent, objects[i])
 			t.resolveColumns(ctx, meta, resource)
@@ -115,15 +157,17 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, r
 				meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver started")
 				if err := t.PostResourceResolver(ctx, meta, resource); err != nil {
 					meta.Logger().Error().Str("table_name", t.Name).Err(err).Msg("post resource resolver finished with error")
+				} else {
+					meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver finished successfully")
 				}
-				meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver finished successfully")
 			}
 			resolvedResources <- resource
 			for _, rel := range t.Relations {
-				rel.Resolve(ctx, meta, resource, resolvedResources)
+				totalResources += rel.Resolve(ctx, meta, resource, resolvedResources)
 			}
 		}
 	}
+	return totalResources
 }
 
 func (t Table) resolveColumns(ctx context.Context, meta ClientMeta, resource *Resource) {
diff --git a/specs/destination.go b/specs/destination.go
index 04876e85f6..e6a032e382 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -2,10 +2,22 @@ package specs
 
 import "gopkg.in/yaml.v3"
 
+type WriteMode int
+
+const (
+	ModeAppendOnly WriteMode = iota
+	ModeOverwrite
+)
+
+func (m WriteMode) String() string {
+	return [...]string{"append-only", "overwrite"}[m]
+}
+
 type DestinationSpec struct {
-	Name     string    `yaml:"name"`
-	Version  string    `yaml:"version"`
-	Path     string    `yaml:"path"`
-	Registry string    `yaml:"registry"`
-	Spec     yaml.Node `yaml:"spec"`
+	Name      string    `yaml:"name"`
+	Version   string    `yaml:"version"`
+	Path      string    `yaml:"path"`
+	Registry  Registry  `yaml:"registry"`
+	WriteMode WriteMode `yaml:"write_mode"`
+	Spec      yaml.Node `yaml:"spec"`
 }
diff --git a/specs/registry.go b/specs/registry.go
new file mode 100644
index 0000000000..6eed7786b0
--- /dev/null
+++ b/specs/registry.go
@@ -0,0 +1,13 @@
+package specs
+
+type Registry int
+
+const (
+	RegistryGithub Registry = iota
+	RegistryLocal
+	RegistryGrpc
+)
+
+func (m Registry) String() string {
+	return [...]string{"github", "local", "grpc"}[m]
+}
diff --git a/specs/source.go b/specs/source.go
index e15ac05d25..31da24b864 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -13,7 +13,7 @@ type SourceSpec struct {
 	// Path is the path in the registry
 	Path string `json:"path" yaml:"path"`
 	// Registry can be github,local,grpc. Might support things like https in the future.
-	Registry      string    `json:"registry" yaml:"registry"`
+	Registry      Registry  `json:"registry" yaml:"registry"`
 	MaxGoRoutines uint64    `json:"max_goroutines" yaml:"max_goroutines"`
 	Tables        []string  `json:"tables" yaml:"tables"`
 	SkipTables    []string  `json:"skip_tables" yaml:"skip_tables"`
@@ -33,8 +33,8 @@ func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
 	}
 
 	// set default
-	if s.Registry == "" {
-		s.Registry = "github"
+	if s.Registry.String() == "" {
+		s.Registry = RegistryGithub
 	}
 	if s.Path == "" {
 		s.Path = s.Name
@@ -42,7 +42,7 @@ func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
 	if s.Version == "" {
 		s.Version = "latest"
 	}
-	if !strings.Contains(s.Path, "/") {
+	if s.Registry == RegistryGithub && !strings.Contains(s.Path, "/") {
 		s.Path = "cloudquery/" + s.Path
 	}
 	return nil

From 83b80e1e3c388c17717da1d176c53c538f82be12 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sun, 14 Aug 2022 18:21:18 +0300
Subject: [PATCH 04/25] added bunch of tests

---
 clients/destination.go        | 45 +++++++++++++++++++
 clients/source.go             | 13 ++++--
 go.mod                        |  1 +
 go.sum                        |  4 ++
 internal/pb/destination.pb.go | 83 +++++++++++++++++++++--------------
 internal/pb/destination.proto |  5 ++-
 internal/servers/source.go    |  5 ++-
 plugins/destination.go        |  2 +-
 plugins/source.go             | 22 +++++-----
 schema/column.go              | 16 +++----
 schema/table.go               | 26 +++++------
 specs/destination.go          | 63 +++++++++++++++++++++-----
 specs/registry.go             | 62 +++++++++++++++++++++++++-
 specs/registry_test.go        | 36 +++++++++++++++
 specs/source.go               | 18 ++++----
 specs/spec.go                 | 12 +++++
 specs/spec_reader.go          | 12 ++---
 specs/spec_reader_test.go     | 16 +------
 specs/spec_test.go            | 60 +++++++++++++++++++++++++
 specs/testdata/aws.cq.yml     |  5 +--
 specs/testdata/pg.cq.yml      |  6 +++
 21 files changed, 394 insertions(+), 118 deletions(-)
 create mode 100644 specs/registry_test.go
 create mode 100644 specs/spec_test.go
 create mode 100644 specs/testdata/pg.cq.yml

diff --git a/clients/destination.go b/clients/destination.go
index ab78a39535..46ca320ae3 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -2,11 +2,13 @@ package clients
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"github.com/cloudquery/plugin-sdk/internal/pb"
 	"github.com/cloudquery/plugin-sdk/plugins"
 	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/cloudquery/plugin-sdk/specs"
 	"google.golang.org/grpc"
 )
 
@@ -28,6 +30,49 @@ func NewLocalDestinationClient(p plugins.DestinationPlugin) *DestinationClient {
 	}
 }
 
+func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error) {
+	if c.localClient != nil {
+		return c.localClient.GetExampleConfig(), nil
+	}
+	res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{})
+	if err != nil {
+		return "", err
+	}
+	return res.Config, nil
+}
+
+func (c *DestinationClient) Configure(ctx context.Context, spec specs.DestinationSpec) error {
+	if c.localClient != nil {
+		return c.localClient.Configure(ctx, spec)
+	}
+	b, err := json.Marshal(spec)
+	if err != nil {
+		return fmt.Errorf("destination configure: failed to marshal spec: %w", err)
+	}
+	_, err = c.pbClient.Configure(ctx, &pb.Configure_Request{
+		Config: b,
+	})
+	if err != nil {
+		return fmt.Errorf("destination configure: failed to configure: %w", err)
+	}
+	return nil
+}
+
+func (c *DestinationClient) Migrate(ctx context.Context, name string, version string, tables []*schema.Table) error {
+	if c.localClient != nil {
+		return c.Migrate(ctx, name, version, tables)
+	}
+	b, err := json.Marshal(tables)
+	if err != nil {
+		return fmt.Errorf("destination migrate: failed to marshal plugin: %w", err)
+	}
+	_, err = c.pbClient.Migrate(ctx, &pb.Migrate_Request{Name: name, Version: version, Tables: b})
+	if err != nil {
+		return fmt.Errorf("destination migrate: failed to migrate: %w", err)
+	}
+	return nil
+}
+
 func (c *DestinationClient) Write(ctx context.Context, resource *schema.Resource) error {
 	// var saveClient pb.Destination_SaveClient
 	// var err error
diff --git a/clients/source.go b/clients/source.go
index 10ccbc1f55..df00a853d3 100644
--- a/clients/source.go
+++ b/clients/source.go
@@ -5,6 +5,7 @@ package clients
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"text/template"
@@ -47,7 +48,7 @@ func (c *SourceClient) GetTables(ctx context.Context) ([]*schema.Table, error) {
 		return nil, err
 	}
 	var tables []*schema.Table
-	if err := msgpack.Unmarshal(res.Tables, &tables); err != nil {
+	if err := json.Unmarshal(res.Tables, &tables); err != nil {
 		return nil, err
 	}
 	return tables, nil
@@ -89,7 +90,7 @@ func (c *SourceClient) GetExampleConfig(ctx context.Context) (string, error) {
 	return tpl.String(), nil
 }
 
-func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res chan<- *FetchResultMessage) error {
+func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res chan<- *schema.Resource) error {
 	stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{})
 	if err != nil {
 		return fmt.Errorf("failed to fetch resources: %w", err)
@@ -102,8 +103,12 @@ func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res cha
 			}
 			return fmt.Errorf("failed to fetch resources from stream: %w", err)
 		}
-		res <- &FetchResultMessage{
-			Resource: r.Resource,
+		var resource schema.Resource
+		err = json.Unmarshal(r.Resource, &resource)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal resource: %w", err)
 		}
+
+		res <- &resource
 	}
 }
diff --git a/go.mod b/go.mod
index 73115be458..9b247c2997 100644
--- a/go.mod
+++ b/go.mod
@@ -46,6 +46,7 @@ require (
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
diff --git a/go.sum b/go.sum
index effc353d40..b3d31a1dab 100644
--- a/go.sum
+++ b/go.sum
@@ -226,6 +226,8 @@ github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
 github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
@@ -438,12 +440,14 @@ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscL
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/internal/pb/destination.pb.go b/internal/pb/destination.pb.go
index 80fbbb0909..0c76315f5e 100644
--- a/internal/pb/destination.pb.go
+++ b/internal/pb/destination.pb.go
@@ -101,8 +101,9 @@ type Migrate_Request struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// marshalled sourcePlugin
-	SourcePlugin []byte `protobuf:"bytes,1,opt,name=source_plugin,json=sourcePlugin,proto3" json:"source_plugin,omitempty"`
+	Name    string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
+	Tables  []byte `protobuf:"bytes,3,opt,name=tables,proto3" json:"tables,omitempty"`
 }
 
 func (x *Migrate_Request) Reset() {
@@ -137,9 +138,23 @@ func (*Migrate_Request) Descriptor() ([]byte, []int) {
 	return file_internal_pb_destination_proto_rawDescGZIP(), []int{0, 0}
 }
 
-func (x *Migrate_Request) GetSourcePlugin() []byte {
+func (x *Migrate_Request) GetName() string {
 	if x != nil {
-		return x.SourcePlugin
+		return x.Name
+	}
+	return ""
+}
+
+func (x *Migrate_Request) GetVersion() string {
+	if x != nil {
+		return x.Version
+	}
+	return ""
+}
+
+func (x *Migrate_Request) GetTables() []byte {
+	if x != nil {
+		return x.Tables
 	}
 	return nil
 }
@@ -284,36 +299,38 @@ var file_internal_pb_destination_proto_rawDesc = []byte{
 	0x0a, 0x1d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65,
 	0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
 	0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
-	0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x45,
-	0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x1a, 0x2e, 0x0a, 0x07, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70,
-	0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x1a, 0x25,
-	0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x9a, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74,
-	0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78,
-	0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70,
+	0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x66,
+	0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x1a, 0x4f, 0x0a, 0x07, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73,
+	0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x1a,
+	0x25, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x9a, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73,
+	0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45,
+	0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70,
 	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40,
-	0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x3a, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72,
-	0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05,
-	0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72,
-	0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x28, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x33,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x12, 0x3a, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67,
+	0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a,
+	0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57,
+	0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/internal/pb/destination.proto b/internal/pb/destination.proto
index f8fe68d1e7..c8de802a32 100644
--- a/internal/pb/destination.proto
+++ b/internal/pb/destination.proto
@@ -17,8 +17,9 @@ service Destination {
 
 message Migrate {
   message Request {
-    // marshalled sourcePlugin
-    bytes source_plugin = 1;
+    string name = 1;
+    string version = 2;
+    bytes tables = 3;
   }
   message Response {
   }
diff --git a/internal/servers/source.go b/internal/servers/source.go
index 0a417e7712..7b8e4705bd 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -2,6 +2,7 @@ package servers
 
 import (
 	"context"
+	"encoding/json"
 
 	"github.com/cloudquery/plugin-sdk/internal/pb"
 	"github.com/cloudquery/plugin-sdk/plugins"
@@ -43,7 +44,7 @@ func (s *SourceServer) Configure(ctx context.Context, req *pb.Configure_Request)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure source")
 	}
-	b, err := msgpack.Marshal(jsonschemaResult)
+	b, err := json.Marshal(jsonschemaResult)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal json schema result")
 	}
@@ -63,7 +64,7 @@ func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer
 	}()
 
 	for resource := range resources {
-		b, err := msgpack.Marshal(schema.WireResource{
+		b, err := json.Marshal(schema.WireResource{
 			Data:      resource.Data,
 			TableName: resource.Table.Name,
 		})
diff --git a/plugins/destination.go b/plugins/destination.go
index ae8e199c9f..4f01e0f792 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -29,7 +29,7 @@ import (
 
 type DestinationPlugin interface {
 	Configure(ctx context.Context, spec specs.DestinationSpec) error
-	Migrate(ctx context.Context, plugin *SourcePlugin) error
+	Migrate(ctx context.Context, sourceName string, version string, tables schema.Tables) error
 	Write(ctx context.Context, resources *schema.Resource) error
 	GetExampleConfig() string
 }
diff --git a/plugins/source.go b/plugins/source.go
index b215e6a4a1..f2d4b96669 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -36,26 +36,26 @@ type SourceConfigureFunc func(context.Context, *SourcePlugin, specs.SourceSpec)
 // We take a similar/declerative approach to API here similar to Cobra
 type SourcePlugin struct {
 	// Name of plugin i.e aws,gcp, azure etc'
-	Name string
+	Name string `json:"name"`
 	// Version of the plugin
-	Version string
+	Version string `json:"version"`
 	// Classify error and return it's severity and type
-	ClassifyError schema.ClassifyErrorFunc
+	ClassifyError schema.ClassifyErrorFunc `json:"-"`
 	// Called upon configure call to validate and init configuration
-	configure SourceConfigureFunc
+	configure SourceConfigureFunc `json:"-"`
 	// Tables is all tables supported by this source plugin
-	Tables schema.Tables
+	Tables schema.Tables `json:"tables"`
 	// JsonSchema for specific source plugin spec
-	JsonSchema string
+	JsonSchema string `json:"json_schema"`
 	// ExampleConfig is the example configuration for this plugin
-	ExampleConfig string
+	ExampleConfig string `json:"example_config"`
 	// Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead.
-	Logger zerolog.Logger
+	Logger zerolog.Logger `json:"-"`
 
 	// Internal fields set by configure
-	clientMeta schema.ClientMeta
-	spec       *specs.SourceSpec
-	m          sync.Mutex
+	clientMeta schema.ClientMeta `json:"-"`
+	spec       *specs.SourceSpec `json:"-"`
+	m          sync.Mutex        `json:"-"`
 }
 
 type SourceOption func(*SourcePlugin)
diff --git a/schema/column.go b/schema/column.go
index f36d0d6487..6d52183d6a 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -43,25 +43,25 @@ type ColumnCreationOptions struct {
 // Column definition for Table
 type Column struct {
 	// Name of column
-	Name string
+	Name string `json:"name"`
 	// Value Type of column i.e String, UUID etc'
-	Type ValueType
+	Type ValueType `json:"type"`
 	// Description about column, this description is added as a comment in the database
-	Description string
+	Description string `json:"description"`
 	// Column Resolver allows to set you own data based on resolving this can be an API call or setting multiple embedded values etc'
-	Resolver ColumnResolver `msgpack:"-"`
+	Resolver ColumnResolver `json:"-"`
 	// Creation options allow modifying how column is defined when table is created
-	CreationOptions ColumnCreationOptions
+	CreationOptions ColumnCreationOptions `json:"creation_options"`
 	// IgnoreInTests is used to skip verifying the column is non-nil in integration tests.
 	// By default, integration tests perform a fetch for all resources in cloudquery's test account, and
 	// verify all columns are non-nil.
 	// If IgnoreInTests is true, verification is skipped for this column.
 	// Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns).
-	IgnoreInTests bool
+	IgnoreInTests bool `json:"ignore_in_tests"`
 	// internal is true if this column is managed by the SDK
-	internal bool
+	internal bool `json:"-"`
 	// meta holds serializable information about the column's resolvers and functions
-	meta *ColumnMeta
+	meta *ColumnMeta `json:"-"`
 }
 
 const (
diff --git a/schema/table.go b/schema/table.go
index 2a97c6f0a1..c8551e0188 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -40,38 +40,38 @@ func (tt Tables) TableNames() []string {
 
 type Table struct {
 	// Name of table
-	Name string
+	Name string `json:"name"`
 	// table description
-	Description string
+	Description string `json:"description"`
 	// Columns are the set of fields that are part of this table
-	Columns ColumnList
+	Columns ColumnList `json:"columns"`
 	// Relations are a set of related tables defines
-	Relations []*Table
+	Relations []*Table `json:"relations"`
 	// Resolver is the main entry point to fetching table data and
-	Resolver TableResolver `msgpack:"-"`
+	Resolver TableResolver `json:"-"`
 	// ClassifyError is a function that classifies error and returns it's severity and type
-	ClassifyError ClassifyErrorFunc `msgpack:"-"`
+	ClassifyError ClassifyErrorFunc `json:"-"`
 	// Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them
-	Multiplex func(meta ClientMeta) []ClientMeta `msgpack:"-"`
+	Multiplex func(meta ClientMeta) []ClientMeta `json:"-"`
 	// Post resource resolver is called after all columns have been resolved, and before resource is inserted to database.
-	PostResourceResolver RowResolver `msgpack:"-"`
+	PostResourceResolver RowResolver `json:"-"`
 	// Options allow modification of how the table is defined when created
-	Options TableCreationOptions
+	Options TableCreationOptions `json:"options"`
 
 	// IgnoreInTests is used to exclude a table from integration tests.
 	// By default, integration tests fetch all resources from cloudquery's test account, and verify all tables
 	// have at least one row.
 	// When IgnoreInTests is true, integration tests won't fetch from this table.
 	// Used when it is hard to create a reproducible environment with a row in this table.
-	IgnoreInTests bool
+	IgnoreInTests bool `json:"ignore_in_tests"`
 
 	// Parent is the parent table in case this table is called via parent table (i.e. relation)
-	Parent *Table
+	Parent *Table `json:"-"`
 
 	// Serial is used to force a signature change, which forces new table creation and cascading removal of old table and relations
-	Serial string
+	Serial string `json:"-"`
 
-	columnsMap map[string]int
+	columnsMap map[string]int `json:"-"`
 }
 
 // TableCreationOptions allow modifying how table is created such as defining primary keys, indices, foreign keys and constraints.
diff --git a/specs/destination.go b/specs/destination.go
index e6a032e382..4fc87fa316 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -1,23 +1,66 @@
 package specs
 
-import "gopkg.in/yaml.v3"
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
 
 type WriteMode int
 
 const (
-	ModeAppendOnly WriteMode = iota
-	ModeOverwrite
+	WriteModeAppendOnly WriteMode = iota
+	WriteModeOverwrite
 )
 
+type DestinationSpec struct {
+	Name      string    `yaml:"name,omitempty" json:"name,omitempty"`
+	Version   string    `yaml:"version,omitempty" json:"version,omitempty"`
+	Path      string    `yaml:"path,omitempty" json:"path,omitempty"`
+	Registry  Registry  `yaml:"registry,omitempty" json:"registry,omitempty"`
+	WriteMode WriteMode `yaml:"write_mode,omitempty" json:"write_mode,omitempty"`
+	Spec      yaml.Node `yaml:"spec,omitempty" json:"spec,omitempty"`
+}
+
 func (m WriteMode) String() string {
 	return [...]string{"append-only", "overwrite"}[m]
 }
 
-type DestinationSpec struct {
-	Name      string    `yaml:"name"`
-	Version   string    `yaml:"version"`
-	Path      string    `yaml:"path"`
-	Registry  Registry  `yaml:"registry"`
-	WriteMode WriteMode `yaml:"write_mode"`
-	Spec      yaml.Node `yaml:"spec"`
+func (m WriteMode) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString(`"`)
+	buffer.WriteString(m.String())
+	buffer.WriteString(`"`)
+	return buffer.Bytes(), nil
+}
+
+func (m *WriteMode) UnmarshalJSON(data []byte) (err error) {
+	var writeMode string
+	if err := json.Unmarshal(data, &writeMode); err != nil {
+		return err
+	}
+	if *m, err = WriteModeFromString(writeMode); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m WriteMode) MarshalYAML() (interface{}, error) {
+	return m.String(), nil
+}
+
+func (r *WriteMode) UnmarshalYAML(n *yaml.Node) (err error) {
+	*r, err = WriteModeFromString(n.Value)
+	return err
+}
+
+func WriteModeFromString(s string) (WriteMode, error) {
+	switch s {
+	case "append-only":
+		return WriteModeAppendOnly, nil
+	case "overwrite":
+		return WriteModeOverwrite, nil
+	}
+	return 0, fmt.Errorf("invalid write mode: %s", s)
 }
diff --git a/specs/registry.go b/specs/registry.go
index 6eed7786b0..ad1ce56360 100644
--- a/specs/registry.go
+++ b/specs/registry.go
@@ -1,5 +1,13 @@
 package specs
 
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
 type Registry int
 
 const (
@@ -8,6 +16,56 @@ const (
 	RegistryGrpc
 )
 
-func (m Registry) String() string {
-	return [...]string{"github", "local", "grpc"}[m]
+func (r Registry) String() string {
+	return [...]string{"github", "local", "grpc"}[r]
+}
+func (r Registry) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString(`"`)
+	buffer.WriteString(r.String())
+	buffer.WriteString(`"`)
+	return buffer.Bytes(), nil
+}
+
+func (r *Registry) UnmarshalJSON(data []byte) (err error) {
+	var registry string
+	if err := json.Unmarshal(data, &registry); err != nil {
+		return err
+	}
+	if *r, err = RegistryFromString(registry); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r Registry) MarshalYAML() (interface{}, error) {
+	return r.String(), nil
+}
+
+func (r *Registry) UnmarshalYAML(n *yaml.Node) (err error) {
+	*r, err = RegistryFromString(n.Value)
+	return err
+}
+
+// func (r *Registry) UnmarshalYaml(data []byte) (err error) {
+// 	var registry string
+// 	if err := yaml.Unmarshal(data, &registry); err != nil {
+// 		return err
+// 	}
+// 	if *r, err = RegistryFromString(registry); err != nil {
+// 		return err
+// 	}
+// 	return nil
+// }
+
+func RegistryFromString(s string) (Registry, error) {
+	switch s {
+	case "github":
+		return RegistryGithub, nil
+	case "local":
+		return RegistryLocal, nil
+	case "grpc":
+		return RegistryGrpc, nil
+	default:
+		return RegistryGithub, fmt.Errorf("unknown registry %s", s)
+	}
 }
diff --git a/specs/registry_test.go b/specs/registry_test.go
new file mode 100644
index 0000000000..d7751a27ca
--- /dev/null
+++ b/specs/registry_test.go
@@ -0,0 +1,36 @@
+package specs
+
+import (
+	"encoding/json"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestRegistryJsonMarshalUnmarshal(t *testing.T) {
+	b, err := json.Marshal(RegistryGrpc)
+	if err != nil {
+		t.Fatal("failed to marshal registry:", err)
+	}
+	var registry Registry
+	if err := json.Unmarshal(b, &registry); err != nil {
+		t.Fatal("failed to unmarshal registry:", err)
+	}
+	if registry != RegistryGrpc {
+		t.Fatal("expected registry to be github, but got:", registry)
+	}
+}
+
+func TestRegistryYamlMarshalUnmarsahl(t *testing.T) {
+	b, err := yaml.Marshal(RegistryGrpc)
+	if err != nil {
+		t.Fatal("failed to marshal registry:", err)
+	}
+	var registry Registry
+	if err := yaml.Unmarshal(b, &registry); err != nil {
+		t.Fatal("failed to unmarshal registry:", err)
+	}
+	if registry != RegistryGrpc {
+		t.Fatal("expected registry to be github, but got:", registry)
+	}
+}
diff --git a/specs/source.go b/specs/source.go
index 31da24b864..6dc6252407 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -8,17 +8,17 @@ import (
 
 // SourceSpec is the shared configuration for all source plugins
 type SourceSpec struct {
-	Name    string `json:"name" yaml:"name"`
-	Version string `json:"version" yaml:"version"`
+	Name    string `json:"name,omitempty" yaml:"name,omitempty"`
+	Version string `json:"version,omitempty" yaml:"version,omitempty"`
 	// Path is the path in the registry
-	Path string `json:"path" yaml:"path"`
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
 	// Registry can be github,local,grpc. Might support things like https in the future.
-	Registry      Registry  `json:"registry" yaml:"registry"`
-	MaxGoRoutines uint64    `json:"max_goroutines" yaml:"max_goroutines"`
-	Tables        []string  `json:"tables" yaml:"tables"`
-	SkipTables    []string  `json:"skip_tables" yaml:"skip_tables"`
-	Destinations  []string  `json:"destinations" yaml:"destinations"`
-	Spec          yaml.Node `json:"spec" yaml:"spec"`
+	Registry      Registry  `json:"registry,omitempty" yaml:"registry,omitempty"`
+	MaxGoRoutines uint64    `json:"max_goroutines,omitempty" yaml:"max_goroutines,omitempty"`
+	Tables        []string  `json:"tables,omitempty" yaml:"tables,omitempty"`
+	SkipTables    []string  `json:"skip_tables,omitempty" yaml:"skip_tables,omitempty"`
+	Destinations  []string  `json:"destinations,omitempty" yaml:"destinations,omitempty"`
+	Spec          yaml.Node `json:"spec,omitempty" yaml:"spec,omitempty"`
 }
 
 func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
diff --git a/specs/spec.go b/specs/spec.go
index aeff7c732e..748b4e90c2 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -32,3 +32,15 @@ func (s *Spec) UnmarshalYAML(n *yaml.Node) error {
 	}
 	return obj.Spec.Decode(s.Spec)
 }
+
+func (s Spec) MarshalYAML() (interface{}, error) {
+	type T struct {
+		Kind string      `yaml:"kind,omitempty"`
+		Spec interface{} `yaml:"spec,omitempty"`
+	}
+	tmp := T{
+		Kind: s.Kind,
+		Spec: s.Spec,
+	}
+	return tmp, nil
+}
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index 687e6c6760..fcb0cb177f 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -56,20 +56,20 @@ func (r *SpecReader) GetSources() []SourceSpec {
 	return sources
 }
 
-func (s *SpecReader) GetSourceByName(name string) SourceSpec {
+func (s *SpecReader) GetSourceByName(name string) *SourceSpec {
 	for _, spec := range s.sources {
 		if spec.Name == name {
-			return spec
+			return &spec
 		}
 	}
-	return SourceSpec{}
+	return nil
 }
 
-func (s *SpecReader) GetDestinatinoByName(name string) DestinationSpec {
+func (s *SpecReader) GetDestinatinoByName(name string) *DestinationSpec {
 	for _, spec := range s.destinations {
 		if spec.Name == name {
-			return spec
+			return &spec
 		}
 	}
-	return DestinationSpec{}
+	return nil
 }
diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go
index a728f407d3..b66f084fa1 100644
--- a/specs/spec_reader_test.go
+++ b/specs/spec_reader_test.go
@@ -5,18 +5,6 @@ import (
 	"testing"
 )
 
-// This test is testing both unmarshalling and LoadSpecs together
-// so to add a new test case add a new file to testdata and an expected unmarshaled objectSpec
-var expectedSpecs = map[string]interface{}{
-	"aws.cq.yml": SourceSpec{
-		Name:          "aws",
-		Path:          "cloudquery/aws",
-		Registry:      "github",
-		Version:       "1.0.0",
-		MaxGoRoutines: 10,
-	},
-}
-
 func TestLoadSpecs(t *testing.T) {
 	specReader, err := NewSpecReader("testdata")
 	if err != nil {
@@ -24,8 +12,8 @@ func TestLoadSpecs(t *testing.T) {
 	}
 	for fileName, spec := range specReader.sources {
 		t.Run(fileName, func(t *testing.T) {
-			if !reflect.DeepEqual(spec, expectedSpecs[fileName]) {
-				t.Fatalf("expected %v, got %v", expectedSpecs[fileName], spec)
+			if !reflect.DeepEqual(spec, testSpecs[fileName]) {
+				t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, testSpecs[fileName].Spec, spec.Spec)
 			}
 		})
 	}
diff --git a/specs/spec_test.go b/specs/spec_test.go
new file mode 100644
index 0000000000..b6271c6829
--- /dev/null
+++ b/specs/spec_test.go
@@ -0,0 +1,60 @@
+package specs
+
+import (
+	"embed"
+	"os"
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+//go:embed testdata/*.cq.yml
+var testSpecsFS embed.FS
+
+var testSpecs = map[string]Spec{
+	"testdata/pg.cq.yml": {
+		Kind: "destination",
+		Spec: &DestinationSpec{
+			Name:      "postgresql",
+			Version:   "v1.0.0",
+			Registry:  RegistryGrpc,
+			WriteMode: WriteModeOverwrite,
+		},
+	},
+	"testdata/aws.cq.yml": {
+		Kind: "source",
+		Spec: &SourceSpec{
+			Name:          "aws",
+			Path:          "aws",
+			Version:       "v1.0.0",
+			MaxGoRoutines: 10,
+			Registry:      RegistryLocal,
+		},
+	},
+}
+
+func TestSpecYamlMarshal(t *testing.T) {
+	for fileName, expectedSpec := range testSpecs {
+		t.Run(fileName, func(t *testing.T) {
+			b, err := os.ReadFile(fileName)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var spec Spec
+			if err := yaml.Unmarshal(b, &spec); err != nil {
+				t.Fatal(err)
+			}
+			b, err = yaml.Marshal(spec)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := yaml.Unmarshal(b, &spec); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(spec, expectedSpec) {
+				t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, expectedSpec.Spec, spec.Spec)
+			}
+		})
+	}
+}
diff --git a/specs/testdata/aws.cq.yml b/specs/testdata/aws.cq.yml
index 3ca996f012..2ef555424e 100644
--- a/specs/testdata/aws.cq.yml
+++ b/specs/testdata/aws.cq.yml
@@ -1,7 +1,6 @@
 kind: source
 spec:
   name: aws
-  version: 1.0.0
+  version: v1.0.0
   max_goroutines: 10
-  spec:
-    regions: ["us-east-1"]
\ No newline at end of file
+  registry: local
\ No newline at end of file
diff --git a/specs/testdata/pg.cq.yml b/specs/testdata/pg.cq.yml
new file mode 100644
index 0000000000..e187ad190f
--- /dev/null
+++ b/specs/testdata/pg.cq.yml
@@ -0,0 +1,6 @@
+kind: destination
+spec:
+  name: postgresql
+  version: v1.0.0
+  registry: grpc
+  write_mode: overwrite

From 0e358948b67b4309336b653d0c4df6cb9d39962e Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sun, 14 Aug 2022 18:50:54 +0300
Subject: [PATCH 05/25] remove limit from sdk

---
 helpers/limit/limits.go                       |  82 -------------
 helpers/limit/limits_test.go                  |  26 ----
 helpers/limit/sysctl_darwin.go                |  27 -----
 helpers/limit/sysctl_freebsd.go               |   9 --
 helpers/limit/sysctl_linux.go                 |  27 -----
 helpers/limit/sysctl_windows.go               |   9 --
 helpers/limit/ulimit_freebsd.go               |  13 --
 helpers/limit/ulimit_unix.go                  |  13 --
 helpers/limit/ulimit_win.go                   |   9 --
 internal/servers/source.go                    |   2 +-
 plugins/source.go                             |   6 +-
 plugins/source_test.go                        | 113 ++++++++++++++++++
 plugins/source_test.go.backup                 |  89 --------------
 plugins/source_testing.go                     |   2 +-
 ...lvers_test.go.backup => resolvers_test.go} |   0
 specs/spec_reader_test.go                     |  31 ++---
 16 files changed, 129 insertions(+), 329 deletions(-)
 delete mode 100644 helpers/limit/limits.go
 delete mode 100644 helpers/limit/limits_test.go
 delete mode 100644 helpers/limit/sysctl_darwin.go
 delete mode 100644 helpers/limit/sysctl_freebsd.go
 delete mode 100644 helpers/limit/sysctl_linux.go
 delete mode 100644 helpers/limit/sysctl_windows.go
 delete mode 100644 helpers/limit/ulimit_freebsd.go
 delete mode 100644 helpers/limit/ulimit_unix.go
 delete mode 100644 helpers/limit/ulimit_win.go
 create mode 100644 plugins/source_test.go
 delete mode 100644 plugins/source_test.go.backup
 rename schema/{resolvers_test.go.backup => resolvers_test.go} (100%)

diff --git a/helpers/limit/limits.go b/helpers/limit/limits.go
deleted file mode 100644
index 519297685b..0000000000
--- a/helpers/limit/limits.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package limit
-
-import (
-	"fmt"
-	"math"
-
-	"github.com/pbnjay/memory"
-)
-
-const (
-	gbInBytes         int     = 1024 * 1024 * 1024
-	goroutinesPerGB   float64 = 250000
-	minimalGoRoutines float64 = 100
-	goroutineReducer          = 0.8
-	// only use 75% of the available file descriptors, so to leave for other processes file descriptors
-	mfoReducer = 0.75
-)
-
-type Rlimit struct {
-	Cur uint64
-	Max uint64
-}
-
-func GetMaxGoRoutines() uint64 {
-	limit := calculateGoRoutines(getMemory())
-	ulimit, err := GetUlimit()
-	if err != nil || ulimit.Cur == 0 {
-		return limit
-	}
-	if ulimit.Cur > limit {
-		return limit
-	}
-	return ulimit.Cur
-}
-
-// DiagnoseLimits verifies if user should increase ulimit or max file descriptors to improve number of expected
-// goroutines in CQ to improve performance
-func DiagnoseLimits() error {
-	// the amount of goroutines we want based on machine memory
-	want := calculateGoRoutines(getMemory())
-	// calculate file descriptor limit
-	fds, err := calculateFileLimit()
-	if err != nil {
-		return err
-	}
-	if fds < want {
-		fmt.Printf("available descriptor capacity is %d want %d to run optimally, consider increasing max file descriptors on machine.", fds, want)
-	}
-	ulimit, err := GetUlimit()
-	if err != nil {
-		return err
-	}
-	if ulimit.Cur < want {
-		fmt.Printf("set ulimit capacity is %d want %d to run optimally, consider increasing ulimit on this machine.", ulimit.Cur, want)
-	}
-	return err
-}
-
-func getMemory() uint64 {
-	return memory.TotalMemory()
-}
-
-func calculateMemoryGoRoutines(totalMemory uint64) uint64 {
-	if totalMemory == 0 {
-		// assume we have 2 GB RAM
-		return uint64(math.Max(minimalGoRoutines, goroutinesPerGB*2*goroutineReducer))
-	}
-	return uint64(math.Max(minimalGoRoutines, (goroutinesPerGB*float64(totalMemory)/float64(gbInBytes))*goroutineReducer))
-}
-
-func calculateGoRoutines(totalMemory uint64) uint64 {
-	total := calculateMemoryGoRoutines(totalMemory)
-	mfo, err := calculateFileLimit()
-	if err != nil {
-		return total
-	}
-
-	if mfo < total {
-		return uint64(float64(mfo) * mfoReducer)
-	}
-	return total
-}
diff --git a/helpers/limit/limits_test.go b/helpers/limit/limits_test.go
deleted file mode 100644
index 1ef11134b3..0000000000
--- a/helpers/limit/limits_test.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package limit
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCalculateGoRoutines(t *testing.T) {
-	cases := []struct {
-		Name       string
-		Memory     uint64
-		GoRoutines uint64
-	}{
-		{Name: "Zero", Memory: uint64(0), GoRoutines: uint64(400000)},
-		{Name: "Below 1073741824", Memory: uint64(990498816), GoRoutines: uint64(184494)},
-		{Name: "At 1073741824", Memory: uint64(1073741824), GoRoutines: uint64(200000)},
-		{Name: "Above 1073741824", Memory: uint64(1573741824), GoRoutines: uint64(293132)},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.Name, func(t *testing.T) {
-			assert.Equal(t, int(tc.GoRoutines), int(calculateGoRoutines(tc.Memory)))
-		})
-	}
-}
diff --git a/helpers/limit/sysctl_darwin.go b/helpers/limit/sysctl_darwin.go
deleted file mode 100644
index 7771571f0d..0000000000
--- a/helpers/limit/sysctl_darwin.go
+++ /dev/null
@@ -1,27 +0,0 @@
-//go:build darwin
-
-package limit
-
-import (
-	"github.com/lorenzosaino/go-sysctl"
-	"github.com/spf13/cast"
-)
-
-func calculateFileLimit() (uint64, error) {
-	maxFileOpen, err := sysctl.Get("kern.maxfilesperproc")
-	if err != nil {
-		return 0, err
-	}
-	mfo, err := cast.ToUint64E(maxFileOpen)
-	if err != nil {
-		return 0, err
-	}
-
-	fileNr, err := sysctl.Get("kern.num_files")
-	if err != nil {
-		return 0, err
-	}
-	fnr := cast.ToUint64(fileNr)
-
-	return uint64(float64(mfo-fnr) * goroutineReducer), nil
-}
diff --git a/helpers/limit/sysctl_freebsd.go b/helpers/limit/sysctl_freebsd.go
deleted file mode 100644
index 00dc5269ff..0000000000
--- a/helpers/limit/sysctl_freebsd.go
+++ /dev/null
@@ -1,9 +0,0 @@
-//go:build freebsd
-
-package limit
-
-import "errors"
-
-func calculateFileLimit() (uint64, error) {
-	return 0, errors.New("file descriptors limiter not supported on this platform")
-}
diff --git a/helpers/limit/sysctl_linux.go b/helpers/limit/sysctl_linux.go
deleted file mode 100644
index 7917e608a8..0000000000
--- a/helpers/limit/sysctl_linux.go
+++ /dev/null
@@ -1,27 +0,0 @@
-//go:build linux
-
-package limit
-
-import (
-	"github.com/lorenzosaino/go-sysctl"
-	"github.com/spf13/cast"
-)
-
-func calculateFileLimit() (uint64, error) {
-	maxFileOpen, err := sysctl.Get("fs.file-max")
-	if err != nil {
-		return 0, err
-	}
-	mfo, err := cast.ToUint64E(maxFileOpen)
-	if err != nil {
-		return 0, err
-	}
-
-	fileNr, err := sysctl.Get("fs.file-nr")
-	if err != nil {
-		return 0, err
-	}
-	fnr := cast.ToUint64(fileNr)
-
-	return uint64(float64(mfo-fnr) * goroutineReducer), nil
-}
diff --git a/helpers/limit/sysctl_windows.go b/helpers/limit/sysctl_windows.go
deleted file mode 100644
index 9924849243..0000000000
--- a/helpers/limit/sysctl_windows.go
+++ /dev/null
@@ -1,9 +0,0 @@
-//go:build windows
-
-package limit
-
-import "errors"
-
-func calculateFileLimit() (uint64, error) {
-	return 0, errors.New("file descriptors limiter not supported in windows")
-}
diff --git a/helpers/limit/ulimit_freebsd.go b/helpers/limit/ulimit_freebsd.go
deleted file mode 100644
index 999b5032fc..0000000000
--- a/helpers/limit/ulimit_freebsd.go
+++ /dev/null
@@ -1,13 +0,0 @@
-//go:build freebsd
-
-package limit
-
-import (
-	"syscall"
-)
-
-func GetUlimit() (Rlimit, error) {
-	var rLimit syscall.Rlimit
-	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-	return Rlimit{uint64(rLimit.Cur), uint64(rLimit.Max)}, err
-}
diff --git a/helpers/limit/ulimit_unix.go b/helpers/limit/ulimit_unix.go
deleted file mode 100644
index a65486569b..0000000000
--- a/helpers/limit/ulimit_unix.go
+++ /dev/null
@@ -1,13 +0,0 @@
-//go:build darwin || linux
-
-package limit
-
-import (
-	"syscall"
-)
-
-func GetUlimit() (Rlimit, error) {
-	var rLimit syscall.Rlimit
-	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-	return Rlimit{rLimit.Cur, rLimit.Max}, err
-}
diff --git a/helpers/limit/ulimit_win.go b/helpers/limit/ulimit_win.go
deleted file mode 100644
index 64f31e291b..0000000000
--- a/helpers/limit/ulimit_win.go
+++ /dev/null
@@ -1,9 +0,0 @@
-//go:build windows
-
-package limit
-
-import "errors"
-
-func GetUlimit() (Rlimit, error) {
-	return Rlimit{0, 0}, errors.New("ulimit not supported in windows")
-}
diff --git a/internal/servers/source.go b/internal/servers/source.go
index 7b8e4705bd..cfe61b3e29 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -58,7 +58,7 @@ func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer
 	var fetchErr error
 	go func() {
 		defer close(resources)
-		if err := s.Plugin.Fetch(stream.Context(), resources); err != nil {
+		if err := s.Plugin.Sync(stream.Context(), resources); err != nil {
 			fetchErr = errors.Wrap(err, "failed to fetch resources")
 		}
 	}()
diff --git a/plugins/source.go b/plugins/source.go
index f2d4b96669..dda3de1f65 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/cloudquery/plugin-sdk/helpers"
-	"github.com/cloudquery/plugin-sdk/helpers/limit"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
@@ -131,16 +130,13 @@ func (p *SourcePlugin) Configure(ctx context.Context, spec specs.SourceSpec) (*g
 }
 
 // Fetch fetches data according to source configuration and
-func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) error {
+func (p *SourcePlugin) Sync(ctx context.Context, res chan<- *schema.Resource) error {
 	if p.spec == nil {
 		return fmt.Errorf("source plugin not configured")
 	}
 
 	// limiter used to limit the amount of resources fetched concurrently
 	maxGoroutines := p.spec.MaxGoRoutines
-	if maxGoroutines == 0 {
-		maxGoroutines = limit.GetMaxGoRoutines()
-	}
 	p.Logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch")
 	goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
 
diff --git a/plugins/source_test.go b/plugins/source_test.go
new file mode 100644
index 0000000000..c2c247ac34
--- /dev/null
+++ b/plugins/source_test.go
@@ -0,0 +1,113 @@
+package plugins
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/rs/zerolog"
+)
+
+type Account struct {
+	Name    string   `yaml:"name"`
+	Regions []string `yaml:"regions"`
+}
+
+type TestConfig struct {
+	Accounts []Account `yaml:"accounts"`
+	Regions  []string  `yaml:"regions"`
+}
+
+func (TestConfig) Example() string {
+	return ""
+}
+
+type testSourcePluginClient struct {
+	logger zerolog.Logger
+}
+
+func (t testSourcePluginClient) Logger() *zerolog.Logger {
+	return &t.logger
+}
+
+type testPluginClient struct {
+	logger zerolog.Logger
+}
+
+func (c *testPluginClient) Logger() *zerolog.Logger {
+	return &c.logger
+}
+
+func testPluginConfigure(ctx context.Context, p *SourcePlugin, spec specs.SourceSpec) (schema.ClientMeta, error) {
+	return &testPluginClient{
+		logger: p.Logger,
+	}, nil
+}
+
+func testTable() *schema.Table {
+	return &schema.Table{
+		Name: "testTable",
+		Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
+			res <- map[string]interface{}{
+				"testColumn": 3,
+			}
+			return nil
+		},
+		Columns: []schema.Column{
+			{
+				Name: "testColumn",
+				Type: schema.TypeInt,
+			},
+		},
+	}
+}
+
+const testSourceCfg = `
+kind: source
+spec:
+  name: testSourcePlugin
+  version: 1.0.0
+  spec:
+    accounts:
+    - name: testAccount
+`
+
+func TestSync(t *testing.T) {
+	// ctx := context.Background()
+	// testSourcePlugin := NewSourcePlugin(
+	// 	"test",
+	// 	"v1.0.0",
+	// 	[]*schema.Table{
+	// 		testTable(),
+	// 	},
+	// 	testPluginConfigure,
+	// 	WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
+	// )
+	// yaml.Unmarshal()
+	// testSourcePlugin.Configure(ctx)
+	// 	cfg := `
+	// tables:
+	//   - "*"
+	// configuration:
+	//   regions:
+	//   - "us-east-1"
+	//   accounts:
+	//   - name: "testAccount"
+	//     regions:
+	//     - "us-east-2"
+	// `
+	// 	resources := make(chan *schema.Resource)
+	// 	var fetchErr error
+	// 	var result *gojsonschema.Result
+	// 	go func() {
+	// 		defer close(resources)
+	// 		result, fetchErr = testSourcePlugin.Fetch(context.Background(), []byte(cfg), resources)
+	// 	}()
+	// 	for resource := range resources {
+	// 		t.Logf("%+v", resource)
+	// 	}
+	// 	if fetchErr != nil {
+	// 		t.Errorf("fetch error: %v", fetchErr)
+	// 	}
+}
diff --git a/plugins/source_test.go.backup b/plugins/source_test.go.backup
deleted file mode 100644
index 632fd163cc..0000000000
--- a/plugins/source_test.go.backup
+++ /dev/null
@@ -1,89 +0,0 @@
-package plugins
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/rs/zerolog"
-	"github.com/xeipuuv/gojsonschema"
-)
-
-type Account struct {
-	Name    string   `yaml:"name"`
-	Regions []string `yaml:"regions"`
-}
-
-type TestConfig struct {
-	Accounts []Account `yaml:"accounts"`
-	Regions  []string  `yaml:"regions"`
-}
-
-func (TestConfig) Example() string {
-	return ""
-}
-
-type testSourcePluginClient struct {
-	logger zerolog.Logger
-}
-
-func (t testSourcePluginClient) Logger() *zerolog.Logger {
-	return &t.logger
-}
-
-var testSourcePlugin = SourcePlugin{
-	Name:    "testSourcePlugin",
-	Version: "1.0.0",
-	Configure: func(l zerolog.Logger, i interface{}) (schema.ClientMeta, error) {
-		return testSourcePluginClient{logger: l}, nil
-	},
-	Tables: []*schema.Table{
-		{
-			Name: "testTable",
-			Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
-				res <- map[string]interface{}{
-					"testColumn": 3,
-				}
-				return nil
-			},
-			Columns: []schema.Column{
-				{
-					Name: "testColumn",
-					Type: schema.TypeInt,
-				},
-			},
-		},
-	},
-	Config: func() interface{} {
-		return &TestConfig{}
-	},
-	Logger: zerolog.New(os.Stderr),
-}
-
-func TestFetch(t *testing.T) {
-	cfg := `
-tables:
-  - "*"
-configuration:
-  regions:
-  - "us-east-1"
-  accounts:
-  - name: "testAccount"
-    regions:
-    - "us-east-2"
-`
-	resources := make(chan *schema.Resource)
-	var fetchErr error
-	var result *gojsonschema.Result
-	go func() {
-		defer close(resources)
-		result, fetchErr = testSourcePlugin.Fetch(context.Background(), []byte(cfg), resources)
-	}()
-	for resource := range resources {
-		t.Logf("%+v", resource)
-	}
-	if fetchErr != nil {
-		t.Errorf("fetch error: %v", fetchErr)
-	}
-}
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index 9528a540d5..a4cfe96e4c 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -65,7 +65,7 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
 	// tc.Plugin.Logger = zerolog.New(zerolog.NewTestWriter(t))
 	go func() {
 		defer close(resources)
-		fetchErr = tc.Plugin.Fetch(context.Background(), resources)
+		fetchErr = tc.Plugin.Sync(context.Background(), resources)
 	}()
 	for resource := range resources {
 		validateResource(t, resource)
diff --git a/schema/resolvers_test.go.backup b/schema/resolvers_test.go
similarity index 100%
rename from schema/resolvers_test.go.backup
rename to schema/resolvers_test.go
diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go
index b66f084fa1..f169033ece 100644
--- a/specs/spec_reader_test.go
+++ b/specs/spec_reader_test.go
@@ -1,20 +1,15 @@
 package specs
 
-import (
-	"reflect"
-	"testing"
-)
-
-func TestLoadSpecs(t *testing.T) {
-	specReader, err := NewSpecReader("testdata")
-	if err != nil {
-		t.Fatal(err)
-	}
-	for fileName, spec := range specReader.sources {
-		t.Run(fileName, func(t *testing.T) {
-			if !reflect.DeepEqual(spec, testSpecs[fileName]) {
-				t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, testSpecs[fileName].Spec, spec.Spec)
-			}
-		})
-	}
-}
+// func TestLoadSpecs(t *testing.T) {
+// 	specReader, err := NewSpecReader("testdata")
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	for fileName, spec := range specReader.sources {
+// 		t.Run(fileName, func(t *testing.T) {
+// 			if !reflect.DeepEqual(spec, testSpecs[fileName]) {
+// 				t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, testSpecs[fileName].Spec, spec.Spec)
+// 			}
+// 		})
+// 	}
+// }

From 3a70d2bed1e58bfc70d70ca34dfef6f2e3a607df Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Mon, 15 Aug 2022 22:04:52 +0300
Subject: [PATCH 06/25] Added more tests

---
 clients/destination.go                        |  24 +--
 clients/source.go                             |   3 +-
 internal/pb/source.pb.go                      |  70 ++++----
 internal/pb/source.proto                      |   3 +-
 internal/servers/destinations.go              |   4 +-
 internal/servers/source.go                    |  36 ++--
 plugins/destination.go                        |  11 +-
 plugins/source.go                             | 146 +++++++++--------
 plugins/source_test.go                        | 119 ++++++--------
 plugins/source_testing.go                     | 154 +-----------------
 plugins/verifiers.go                          | 103 ------------
 schema/column.go                              |  14 +-
 schema/column_test.go                         |  22 +++
 ...lvers_test.go => resolvers_test.go.backup} |   0
 schema/table.go                               |  13 +-
 serve/doc.go                                  |   2 +-
 serve/serve.go                                |  23 ++-
 serve/serve_test.go                           | 107 ++++++++++++
 18 files changed, 357 insertions(+), 497 deletions(-)
 delete mode 100644 plugins/verifiers.go
 rename schema/{resolvers_test.go => resolvers_test.go.backup} (100%)
 create mode 100644 serve/serve_test.go

diff --git a/clients/destination.go b/clients/destination.go
index 46ca320ae3..28b33891c8 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -32,7 +32,7 @@ func NewLocalDestinationClient(p plugins.DestinationPlugin) *DestinationClient {
 
 func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error) {
 	if c.localClient != nil {
-		return c.localClient.GetExampleConfig(), nil
+		return c.localClient.ExampleConfig(), nil
 	}
 	res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{})
 	if err != nil {
@@ -43,7 +43,7 @@ func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error
 
 func (c *DestinationClient) Configure(ctx context.Context, spec specs.DestinationSpec) error {
 	if c.localClient != nil {
-		return c.localClient.Configure(ctx, spec)
+		return c.localClient.Initialize(ctx, spec)
 	}
 	b, err := json.Marshal(spec)
 	if err != nil {
@@ -58,15 +58,15 @@ func (c *DestinationClient) Configure(ctx context.Context, spec specs.Destinatio
 	return nil
 }
 
-func (c *DestinationClient) Migrate(ctx context.Context, name string, version string, tables []*schema.Table) error {
+func (c *DestinationClient) Migrate(ctx context.Context, tables []*schema.Table) error {
 	if c.localClient != nil {
-		return c.Migrate(ctx, name, version, tables)
+		return c.localClient.Migrate(ctx, tables)
 	}
 	b, err := json.Marshal(tables)
 	if err != nil {
 		return fmt.Errorf("destination migrate: failed to marshal plugin: %w", err)
 	}
-	_, err = c.pbClient.Migrate(ctx, &pb.Migrate_Request{Name: name, Version: version, Tables: b})
+	_, err = c.pbClient.Migrate(ctx, &pb.Migrate_Request{Tables: b})
 	if err != nil {
 		return fmt.Errorf("destination migrate: failed to migrate: %w", err)
 	}
@@ -90,17 +90,3 @@ func (c *DestinationClient) Write(ctx context.Context, resource *schema.Resource
 
 	return nil
 }
-
-// func (c *DestinationClient) CreateTables(ctx context.Context, tables []*schema.Table) error {
-// 	if c.localClient != nil {
-// 		return c.localClient.CreateTables(ctx, tables)
-// 	}
-// 	b, err := yaml.Marshal(tables)
-// 	if err != nil {
-// 		return fmt.Errorf("failed to marshal tables: %w", err)
-// 	}
-// 	if _, err := c.pbClient.CreateTables(ctx, &pb.CreateTables_Request{Tables: b}); err != nil {
-// 		return err
-// 	}
-// 	return nil
-// }
diff --git a/clients/source.go b/clients/source.go
index df00a853d3..0847c1e902 100644
--- a/clients/source.go
+++ b/clients/source.go
@@ -14,7 +14,6 @@ import (
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/pkg/errors"
-	"github.com/vmihailenco/msgpack/v5"
 	"github.com/xeipuuv/gojsonschema"
 	"google.golang.org/grpc"
 	"gopkg.in/yaml.v3"
@@ -64,7 +63,7 @@ func (c *SourceClient) Configure(ctx context.Context, spec specs.SourceSpec) (*g
 		return nil, errors.Wrap(err, "failed to configure source")
 	}
 	var validationResult gojsonschema.Result
-	if err := msgpack.Unmarshal(res.JsonschemaResult, &validationResult); err != nil {
+	if err := json.Unmarshal(res.JsonschemaResult, &validationResult); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal validation result")
 	}
 	return &validationResult, nil
diff --git a/internal/pb/source.pb.go b/internal/pb/source.pb.go
index 248c0730a6..0d1de5d75f 100644
--- a/internal/pb/source.pb.go
+++ b/internal/pb/source.pb.go
@@ -100,6 +100,8 @@ type Fetch_Request struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
+
+	Spec []byte `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"`
 }
 
 func (x *Fetch_Request) Reset() {
@@ -134,6 +136,13 @@ func (*Fetch_Request) Descriptor() ([]byte, []int) {
 	return file_internal_pb_source_proto_rawDescGZIP(), []int{0, 0}
 }
 
+func (x *Fetch_Request) GetSpec() []byte {
+	if x != nil {
+		return x.Spec
+	}
+	return nil
+}
+
 type Fetch_Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -290,36 +299,37 @@ var file_internal_pb_source_proto_rawDesc = []byte{
 	0x0a, 0x18, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f,
 	0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74,
 	0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x62,
-	0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3a, 0x0a, 0x05, 0x46, 0x65, 0x74,
-	0x63, 0x68, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x0a,
-	0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c,
-	0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x50, 0x0a,
-	0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
-	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a,
-	0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
-	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65,
-	0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x32,
-	0x9b, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65,
-	0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
-	0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62,
-	0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x10,
-	0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d,
-	0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61,
-	0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
-	0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
-	0x72, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x14,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74,
-	0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x05, 0x5a,
-	0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4e, 0x0a, 0x05, 0x46, 0x65, 0x74,
+	0x63, 0x68, 0x1a, 0x1d, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a,
+	0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x70, 0x65,
+	0x63, 0x1a, 0x26, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a,
+	0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74,
+	0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74,
+	0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62,
+	0x6c, 0x65, 0x73, 0x32, 0x9b, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40,
+	0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
+	0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74,
+	0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
+	0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
+	0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74,
+	0x63, 0x68, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68,
+	0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30,
+	0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/internal/pb/source.proto b/internal/pb/source.proto
index 6846fddcc1..5d3d0165aa 100644
--- a/internal/pb/source.proto
+++ b/internal/pb/source.proto
@@ -9,14 +9,13 @@ service Source {
   rpc GetTables(GetTables.Request) returns (GetTables.Response);
   // Get an example configuration for the source plugin
   rpc GetExampleConfig(GetExampleConfig.Request) returns (GetExampleConfig.Response);
-  // Configure the source plugin with the given spec
-  rpc Configure(Configure.Request) returns (Configure.Response);
   // Fetch resources
   rpc Fetch(Fetch.Request) returns (stream Fetch.Response);
 }
 
 message Fetch {
   message Request {
+    bytes spec = 1;
   }
   message Response {
     // marshalled *schema.Resources
diff --git a/internal/servers/destinations.go b/internal/servers/destinations.go
index f93b7d6fa1..6a0b9f63b9 100644
--- a/internal/servers/destinations.go
+++ b/internal/servers/destinations.go
@@ -25,12 +25,12 @@ func (s *DestinationServer) Configure(ctx context.Context, req *pb.Configure_Req
 	if err := yaml.Unmarshal(req.Config, &spec); err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err)
 	}
-	return &pb.Configure_Response{}, s.Plugin.Configure(ctx, spec)
+	return &pb.Configure_Response{}, s.Plugin.Initialize(ctx, spec)
 }
 
 func (s *DestinationServer) GetExampleConfig(ctx context.Context, req *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
 	return &pb.GetExampleConfig_Response{
-		Config: s.Plugin.GetExampleConfig(),
+		Config: s.Plugin.ExampleConfig(),
 	}, nil
 }
 
diff --git a/internal/servers/source.go b/internal/servers/source.go
index cfe61b3e29..e9ac345671 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -3,13 +3,13 @@ package servers
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/cloudquery/plugin-sdk/internal/pb"
 	"github.com/cloudquery/plugin-sdk/plugins"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/pkg/errors"
-	"github.com/vmihailenco/msgpack/v5"
 	"gopkg.in/yaml.v3"
 )
 
@@ -19,7 +19,7 @@ type SourceServer struct {
 }
 
 func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.GetTables_Response, error) {
-	b, err := msgpack.Marshal(s.Plugin.Tables)
+	b, err := json.Marshal(s.Plugin.Tables())
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal tables")
 	}
@@ -30,35 +30,23 @@ func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.Ge
 
 func (s *SourceServer) GetExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
 	return &pb.GetExampleConfig_Response{
-		Name:    s.Plugin.Name,
-		Version: s.Plugin.Version,
-		Config:  s.Plugin.ExampleConfig}, nil
-}
-
-func (s *SourceServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) {
-	var spec specs.SourceSpec
-	if err := yaml.Unmarshal(req.Config, &spec); err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal config")
-	}
-	jsonschemaResult, err := s.Plugin.Configure(ctx, spec)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure source")
-	}
-	b, err := json.Marshal(jsonschemaResult)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal json schema result")
-	}
-	return &pb.Configure_Response{
-		JsonschemaResult: b,
-	}, nil
+		Name:    s.Plugin.Name(),
+		Version: s.Plugin.Version(),
+		Config:  s.Plugin.ExampleConfig()}, nil
 }
 
 func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer) error {
 	resources := make(chan *schema.Resource)
 	var fetchErr error
+
+	var spec specs.SourceSpec
+	if err := yaml.Unmarshal(req.Spec, &spec); err != nil {
+		return fmt.Errorf("failed to unmarshal source spec: %w", err)
+	}
+
 	go func() {
 		defer close(resources)
-		if err := s.Plugin.Sync(stream.Context(), resources); err != nil {
+		if err := s.Plugin.Sync(stream.Context(), spec, resources); err != nil {
 			fetchErr = errors.Wrap(err, "failed to fetch resources")
 		}
 	}()
diff --git a/plugins/destination.go b/plugins/destination.go
index 4f01e0f792..b7a17a32f9 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/rs/zerolog"
 )
 
 // type DestinationOption func(*DestinationPlugin)
@@ -28,10 +29,14 @@ import (
 // }
 
 type DestinationPlugin interface {
-	Configure(ctx context.Context, spec specs.DestinationSpec) error
-	Migrate(ctx context.Context, sourceName string, version string, tables schema.Tables) error
+	Name() string
+	Version() string
+	ExampleConfig() string
+	JsonSchema() string
+	Initialize(ctx context.Context, spec specs.DestinationSpec) error
+	Migrate(ctx context.Context, tables schema.Tables) error
 	Write(ctx context.Context, resources *schema.Resource) error
-	GetExampleConfig() string
+	SetLogger(logger zerolog.Logger)
 }
 
 // func WithExampleConfig(exampleConfig string) DestinationOption {
diff --git a/plugins/source.go b/plugins/source.go
index dda3de1f65..5cf2bba6ea 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -11,7 +11,6 @@ import (
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
 	"github.com/thoas/go-funk"
-	"github.com/xeipuuv/gojsonschema"
 	"golang.org/x/sync/semaphore"
 
 	_ "embed"
@@ -29,69 +28,64 @@ const ExampleSourceConfig = `
 # skip_tables: []
 `
 
-type SourceConfigureFunc func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
+type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
 
 // SourcePlugin is the base structure required to pass to sdk.serve
 // We take a similar/declerative approach to API here similar to Cobra
 type SourcePlugin struct {
 	// Name of plugin i.e aws,gcp, azure etc'
-	Name string `json:"name"`
+	name string
 	// Version of the plugin
-	Version string `json:"version"`
+	version string
 	// Classify error and return it's severity and type
-	ClassifyError schema.ClassifyErrorFunc `json:"-"`
+	ignoreError schema.IgnoreErrorFunc
 	// Called upon configure call to validate and init configuration
-	configure SourceConfigureFunc `json:"-"`
+	newExecutionClient SourceNewExecutionClientFunc
 	// Tables is all tables supported by this source plugin
-	Tables schema.Tables `json:"tables"`
+	tables schema.Tables
 	// JsonSchema for specific source plugin spec
-	JsonSchema string `json:"json_schema"`
+	jsonSchema string
 	// ExampleConfig is the example configuration for this plugin
-	ExampleConfig string `json:"example_config"`
+	exampleConfig string
 	// Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead.
-	Logger zerolog.Logger `json:"-"`
-
-	// Internal fields set by configure
-	clientMeta schema.ClientMeta `json:"-"`
-	spec       *specs.SourceSpec `json:"-"`
-	m          sync.Mutex        `json:"-"`
+	logger zerolog.Logger
 }
 
 type SourceOption func(*SourcePlugin)
 
 func WithSourceExampleConfig(exampleConfig string) SourceOption {
 	return func(p *SourcePlugin) {
-		p.ExampleConfig = exampleConfig
+		p.exampleConfig = exampleConfig
 	}
 }
 
 func WithSourceJsonSchema(jsonSchema string) SourceOption {
 	return func(p *SourcePlugin) {
-		p.JsonSchema = jsonSchema
+		p.jsonSchema = jsonSchema
 	}
 }
 
 func WithSourceLogger(logger zerolog.Logger) SourceOption {
 	return func(p *SourcePlugin) {
-		p.Logger = logger
+		p.logger = logger
 	}
 }
 
-func WithClassifyError(classifyError schema.ClassifyErrorFunc) SourceOption {
+func WithClassifyError(ignoreError schema.IgnoreErrorFunc) SourceOption {
 	return func(p *SourcePlugin) {
-		p.ClassifyError = classifyError
+		p.ignoreError = ignoreError
 	}
 }
 
-func NewSourcePlugin(name string, version string, tables []*schema.Table, configure SourceConfigureFunc, opts ...SourceOption) *SourcePlugin {
+func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, opts ...SourceOption) *SourcePlugin {
 	p := SourcePlugin{
-		Name:      name,
-		Version:   version,
-		Tables:    tables,
-		configure: configure,
+		name:               name,
+		version:            version,
+		tables:             tables,
+		newExecutionClient: newExecutionClient,
 	}
-	if configure == nil {
-		panic("configure function not defined for source plugin:" + name)
+	if newExecutionClient == nil {
+		panic("newExecutionClient function not defined for source plugin:" + name)
 	}
 	for _, opt := range opts {
 		opt(&p)
@@ -99,64 +93,69 @@ func NewSourcePlugin(name string, version string, tables []*schema.Table, config
 	return &p
 }
 
-func (p *SourcePlugin) Configure(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
-	// we permit only one configure per source plugin for security reasons.
-	// in the grpc layer this will behave similarly and for every new configuration/creds the cli will have to create a new process.
-	p.m.Lock()
-	defer p.m.Unlock()
-	if p.spec != nil {
-		return nil, fmt.Errorf("source plugin %s already configured", p.Name)
-	}
-	res, err := specs.ValidateSpec(sourceSchema, spec)
-	if err != nil {
-		return nil, err
-	}
-	if !res.Valid() {
-		return res, nil
-	}
+func (p *SourcePlugin) Tables() schema.Tables {
+	return p.tables
+}
 
-	// if resources ["*"] is requested we will fetch all resources
-	p.spec.Tables, err = p.interpolateAllResources(p.spec.Tables)
-	if err != nil {
-		return res, fmt.Errorf("failed to interpolate resources: %w", err)
-	}
+func (p *SourcePlugin) ExampleConfig() string {
+	return p.exampleConfig
+}
 
-	p.clientMeta, err = p.configure(ctx, p, spec)
-	if err != nil {
-		return res, fmt.Errorf("failed to configure source plugin: %w", err)
-	}
-	p.spec = &spec
-	return res, nil
+func (p *SourcePlugin) GetJsonSchema() string {
+	return p.jsonSchema
+}
+
+func (p *SourcePlugin) Name() string {
+	return p.name
 }
 
+func (p *SourcePlugin) Version() string {
+	return p.version
+}
+
+func (p *SourcePlugin) SetLogger(log zerolog.Logger) {
+	p.logger = log
+}
+
+const minGoRoutines = 5
+
 // Fetch fetches data according to source configuration and
-func (p *SourcePlugin) Sync(ctx context.Context, res chan<- *schema.Resource) error {
-	if p.spec == nil {
-		return fmt.Errorf("source plugin not configured")
+func (p *SourcePlugin) Sync(ctx context.Context, spec specs.SourceSpec, res chan<- *schema.Resource) error {
+
+	c, err := p.newExecutionClient(ctx, p, spec)
+	if err != nil {
+		return fmt.Errorf("failed to create execution client for source plugin %s: %w", p.name, err)
 	}
 
 	// limiter used to limit the amount of resources fetched concurrently
-	maxGoroutines := p.spec.MaxGoRoutines
-	p.Logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch")
+	maxGoroutines := spec.MaxGoRoutines
+	if maxGoroutines == 0 {
+		maxGoroutines = minGoRoutines
+	}
+	p.logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch")
+
 	goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
 
 	w := sync.WaitGroup{}
 	totalResources := 0
 	startTime := time.Now()
-	tableNames := p.Tables.TableNames()
-	for _, table := range p.Tables {
+	tableNames, err := p.interpolateAllResources(p.tables.TableNames())
+	if err != nil {
+		return err
+	}
+	for _, table := range p.tables {
 		table := table
-		if funk.ContainsString(p.spec.SkipTables, table.Name) || !funk.ContainsString(tableNames, table.Name) {
-			p.Logger.Info().Str("table", table.Name).Msg("skipping table")
+		if funk.ContainsString(spec.SkipTables, table.Name) || !funk.ContainsString(tableNames, table.Name) {
+			p.logger.Info().Str("table", table.Name).Msg("skipping table")
 			continue
 		}
-		clients := []schema.ClientMeta{p.clientMeta}
+		clients := []schema.ClientMeta{c}
 		if table.Multiplex != nil {
-			clients = table.Multiplex(p.clientMeta)
+			clients = table.Multiplex(c)
 		}
 		// because table can't import sourceplugin we need to set classifyError if it is not set by table
-		if table.ClassifyError == nil {
-			table.ClassifyError = p.ClassifyError
+		if table.IgnoreError == nil {
+			table.IgnoreError = p.ignoreError
 		}
 		// we call this here because we dont know when the following goroutine will be called and we do want an order
 		// of table by table
@@ -168,7 +167,7 @@ func (p *SourcePlugin) Sync(ctx context.Context, res chan<- *schema.Resource) er
 			defer w.Done()
 			defer goroutinesSem.Release(int64(totalClients) - newN)
 			wg := sync.WaitGroup{}
-			p.Logger.Info().Str("table", table.Name).Msg("fetch start")
+			p.logger.Info().Str("table", table.Name).Msg("fetch start")
 			tableStartTime := time.Now()
 			totalTableResources := 0
 			for i, client := range clients {
@@ -178,7 +177,7 @@ func (p *SourcePlugin) Sync(ctx context.Context, res chan<- *schema.Resource) er
 				if newN > 0 && i >= (totalClients-int(newN)) {
 					if err := goroutinesSem.Acquire(ctx, 1); err != nil {
 						// this can happen if context was cancelled so we just break out of the loop
-						p.Logger.Error().Err(err).Msg("failed to acquire semaphore")
+						p.logger.Error().Err(err).Msg("failed to acquire semaphore")
 						return
 					}
 				}
@@ -194,15 +193,18 @@ func (p *SourcePlugin) Sync(ctx context.Context, res chan<- *schema.Resource) er
 			}
 			wg.Wait()
 			totalResources += totalTableResources
-			p.Logger.Info().Str("table", table.Name).Int("total_resources", totalTableResources).TimeDiff("duration", time.Now(), tableStartTime).Msg("fetch table finished")
+			p.logger.Info().Str("table", table.Name).Int("total_resources", totalTableResources).TimeDiff("duration", time.Now(), tableStartTime).Msg("fetch table finished")
 		}()
 	}
 	w.Wait()
-	p.Logger.Info().Int("total_resources", totalResources).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished")
+	p.logger.Info().Int("total_resources", totalResources).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished")
 	return nil
 }
 
 func (p *SourcePlugin) interpolateAllResources(tables []string) ([]string, error) {
+	if tables == nil {
+		return make([]string, 0), nil
+	}
 	if !funk.ContainsString(tables, "*") {
 		return tables, nil
 	}
@@ -211,8 +213,8 @@ func (p *SourcePlugin) interpolateAllResources(tables []string) ([]string, error
 		return nil, fmt.Errorf("invalid \"*\" resource, with explicit resources")
 	}
 
-	allResources := make([]string, 0, len(p.Tables))
-	for _, k := range p.Tables {
+	allResources := make([]string, 0, len(p.tables))
+	for _, k := range p.tables {
 		allResources = append(allResources, k.Name)
 	}
 	return allResources, nil
diff --git a/plugins/source_test.go b/plugins/source_test.go
index c2c247ac34..7d7a3fd6f8 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -7,6 +7,7 @@ import (
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
 )
 
 type Account struct {
@@ -23,91 +24,71 @@ func (TestConfig) Example() string {
 	return ""
 }
 
-type testSourcePluginClient struct {
-	logger zerolog.Logger
-}
-
-func (t testSourcePluginClient) Logger() *zerolog.Logger {
-	return &t.logger
-}
-
-type testPluginClient struct {
-	logger zerolog.Logger
-}
-
-func (c *testPluginClient) Logger() *zerolog.Logger {
-	return &c.logger
-}
-
-func testPluginConfigure(ctx context.Context, p *SourcePlugin, spec specs.SourceSpec) (schema.ClientMeta, error) {
-	return &testPluginClient{
-		logger: p.Logger,
-	}, nil
-}
-
 func testTable() *schema.Table {
 	return &schema.Table{
 		Name: "testTable",
 		Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
 			res <- map[string]interface{}{
-				"testColumn": 3,
+				"TestColumn": 3,
 			}
 			return nil
 		},
 		Columns: []schema.Column{
 			{
-				Name: "testColumn",
+				Name: "test_column",
 				Type: schema.TypeInt,
 			},
 		},
 	}
 }
 
-const testSourceCfg = `
-kind: source
-spec:
-  name: testSourcePlugin
-  version: 1.0.0
-  spec:
-    accounts:
-    - name: testAccount
-`
+var _ schema.ClientMeta = &testExecutionClient{}
+
+type testExecutionClient struct {
+	logger zerolog.Logger
+}
+
+func (c *testExecutionClient) Logger() *zerolog.Logger {
+	return &c.logger
+}
+
+func newTestExecutionClient(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error) {
+	return &testExecutionClient{}, nil
+}
 
 func TestSync(t *testing.T) {
-	// ctx := context.Background()
-	// testSourcePlugin := NewSourcePlugin(
-	// 	"test",
-	// 	"v1.0.0",
-	// 	[]*schema.Table{
-	// 		testTable(),
-	// 	},
-	// 	testPluginConfigure,
-	// 	WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
-	// )
-	// yaml.Unmarshal()
-	// testSourcePlugin.Configure(ctx)
-	// 	cfg := `
-	// tables:
-	//   - "*"
-	// configuration:
-	//   regions:
-	//   - "us-east-1"
-	//   accounts:
-	//   - name: "testAccount"
-	//     regions:
-	//     - "us-east-2"
-	// `
-	// 	resources := make(chan *schema.Resource)
-	// 	var fetchErr error
-	// 	var result *gojsonschema.Result
-	// 	go func() {
-	// 		defer close(resources)
-	// 		result, fetchErr = testSourcePlugin.Fetch(context.Background(), []byte(cfg), resources)
-	// 	}()
-	// 	for resource := range resources {
-	// 		t.Logf("%+v", resource)
-	// 	}
-	// 	if fetchErr != nil {
-	// 		t.Errorf("fetch error: %v", fetchErr)
-	// 	}
+	ctx := context.Background()
+	plugin := NewSourcePlugin(
+		"testSourcePlugin",
+		"1.0.0",
+		[]*schema.Table{testTable()},
+		newTestExecutionClient,
+		WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))))
+
+	resources := make(chan *schema.Resource)
+	g, ctx := errgroup.WithContext(ctx)
+	g.Go(func() error {
+		defer close(resources)
+		return plugin.Sync(ctx,
+			specs.SourceSpec{},
+			resources)
+	})
+
+	for resource := range resources {
+		if resource.Table.Name != "testTable" {
+			t.Fatalf("unexpected resource table name: %s", resource.Table.Name)
+		}
+		obj := resource.Get("test_column")
+		val, ok := obj.(int)
+		if !ok {
+			t.Fatalf("unexpected resource column value (expected int): %v", obj)
+		}
+
+		if val != 3 {
+			t.Fatalf("unexpected resource column value: %v", val)
+		}
+	}
+	if err := g.Wait(); err != nil {
+		t.Fatal(err)
+	}
 }
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index a4cfe96e4c..31828fd579 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -8,15 +8,11 @@ import (
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/georgysavva/scany/pgxscan"
-	"github.com/xeipuuv/gojsonschema"
-	"gopkg.in/yaml.v3"
 )
 
 type ResourceTestCase struct {
 	Plugin *SourcePlugin
-	Config string
-	// we want it to be parallel by default
-	NotParallel bool
+	Spec   specs.SourceSpec
 	// ParallelFetchingLimit limits parallel resources fetch at a time
 	ParallelFetchingLimit uint64
 	// SkipIgnoreInTest flag which detects if schema.Table or schema.Column should be ignored
@@ -49,33 +45,20 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
 	// resource.Plugin.Logger = l
 	resources := make(chan *schema.Resource)
 	var fetchErr error
-	var result *gojsonschema.Result
-	var spec specs.SourceSpec
-	if err := yaml.Unmarshal([]byte(tc.Config), &spec); err != nil {
-		t.Fatal("failed to unmarshal source spec:", err)
-	}
-	validationResult, err := tc.Plugin.Configure(context.Background(), spec)
-	if err != nil {
-		t.Fatal("failed to init plugin:", err)
-	}
-	if !validationResult.Valid() {
-		t.Fatal("failed to validate plugin config:", validationResult.Errors())
-	}
 
 	// tc.Plugin.Logger = zerolog.New(zerolog.NewTestWriter(t))
 	go func() {
 		defer close(resources)
-		fetchErr = tc.Plugin.Sync(context.Background(), resources)
+		fetchErr = tc.Plugin.Sync(context.Background(), tc.Spec, resources)
 	}()
+
 	for resource := range resources {
 		validateResource(t, resource)
 	}
+
 	if fetchErr != nil {
 		t.Fatal(fetchErr)
 	}
-	if result != nil && !result.Valid() {
-		t.Errorf("invalid schema: %v", result.Errors())
-	}
 }
 
 func validateResource(t *testing.T, resource *schema.Resource) {
@@ -86,132 +69,3 @@ func validateResource(t *testing.T, resource *schema.Resource) {
 		}
 	}
 }
-
-// func testResource(t *testing.T, resource ResourceTestCase, name string, table *schema.Table, conn execution.QueryExecer) error {
-// 	t.Helper()
-
-// 	if createErr := dropAndCreateTable(context.Background(), conn, table); createErr != nil {
-// 		assert.FailNow(t, fmt.Sprintf("failed to create table %s", table.Name), createErr)
-// 	}
-
-// 	if !resource.SkipIgnoreInTest && table.IgnoreInTests {
-// 		t.Logf("skipping fetch of resource: %s in tests", name)
-// 	} else {
-// 		if err := fetchResource(t, &resource, name); err != nil {
-// 			return err
-// 		}
-// 	}
-
-// 	if verifiers, ok := resource.Verifiers[name]; ok {
-// 		for _, verifier := range verifiers {
-// 			verifier(t, table, conn, resource.SkipIgnoreInTest)
-// 		}
-// 	} else {
-// 		// fallback to default verification
-// 		verifyNoEmptyColumns(t, table, conn, resource.SkipIgnoreInTest)
-// 	}
-
-// 	return nil
-// }
-
-// // fetchResource - fetches a resource from the cloud and puts them into database. database config can be specified via DATABASE_URL env variable
-// func fetchResource(t *testing.T, resource *ResourceTestCase, resourceName string) error {
-// 	t.Helper()
-
-// 	t.Logf("fetch resource %v", resourceName)
-
-// 	var resourceSender = &testResourceSender{
-// 		Errors: []string{},
-// 	}
-
-// 	if err := resource.Provider.FetchResources(context.Background(),
-// 		&cqproto.FetchResourcesRequest{
-// 			Resources:             []string{resourceName},
-// 			ParallelFetchingLimit: resource.ParallelFetchingLimit,
-// 		},
-// 		resourceSender,
-// 	); err != nil {
-// 		return err
-// 	}
-
-// 	if len(resourceSender.Errors) > 0 {
-// 		return fmt.Errorf("error/s occurred during test, %s", strings.Join(resourceSender.Errors, ", "))
-// 	}
-
-// 	return nil
-// }
-
-// func verifyNoEmptyColumns(t *testing.T, table *schema.Table, conn pgxscan.Querier, shouldSkipIgnoreInTest bool) {
-// 	t.Helper()
-// 	t.Run(table.Name, func(t *testing.T) {
-// 		t.Helper()
-
-// 		if !shouldSkipIgnoreInTest && table.IgnoreInTests {
-// 			t.Skipf("table %s marked as IgnoreInTest. Skipping...", table.Name)
-// 		}
-// 		s := sq.StatementBuilder.
-// 			PlaceholderFormat(sq.Dollar).
-// 			Select(fmt.Sprintf("json_agg(%s)", table.Name)).
-// 			From(table.Name)
-// 		query, args, err := s.ToSql()
-// 		if err != nil {
-// 			t.Fatal(err)
-// 		}
-// 		var data []map[string]interface{}
-// 		if err := pgxscan.Get(context.Background(), conn, &data, query, args...); err != nil {
-// 			t.Fatal(err)
-// 		}
-
-// 		if len(data) == 0 {
-// 			t.Errorf("expected to have at least 1 entry at table %s got zero", table.Name)
-// 			return
-// 		}
-
-// 		nilColumns := map[string]bool{}
-// 		// mark all columns as nil
-// 		for _, c := range table.Columns {
-// 			if shouldSkipIgnoreInTest || !c.IgnoreInTests {
-// 				nilColumns[c.Name] = true
-// 			}
-// 		}
-
-// 		for _, row := range data {
-// 			for c, v := range row {
-// 				if v != nil {
-// 					// as long as we had one row or result with this column not nil it means the resolver worked
-// 					nilColumns[c] = false
-// 				}
-// 			}
-// 		}
-
-// 		var nilColumnsArr []string
-// 		for c, v := range nilColumns {
-// 			if v {
-// 				nilColumnsArr = append(nilColumnsArr, c)
-// 			}
-// 		}
-
-// 		if len(nilColumnsArr) != 0 {
-// 			t.Errorf("found nil column in table %s. columns=%s", table.Name, strings.Join(nilColumnsArr, ","))
-// 		}
-// 		for _, childTable := range table.Relations {
-// 			verifyNoEmptyColumns(t, childTable, conn, shouldSkipIgnoreInTest)
-// 		}
-// 	})
-// }
-
-// func (f *testResourceSender) Send(r *cqproto.FetchResourcesResponse) error {
-// 	if r.Error != "" {
-// 		fmt.Printf(r.Error)
-// 		f.Errors = append(f.Errors, r.Error)
-// 	}
-
-// 	return nil
-// }
-
-// func getEnv(key, fallback string) string {
-// 	if value, ok := os.LookupEnv(key); ok {
-// 		return value
-// 	}
-// 	return fallback
-// }
diff --git a/plugins/verifiers.go b/plugins/verifiers.go
deleted file mode 100644
index ff118b0040..0000000000
--- a/plugins/verifiers.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package plugins
-
-import (
-	"context"
-	"fmt"
-	"testing"
-
-	"github.com/cloudquery/faker/v3/support/slice"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/georgysavva/scany/pgxscan"
-)
-
-type Row map[string]interface{}
-
-// VerifyRowPredicateInTable is a base verifier accepting single row verifier for specific table from schema
-func VerifyRowPredicateInTable(tableName string, rowVerifier func(*testing.T, Row)) Verifier {
-	var verifier Verifier
-	verifier = func(t *testing.T, table *schema.Table, conn pgxscan.Querier, shouldSkipIgnoreInTest bool) {
-		if tableName == table.Name {
-			rows := getRows(t, conn, table, shouldSkipIgnoreInTest)
-			for _, row := range rows {
-				rowVerifier(t, row)
-			}
-		}
-		for _, r := range table.Relations {
-			verifier(t, r, conn, shouldSkipIgnoreInTest)
-		}
-	}
-	return verifier
-}
-
-func getRows(t *testing.T, conn pgxscan.Querier, table *schema.Table, shouldSkipIgnoreInTest bool) []Row {
-	if shouldSkipIgnoreInTest && table.IgnoreInTests {
-		t.Skipf("table %s marked as IgnoreInTest. Skipping...", table.Name)
-	}
-
-	var rows []Row
-	err := pgxscan.Get(
-		context.Background(),
-		conn,
-		&rows,
-		fmt.Sprintf("select json_agg(%[1]s) from %[1]s", table.Name),
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, c := range table.Columns {
-		if shouldSkipIgnoreInTest && c.IgnoreInTests {
-			for _, row := range rows {
-				delete(row, c.Name)
-			}
-		}
-	}
-
-	return rows
-}
-
-// VerifyNoEmptyColumnsExcept verifies that for each row in table its columns are not empty except passed
-func VerifyNoEmptyColumnsExcept(tableName string, except ...string) Verifier {
-	return VerifyRowPredicateInTable(tableName, func(t *testing.T, row Row) {
-		for k, v := range row {
-			if !slice.Contains(except, k) && v == nil {
-				t.Fatal("VerifyNoEmptyColumnsExcept failed: illegal row found")
-			}
-		}
-	})
-}
-
-// VerifyAtMostOneOf verifies that for each row in table at most one column from oneof is not empty
-func VerifyAtMostOneOf(tableName string, oneof ...string) Verifier {
-	return VerifyRowPredicateInTable(tableName, func(t *testing.T, row Row) {
-		cnt := 0
-		for _, k := range oneof {
-			v, ok := row[k]
-			if !ok {
-				t.Fatalf("VerifyAtMostOneOf failed: column %s doesn't exist", k)
-			}
-			if v != nil {
-				cnt++
-			}
-		}
-
-		if cnt > 1 {
-			t.Fatal("VerifyAtMostOneOf failed: illegal row found")
-		}
-	})
-}
-
-// VerifyAtLeastOneRow verifies that main table from schema has at least one row
-func VerifyAtLeastOneRow() Verifier {
-	return func(t *testing.T, table *schema.Table, conn pgxscan.Querier, _ bool) {
-		rows, err := conn.Query(context.Background(), fmt.Sprintf("select * from %s;", table.Name))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !rows.Next() {
-			t.Fatal("VerifyAtLeastOneRow failed: table is empty")
-		}
-
-		rows.Close()
-	}
-}
diff --git a/schema/column.go b/schema/column.go
index 6d52183d6a..f87aa4304e 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -36,28 +36,28 @@ type ColumnResolver func(ctx context.Context, meta ClientMeta, resource *Resourc
 
 // ColumnCreationOptions allow modification of how column is defined when table is created
 type ColumnCreationOptions struct {
-	Unique  bool
-	NotNull bool
+	Unique  bool `json:"unique,omitempty"`
+	NotNull bool `json:"notnull,omitempty"`
 }
 
 // Column definition for Table
 type Column struct {
 	// Name of column
-	Name string `json:"name"`
+	Name string `json:"name,omitempty"`
 	// Value Type of column i.e String, UUID etc'
-	Type ValueType `json:"type"`
+	Type ValueType `json:"type,omitempty"`
 	// Description about column, this description is added as a comment in the database
-	Description string `json:"description"`
+	Description string `json:"description,omitempty"`
 	// Column Resolver allows to set you own data based on resolving this can be an API call or setting multiple embedded values etc'
 	Resolver ColumnResolver `json:"-"`
 	// Creation options allow modifying how column is defined when table is created
-	CreationOptions ColumnCreationOptions `json:"creation_options"`
+	CreationOptions ColumnCreationOptions `json:"creation_options,omitempty"`
 	// IgnoreInTests is used to skip verifying the column is non-nil in integration tests.
 	// By default, integration tests perform a fetch for all resources in cloudquery's test account, and
 	// verify all columns are non-nil.
 	// If IgnoreInTests is true, verification is skipped for this column.
 	// Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns).
-	IgnoreInTests bool `json:"ignore_in_tests"`
+	IgnoreInTests bool `json:"-"`
 	// internal is true if this column is managed by the SDK
 	internal bool `json:"-"`
 	// meta holds serializable information about the column's resolvers and functions
diff --git a/schema/column_test.go b/schema/column_test.go
index b71e58d7c6..66d2e9d3cc 100644
--- a/schema/column_test.go
+++ b/schema/column_test.go
@@ -1,9 +1,11 @@
 package schema
 
 import (
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net"
+	"reflect"
 	"testing"
 	"time"
 
@@ -176,3 +178,23 @@ func BenchmarkColumn_ValidateTypeMap(b *testing.B) {
 		_ = col.ValidateType(m)
 	}
 }
+
+func TestColumnJsonMarshal(t *testing.T) {
+	// we are testing column json marshalling to make sure
+	// this can be easily sent over the wire
+	expected := Column{
+		Name: "test",
+		Type: TypeJSON,
+	}
+	b, err := json.Marshal(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := Column{}
+	if err := json.Unmarshal(b, &got); err != nil {
+		t.Fatal(err)
+	}
+	if reflect.DeepEqual(expected, got) == false {
+		t.Fatalf("expected %v got %v", expected, got)
+	}
+}
diff --git a/schema/resolvers_test.go b/schema/resolvers_test.go.backup
similarity index 100%
rename from schema/resolvers_test.go
rename to schema/resolvers_test.go.backup
diff --git a/schema/table.go b/schema/table.go
index c8551e0188..84ed1ade91 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -20,13 +20,10 @@ import (
 //
 type TableResolver func(ctx context.Context, meta ClientMeta, parent *Resource, res chan<- interface{}) error
 
-// IgnoreErrorFunc checks if returned error from table resolver should be ignored.
-type IgnoreErrorFunc func(err error) bool
-
 type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error
 
 // Classify error and return it's severity and type
-type ClassifyErrorFunc func(err error) (bool, string)
+type IgnoreErrorFunc func(err error) (bool, string)
 
 type Tables []*Table
 
@@ -49,8 +46,8 @@ type Table struct {
 	Relations []*Table `json:"relations"`
 	// Resolver is the main entry point to fetching table data and
 	Resolver TableResolver `json:"-"`
-	// ClassifyError is a function that classifies error and returns it's severity and type
-	ClassifyError ClassifyErrorFunc `json:"-"`
+	// IgnoreError is a function that classifies error and returns it's severity and type
+	IgnoreError IgnoreErrorFunc `json:"-"`
 	// Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them
 	Multiplex func(meta ClientMeta) []ClientMeta `json:"-"`
 	// Post resource resolver is called after all columns have been resolved, and before resource is inserted to database.
@@ -131,8 +128,8 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, r
 		}()
 		meta.Logger().Debug().Str("table_name", t.Name).Msg("table resolver started")
 		if err := t.Resolver(ctx, meta, parent, res); err != nil {
-			if t.ClassifyError != nil {
-				if classify, errType := t.ClassifyError(err); classify {
+			if t.IgnoreError != nil {
+				if ignore, errType := t.IgnoreError(err); ignore {
 					meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Str("error_type", errType).Msg("table resolver finished with error")
 					return
 				}
diff --git a/serve/doc.go b/serve/doc.go
index cc5de916bf..4a3eda2cf6 100644
--- a/serve/doc.go
+++ b/serve/doc.go
@@ -22,7 +22,7 @@ func newCmdDoc(opts Options) *cobra.Command {
 				return fmt.Errorf("doc generation is only supported for source plugins")
 			}
 
-			return schema.GenerateMarkdownTree(opts.SourcePlugin.Tables, args[0])
+			return schema.GenerateMarkdownTree(opts.SourcePlugin.Tables(), args[0])
 		},
 	}
 }
diff --git a/serve/serve.go b/serve/serve.go
index 6c6c148b12..9611cbdf24 100644
--- a/serve/serve.go
+++ b/serve/serve.go
@@ -16,8 +16,15 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
 )
 
+// bufSize used for unit testing grpc server and client
+const testBufSize = 1024 * 1024
+
+// lis used for unit testing grpc server and client
+var testListener *bufconn.Listener
+
 type Options struct {
 	// Required: Source or destination plugin to serve.
 	SourcePlugin      *plugins.SourcePlugin
@@ -50,9 +57,15 @@ func newCmdServe(opts Options) *cobra.Command {
 				logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}).Level(zerologLevel)
 			}
 			// opts.Plugin.Logger = logger
-			listener, err := net.Listen(network, address)
-			if err != nil {
-				return fmt.Errorf("failed to listen: %w", err)
+			var listener net.Listener
+			if network == "test" {
+				listener = bufconn.Listen(testBufSize)
+				testListener = listener.(*bufconn.Listener)
+			} else {
+				listener, err = net.Listen(network, address)
+				if err != nil {
+					return fmt.Errorf("failed to listen %s:%s: %w", network, address, err)
+				}
 			}
 			// See logging pattern https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v2/providers/zerolog/examples_test.go
 			s := grpc.NewServer(
@@ -67,11 +80,11 @@ func newCmdServe(opts Options) *cobra.Command {
 			)
 
 			if opts.SourcePlugin != nil {
-				opts.SourcePlugin.Logger = logger
+				opts.SourcePlugin.SetLogger(logger)
 				pb.RegisterSourceServer(s, &servers.SourceServer{Plugin: opts.SourcePlugin})
 			}
 			if opts.DestinationPlugin != nil {
-				opts.SourcePlugin.Logger = logger
+				// opts.DestinationPlugin.Logger = logger
 				pb.RegisterDestinationServer(s, &servers.DestinationServer{Plugin: opts.DestinationPlugin})
 			}
 
diff --git a/serve/serve_test.go b/serve/serve_test.go
new file mode 100644
index 0000000000..aa611dd561
--- /dev/null
+++ b/serve/serve_test.go
@@ -0,0 +1,107 @@
+package serve
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/cloudquery/plugin-sdk/plugins"
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc"
+)
+
+func testTable() *schema.Table {
+	return &schema.Table{
+		Name: "testTable",
+		Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
+			res <- map[string]interface{}{
+				"TestColumn": 3,
+			}
+			return nil
+		},
+		Columns: []schema.Column{
+			{
+				Name: "test_column",
+				Type: schema.TypeInt,
+			},
+		},
+	}
+}
+
+var _ schema.ClientMeta = &testExecutionClient{}
+
+type testExecutionClient struct {
+	logger zerolog.Logger
+}
+
+func (c *testExecutionClient) Logger() *zerolog.Logger {
+	return &c.logger
+}
+
+func newTestExecutionClient(context.Context, *plugins.SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error) {
+	return &testExecutionClient{}, nil
+}
+
+// https://stackoverflow.com/questions/32840687/timeout-for-waitgroup-wait
+func waitTimeout(wg *errgroup.Group, timeout time.Duration) (bool, error) {
+	c := make(chan struct{})
+	var err error
+	go func() {
+		defer close(c)
+		err = wg.Wait()
+	}()
+	select {
+	case <-c:
+		return false, err // completed normally
+	case <-time.After(timeout):
+		return true, err // timed out
+	}
+}
+
+func bufDialer(context.Context, string) (net.Conn, error) {
+	return testListener.Dial()
+}
+
+func TestServe(t *testing.T) {
+	plugin := plugins.NewSourcePlugin(
+		"testSourcePlugin",
+		"1.0.0",
+		[]*schema.Table{testTable()},
+		newTestExecutionClient,
+		plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))))
+
+	cmd := newCmdRoot(Options{
+		SourcePlugin: plugin,
+	})
+	cmd.SetArgs([]string{"serve", "--network", "test"})
+
+	go func() {
+		cmd.Execute()
+	}()
+
+	// https://stackoverflow.com/questions/42102496/testing-a-grpc-service
+	ctx := context.Background()
+	_, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Failed to dial bufnet: %v", err)
+	}
+	// c := clients.NewSourceClient(conn)
+	// c.
+
+	// g := errgroup.Group{}
+	// g.Go(func() error {
+	// 	return cmd.Execute()
+	// })
+
+	// // there is no programmatic way to shutdown server so we just check if returned an
+	// if waitTimeout(&g, time.Second*3) {
+	// 	t.Fatal("timed out")
+	// }
+	// if err := g.Wait(); err != nil {
+	// 	t.Fatal(err)
+	// }
+}

From 2e9ed81902f46d3e62207ddba659ce1ca3a7db94 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 16 Aug 2022 10:32:10 +0300
Subject: [PATCH 07/25] remove dead code

---
 .gitignore          |  1 +
 helpers/integers.go | 19 -------------------
 2 files changed, 1 insertion(+), 19 deletions(-)
 delete mode 100644 helpers/integers.go

diff --git a/.gitignore b/.gitignore
index da59a40e6f..66f4ce53f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ config.hcl
 
 .vscode
 vendor
+cover.out
\ No newline at end of file
diff --git a/helpers/integers.go b/helpers/integers.go
deleted file mode 100644
index 22d2ab5707..0000000000
--- a/helpers/integers.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package helpers
-
-import "math"
-
-// Uint64ToInt64 if value is bigger than math.MaxInt64 return math.MaxInt64
-// otherwise returns original value casted to int64
-func Uint64ToInt64(i uint64) int64 {
-	if i > math.MaxInt64 {
-		return math.MaxInt64
-	}
-	return int64(i)
-}
-
-func Uint64ToInt(i uint64) int {
-	if i > math.MaxInt {
-		return math.MaxInt
-	}
-	return int(i)
-}

From 2f8c3198c76be994c09410b3a7c22361c5e4e569 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 16 Aug 2022 10:46:11 +0300
Subject: [PATCH 08/25] fix some linters

---
 helpers/integers.go    | 19 +++++++++++++++++++
 plugins/source.go      |  1 -
 plugins/source_test.go | 12 ++++++------
 schema/column.go       |  4 ++--
 schema/column_test.go  |  2 +-
 schema/resource.go     |  1 -
 schema/table.go        | 18 +++++++++---------
 serve/serve_test.go    | 15 ++++++++-------
 specs/destination.go   |  4 ++--
 specs/spec_reader.go   |  6 +++---
 specs/spec_test.go     |  4 ----
 11 files changed, 50 insertions(+), 36 deletions(-)
 create mode 100644 helpers/integers.go

diff --git a/helpers/integers.go b/helpers/integers.go
new file mode 100644
index 0000000000..22d2ab5707
--- /dev/null
+++ b/helpers/integers.go
@@ -0,0 +1,19 @@
+package helpers
+
+import "math"
+
+// Uint64ToInt64 if value is bigger than math.MaxInt64 return math.MaxInt64
+// otherwise returns original value casted to int64
+func Uint64ToInt64(i uint64) int64 {
+	if i > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(i)
+}
+
+func Uint64ToInt(i uint64) int {
+	if i > math.MaxInt {
+		return math.MaxInt
+	}
+	return int(i)
+}
diff --git a/plugins/source.go b/plugins/source.go
index 5cf2bba6ea..646fa794bd 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -121,7 +121,6 @@ const minGoRoutines = 5
 
 // Fetch fetches data according to source configuration and
 func (p *SourcePlugin) Sync(ctx context.Context, spec specs.SourceSpec, res chan<- *schema.Resource) error {
-
 	c, err := p.newExecutionClient(ctx, p, spec)
 	if err != nil {
 		return fmt.Errorf("failed to create execution client for source plugin %s: %w", p.name, err)
diff --git a/plugins/source_test.go b/plugins/source_test.go
index 7d7a3fd6f8..dcffbcd357 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -10,6 +10,12 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+var _ schema.ClientMeta = &testExecutionClient{}
+
+type testExecutionClient struct {
+	logger zerolog.Logger
+}
+
 type Account struct {
 	Name    string   `yaml:"name"`
 	Regions []string `yaml:"regions"`
@@ -42,12 +48,6 @@ func testTable() *schema.Table {
 	}
 }
 
-var _ schema.ClientMeta = &testExecutionClient{}
-
-type testExecutionClient struct {
-	logger zerolog.Logger
-}
-
 func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
diff --git a/schema/column.go b/schema/column.go
index f87aa4304e..2554310b8c 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -59,9 +59,9 @@ type Column struct {
 	// Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns).
 	IgnoreInTests bool `json:"-"`
 	// internal is true if this column is managed by the SDK
-	internal bool `json:"-"`
+	internal bool
 	// meta holds serializable information about the column's resolvers and functions
-	meta *ColumnMeta `json:"-"`
+	meta *ColumnMeta
 }
 
 const (
diff --git a/schema/column_test.go b/schema/column_test.go
index 66d2e9d3cc..b458c93e90 100644
--- a/schema/column_test.go
+++ b/schema/column_test.go
@@ -194,7 +194,7 @@ func TestColumnJsonMarshal(t *testing.T) {
 	if err := json.Unmarshal(b, &got); err != nil {
 		t.Fatal(err)
 	}
-	if reflect.DeepEqual(expected, got) == false {
+	if !reflect.DeepEqual(expected, got) {
 		t.Fatalf("expected %v got %v", expected, got)
 	}
 }
diff --git a/schema/resource.go b/schema/resource.go
index 6e5e40f292..7c9303d515 100644
--- a/schema/resource.go
+++ b/schema/resource.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/google/uuid"
-	// "github.com/segmentio/objconv/json"
 )
 
 type Resources []*Resource
diff --git a/schema/table.go b/schema/table.go
index 84ed1ade91..4da9660683 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -27,14 +27,6 @@ type IgnoreErrorFunc func(err error) (bool, string)
 
 type Tables []*Table
 
-func (tt Tables) TableNames() []string {
-	ret := []string{}
-	for _, t := range tt {
-		ret = append(ret, t.TableNames()...)
-	}
-	return ret
-}
-
 type Table struct {
 	// Name of table
 	Name string `json:"name"`
@@ -68,7 +60,7 @@ type Table struct {
 	// Serial is used to force a signature change, which forces new table creation and cascading removal of old table and relations
 	Serial string `json:"-"`
 
-	columnsMap map[string]int `json:"-"`
+	columnsMap map[string]int
 }
 
 // TableCreationOptions allow modifying how table is created such as defining primary keys, indices, foreign keys and constraints.
@@ -77,6 +69,14 @@ type TableCreationOptions struct {
 	PrimaryKeys []string
 }
 
+func (tt Tables) TableNames() []string {
+	ret := []string{}
+	for _, t := range tt {
+		ret = append(ret, t.TableNames()...)
+	}
+	return ret
+}
+
 func (t Table) Column(name string) *Column {
 	for _, c := range t.Columns {
 		if c.Name == name {
diff --git a/serve/serve_test.go b/serve/serve_test.go
index aa611dd561..c7182c4176 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -12,8 +12,15 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
+var _ schema.ClientMeta = &testExecutionClient{}
+
+type testExecutionClient struct {
+	logger zerolog.Logger
+}
+
 func testTable() *schema.Table {
 	return &schema.Table{
 		Name: "testTable",
@@ -32,12 +39,6 @@ func testTable() *schema.Table {
 	}
 }
 
-var _ schema.ClientMeta = &testExecutionClient{}
-
-type testExecutionClient struct {
-	logger zerolog.Logger
-}
-
 func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
@@ -85,7 +86,7 @@ func TestServe(t *testing.T) {
 
 	// https://stackoverflow.com/questions/42102496/testing-a-grpc-service
 	ctx := context.Background()
-	_, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
+	_, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatalf("Failed to dial bufnet: %v", err)
 	}
diff --git a/specs/destination.go b/specs/destination.go
index 4fc87fa316..c32274cc10 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -50,8 +50,8 @@ func (m WriteMode) MarshalYAML() (interface{}, error) {
 	return m.String(), nil
 }
 
-func (r *WriteMode) UnmarshalYAML(n *yaml.Node) (err error) {
-	*r, err = WriteModeFromString(n.Value)
+func (w *WriteMode) UnmarshalYAML(n *yaml.Node) (err error) {
+	*w, err = WriteModeFromString(n.Value)
 	return err
 }
 
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index fcb0cb177f..ef65b09aa3 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -48,9 +48,9 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 	return &reader, nil
 }
 
-func (r *SpecReader) GetSources() []SourceSpec {
-	sources := make([]SourceSpec, 0, len(r.sources))
-	for _, spec := range r.sources {
+func (s *SpecReader) GetSources() []SourceSpec {
+	sources := make([]SourceSpec, 0, len(s.sources))
+	for _, spec := range s.sources {
 		sources = append(sources, spec)
 	}
 	return sources
diff --git a/specs/spec_test.go b/specs/spec_test.go
index b6271c6829..a335e54298 100644
--- a/specs/spec_test.go
+++ b/specs/spec_test.go
@@ -1,7 +1,6 @@
 package specs
 
 import (
-	"embed"
 	"os"
 	"reflect"
 	"testing"
@@ -9,9 +8,6 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-//go:embed testdata/*.cq.yml
-var testSpecsFS embed.FS
-
 var testSpecs = map[string]Spec{
 	"testdata/pg.cq.yml": {
 		Kind: "destination",

From d3494e77e6b3bed1946452de6bf3e6a3de19fa4e Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Wed, 17 Aug 2022 02:00:21 +0300
Subject: [PATCH 09/25] feat: Using json everywhere apart from yaml for the
 user

We are taking a similar approach to k8s to use json internally
for marshalling/unmarshalling and yaml only for using facing stuff.

yaml parsers are much more complex and also have tons of vulnerabilities
so we want to use json everywhere where there is a machine reading those configurations.
---
 Makefile                              |  4 +-
 clients/destination.go                |  2 +-
 clients/source.go                     | 55 ++++---------------
 internal/pb/source.pb.go              | 34 +++++-------
 internal/pb/source_grpc.pb.go         | 38 -------------
 internal/servers/destinations.go      |  5 +-
 internal/servers/source.go            | 26 +++++----
 plugins/destination.go                |  2 +-
 plugins/source.go                     | 68 +++++++++++++++++------
 plugins/source_test.go                | 45 +++++++++++++---
 plugins/source_testing.go             |  2 +-
 {clients => plugins}/template.go      |  2 +-
 schema/resource.go                    | 51 +++++-------------
 serve/serve_test.go                   | 77 ++++++++++++++++++++-------
 specs/destination.go                  | 25 +++------
 specs/registry.go                     | 22 --------
 specs/source.go                       | 54 ++++++++++---------
 {plugins => specs}/source_schema.json |  0
 specs/spec.go                         | 61 +++++++++++++++++----
 specs/spec_reader.go                  | 20 +++----
 20 files changed, 304 insertions(+), 289 deletions(-)
 rename {clients => plugins}/template.go (94%)
 rename {plugins => specs}/source_schema.json (100%)

diff --git a/Makefile b/Makefile
index 7c150743e4..474bf5f2cd 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,6 @@ test:
 lint:
 	golangci-lint run
 
-.PHONY: generate-protobuf
-generate-protobuf:
+.PHONY: gen-proto
+gen-proto:
 	protoc --proto_path=. --go_out . --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative internal/pb/base.proto internal/pb/source.proto internal/pb/destination.proto
\ No newline at end of file
diff --git a/clients/destination.go b/clients/destination.go
index 28b33891c8..117d10e91c 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -41,7 +41,7 @@ func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error
 	return res.Config, nil
 }
 
-func (c *DestinationClient) Configure(ctx context.Context, spec specs.DestinationSpec) error {
+func (c *DestinationClient) Configure(ctx context.Context, spec specs.Destination) error {
 	if c.localClient != nil {
 		return c.localClient.Initialize(ctx, spec)
 	}
diff --git a/clients/source.go b/clients/source.go
index 0847c1e902..8332069e5a 100644
--- a/clients/source.go
+++ b/clients/source.go
@@ -3,20 +3,15 @@
 package clients
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
-	"text/template"
 
 	"github.com/cloudquery/plugin-sdk/internal/pb"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
-	"github.com/pkg/errors"
-	"github.com/xeipuuv/gojsonschema"
 	"google.golang.org/grpc"
-	"gopkg.in/yaml.v3"
 )
 
 type SourceClient struct {
@@ -27,14 +22,6 @@ type FetchResultMessage struct {
 	Resource []byte
 }
 
-const sourcePluginExampleConfigTemplate = `kind: source
-spec:
-  name: {{.Name}}
-  version: {{.Version}}
-  configuration:
-  {{.PluginExampleConfig | indent 4}}
-`
-
 func NewSourceClient(cc grpc.ClientConnInterface) *SourceClient {
 	return &SourceClient{
 		pbClient: pb.NewSourceClient(cc),
@@ -53,44 +40,22 @@ func (c *SourceClient) GetTables(ctx context.Context) ([]*schema.Table, error) {
 	return tables, nil
 }
 
-func (c *SourceClient) Configure(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) {
-	b, err := yaml.Marshal(spec)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal source spec")
-	}
-	res, err := c.pbClient.Configure(ctx, &pb.Configure_Request{Config: b})
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure source")
-	}
-	var validationResult gojsonschema.Result
-	if err := json.Unmarshal(res.JsonschemaResult, &validationResult); err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal validation result")
-	}
-	return &validationResult, nil
-}
-
-func (c *SourceClient) GetExampleConfig(ctx context.Context) (string, error) {
+func (c *SourceClient) ExampleConfig(ctx context.Context) (string, error) {
 	res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{})
 	if err != nil {
 		return "", fmt.Errorf("failed to get example config: %w", err)
 	}
-	t, err := template.New("source_plugin").Funcs(templateFuncMap()).Parse(sourcePluginExampleConfigTemplate)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse template: %w", err)
-	}
-	var tpl bytes.Buffer
-	if err := t.Execute(&tpl, map[string]interface{}{
-		"Name":                res.Name,
-		"Version":             res.Version,
-		"PluginExampleConfig": res.Config,
-	}); err != nil {
-		return "", fmt.Errorf("failed to generate example config: %w", err)
-	}
-	return tpl.String(), nil
+	return res.Config, nil
 }
 
-func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res chan<- *schema.Resource) error {
-	stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{})
+func (c *SourceClient) Fetch(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error {
+	b, err := json.Marshal(spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal source spec: %w", err)
+	}
+	stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{
+		Spec: b,
+	})
 	if err != nil {
 		return fmt.Errorf("failed to fetch resources: %w", err)
 	}
diff --git a/internal/pb/source.pb.go b/internal/pb/source.pb.go
index 0d1de5d75f..43ddc57355 100644
--- a/internal/pb/source.pb.go
+++ b/internal/pb/source.pb.go
@@ -311,7 +311,7 @@ var file_internal_pb_source_proto_rawDesc = []byte{
 	0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01,
 	0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74,
 	0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62,
-	0x6c, 0x65, 0x73, 0x32, 0x9b, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40,
+	0x6c, 0x65, 0x73, 0x32, 0xd9, 0x01, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40,
 	0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72,
 	0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65,
 	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
@@ -321,15 +321,11 @@ var file_internal_pb_source_proto_rawDesc = []byte{
 	0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65,
 	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
 	0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69,
-	0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65,
-	0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74,
-	0x63, 0x68, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68,
-	0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30,
-	0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68,
+	0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46,
+	0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42,
+	0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -353,21 +349,17 @@ var file_internal_pb_source_proto_goTypes = []interface{}{
 	(*GetTables_Request)(nil),         // 4: proto.GetTables.Request
 	(*GetTables_Response)(nil),        // 5: proto.GetTables.Response
 	(*GetExampleConfig_Request)(nil),  // 6: proto.GetExampleConfig.Request
-	(*Configure_Request)(nil),         // 7: proto.Configure.Request
-	(*GetExampleConfig_Response)(nil), // 8: proto.GetExampleConfig.Response
-	(*Configure_Response)(nil),        // 9: proto.Configure.Response
+	(*GetExampleConfig_Response)(nil), // 7: proto.GetExampleConfig.Response
 }
 var file_internal_pb_source_proto_depIdxs = []int32{
 	4, // 0: proto.Source.GetTables:input_type -> proto.GetTables.Request
 	6, // 1: proto.Source.GetExampleConfig:input_type -> proto.GetExampleConfig.Request
-	7, // 2: proto.Source.Configure:input_type -> proto.Configure.Request
-	2, // 3: proto.Source.Fetch:input_type -> proto.Fetch.Request
-	5, // 4: proto.Source.GetTables:output_type -> proto.GetTables.Response
-	8, // 5: proto.Source.GetExampleConfig:output_type -> proto.GetExampleConfig.Response
-	9, // 6: proto.Source.Configure:output_type -> proto.Configure.Response
-	3, // 7: proto.Source.Fetch:output_type -> proto.Fetch.Response
-	4, // [4:8] is the sub-list for method output_type
-	0, // [0:4] is the sub-list for method input_type
+	2, // 2: proto.Source.Fetch:input_type -> proto.Fetch.Request
+	5, // 3: proto.Source.GetTables:output_type -> proto.GetTables.Response
+	7, // 4: proto.Source.GetExampleConfig:output_type -> proto.GetExampleConfig.Response
+	3, // 5: proto.Source.Fetch:output_type -> proto.Fetch.Response
+	3, // [3:6] is the sub-list for method output_type
+	0, // [0:3] is the sub-list for method input_type
 	0, // [0:0] is the sub-list for extension type_name
 	0, // [0:0] is the sub-list for extension extendee
 	0, // [0:0] is the sub-list for field type_name
diff --git a/internal/pb/source_grpc.pb.go b/internal/pb/source_grpc.pb.go
index 22aa170c0b..ddcf670d61 100644
--- a/internal/pb/source_grpc.pb.go
+++ b/internal/pb/source_grpc.pb.go
@@ -26,8 +26,6 @@ type SourceClient interface {
 	GetTables(ctx context.Context, in *GetTables_Request, opts ...grpc.CallOption) (*GetTables_Response, error)
 	// Get an example configuration for the source plugin
 	GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error)
-	// Configure the source plugin with the given spec
-	Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error)
 	// Fetch resources
 	Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error)
 }
@@ -58,15 +56,6 @@ func (c *sourceClient) GetExampleConfig(ctx context.Context, in *GetExampleConfi
 	return out, nil
 }
 
-func (c *sourceClient) Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) {
-	out := new(Configure_Response)
-	err := c.cc.Invoke(ctx, "/proto.Source/Configure", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 func (c *sourceClient) Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error) {
 	stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/proto.Source/Fetch", opts...)
 	if err != nil {
@@ -107,8 +96,6 @@ type SourceServer interface {
 	GetTables(context.Context, *GetTables_Request) (*GetTables_Response, error)
 	// Get an example configuration for the source plugin
 	GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error)
-	// Configure the source plugin with the given spec
-	Configure(context.Context, *Configure_Request) (*Configure_Response, error)
 	// Fetch resources
 	Fetch(*Fetch_Request, Source_FetchServer) error
 	mustEmbedUnimplementedSourceServer()
@@ -124,9 +111,6 @@ func (UnimplementedSourceServer) GetTables(context.Context, *GetTables_Request)
 func (UnimplementedSourceServer) GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetExampleConfig not implemented")
 }
-func (UnimplementedSourceServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
-}
 func (UnimplementedSourceServer) Fetch(*Fetch_Request, Source_FetchServer) error {
 	return status.Errorf(codes.Unimplemented, "method Fetch not implemented")
 }
@@ -179,24 +163,6 @@ func _Source_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Source_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Configure_Request)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(SourceServer).Configure(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/proto.Source/Configure",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SourceServer).Configure(ctx, req.(*Configure_Request))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
 func _Source_Fetch_Handler(srv interface{}, stream grpc.ServerStream) error {
 	m := new(Fetch_Request)
 	if err := stream.RecvMsg(m); err != nil {
@@ -233,10 +199,6 @@ var Source_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "GetExampleConfig",
 			Handler:    _Source_GetExampleConfig_Handler,
 		},
-		{
-			MethodName: "Configure",
-			Handler:    _Source_Configure_Handler,
-		},
 	},
 	Streams: []grpc.StreamDesc{
 		{
diff --git a/internal/servers/destinations.go b/internal/servers/destinations.go
index 6a0b9f63b9..e62c490415 100644
--- a/internal/servers/destinations.go
+++ b/internal/servers/destinations.go
@@ -12,7 +12,6 @@ import (
 	"github.com/cloudquery/plugin-sdk/specs"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"gopkg.in/yaml.v3"
 )
 
 type DestinationServer struct {
@@ -21,8 +20,8 @@ type DestinationServer struct {
 }
 
 func (s *DestinationServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) {
-	var spec specs.DestinationSpec
-	if err := yaml.Unmarshal(req.Config, &spec); err != nil {
+	var spec specs.Destination
+	if err := json.Unmarshal(req.Config, &spec); err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err)
 	}
 	return &pb.Configure_Response{}, s.Plugin.Initialize(ctx, spec)
diff --git a/internal/servers/source.go b/internal/servers/source.go
index e9ac345671..f1f167338f 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -1,6 +1,7 @@
 package servers
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -10,7 +11,6 @@ import (
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/pkg/errors"
-	"gopkg.in/yaml.v3"
 )
 
 type SourceServer struct {
@@ -28,20 +28,27 @@ func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.Ge
 	}, nil
 }
 
-func (s *SourceServer) GetExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
+func (s *SourceServer) ExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
+	exampleConfig, err := s.Plugin.ExampleConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get example config: %w", err)
+	}
 	return &pb.GetExampleConfig_Response{
 		Name:    s.Plugin.Name(),
 		Version: s.Plugin.Version(),
-		Config:  s.Plugin.ExampleConfig()}, nil
+		Config:  exampleConfig}, nil
 }
 
-func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer) error {
+func (s *SourceServer) Sync(req *pb.Fetch_Request, stream pb.Source_FetchServer) error {
 	resources := make(chan *schema.Resource)
 	var fetchErr error
 
-	var spec specs.SourceSpec
-	if err := yaml.Unmarshal(req.Spec, &spec); err != nil {
-		return fmt.Errorf("failed to unmarshal source spec: %w", err)
+	var spec specs.Source
+	dec := json.NewDecoder(bytes.NewReader(req.Spec))
+	dec.UseNumber()
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&spec); err != nil {
+		return fmt.Errorf("failed to decode source spec: %w", err)
 	}
 
 	go func() {
@@ -52,10 +59,7 @@ func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer
 	}()
 
 	for resource := range resources {
-		b, err := json.Marshal(schema.WireResource{
-			Data:      resource.Data,
-			TableName: resource.Table.Name,
-		})
+		b, err := json.Marshal(resource)
 		if err != nil {
 			return errors.Wrap(err, "failed to marshal resource")
 		}
diff --git a/plugins/destination.go b/plugins/destination.go
index b7a17a32f9..cf83fd7b4f 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -33,7 +33,7 @@ type DestinationPlugin interface {
 	Version() string
 	ExampleConfig() string
 	JsonSchema() string
-	Initialize(ctx context.Context, spec specs.DestinationSpec) error
+	Initialize(ctx context.Context, spec specs.Destination) error
 	Migrate(ctx context.Context, tables schema.Tables) error
 	Write(ctx context.Context, resources *schema.Resource) error
 	SetLogger(logger zerolog.Logger)
diff --git a/plugins/source.go b/plugins/source.go
index 646fa794bd..40325a76b0 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -1,6 +1,7 @@
 package plugins
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"sync"
@@ -12,13 +13,11 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/thoas/go-funk"
 	"golang.org/x/sync/semaphore"
+	"gopkg.in/yaml.v3"
 
 	_ "embed"
 )
 
-//go:embed source_schema.json
-var sourceSchema string
-
 const ExampleSourceConfig = `
 # max_goroutines to use when fetching. 0 means default and calculated by CloudQuery
 # max_goroutines: 0
@@ -28,7 +27,28 @@ const ExampleSourceConfig = `
 # skip_tables: []
 `
 
-type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error)
+const sourcePluginExampleConfigTemplate = `kind: source
+spec:
+  name: {{.Name}}
+  version: {{.Version}}
+  # path: Path to the plugin. by default it is the same as the name of the plugin.
+  # registry can be local, github, grpc (default github)
+  # registry: github
+  # max_goroutines used for sync, by default calculated automatically depending on
+  # memory and cpu avaialble
+  # max_goroutines: 0
+  tables: ["*"]
+  # skip_tables is useful if you want to fetch all tables apart from specific ones
+  # skip_tables: []
+  # name of destinations to sync the data
+  destinations: []
+  configuration:
+  {{.PluginExampleConfig | indent 4}}
+`
+
+type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error)
+
+type SourceNewSpecFunc func() interface{}
 
 // SourcePlugin is the base structure required to pass to sdk.serve
 // We take a similar/declerative approach to API here similar to Cobra
@@ -43,22 +63,16 @@ type SourcePlugin struct {
 	newExecutionClient SourceNewExecutionClientFunc
 	// Tables is all tables supported by this source plugin
 	tables schema.Tables
+	// newSpec return a new struct to be pupolated by the passed configuration
+	newSpec SourceNewSpecFunc
 	// JsonSchema for specific source plugin spec
 	jsonSchema string
-	// ExampleConfig is the example configuration for this plugin
-	exampleConfig string
 	// Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead.
 	logger zerolog.Logger
 }
 
 type SourceOption func(*SourcePlugin)
 
-func WithSourceExampleConfig(exampleConfig string) SourceOption {
-	return func(p *SourcePlugin) {
-		p.exampleConfig = exampleConfig
-	}
-}
-
 func WithSourceJsonSchema(jsonSchema string) SourceOption {
 	return func(p *SourcePlugin) {
 		p.jsonSchema = jsonSchema
@@ -77,16 +91,20 @@ func WithClassifyError(ignoreError schema.IgnoreErrorFunc) SourceOption {
 	}
 }
 
-func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, opts ...SourceOption) *SourcePlugin {
+func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, newSpec SourceNewSpecFunc, opts ...SourceOption) *SourcePlugin {
 	p := SourcePlugin{
 		name:               name,
 		version:            version,
 		tables:             tables,
 		newExecutionClient: newExecutionClient,
+		newSpec:            newSpec,
 	}
 	if newExecutionClient == nil {
 		panic("newExecutionClient function not defined for source plugin:" + name)
 	}
+	if newSpec == nil {
+		panic("newConfig function not defined for source plugin:" + name)
+	}
 	for _, opt := range opts {
 		opt(&p)
 	}
@@ -97,8 +115,24 @@ func (p *SourcePlugin) Tables() schema.Tables {
 	return p.tables
 }
 
-func (p *SourcePlugin) ExampleConfig() string {
-	return p.exampleConfig
+func (p *SourcePlugin) ExampleConfig() (string, error) {
+	spec := specs.Spec{
+		Kind: "source",
+		Spec: specs.Source{
+			Name:         p.name,
+			Version:      p.version,
+			Tables:       []string{"*"},
+			Destinations: []string{},
+			Spec:         p.newSpec(),
+		},
+	}
+	bytes := bytes.NewBuffer([]byte(""))
+	enc := yaml.NewEncoder(bytes)
+	enc.SetIndent(2)
+	if err := enc.Encode(spec); err != nil {
+		return "", err
+	}
+	return bytes.String(), nil
 }
 
 func (p *SourcePlugin) GetJsonSchema() string {
@@ -119,8 +153,8 @@ func (p *SourcePlugin) SetLogger(log zerolog.Logger) {
 
 const minGoRoutines = 5
 
-// Fetch fetches data according to source configuration and
-func (p *SourcePlugin) Sync(ctx context.Context, spec specs.SourceSpec, res chan<- *schema.Resource) error {
+// Sync data from source to the given channel
+func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error {
 	c, err := p.newExecutionClient(ctx, p, spec)
 	if err != nil {
 		return fmt.Errorf("failed to create execution client for source plugin %s: %w", p.name, err)
diff --git a/plugins/source_test.go b/plugins/source_test.go
index dcffbcd357..746df3ecb4 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -2,28 +2,35 @@ package plugins
 
 import (
 	"context"
+	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
+	"gopkg.in/yaml.v3"
 )
 
 var _ schema.ClientMeta = &testExecutionClient{}
 
+const testSourcePluginExampleConfig = `# specify all accounts you want to sync
+accounts: []
+`
+
 type testExecutionClient struct {
 	logger zerolog.Logger
 }
 
 type Account struct {
-	Name    string   `yaml:"name"`
-	Regions []string `yaml:"regions"`
+	Name    string   `json:"name,omitempty"`
+	Regions []string `json:"regions"`
 }
 
 type TestConfig struct {
-	Accounts []Account `yaml:"accounts"`
-	Regions  []string  `yaml:"regions"`
+	Accounts []Account `json:"accounts"`
+	Regions  []string  `json:"regions"`
 }
 
 func (TestConfig) Example() string {
@@ -52,7 +59,7 @@ func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
 
-func newTestExecutionClient(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error) {
+func newTestExecutionClient(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error) {
 	return &testExecutionClient{}, nil
 }
 
@@ -63,15 +70,37 @@ func TestSync(t *testing.T) {
 		"1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
-		WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))))
+		WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
+		WithSourceExampleConfig(testSourcePluginExampleConfig),
+	)
+
+	// test round trip: get example config -> sync with example config -> success
+	exampleConfig, err := plugin.ExampleConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var spec specs.Source
+	if err := yaml.Unmarshal([]byte(exampleConfig), &spec); err != nil {
+		t.Fatal(err)
+	}
+
+	a := json.NewDecoder(strings.NewReader(exampleConfig))
+	json.Strin
+
+	d := yaml.NewDecoder(strings.NewReader(exampleConfig))
+	d.KnownFields(true)
+	if err := d.Decode(&spec); err != nil {
+		t.Fatal(err)
+	}
 
 	resources := make(chan *schema.Resource)
 	g, ctx := errgroup.WithContext(ctx)
 	g.Go(func() error {
 		defer close(resources)
-		return plugin.Sync(ctx,
-			specs.SourceSpec{},
+		_, err = plugin.Sync(ctx,
+			*spec.Spec.(*specs.Source),
 			resources)
+		return err
 	})
 
 	for resource := range resources {
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index 31828fd579..2186d4e8ec 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -12,7 +12,7 @@ import (
 
 type ResourceTestCase struct {
 	Plugin *SourcePlugin
-	Spec   specs.SourceSpec
+	Spec   specs.Source
 	// ParallelFetchingLimit limits parallel resources fetch at a time
 	ParallelFetchingLimit uint64
 	// SkipIgnoreInTest flag which detects if schema.Table or schema.Column should be ignored
diff --git a/clients/template.go b/plugins/template.go
similarity index 94%
rename from clients/template.go
rename to plugins/template.go
index 3a4be441f4..abc7829324 100644
--- a/clients/template.go
+++ b/plugins/template.go
@@ -1,4 +1,4 @@
-package clients
+package plugins
 
 import (
 	"strings"
diff --git a/schema/resource.go b/schema/resource.go
index 7c9303d515..caee2bdfee 100644
--- a/schema/resource.go
+++ b/schema/resource.go
@@ -1,41 +1,34 @@
 package schema
 
 import (
-	"fmt"
-
 	"github.com/google/uuid"
 )
 
 type Resources []*Resource
 
-// this is sent back to the cli "over the wire" and we want to keep this as small as possible and have specific marshal/unmarshal
-// for this struct
-type WireResource struct {
-	Data      []interface{} `json:"data"`
-	TableName string        `json:"table_name"`
-}
-
 // Resource represents a row in it's associated table, it carries a reference to the original item, and automatically
 // generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods
 type Resource struct {
 	// Original resource item that wa from prior resolve
-	Item interface{}
+	Item interface{} `json:"-"`
 	// Set if this is an embedded table
-	Parent *Resource
+	Parent *Resource `json:"-"`
 	// internal fields
-	Table *Table
+	Table *Table `json:"-"`
 	// This is sorted result data by column name
-	Data []interface{}
-	cqId uuid.UUID
+	Data      map[string]interface{} `json:"data"`
+	TableName string                 `json:"table_name"`
+	cqId      uuid.UUID
 }
 
 func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource {
 	return &Resource{
-		Item:   item,
-		Parent: parent,
-		Table:  t,
-		Data:   make([]interface{}, len(t.Columns)),
-		cqId:   uuid.New(),
+		Item:      item,
+		Parent:    parent,
+		Table:     t,
+		Data:      make(map[string]interface{}, len(t.Columns)),
+		cqId:      uuid.New(),
+		TableName: t.Name,
 	}
 }
 
@@ -67,20 +60,11 @@ func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource {
 // }
 
 func (r *Resource) Get(key string) interface{} {
-	i := r.Table.ColumnIndex(key)
-	if i == -1 {
-		return nil
-	}
-	return r.Data[i]
+	return r.Data[key]
 }
 
 func (r *Resource) Set(key string, value interface{}) error {
-	i := r.Table.ColumnIndex(key)
-	if i == -1 {
-		return fmt.Errorf("column %s does not exist", key)
-	}
-
-	r.Data[i] = value
+	r.Data[key] = value
 	return nil
 }
 
@@ -131,13 +115,6 @@ func (r *Resource) Columns() []string {
 // 	return nil
 // }
 
-func (r *Resource) TableName() string {
-	if r.Table == nil {
-		return ""
-	}
-	return r.Table.Name
-}
-
 // func (r Resource) GetMeta(key string) (interface{}, bool) {
 // 	if r.metadata == nil {
 // 		return nil, false
diff --git a/serve/serve_test.go b/serve/serve_test.go
index c7182c4176..b86763b5ca 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cloudquery/plugin-sdk/clients"
 	"github.com/cloudquery/plugin-sdk/plugins"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
@@ -17,13 +18,18 @@ import (
 
 var _ schema.ClientMeta = &testExecutionClient{}
 
+const testSourcePluginExampleConfig = `
+# specify all accounts you want to sync
+accounts: []
+`
+
 type testExecutionClient struct {
 	logger zerolog.Logger
 }
 
 func testTable() *schema.Table {
 	return &schema.Table{
-		Name: "testTable",
+		Name: "test_table",
 		Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
 			res <- map[string]interface{}{
 				"TestColumn": 3,
@@ -39,6 +45,10 @@ func testTable() *schema.Table {
 	}
 }
 
+type TestSourcePluginSpec struct {
+	Accounts []string `json:"accounts,omitempty" yaml:"accounts,omitempty"`
+}
+
 func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
@@ -73,7 +83,9 @@ func TestServe(t *testing.T) {
 		"1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
-		plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))))
+		plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
+		plugins.WithSourceExampleConfig(testSourcePluginExampleConfig),
+	)
 
 	cmd := newCmdRoot(Options{
 		SourcePlugin: plugin,
@@ -84,25 +96,54 @@ func TestServe(t *testing.T) {
 		cmd.Execute()
 	}()
 
+	// wait for the server to start
+	for {
+		if testListener != nil {
+			break
+		}
+		t.Log("waiting for grpc server to start")
+		time.Sleep(time.Millisecond * 200)
+	}
+
 	// https://stackoverflow.com/questions/42102496/testing-a-grpc-service
 	ctx := context.Background()
-	_, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
 	if err != nil {
 		t.Fatalf("Failed to dial bufnet: %v", err)
 	}
-	// c := clients.NewSourceClient(conn)
-	// c.
-
-	// g := errgroup.Group{}
-	// g.Go(func() error {
-	// 	return cmd.Execute()
-	// })
-
-	// // there is no programmatic way to shutdown server so we just check if returned an
-	// if waitTimeout(&g, time.Second*3) {
-	// 	t.Fatal("timed out")
-	// }
-	// if err := g.Wait(); err != nil {
-	// 	t.Fatal(err)
-	// }
+	c := clients.NewSourceClient(conn)
+	resources := make(chan *schema.Resource)
+	wg := errgroup.Group{}
+	wg.Go(func() error {
+		defer close(resources)
+		return c.Fetch(ctx,
+			specs.SourceSpec{
+				Name:     "testSourcePlugin",
+				Version:  "1.0.0",
+				Registry: specs.RegistryGithub,
+				Spec:     TestSourcePluginSpec{Accounts: []string{"cloudquery/plugin-sdk"}},
+			},
+			resources)
+	})
+	for resource := range resources {
+		if resource.TableName != "test_table" {
+			t.Fatalf("Expected resource with table name test: %s", resource.TableName)
+		}
+		if int(resource.Data["test_column"].(float64)) != 3 {
+			t.Fatalf("Expected resource {'test_column':3} got: %v", resource.Data)
+		}
+	}
+	if err := wg.Wait(); err != nil {
+		t.Fatalf("Failed to fetch resources: %v", err)
+	}
+
+	exampleConfig, err := c.GetExampleConfig(ctx)
+	if err != nil {
+		t.Fatalf("Failed to get example config: %v", err)
+	}
+
+	if exampleConfig != testSourcePluginExampleConfig {
+		t.Fatalf("Expected example config:\n%s got:\n%s", testSourcePluginExampleConfig, exampleConfig)
+	}
+
 }
diff --git a/specs/destination.go b/specs/destination.go
index c32274cc10..e56109ea0a 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-
-	"gopkg.in/yaml.v3"
 )
 
 type WriteMode int
@@ -15,13 +13,13 @@ const (
 	WriteModeOverwrite
 )
 
-type DestinationSpec struct {
-	Name      string    `yaml:"name,omitempty" json:"name,omitempty"`
-	Version   string    `yaml:"version,omitempty" json:"version,omitempty"`
-	Path      string    `yaml:"path,omitempty" json:"path,omitempty"`
-	Registry  Registry  `yaml:"registry,omitempty" json:"registry,omitempty"`
-	WriteMode WriteMode `yaml:"write_mode,omitempty" json:"write_mode,omitempty"`
-	Spec      yaml.Node `yaml:"spec,omitempty" json:"spec,omitempty"`
+type Destination struct {
+	Name      string      `json:"name,omitempty"`
+	Version   string      `json:"version,omitempty"`
+	Path      string      `json:"path,omitempty"`
+	Registry  Registry    `json:"registry,omitempty"`
+	WriteMode WriteMode   `json:"write_mode,omitempty"`
+	Spec      interface{} `json:"spec,omitempty"`
 }
 
 func (m WriteMode) String() string {
@@ -46,15 +44,6 @@ func (m *WriteMode) UnmarshalJSON(data []byte) (err error) {
 	return nil
 }
 
-func (m WriteMode) MarshalYAML() (interface{}, error) {
-	return m.String(), nil
-}
-
-func (w *WriteMode) UnmarshalYAML(n *yaml.Node) (err error) {
-	*w, err = WriteModeFromString(n.Value)
-	return err
-}
-
 func WriteModeFromString(s string) (WriteMode, error) {
 	switch s {
 	case "append-only":
diff --git a/specs/registry.go b/specs/registry.go
index ad1ce56360..b18bf3bc39 100644
--- a/specs/registry.go
+++ b/specs/registry.go
@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-
-	"gopkg.in/yaml.v3"
 )
 
 type Registry int
@@ -37,26 +35,6 @@ func (r *Registry) UnmarshalJSON(data []byte) (err error) {
 	return nil
 }
 
-func (r Registry) MarshalYAML() (interface{}, error) {
-	return r.String(), nil
-}
-
-func (r *Registry) UnmarshalYAML(n *yaml.Node) (err error) {
-	*r, err = RegistryFromString(n.Value)
-	return err
-}
-
-// func (r *Registry) UnmarshalYaml(data []byte) (err error) {
-// 	var registry string
-// 	if err := yaml.Unmarshal(data, &registry); err != nil {
-// 		return err
-// 	}
-// 	if *r, err = RegistryFromString(registry); err != nil {
-// 		return err
-// 	}
-// 	return nil
-// }
-
 func RegistryFromString(s string) (Registry, error) {
 	switch s {
 	case "github":
diff --git a/specs/source.go b/specs/source.go
index 6dc6252407..45c8d958f4 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -1,38 +1,28 @@
 package specs
 
 import (
+	"encoding/json"
 	"strings"
 
-	"gopkg.in/yaml.v3"
+	"github.com/xeipuuv/gojsonschema"
 )
 
-// SourceSpec is the shared configuration for all source plugins
-type SourceSpec struct {
-	Name    string `json:"name,omitempty" yaml:"name,omitempty"`
-	Version string `json:"version,omitempty" yaml:"version,omitempty"`
+// Source is the shared configuration for all source plugins
+type Source struct {
+	Name    string `json:"name,omitempty"`
+	Version string `json:"version,omitempty"`
 	// Path is the path in the registry
-	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+	Path string `json:"path,omitempty"`
 	// Registry can be github,local,grpc. Might support things like https in the future.
-	Registry      Registry  `json:"registry,omitempty" yaml:"registry,omitempty"`
-	MaxGoRoutines uint64    `json:"max_goroutines,omitempty" yaml:"max_goroutines,omitempty"`
-	Tables        []string  `json:"tables,omitempty" yaml:"tables,omitempty"`
-	SkipTables    []string  `json:"skip_tables,omitempty" yaml:"skip_tables,omitempty"`
-	Destinations  []string  `json:"destinations,omitempty" yaml:"destinations,omitempty"`
-	Spec          yaml.Node `json:"spec,omitempty" yaml:"spec,omitempty"`
+	Registry      Registry    `json:"registry,omitempty"`
+	MaxGoRoutines uint64      `json:"max_goroutines,omitempty"`
+	Tables        []string    `json:"tables,omitempty"`
+	SkipTables    []string    `json:"skip_tables,omitempty"`
+	Destinations  []string    `json:"destinations,omitempty"`
+	Spec          interface{} `json:"spec,omitempty"`
 }
 
-func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
-	type S SourceSpec
-	type T struct {
-		*S `yaml:",inline"`
-	}
-	// This is a neat trick to avoid recursion and use unmarshal as a one stop shop for default setting
-	obj := &T{S: (*S)(s)}
-	if err := n.Decode(&obj); err != nil {
-		return err
-	}
-
-	// set default
+func (s *Source) SetDefaults() {
 	if s.Registry.String() == "" {
 		s.Registry = RegistryGithub
 	}
@@ -45,5 +35,19 @@ func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error {
 	if s.Registry == RegistryGithub && !strings.Contains(s.Path, "/") {
 		s.Path = "cloudquery/" + s.Path
 	}
-	return nil
+}
+
+func (s *Source) UnmarshalSpec(out interface{}) error {
+	b, err := json.Marshal(s.Spec)
+	if err != nil {
+		return err
+	}
+	dec := json.NewDecoder(nil)
+	dec.UseNumber()
+	dec.DisallowUnknownFields()
+	return json.Unmarshal(b, out)
+}
+
+func (s *Source) Validate() (*gojsonschema.Result, error) {
+	return nil, nil
 }
diff --git a/plugins/source_schema.json b/specs/source_schema.json
similarity index 100%
rename from plugins/source_schema.json
rename to specs/source_schema.json
diff --git a/specs/spec.go b/specs/spec.go
index 748b4e90c2..c0a6dd31f5 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -1,36 +1,77 @@
 package specs
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+)
+
+type Kind int
 
-	"gopkg.in/yaml.v3"
+const (
+	KindSource Kind = iota
+	KindDestination
 )
 
+func (k Kind) String() string {
+	return [...]string{"source", "destination"}[k]
+}
+
+func (k Kind) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString(`"`)
+	buffer.WriteString(k.String())
+	buffer.WriteString(`"`)
+	return buffer.Bytes(), nil
+}
+
+func (k *Kind) UnmarshalJSON(data []byte) (err error) {
+	var kind string
+	if err := json.Unmarshal(data, &kind); err != nil {
+		return err
+	}
+	if *k, err = KindFromString(kind); err != nil {
+		return err
+	}
+	return nil
+}
+
+func KindFromString(s string) (Kind, error) {
+	switch s {
+	case "source":
+		return KindSource, nil
+	case "destination":
+		return KindDestination, nil
+	default:
+		return KindSource, fmt.Errorf("unknown registry %s", s)
+	}
+}
+
 type Spec struct {
-	Kind string      `yaml:"kind"`
-	Spec interface{} `yaml:"-"`
+	Kind string      `json:"kind"`
+	Spec interface{} `json:"-"`
 }
 
-func (s *Spec) UnmarshalYAML(n *yaml.Node) error {
+func (s *Spec) UnmarshalJSON(data []byte) error {
 	type S Spec
 	type T struct {
-		*S   `yaml:",inline"`
-		Spec yaml.Node `yaml:"spec"`
+		*S `json:",inline"`
+		// Spec yaml.Node `yaml:"spec"`
 	}
 
 	obj := &T{S: (*S)(s)}
-	if err := n.Decode(obj); err != nil {
+	if err := json.Unmarshal(data, obj); err != nil {
 		return err
 	}
+
 	switch s.Kind {
 	case "source":
-		s.Spec = new(SourceSpec)
+		s.Spec = new(Source)
 	case "destination":
-		s.Spec = new(DestinationSpec)
+		s.Spec = new(Destination)
 	default:
 		return fmt.Errorf("unknown kind %s", s.Kind)
 	}
-	return obj.Spec.Decode(s.Spec)
+	return json.Unmarshal(data, s.Spec)
 }
 
 func (s Spec) MarshalYAML() (interface{}, error) {
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index ef65b09aa3..ad3949a847 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -11,14 +11,14 @@ import (
 )
 
 type SpecReader struct {
-	sources      map[string]SourceSpec
-	destinations map[string]DestinationSpec
+	sources      map[string]Source
+	destinations map[string]Destination
 }
 
 func NewSpecReader(directory string) (*SpecReader, error) {
 	reader := SpecReader{
-		sources:      make(map[string]SourceSpec),
-		destinations: make(map[string]DestinationSpec),
+		sources:      make(map[string]Source),
+		destinations: make(map[string]Destination),
 	}
 	files, err := ioutil.ReadDir(directory)
 	if err != nil {
@@ -37,9 +37,9 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 			}
 			switch s.Kind {
 			case "source":
-				reader.sources[file.Name()] = *s.Spec.(*SourceSpec)
+				reader.sources[file.Name()] = *s.Spec.(*Source)
 			case "destination":
-				reader.destinations[file.Name()] = *s.Spec.(*DestinationSpec)
+				reader.destinations[file.Name()] = *s.Spec.(*Destination)
 			default:
 				return nil, fmt.Errorf("unknown kind %s", s.Kind)
 			}
@@ -48,15 +48,15 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 	return &reader, nil
 }
 
-func (s *SpecReader) GetSources() []SourceSpec {
-	sources := make([]SourceSpec, 0, len(s.sources))
+func (s *SpecReader) GetSources() []Source {
+	sources := make([]Source, 0, len(s.sources))
 	for _, spec := range s.sources {
 		sources = append(sources, spec)
 	}
 	return sources
 }
 
-func (s *SpecReader) GetSourceByName(name string) *SourceSpec {
+func (s *SpecReader) GetSourceByName(name string) *Source {
 	for _, spec := range s.sources {
 		if spec.Name == name {
 			return &spec
@@ -65,7 +65,7 @@ func (s *SpecReader) GetSourceByName(name string) *SourceSpec {
 	return nil
 }
 
-func (s *SpecReader) GetDestinatinoByName(name string) *DestinationSpec {
+func (s *SpecReader) GetDestinatinoByName(name string) *Destination {
 	for _, spec := range s.destinations {
 		if spec.Name == name {
 			return &spec

From 205b82e296b23cb4c4c12a70d4f046217bd2ad60 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Wed, 17 Aug 2022 11:53:24 +0300
Subject: [PATCH 10/25] more work around configuration

---
 go.mod                    |  2 ++
 go.sum                    |  2 ++
 plugins/source.go         |  2 +-
 plugins/source_test.go    | 26 ++++----------
 serve/serve_test.go       | 25 ++++++++------
 specs/destination.go      | 16 +++++++++
 specs/spec.go             | 72 ++++++++++++++++++++++++++++-----------
 specs/spec_reader.go      |  4 +--
 specs/spec_test.go        | 40 +++++++++-------------
 specs/testdata/aws.cq.yml |  4 ++-
 10 files changed, 118 insertions(+), 75 deletions(-)

diff --git a/go.mod b/go.mod
index 9b247c2997..27c732a6b1 100644
--- a/go.mod
+++ b/go.mod
@@ -30,6 +30,7 @@ require (
 require (
 	github.com/BurntSushi/toml v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -61,5 +62,6 @@ require (
 	golang.org/x/tools v0.1.11 // indirect
 	google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	honnef.co/go/tools v0.3.2 // indirect
 )
diff --git a/go.sum b/go.sum
index b3d31a1dab..325c7656db 100644
--- a/go.sum
+++ b/go.sum
@@ -42,6 +42,7 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/georgysavva/scany v1.0.0 h1:9ar4458sgkWehk8bRsEe128FQV3pVKxdN4ytmCK6BEY=
 github.com/georgysavva/scany v1.0.0/go.mod h1:q8QyrfXjmBk9iJD00igd4lbkAKEXAH/zIYoZ0z/Wan4=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
@@ -448,6 +449,7 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/plugins/source.go b/plugins/source.go
index 40325a76b0..892d91a076 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -117,7 +117,7 @@ func (p *SourcePlugin) Tables() schema.Tables {
 
 func (p *SourcePlugin) ExampleConfig() (string, error) {
 	spec := specs.Spec{
-		Kind: "source",
+		Kind: specs.KindSource,
 		Spec: specs.Source{
 			Name:         p.name,
 			Version:      p.version,
diff --git a/plugins/source_test.go b/plugins/source_test.go
index 746df3ecb4..88b599ed3f 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -2,15 +2,12 @@ package plugins
 
 import (
 	"context"
-	"encoding/json"
-	"strings"
 	"testing"
 
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
-	"gopkg.in/yaml.v3"
 )
 
 var _ schema.ClientMeta = &testExecutionClient{}
@@ -28,13 +25,13 @@ type Account struct {
 	Regions []string `json:"regions"`
 }
 
-type TestConfig struct {
+type testSourceSpec struct {
 	Accounts []Account `json:"accounts"`
 	Regions  []string  `json:"regions"`
 }
 
-func (TestConfig) Example() string {
-	return ""
+func newTestSourceSpec() interface{} {
+	return &testSourceSpec{}
 }
 
 func testTable() *schema.Table {
@@ -70,8 +67,8 @@ func TestSync(t *testing.T) {
 		"1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
+		newTestSourceSpec,
 		WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
-		WithSourceExampleConfig(testSourcePluginExampleConfig),
 	)
 
 	// test round trip: get example config -> sync with example config -> success
@@ -79,17 +76,8 @@ func TestSync(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	var spec specs.Source
-	if err := yaml.Unmarshal([]byte(exampleConfig), &spec); err != nil {
-		t.Fatal(err)
-	}
-
-	a := json.NewDecoder(strings.NewReader(exampleConfig))
-	json.Strin
-
-	d := yaml.NewDecoder(strings.NewReader(exampleConfig))
-	d.KnownFields(true)
-	if err := d.Decode(&spec); err != nil {
+	var spec specs.Spec
+	if err := specs.SpecUnmarshalYamlStrict([]byte(exampleConfig), &spec); err != nil {
 		t.Fatal(err)
 	}
 
@@ -97,7 +85,7 @@ func TestSync(t *testing.T) {
 	g, ctx := errgroup.WithContext(ctx)
 	g.Go(func() error {
 		defer close(resources)
-		_, err = plugin.Sync(ctx,
+		err = plugin.Sync(ctx,
 			*spec.Spec.(*specs.Source),
 			resources)
 		return err
diff --git a/serve/serve_test.go b/serve/serve_test.go
index b86763b5ca..a45b759814 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -18,10 +18,15 @@ import (
 
 var _ schema.ClientMeta = &testExecutionClient{}
 
-const testSourcePluginExampleConfig = `
-# specify all accounts you want to sync
-accounts: []
-`
+type testSourceSpec struct {
+	accounts []string `json:"accounts,omitempty"`
+}
+
+func newTestSourceSpec() interface{} {
+	return &testSourceSpec{
+		accounts: []string{"all"},
+	}
+}
 
 type testExecutionClient struct {
 	logger zerolog.Logger
@@ -53,7 +58,7 @@ func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
 
-func newTestExecutionClient(context.Context, *plugins.SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error) {
+func newTestExecutionClient(context.Context, *plugins.SourcePlugin, specs.Source) (schema.ClientMeta, error) {
 	return &testExecutionClient{}, nil
 }
 
@@ -83,8 +88,8 @@ func TestServe(t *testing.T) {
 		"1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
+		newTestSourceSpec,
 		plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
-		plugins.WithSourceExampleConfig(testSourcePluginExampleConfig),
 	)
 
 	cmd := newCmdRoot(Options{
@@ -117,7 +122,7 @@ func TestServe(t *testing.T) {
 	wg.Go(func() error {
 		defer close(resources)
 		return c.Fetch(ctx,
-			specs.SourceSpec{
+			specs.Source{
 				Name:     "testSourcePlugin",
 				Version:  "1.0.0",
 				Registry: specs.RegistryGithub,
@@ -137,13 +142,13 @@ func TestServe(t *testing.T) {
 		t.Fatalf("Failed to fetch resources: %v", err)
 	}
 
-	exampleConfig, err := c.GetExampleConfig(ctx)
+	exampleConfig, err := c.ExampleConfig(ctx)
 	if err != nil {
 		t.Fatalf("Failed to get example config: %v", err)
 	}
 
-	if exampleConfig != testSourcePluginExampleConfig {
-		t.Fatalf("Expected example config:\n%s got:\n%s", testSourcePluginExampleConfig, exampleConfig)
+	if exampleConfig != "" {
+		t.Fatalf("Expected example config:\n%s got:\n%s", "", exampleConfig)
 	}
 
 }
diff --git a/specs/destination.go b/specs/destination.go
index e56109ea0a..047089c9f5 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"strings"
 )
 
 type WriteMode int
@@ -22,6 +23,21 @@ type Destination struct {
 	Spec      interface{} `json:"spec,omitempty"`
 }
 
+func (d *Destination) SetDefaults() {
+	if d.Registry.String() == "" {
+		d.Registry = RegistryGithub
+	}
+	if d.Path == "" {
+		d.Path = d.Name
+	}
+	if d.Version == "" {
+		d.Version = "latest"
+	}
+	if d.Registry == RegistryGithub && !strings.Contains(d.Path, "/") {
+		d.Path = "cloudquery/" + d.Path
+	}
+}
+
 func (m WriteMode) String() string {
 	return [...]string{"append-only", "overwrite"}[m]
 }
diff --git a/specs/spec.go b/specs/spec.go
index c0a6dd31f5..30cc749900 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+
+	"github.com/ghodss/yaml"
 )
 
 type Kind int
@@ -47,41 +49,73 @@ func KindFromString(s string) (Kind, error) {
 }
 
 type Spec struct {
-	Kind string      `json:"kind"`
-	Spec interface{} `json:"-"`
+	Kind Kind        `json:"kind"`
+	Spec interface{} `json:"spec"`
 }
 
 func (s *Spec) UnmarshalJSON(data []byte) error {
-	type S Spec
-	type T struct {
-		*S `json:",inline"`
+	var t struct {
+		Kind Kind        `json:"kind"`
+		Spec interface{} `json:"spec"`
 		// Spec yaml.Node `yaml:"spec"`
 	}
 
-	obj := &T{S: (*S)(s)}
-	if err := json.Unmarshal(data, obj); err != nil {
+	if err := json.Unmarshal(data, &t); err != nil {
 		return err
 	}
-
+	s.Kind = t.Kind
 	switch s.Kind {
-	case "source":
+	case KindSource:
 		s.Spec = new(Source)
-	case "destination":
+	case KindDestination:
 		s.Spec = new(Destination)
 	default:
 		return fmt.Errorf("unknown kind %s", s.Kind)
 	}
-	return json.Unmarshal(data, s.Spec)
+	b, err := json.Marshal(t.Spec)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, s.Spec)
 }
 
-func (s Spec) MarshalYAML() (interface{}, error) {
-	type T struct {
-		Kind string      `yaml:"kind,omitempty"`
-		Spec interface{} `yaml:"spec,omitempty"`
+func UnmarshalJsonStrict(b []byte, out interface{}) error {
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.DisallowUnknownFields()
+	dec.UseNumber()
+	return dec.Decode(out)
+}
+
+func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error {
+	jb, err := yaml.YAMLToJSON(b)
+	if err != nil {
+		return fmt.Errorf("failed to convert yaml to json: %w", err)
+	}
+	dec := json.NewDecoder(bytes.NewReader(jb))
+	dec.DisallowUnknownFields()
+	dec.UseNumber()
+	if err := dec.Decode(spec); err != nil {
+		return fmt.Errorf("failed to decode json: %w", err)
 	}
-	tmp := T{
-		Kind: s.Kind,
-		Spec: s.Spec,
+	switch spec.Kind {
+	case KindSource:
+		spec.Spec.(*Source).SetDefaults()
+	case KindDestination:
+		spec.Spec.(*Destination).SetDefaults()
+	default:
+		return fmt.Errorf("unknown kind %s", spec.Kind)
 	}
-	return tmp, nil
+	return nil
 }
+
+// func (s Spec) MarshalYAML() (interface{}, error) {
+// 	type T struct {
+// 		Kind Kind        `json:"kind,omitempty"`
+// 		Spec interface{} `json:"spec,omitempty"`
+// 	}
+// 	tmp := T{
+// 		Kind: s.Kind,
+// 		Spec: s.Spec,
+// 	}
+// 	return tmp, nil
+// }
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index ad3949a847..772198a761 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -36,9 +36,9 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 				return nil, fmt.Errorf("failed to unmarshal file %s: %w", file.Name(), err)
 			}
 			switch s.Kind {
-			case "source":
+			case KindSource:
 				reader.sources[file.Name()] = *s.Spec.(*Source)
-			case "destination":
+			case KindDestination:
 				reader.destinations[file.Name()] = *s.Spec.(*Destination)
 			default:
 				return nil, fmt.Errorf("unknown kind %s", s.Kind)
diff --git a/specs/spec_test.go b/specs/spec_test.go
index a335e54298..789ba0e385 100644
--- a/specs/spec_test.go
+++ b/specs/spec_test.go
@@ -4,30 +4,29 @@ import (
 	"os"
 	"reflect"
 	"testing"
-
-	"gopkg.in/yaml.v3"
 )
 
 var testSpecs = map[string]Spec{
 	"testdata/pg.cq.yml": {
-		Kind: "destination",
-		Spec: &DestinationSpec{
+		Kind: KindDestination,
+		Spec: &Destination{
 			Name:      "postgresql",
+			Path:      "postgresql",
 			Version:   "v1.0.0",
 			Registry:  RegistryGrpc,
 			WriteMode: WriteModeOverwrite,
 		},
 	},
-	"testdata/aws.cq.yml": {
-		Kind: "source",
-		Spec: &SourceSpec{
-			Name:          "aws",
-			Path:          "aws",
-			Version:       "v1.0.0",
-			MaxGoRoutines: 10,
-			Registry:      RegistryLocal,
-		},
-	},
+	// "testdata/aws.cq.yml": {
+	// 	Kind: KindSource,
+	// 	Spec: &Source{
+	// 		Name:          "aws",
+	// 		Path:          "aws",
+	// 		Version:       "v1.0.0",
+	// 		MaxGoRoutines: 10,
+	// 		Registry:      RegistryLocal,
+	// 	},
+	// },
 }
 
 func TestSpecYamlMarshal(t *testing.T) {
@@ -37,19 +36,14 @@ func TestSpecYamlMarshal(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
+
 			var spec Spec
-			if err := yaml.Unmarshal(b, &spec); err != nil {
-				t.Fatal(err)
-			}
-			b, err = yaml.Marshal(spec)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if err := yaml.Unmarshal(b, &spec); err != nil {
+			if err := SpecUnmarshalYamlStrict(b, &spec); err != nil {
 				t.Fatal(err)
 			}
+
 			if !reflect.DeepEqual(spec, expectedSpec) {
-				t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, expectedSpec.Spec, spec.Spec)
+				t.Errorf("expected spec %s to be:\n%+v\nbut got:\n%+v", fileName, expectedSpec.Spec, spec.Spec)
 			}
 		})
 	}
diff --git a/specs/testdata/aws.cq.yml b/specs/testdata/aws.cq.yml
index 2ef555424e..39c9736c5b 100644
--- a/specs/testdata/aws.cq.yml
+++ b/specs/testdata/aws.cq.yml
@@ -3,4 +3,6 @@ spec:
   name: aws
   version: v1.0.0
   max_goroutines: 10
-  registry: local
\ No newline at end of file
+  registry: local
+
+  
\ No newline at end of file

From 03e5f77d43d995811bc12f3fe17bd7fe35b53a86 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Wed, 17 Aug 2022 13:47:15 +0300
Subject: [PATCH 11/25] tests working again

---
 clients/source.go             |   6 +-
 go.mod                        |   1 +
 internal/pb/source.pb.go      | 120 +++++++++++++++++-----------------
 internal/pb/source.proto      |   4 +-
 internal/pb/source_grpc.pb.go |  42 ++++++------
 internal/servers/source.go    |   6 +-
 plugins/source.go             |  53 ++++++---------
 plugins/source_test.go        |  12 ++--
 serve/serve_test.go           |  31 +++++++--
 9 files changed, 138 insertions(+), 137 deletions(-)

diff --git a/clients/source.go b/clients/source.go
index 8332069e5a..99f84a09fc 100644
--- a/clients/source.go
+++ b/clients/source.go
@@ -48,16 +48,16 @@ func (c *SourceClient) ExampleConfig(ctx context.Context) (string, error) {
 	return res.Config, nil
 }
 
-func (c *SourceClient) Fetch(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error {
+func (c *SourceClient) Sync(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error {
 	b, err := json.Marshal(spec)
 	if err != nil {
 		return fmt.Errorf("failed to marshal source spec: %w", err)
 	}
-	stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{
+	stream, err := c.pbClient.Sync(ctx, &pb.Sync_Request{
 		Spec: b,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to fetch resources: %w", err)
+		return fmt.Errorf("failed to sync resources: %w", err)
 	}
 	for {
 		r, err := stream.Recv()
diff --git a/go.mod b/go.mod
index 27c732a6b1..221ff4861e 100644
--- a/go.mod
+++ b/go.mod
@@ -33,6 +33,7 @@ require (
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.12.1 // indirect
diff --git a/internal/pb/source.pb.go b/internal/pb/source.pb.go
index 43ddc57355..fa25af4285 100644
--- a/internal/pb/source.pb.go
+++ b/internal/pb/source.pb.go
@@ -20,14 +20,14 @@ const (
 	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )
 
-type Fetch struct {
+type Sync struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *Fetch) Reset() {
-	*x = Fetch{}
+func (x *Sync) Reset() {
+	*x = Sync{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_source_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -35,13 +35,13 @@ func (x *Fetch) Reset() {
 	}
 }
 
-func (x *Fetch) String() string {
+func (x *Sync) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Fetch) ProtoMessage() {}
+func (*Sync) ProtoMessage() {}
 
-func (x *Fetch) ProtoReflect() protoreflect.Message {
+func (x *Sync) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_source_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -53,8 +53,8 @@ func (x *Fetch) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Fetch.ProtoReflect.Descriptor instead.
-func (*Fetch) Descriptor() ([]byte, []int) {
+// Deprecated: Use Sync.ProtoReflect.Descriptor instead.
+func (*Sync) Descriptor() ([]byte, []int) {
 	return file_internal_pb_source_proto_rawDescGZIP(), []int{0}
 }
 
@@ -96,7 +96,7 @@ func (*GetTables) Descriptor() ([]byte, []int) {
 	return file_internal_pb_source_proto_rawDescGZIP(), []int{1}
 }
 
-type Fetch_Request struct {
+type Sync_Request struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
@@ -104,8 +104,8 @@ type Fetch_Request struct {
 	Spec []byte `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"`
 }
 
-func (x *Fetch_Request) Reset() {
-	*x = Fetch_Request{}
+func (x *Sync_Request) Reset() {
+	*x = Sync_Request{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_source_proto_msgTypes[2]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -113,13 +113,13 @@ func (x *Fetch_Request) Reset() {
 	}
 }
 
-func (x *Fetch_Request) String() string {
+func (x *Sync_Request) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Fetch_Request) ProtoMessage() {}
+func (*Sync_Request) ProtoMessage() {}
 
-func (x *Fetch_Request) ProtoReflect() protoreflect.Message {
+func (x *Sync_Request) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_source_proto_msgTypes[2]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -131,19 +131,19 @@ func (x *Fetch_Request) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Fetch_Request.ProtoReflect.Descriptor instead.
-func (*Fetch_Request) Descriptor() ([]byte, []int) {
+// Deprecated: Use Sync_Request.ProtoReflect.Descriptor instead.
+func (*Sync_Request) Descriptor() ([]byte, []int) {
 	return file_internal_pb_source_proto_rawDescGZIP(), []int{0, 0}
 }
 
-func (x *Fetch_Request) GetSpec() []byte {
+func (x *Sync_Request) GetSpec() []byte {
 	if x != nil {
 		return x.Spec
 	}
 	return nil
 }
 
-type Fetch_Response struct {
+type Sync_Response struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
@@ -152,8 +152,8 @@ type Fetch_Response struct {
 	Resource []byte `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"`
 }
 
-func (x *Fetch_Response) Reset() {
-	*x = Fetch_Response{}
+func (x *Sync_Response) Reset() {
+	*x = Sync_Response{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_internal_pb_source_proto_msgTypes[3]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -161,13 +161,13 @@ func (x *Fetch_Response) Reset() {
 	}
 }
 
-func (x *Fetch_Response) String() string {
+func (x *Sync_Response) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Fetch_Response) ProtoMessage() {}
+func (*Sync_Response) ProtoMessage() {}
 
-func (x *Fetch_Response) ProtoReflect() protoreflect.Message {
+func (x *Sync_Response) ProtoReflect() protoreflect.Message {
 	mi := &file_internal_pb_source_proto_msgTypes[3]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -179,12 +179,12 @@ func (x *Fetch_Response) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Fetch_Response.ProtoReflect.Descriptor instead.
-func (*Fetch_Response) Descriptor() ([]byte, []int) {
+// Deprecated: Use Sync_Response.ProtoReflect.Descriptor instead.
+func (*Sync_Response) Descriptor() ([]byte, []int) {
 	return file_internal_pb_source_proto_rawDescGZIP(), []int{0, 1}
 }
 
-func (x *Fetch_Response) GetResource() []byte {
+func (x *Sync_Response) GetResource() []byte {
 	if x != nil {
 		return x.Resource
 	}
@@ -299,33 +299,33 @@ var file_internal_pb_source_proto_rawDesc = []byte{
 	0x0a, 0x18, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f,
 	0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74,
 	0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x62,
-	0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4e, 0x0a, 0x05, 0x46, 0x65, 0x74,
-	0x63, 0x68, 0x1a, 0x1d, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a,
-	0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x70, 0x65,
-	0x63, 0x1a, 0x26, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a,
-	0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
-	0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74,
-	0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a,
-	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
-	0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74,
-	0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62,
-	0x6c, 0x65, 0x73, 0x32, 0xd9, 0x01, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40,
-	0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
-	0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74,
+	0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4d, 0x0a, 0x04, 0x53, 0x79, 0x6e,
+	0x63, 0x1a, 0x1d, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04,
+	0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63,
+	0x1a, 0x26, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08,
+	0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08,
+	0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x54,
+	0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04,
+	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61,
+	0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c,
+	0x65, 0x73, 0x32, 0xd6, 0x01, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a,
+	0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74,
+	0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45,
+	0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74,
 	0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
-	0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68,
-	0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46,
-	0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42,
-	0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x13,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x79, 0x6e, 0x63,
+	0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f,
+	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -342,10 +342,10 @@ func file_internal_pb_source_proto_rawDescGZIP() []byte {
 
 var file_internal_pb_source_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
 var file_internal_pb_source_proto_goTypes = []interface{}{
-	(*Fetch)(nil),                     // 0: proto.Fetch
+	(*Sync)(nil),                      // 0: proto.Sync
 	(*GetTables)(nil),                 // 1: proto.GetTables
-	(*Fetch_Request)(nil),             // 2: proto.Fetch.Request
-	(*Fetch_Response)(nil),            // 3: proto.Fetch.Response
+	(*Sync_Request)(nil),              // 2: proto.Sync.Request
+	(*Sync_Response)(nil),             // 3: proto.Sync.Response
 	(*GetTables_Request)(nil),         // 4: proto.GetTables.Request
 	(*GetTables_Response)(nil),        // 5: proto.GetTables.Response
 	(*GetExampleConfig_Request)(nil),  // 6: proto.GetExampleConfig.Request
@@ -354,10 +354,10 @@ var file_internal_pb_source_proto_goTypes = []interface{}{
 var file_internal_pb_source_proto_depIdxs = []int32{
 	4, // 0: proto.Source.GetTables:input_type -> proto.GetTables.Request
 	6, // 1: proto.Source.GetExampleConfig:input_type -> proto.GetExampleConfig.Request
-	2, // 2: proto.Source.Fetch:input_type -> proto.Fetch.Request
+	2, // 2: proto.Source.Sync:input_type -> proto.Sync.Request
 	5, // 3: proto.Source.GetTables:output_type -> proto.GetTables.Response
 	7, // 4: proto.Source.GetExampleConfig:output_type -> proto.GetExampleConfig.Response
-	3, // 5: proto.Source.Fetch:output_type -> proto.Fetch.Response
+	3, // 5: proto.Source.Sync:output_type -> proto.Sync.Response
 	3, // [3:6] is the sub-list for method output_type
 	0, // [0:3] is the sub-list for method input_type
 	0, // [0:0] is the sub-list for extension type_name
@@ -373,7 +373,7 @@ func file_internal_pb_source_proto_init() {
 	file_internal_pb_base_proto_init()
 	if !protoimpl.UnsafeEnabled {
 		file_internal_pb_source_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Fetch); i {
+			switch v := v.(*Sync); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -397,7 +397,7 @@ func file_internal_pb_source_proto_init() {
 			}
 		}
 		file_internal_pb_source_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Fetch_Request); i {
+			switch v := v.(*Sync_Request); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -409,7 +409,7 @@ func file_internal_pb_source_proto_init() {
 			}
 		}
 		file_internal_pb_source_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Fetch_Response); i {
+			switch v := v.(*Sync_Response); i {
 			case 0:
 				return &v.state
 			case 1:
diff --git a/internal/pb/source.proto b/internal/pb/source.proto
index 5d3d0165aa..a08f87a50d 100644
--- a/internal/pb/source.proto
+++ b/internal/pb/source.proto
@@ -10,10 +10,10 @@ service Source {
   // Get an example configuration for the source plugin
   rpc GetExampleConfig(GetExampleConfig.Request) returns (GetExampleConfig.Response);
   // Fetch resources
-  rpc Fetch(Fetch.Request) returns (stream Fetch.Response);
+  rpc Sync(Sync.Request) returns (stream Sync.Response);
 }
 
-message Fetch {
+message Sync {
   message Request {
     bytes spec = 1;
   }
diff --git a/internal/pb/source_grpc.pb.go b/internal/pb/source_grpc.pb.go
index ddcf670d61..06c3185e59 100644
--- a/internal/pb/source_grpc.pb.go
+++ b/internal/pb/source_grpc.pb.go
@@ -27,7 +27,7 @@ type SourceClient interface {
 	// Get an example configuration for the source plugin
 	GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error)
 	// Fetch resources
-	Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error)
+	Sync(ctx context.Context, in *Sync_Request, opts ...grpc.CallOption) (Source_SyncClient, error)
 }
 
 type sourceClient struct {
@@ -56,12 +56,12 @@ func (c *sourceClient) GetExampleConfig(ctx context.Context, in *GetExampleConfi
 	return out, nil
 }
 
-func (c *sourceClient) Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/proto.Source/Fetch", opts...)
+func (c *sourceClient) Sync(ctx context.Context, in *Sync_Request, opts ...grpc.CallOption) (Source_SyncClient, error) {
+	stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/proto.Source/Sync", opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &sourceFetchClient{stream}
+	x := &sourceSyncClient{stream}
 	if err := x.ClientStream.SendMsg(in); err != nil {
 		return nil, err
 	}
@@ -71,17 +71,17 @@ func (c *sourceClient) Fetch(ctx context.Context, in *Fetch_Request, opts ...grp
 	return x, nil
 }
 
-type Source_FetchClient interface {
-	Recv() (*Fetch_Response, error)
+type Source_SyncClient interface {
+	Recv() (*Sync_Response, error)
 	grpc.ClientStream
 }
 
-type sourceFetchClient struct {
+type sourceSyncClient struct {
 	grpc.ClientStream
 }
 
-func (x *sourceFetchClient) Recv() (*Fetch_Response, error) {
-	m := new(Fetch_Response)
+func (x *sourceSyncClient) Recv() (*Sync_Response, error) {
+	m := new(Sync_Response)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
@@ -97,7 +97,7 @@ type SourceServer interface {
 	// Get an example configuration for the source plugin
 	GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error)
 	// Fetch resources
-	Fetch(*Fetch_Request, Source_FetchServer) error
+	Sync(*Sync_Request, Source_SyncServer) error
 	mustEmbedUnimplementedSourceServer()
 }
 
@@ -111,8 +111,8 @@ func (UnimplementedSourceServer) GetTables(context.Context, *GetTables_Request)
 func (UnimplementedSourceServer) GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetExampleConfig not implemented")
 }
-func (UnimplementedSourceServer) Fetch(*Fetch_Request, Source_FetchServer) error {
-	return status.Errorf(codes.Unimplemented, "method Fetch not implemented")
+func (UnimplementedSourceServer) Sync(*Sync_Request, Source_SyncServer) error {
+	return status.Errorf(codes.Unimplemented, "method Sync not implemented")
 }
 func (UnimplementedSourceServer) mustEmbedUnimplementedSourceServer() {}
 
@@ -163,24 +163,24 @@ func _Source_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Source_Fetch_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(Fetch_Request)
+func _Source_Sync_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(Sync_Request)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(SourceServer).Fetch(m, &sourceFetchServer{stream})
+	return srv.(SourceServer).Sync(m, &sourceSyncServer{stream})
 }
 
-type Source_FetchServer interface {
-	Send(*Fetch_Response) error
+type Source_SyncServer interface {
+	Send(*Sync_Response) error
 	grpc.ServerStream
 }
 
-type sourceFetchServer struct {
+type sourceSyncServer struct {
 	grpc.ServerStream
 }
 
-func (x *sourceFetchServer) Send(m *Fetch_Response) error {
+func (x *sourceSyncServer) Send(m *Sync_Response) error {
 	return x.ServerStream.SendMsg(m)
 }
 
@@ -202,8 +202,8 @@ var Source_ServiceDesc = grpc.ServiceDesc{
 	},
 	Streams: []grpc.StreamDesc{
 		{
-			StreamName:    "Fetch",
-			Handler:       _Source_Fetch_Handler,
+			StreamName:    "Sync",
+			Handler:       _Source_Sync_Handler,
 			ServerStreams: true,
 		},
 	},
diff --git a/internal/servers/source.go b/internal/servers/source.go
index f1f167338f..b22de4fab8 100644
--- a/internal/servers/source.go
+++ b/internal/servers/source.go
@@ -28,7 +28,7 @@ func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.Ge
 	}, nil
 }
 
-func (s *SourceServer) ExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
+func (s *SourceServer) GetExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) {
 	exampleConfig, err := s.Plugin.ExampleConfig()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get example config: %w", err)
@@ -39,7 +39,7 @@ func (s *SourceServer) ExampleConfig(context.Context, *pb.GetExampleConfig_Reque
 		Config:  exampleConfig}, nil
 }
 
-func (s *SourceServer) Sync(req *pb.Fetch_Request, stream pb.Source_FetchServer) error {
+func (s *SourceServer) Sync(req *pb.Sync_Request, stream pb.Source_SyncServer) error {
 	resources := make(chan *schema.Resource)
 	var fetchErr error
 
@@ -63,7 +63,7 @@ func (s *SourceServer) Sync(req *pb.Fetch_Request, stream pb.Source_FetchServer)
 		if err != nil {
 			return errors.Wrap(err, "failed to marshal resource")
 		}
-		if err := stream.Send(&pb.Fetch_Response{
+		if err := stream.Send(&pb.Sync_Response{
 			Resource: b,
 		}); err != nil {
 			return errors.Wrap(err, "failed to send resource")
diff --git a/plugins/source.go b/plugins/source.go
index 892d91a076..2982210739 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -1,8 +1,8 @@
 package plugins
 
 import (
-	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
@@ -10,10 +10,10 @@ import (
 	"github.com/cloudquery/plugin-sdk/helpers"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/ghodss/yaml"
 	"github.com/rs/zerolog"
 	"github.com/thoas/go-funk"
 	"golang.org/x/sync/semaphore"
-	"gopkg.in/yaml.v3"
 
 	_ "embed"
 )
@@ -27,25 +27,6 @@ const ExampleSourceConfig = `
 # skip_tables: []
 `
 
-const sourcePluginExampleConfigTemplate = `kind: source
-spec:
-  name: {{.Name}}
-  version: {{.Version}}
-  # path: Path to the plugin. by default it is the same as the name of the plugin.
-  # registry can be local, github, grpc (default github)
-  # registry: github
-  # max_goroutines used for sync, by default calculated automatically depending on
-  # memory and cpu avaialble
-  # max_goroutines: 0
-  tables: ["*"]
-  # skip_tables is useful if you want to fetch all tables apart from specific ones
-  # skip_tables: []
-  # name of destinations to sync the data
-  destinations: []
-  configuration:
-  {{.PluginExampleConfig | indent 4}}
-`
-
 type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error)
 
 type SourceNewSpecFunc func() interface{}
@@ -116,23 +97,27 @@ func (p *SourcePlugin) Tables() schema.Tables {
 }
 
 func (p *SourcePlugin) ExampleConfig() (string, error) {
+	sourceSpec := specs.Source{
+		Name:         p.name,
+		Version:      p.version,
+		Tables:       []string{"*"},
+		Destinations: []string{},
+		Spec:         p.newSpec(),
+	}
+	sourceSpec.SetDefaults()
 	spec := specs.Spec{
 		Kind: specs.KindSource,
-		Spec: specs.Source{
-			Name:         p.name,
-			Version:      p.version,
-			Tables:       []string{"*"},
-			Destinations: []string{},
-			Spec:         p.newSpec(),
-		},
+		Spec: &sourceSpec,
 	}
-	bytes := bytes.NewBuffer([]byte(""))
-	enc := yaml.NewEncoder(bytes)
-	enc.SetIndent(2)
-	if err := enc.Encode(spec); err != nil {
-		return "", err
+	b, err := json.Marshal(spec)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal example config to json: %w", err)
+	}
+	b, err = yaml.JSONToYAML(b)
+	if err != nil {
+		return "", fmt.Errorf("failed to convert json to yaml: %w", err)
 	}
-	return bytes.String(), nil
+	return string(b), nil
 }
 
 func (p *SourcePlugin) GetJsonSchema() string {
diff --git a/plugins/source_test.go b/plugins/source_test.go
index 88b599ed3f..eee6251c64 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -2,6 +2,7 @@ package plugins
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/cloudquery/plugin-sdk/schema"
@@ -12,22 +13,18 @@ import (
 
 var _ schema.ClientMeta = &testExecutionClient{}
 
-const testSourcePluginExampleConfig = `# specify all accounts you want to sync
-accounts: []
-`
-
 type testExecutionClient struct {
 	logger zerolog.Logger
 }
 
 type Account struct {
 	Name    string   `json:"name,omitempty"`
-	Regions []string `json:"regions"`
+	Regions []string `json:"regions,omitempty"`
 }
 
 type testSourceSpec struct {
-	Accounts []Account `json:"accounts"`
-	Regions  []string  `json:"regions"`
+	Accounts []Account `json:"accounts,omitempty"`
+	Regions  []string  `json:"regions,omitempty"`
 }
 
 func newTestSourceSpec() interface{} {
@@ -76,6 +73,7 @@ func TestSync(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	fmt.Println(exampleConfig)
 	var spec specs.Spec
 	if err := specs.SpecUnmarshalYamlStrict([]byte(exampleConfig), &spec); err != nil {
 		t.Fatal(err)
diff --git a/serve/serve_test.go b/serve/serve_test.go
index a45b759814..ef91314699 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/cloudquery/plugin-sdk/plugins"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/google/go-cmp/cmp"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
@@ -19,12 +20,23 @@ import (
 var _ schema.ClientMeta = &testExecutionClient{}
 
 type testSourceSpec struct {
-	accounts []string `json:"accounts,omitempty"`
+	Accounts []string `json:"accounts,omitempty"`
+}
+
+var expectedExampleSpecConfig = specs.Spec{
+	Kind: specs.KindSource,
+	Spec: &specs.Source{
+		Name:    "testSourcePlugin",
+		Path:    "cloudquery/testSourcePlugin",
+		Version: "v1.0.0",
+		Tables:  []string{"*"},
+		Spec:    map[string]interface{}{"accounts": []interface{}{"all"}},
+	},
 }
 
 func newTestSourceSpec() interface{} {
 	return &testSourceSpec{
-		accounts: []string{"all"},
+		Accounts: []string{"all"},
 	}
 }
 
@@ -85,7 +97,7 @@ func bufDialer(context.Context, string) (net.Conn, error) {
 func TestServe(t *testing.T) {
 	plugin := plugins.NewSourcePlugin(
 		"testSourcePlugin",
-		"1.0.0",
+		"v1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
 		newTestSourceSpec,
@@ -121,10 +133,10 @@ func TestServe(t *testing.T) {
 	wg := errgroup.Group{}
 	wg.Go(func() error {
 		defer close(resources)
-		return c.Fetch(ctx,
+		return c.Sync(ctx,
 			specs.Source{
 				Name:     "testSourcePlugin",
-				Version:  "1.0.0",
+				Version:  "v1.0.0",
 				Registry: specs.RegistryGithub,
 				Spec:     TestSourcePluginSpec{Accounts: []string{"cloudquery/plugin-sdk"}},
 			},
@@ -146,9 +158,14 @@ func TestServe(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to get example config: %v", err)
 	}
+	var exampleSpec specs.Spec
+	if err := specs.SpecUnmarshalYamlStrict([]byte(exampleConfig), &exampleSpec); err != nil {
+		t.Fatalf("Failed to unmarshal example config: %v", err)
+	}
+	// skip internal validation for now
 
-	if exampleConfig != "" {
-		t.Fatalf("Expected example config:\n%s got:\n%s", "", exampleConfig)
+	if diff := cmp.Diff(expectedExampleSpecConfig, exampleSpec); diff != "" {
+		t.Fatalf("Spec mismatch (-want +got):\n%s", diff)
 	}
 
 }

From ba692d105a4cd1114730f4b907af761c990b6a73 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sat, 20 Aug 2022 17:34:55 +0300
Subject: [PATCH 12/25] Implement code generation helpers

---
 clients/destination.go          |   2 +-
 codegen/doc.go                  |   2 +
 codegen/golang.go               | 101 ++++++++++++++++++++++++++++++++
 codegen/golang_test.go          |  72 +++++++++++++++++++++++
 codegen/table.go                |  32 ++++++++++
 codegen/templates/column.go.tpl |   4 ++
 codegen/templates/table.go.tpl  |  20 +++++++
 helpers/sync.go                 |  16 ++++-
 plugins/destination.go          |   1 -
 plugins/source.go               |  93 ++++++++++-------------------
 plugins/source_test.go          |   1 -
 schema/column.go                |  20 ++-----
 schema/meta.go                  |   7 ++-
 schema/resolvers.go             |  13 +++-
 schema/resource.go              |  15 +++--
 schema/table.go                 |   8 +--
 serve/serve_test.go             |   2 +-
 specs/destination.go            |  11 ++++
 specs/spec_reader.go            |   5 +-
 19 files changed, 324 insertions(+), 101 deletions(-)
 create mode 100644 codegen/doc.go
 create mode 100644 codegen/golang.go
 create mode 100644 codegen/golang_test.go
 create mode 100644 codegen/table.go
 create mode 100644 codegen/templates/column.go.tpl
 create mode 100644 codegen/templates/table.go.tpl

diff --git a/clients/destination.go b/clients/destination.go
index 117d10e91c..3f98a231db 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -41,7 +41,7 @@ func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error
 	return res.Config, nil
 }
 
-func (c *DestinationClient) Configure(ctx context.Context, spec specs.Destination) error {
+func (c *DestinationClient) Initialize(ctx context.Context, spec specs.Destination) error {
 	if c.localClient != nil {
 		return c.localClient.Initialize(ctx, spec)
 	}
diff --git a/codegen/doc.go b/codegen/doc.go
new file mode 100644
index 0000000000..beb54c81b4
--- /dev/null
+++ b/codegen/doc.go
@@ -0,0 +1,2 @@
+//codgen helps autogenerate cloudquery plugins configured by definition
+package codegen
diff --git a/codegen/golang.go b/codegen/golang.go
new file mode 100644
index 0000000000..0f67d10920
--- /dev/null
+++ b/codegen/golang.go
@@ -0,0 +1,101 @@
+package codegen
+
+import (
+	"embed"
+	"fmt"
+	"io"
+	"reflect"
+	"text/template"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/iancoleman/strcase"
+)
+
+//go:embed templates/*.go.tpl
+var TemplatesFS embed.FS
+
+func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
+	k := v.Kind()
+	switch k {
+	case reflect.String:
+		return schema.TypeString, nil
+	case reflect.Bool:
+		return schema.TypeBool, nil
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return schema.TypeInt, nil
+	case reflect.Float32, reflect.Float64:
+		return schema.TypeFloat, nil
+	case reflect.Struct, reflect.Map:
+		return schema.TypeJSON, nil
+	case reflect.Pointer:
+		return valueToSchemaType(v.Elem())
+	case reflect.Slice:
+		switch v.Elem().Kind() {
+		case reflect.String:
+			return schema.TypeStringArray, nil
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			return schema.TypeIntArray, nil
+		default:
+			return schema.TypeJSON, nil
+		}
+	default:
+		return schema.TypeInvalid, fmt.Errorf("unsupported type: %s", k)
+	}
+}
+
+type TableOptions func(*TableDefinition)
+
+func WithNameTransformer(transformer func(string) string) TableOptions {
+	return func(t *TableDefinition) {
+		t.Name = transformer(t.Name)
+	}
+}
+
+func defaultTransformer(name string) string {
+	return strcase.ToSnake(name)
+}
+
+func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*TableDefinition, error) {
+	t := TableDefinition{
+		Name:            name,
+		nameTransformer: defaultTransformer,
+	}
+	for _, opt := range opts {
+		opt(&t)
+	}
+
+	e := reflect.ValueOf(obj)
+	if e.Kind() == reflect.Pointer {
+		e = e.Elem()
+	}
+	if e.Kind() != reflect.Struct {
+		return nil, fmt.Errorf("expected struct, got %s", e.Kind())
+	}
+	for i := 0; i < e.NumField(); i++ {
+		field := e.Type().Field(i)
+		columnType, err := valueToSchemaType(field.Type)
+		if err != nil {
+			return nil, err
+		}
+		column := ColumnDefinition{
+			Name: t.nameTransformer(field.Name),
+			Type: columnType,
+		}
+		t.Columns = append(t.Columns, column)
+	}
+	return &t, nil
+}
+
+func (t *TableDefinition) GenerateTemplate(wr io.Writer) error {
+	tpl, err := template.New("table.go.tpl").ParseFS(TemplatesFS, "templates/*")
+	if err != nil {
+		return fmt.Errorf("failed to parse template: %w", err)
+	}
+
+	if err := tpl.Execute(wr, t); err != nil {
+		return fmt.Errorf("failed to execute template: %w", err)
+	}
+	return nil
+}
diff --git a/codegen/golang_test.go b/codegen/golang_test.go
new file mode 100644
index 0000000000..076ebb21ce
--- /dev/null
+++ b/codegen/golang_test.go
@@ -0,0 +1,72 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+)
+
+type testStruct struct {
+	IntCol    int     `json:"int_col,omitempty"`
+	StringCol string  `json:"string_col,omitempty"`
+	FloatCol  float64 `json:"float_col,omitempty"`
+	BoolCol   bool    `json:"bool_col,omitempty"`
+	JsonCol   struct {
+		IntCol    int    `json:"int_col,omitempty"`
+		StringCol string `json:"string_col,omitempty"`
+	}
+	IntArrayCol    []int    `json:"int_array_col,omitempty"`
+	StringArrayCol []string `json:"string_array_col,omitempty"`
+}
+
+var expectedTestTable = TableDefinition{
+	Name: "test_struct",
+	Columns: []ColumnDefinition{
+		{
+			Name: "int_col",
+			Type: schema.TypeInt,
+		},
+		{
+			Name: "string_col",
+			Type: schema.TypeString,
+		},
+		{
+			Name: "float_col",
+			Type: schema.TypeFloat,
+		},
+		{
+			Name: "bool_col",
+			Type: schema.TypeBool,
+		},
+		{
+			Name: "json_col",
+			Type: schema.TypeJSON,
+		},
+		{
+			Name: "int_array_col",
+			Type: schema.TypeIntArray,
+		},
+		{
+			Name: "string_array_col",
+			Type: schema.TypeStringArray,
+		},
+	},
+	nameTransformer: defaultTransformer,
+}
+
+func TestTableFromGoStruct(t *testing.T) {
+	table, err := NewTableFromStruct("test_struct", testStruct{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// if !reflect.DeepEqual(table, &expectedTestTable) {
+	// 	t.Fatalf("expected:\n%+v, got:\n%+v", expectedTestTable, table)
+	// }
+	buf := bytes.NewBufferString("")
+	if err := table.GenerateTemplate(buf); err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(buf.String())
+}
diff --git a/codegen/table.go b/codegen/table.go
new file mode 100644
index 0000000000..755da55764
--- /dev/null
+++ b/codegen/table.go
@@ -0,0 +1,32 @@
+package codegen
+
+import (
+	"github.com/cloudquery/plugin-sdk/schema"
+)
+
+type ResourceDefinition struct {
+	Name  string
+	Table *TableDefinition
+}
+
+type TableDefinition struct {
+	Name        string
+	Description string
+	Columns     []ColumnDefinition
+	Relations   []*TableDefinition
+
+	Resolver             string
+	IgnoreError          string
+	Multiplex            string
+	PostResourceResolver string
+	Options              schema.TableCreationOptions
+	nameTransformer      func(string) string
+}
+
+type ColumnDefinition struct {
+	Name          string
+	Type          schema.ValueType
+	Resolver      string
+	Description   string
+	IgnoreInTests bool
+}
diff --git a/codegen/templates/column.go.tpl b/codegen/templates/column.go.tpl
new file mode 100644
index 0000000000..b9e89e26c2
--- /dev/null
+++ b/codegen/templates/column.go.tpl
@@ -0,0 +1,4 @@
+{
+  Name:        "{{.Name}}",
+  Type:        schema.{{.Type}},
+},
diff --git a/codegen/templates/table.go.tpl b/codegen/templates/table.go.tpl
new file mode 100644
index 0000000000..cfc52743f8
--- /dev/null
+++ b/codegen/templates/table.go.tpl
@@ -0,0 +1,20 @@
+{
+		Name:         "{{.Name}}",
+		{{- if .Resolver}}
+    Resolver:     {{.Resolver}},
+    {{- end}}
+		{{- if .Multiplex}}
+    Multiplex:    {{.Multiplex}},
+    {{- end}}
+		{{- if .IgnoreError}}
+    IgnoreError:  {{.IgnoreError}},
+    {{- end}}
+		Columns: []schema.Column{
+{{range .Columns}}{{template "column.go.tpl" .}}{{end}}
+		},
+{{with .Relations}}
+		Relations: []*schema.Table{
+{{range .}}{{template "table.go.tpl" .}}{{end}}
+		},
+{{end}}
+}
\ No newline at end of file
diff --git a/helpers/sync.go b/helpers/sync.go
index b5aa6c5375..acd435f7dd 100644
--- a/helpers/sync.go
+++ b/helpers/sync.go
@@ -1,18 +1,28 @@
 package helpers
 
 import (
+	"context"
+
 	"golang.org/x/sync/semaphore"
 )
 
 // SemaphoreAcauireMax is calling TryAcquire with 1 until it fails and return n-TryAcquireSucess
-func TryAcquireMax(sem *semaphore.Weighted, n int64) int64 {
+func TryAcquireMax(ctx context.Context, sem *semaphore.Weighted, n int64) (int64, error) {
 	newN := n
 	for {
 		if newN <= 0 {
-			return newN
+			return newN, nil
+		}
+		// first try will be blocking
+		if newN == n {
+			if err := sem.Acquire(ctx, 1); err != nil {
+				return newN, err
+			}
+			newN--
+			continue
 		}
 		if !sem.TryAcquire(1) {
-			return newN
+			return newN, nil
 		}
 		newN--
 	}
diff --git a/plugins/destination.go b/plugins/destination.go
index cf83fd7b4f..9949c25b47 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -32,7 +32,6 @@ type DestinationPlugin interface {
 	Name() string
 	Version() string
 	ExampleConfig() string
-	JsonSchema() string
 	Initialize(ctx context.Context, spec specs.Destination) error
 	Migrate(ctx context.Context, tables schema.Tables) error
 	Write(ctx context.Context, resources *schema.Resource) error
diff --git a/plugins/source.go b/plugins/source.go
index 2982210739..f2cfcf802f 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -2,18 +2,14 @@ package plugins
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/cloudquery/plugin-sdk/helpers"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
-	"github.com/ghodss/yaml"
 	"github.com/rs/zerolog"
 	"github.com/thoas/go-funk"
-	"golang.org/x/sync/semaphore"
 
 	_ "embed"
 )
@@ -29,8 +25,6 @@ const ExampleSourceConfig = `
 
 type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error)
 
-type SourceNewSpecFunc func() interface{}
-
 // SourcePlugin is the base structure required to pass to sdk.serve
 // We take a similar/declerative approach to API here similar to Cobra
 type SourcePlugin struct {
@@ -44,19 +38,17 @@ type SourcePlugin struct {
 	newExecutionClient SourceNewExecutionClientFunc
 	// Tables is all tables supported by this source plugin
 	tables schema.Tables
-	// newSpec return a new struct to be pupolated by the passed configuration
-	newSpec SourceNewSpecFunc
-	// JsonSchema for specific source plugin spec
-	jsonSchema string
+	// exampleConfig
+	exampleConfig string
 	// Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead.
 	logger zerolog.Logger
 }
 
 type SourceOption func(*SourcePlugin)
 
-func WithSourceJsonSchema(jsonSchema string) SourceOption {
+func WithSourceExampleConfig(exampleConfig string) SourceOption {
 	return func(p *SourcePlugin) {
-		p.jsonSchema = jsonSchema
+		p.exampleConfig = exampleConfig
 	}
 }
 
@@ -72,23 +64,29 @@ func WithClassifyError(ignoreError schema.IgnoreErrorFunc) SourceOption {
 	}
 }
 
-func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, newSpec SourceNewSpecFunc, opts ...SourceOption) *SourcePlugin {
+// Add internal columns
+func addInternalColumns(tables []*schema.Table) {
+	for _, table := range tables {
+		table.Columns = append(schema.CqColumns, table.Columns...)
+		addInternalColumns(table.Relations)
+	}
+}
+
+func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, opts ...SourceOption) *SourcePlugin {
 	p := SourcePlugin{
 		name:               name,
 		version:            version,
 		tables:             tables,
 		newExecutionClient: newExecutionClient,
-		newSpec:            newSpec,
 	}
 	if newExecutionClient == nil {
 		panic("newExecutionClient function not defined for source plugin:" + name)
 	}
-	if newSpec == nil {
-		panic("newConfig function not defined for source plugin:" + name)
-	}
 	for _, opt := range opts {
 		opt(&p)
 	}
+	addInternalColumns(p.tables)
+	// add default columns to tables
 	return &p
 }
 
@@ -97,31 +95,7 @@ func (p *SourcePlugin) Tables() schema.Tables {
 }
 
 func (p *SourcePlugin) ExampleConfig() (string, error) {
-	sourceSpec := specs.Source{
-		Name:         p.name,
-		Version:      p.version,
-		Tables:       []string{"*"},
-		Destinations: []string{},
-		Spec:         p.newSpec(),
-	}
-	sourceSpec.SetDefaults()
-	spec := specs.Spec{
-		Kind: specs.KindSource,
-		Spec: &sourceSpec,
-	}
-	b, err := json.Marshal(spec)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal example config to json: %w", err)
-	}
-	b, err = yaml.JSONToYAML(b)
-	if err != nil {
-		return "", fmt.Errorf("failed to convert json to yaml: %w", err)
-	}
-	return string(b), nil
-}
-
-func (p *SourcePlugin) GetJsonSchema() string {
-	return p.jsonSchema
+	return p.exampleConfig, nil
 }
 
 func (p *SourcePlugin) Name() string {
@@ -152,7 +126,7 @@ func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *
 	}
 	p.logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch")
 
-	goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
+	// goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines))
 
 	w := sync.WaitGroup{}
 	totalResources := 0
@@ -161,6 +135,10 @@ func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *
 	if err != nil {
 		return err
 	}
+
+	// this is the same fetchtime for all resources
+	fetchTime := time.Now()
+
 	for _, table := range p.tables {
 		table := table
 		if funk.ContainsString(spec.SkipTables, table.Name) || !funk.ContainsString(tableNames, table.Name) {
@@ -177,36 +155,29 @@ func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *
 		}
 		// we call this here because we dont know when the following goroutine will be called and we do want an order
 		// of table by table
-		totalClients := len(clients)
-		newN := helpers.TryAcquireMax(goroutinesSem, int64(totalClients))
+		// totalClients := len(clients)
+		// newN, err := helpers.TryAcquireMax(ctx, goroutinesSem, int64(totalClients))
+		// if err != nil {
+		// 	p.logger.Error().Err(err).Msg("failed to TryAcquireMax semaphore. exiting")
+		// 	break
+		// }
 		// goroutinesSem.TryAcquire()
 		w.Add(1)
 		go func() {
 			defer w.Done()
-			defer goroutinesSem.Release(int64(totalClients) - newN)
 			wg := sync.WaitGroup{}
 			p.logger.Info().Str("table", table.Name).Msg("fetch start")
 			tableStartTime := time.Now()
 			totalTableResources := 0
-			for i, client := range clients {
+			for _, client := range clients {
 				client := client
-				i := i
-				// acquire semaphore only if we couldn't acquire it earlier
-				if newN > 0 && i >= (totalClients-int(newN)) {
-					if err := goroutinesSem.Acquire(ctx, 1); err != nil {
-						// this can happen if context was cancelled so we just break out of the loop
-						p.logger.Error().Err(err).Msg("failed to acquire semaphore")
-						return
-					}
-				}
+
+				// i := i
 				wg.Add(1)
 				go func() {
 					defer wg.Done()
-					if newN > 0 && i >= (totalClients-int(newN)) {
-						defer goroutinesSem.Release(1)
-					}
-
-					totalTableResources += table.Resolve(ctx, client, nil, res)
+					// defer goroutinesSem.Release(1)
+					totalTableResources += table.Resolve(ctx, client, fetchTime, nil, res)
 				}()
 			}
 			wg.Wait()
diff --git a/plugins/source_test.go b/plugins/source_test.go
index eee6251c64..6af9fba097 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -64,7 +64,6 @@ func TestSync(t *testing.T) {
 		"1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
-		newTestSourceSpec,
 		WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
 	)
 
diff --git a/schema/column.go b/schema/column.go
index 2554310b8c..1e70cf743f 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -67,9 +67,7 @@ type Column struct {
 const (
 	TypeInvalid ValueType = iota
 	TypeBool
-	TypeSmallInt
 	TypeInt
-	TypeBigInt
 	TypeFloat
 	TypeUUID
 	TypeString
@@ -91,8 +89,8 @@ func (v ValueType) String() string {
 	switch v {
 	case TypeBool:
 		return "TypeBool"
-	case TypeInt, TypeBigInt, TypeSmallInt:
-		return "TypeBigInt"
+	case TypeInt:
+		return "TypeInt"
 	case TypeFloat:
 		return "TypeFloat"
 	case TypeUUID:
@@ -136,7 +134,7 @@ func ValueTypeFromString(s string) ValueType {
 	case "bool":
 		return TypeBool
 	case "int", "bigint", "smallint":
-		return TypeBigInt
+		return TypeInt
 	case "float":
 		return TypeFloat
 	case "uuid":
@@ -202,7 +200,7 @@ func (c Column) checkType(v interface{}) bool {
 	switch val := v.(type) {
 	case int8, *int8, uint8, *uint8, int16, *int16, uint16, *uint16, int32, *int32, int, *int, uint32, *uint32, int64, *int64:
 		// TODO: Deprecate all Int Types in favour of BigInt
-		return c.Type == TypeBigInt || c.Type == TypeSmallInt || c.Type == TypeInt
+		return c.Type == TypeInt
 	case []byte:
 		if c.Type == TypeUUID {
 			if _, err := uuid.FromBytes(val); err != nil {
@@ -278,16 +276,6 @@ func (c Column) checkType(v interface{}) bool {
 		if kindName == reflect.Struct {
 			return c.Type == TypeJSON
 		}
-		if c.Type == TypeSmallInt && (kindName == reflect.Int8 || kindName == reflect.Int16 || kindName == reflect.Uint8) {
-			return true
-		}
-
-		if c.Type == TypeInt && (kindName == reflect.Uint16 || kindName == reflect.Int32) {
-			return true
-		}
-		if c.Type == TypeBigInt && (kindName == reflect.Int || kindName == reflect.Int64 || kindName == reflect.Uint || kindName == reflect.Uint32 || kindName == reflect.Uint64) {
-			return true
-		}
 	}
 
 	return false
diff --git a/schema/meta.go b/schema/meta.go
index f55e2dd44a..cc91268f8c 100644
--- a/schema/meta.go
+++ b/schema/meta.go
@@ -17,9 +17,12 @@ type Meta struct {
 
 const FetchIdMetaKey = "cq_fetch_id"
 
+var CqIdColumn = Column{Name: "_cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", CreationOptions: ColumnCreationOptions{Unique: true}, Resolver: CQUUIDResolver()}
+var CqFetchTime = Column{Name: "_cq_fetch_time", Type: TypeTimestamp, Description: "Internal CQ row of when fetch was started (this will be the same for all rows in a single fetch)"}
+
 var CqColumns = []Column{
-	{Name: "cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", CreationOptions: ColumnCreationOptions{Unique: true}},
-	{Name: "cq_fetch_time", Type: TypeTimestamp, Description: "Internal CQ row of when fetch was started (this will be the same for all rows in a single fetch)"},
+	CqIdColumn,
+	CqFetchTime,
 }
 
 var (
diff --git a/schema/resolvers.go b/schema/resolvers.go
index 3c7489330d..a46cb60583 100644
--- a/schema/resolvers.go
+++ b/schema/resolvers.go
@@ -7,7 +7,8 @@ import (
 	"time"
 
 	"github.com/cloudquery/plugin-sdk/helpers"
-	"github.com/gofrs/uuid"
+	// "github.com/gofrs/uuid"
+	"github.com/google/uuid"
 	"github.com/spf13/cast"
 	"github.com/thoas/go-funk"
 )
@@ -24,6 +25,13 @@ func PathResolver(path string) ColumnResolver {
 	}
 }
 
+func CQUUIDResolver() ColumnResolver {
+	return func(ctx context.Context, meta ClientMeta, r *Resource, c Column) error {
+		uuidGen := uuid.New()
+		return r.Set(c.Name, uuidGen)
+	}
+}
+
 // ParentIdResolver resolves the cq_id from the parent
 // if you want to reference the parent's primary keys use ParentResourceFieldResolver as required.
 func ParentIdResolver(_ context.Context, _ ClientMeta, r *Resource, c Column) error {
@@ -198,8 +206,7 @@ func UUIDResolver(path string) ColumnResolver {
 		if err != nil {
 			return err
 		}
-
-		id, err := uuid.FromString(uuidString)
+		id, err := uuid.FromBytes([]byte(uuidString))
 		if err != nil {
 			return err
 		}
diff --git a/schema/resource.go b/schema/resource.go
index caee2bdfee..bf0857edab 100644
--- a/schema/resource.go
+++ b/schema/resource.go
@@ -1,6 +1,8 @@
 package schema
 
 import (
+	"time"
+
 	"github.com/google/uuid"
 )
 
@@ -18,18 +20,18 @@ type Resource struct {
 	// This is sorted result data by column name
 	Data      map[string]interface{} `json:"data"`
 	TableName string                 `json:"table_name"`
-	cqId      uuid.UUID
 }
 
-func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource {
-	return &Resource{
+func NewResourceData(t *Table, parent *Resource, fetchTime time.Time, item interface{}) *Resource {
+	r := Resource{
 		Item:      item,
 		Parent:    parent,
 		Table:     t,
 		Data:      make(map[string]interface{}, len(t.Columns)),
-		cqId:      uuid.New(),
 		TableName: t.Name,
 	}
+	r.Data[CqFetchTime.Name] = fetchTime
+	return &r
 }
 
 // func (r *Resource) PrimaryKeyValues() []string {
@@ -69,7 +71,10 @@ func (r *Resource) Set(key string, value interface{}) error {
 }
 
 func (r *Resource) Id() uuid.UUID {
-	return r.cqId
+	if r.Data[cqIdColumn.Name] == nil {
+		return uuid.UUID{}
+	}
+	return r.Data[cqIdColumn.Name].(uuid.UUID)
 }
 
 func (r *Resource) Columns() []string {
diff --git a/schema/table.go b/schema/table.go
index 4da9660683..3ae7237ca3 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -35,7 +35,7 @@ type Table struct {
 	// Columns are the set of fields that are part of this table
 	Columns ColumnList `json:"columns"`
 	// Relations are a set of related tables defines
-	Relations []*Table `json:"relations"`
+	Relations Tables `json:"relations"`
 	// Resolver is the main entry point to fetching table data and
 	Resolver TableResolver `json:"-"`
 	// IgnoreError is a function that classifies error and returns it's severity and type
@@ -115,7 +115,7 @@ func (t Table) TableNames() []string {
 }
 
 // Call the table resolver with with all of it's relation for every reolved resource
-func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, resolvedResources chan<- *Resource) int {
+func (t Table) Resolve(ctx context.Context, meta ClientMeta, fetchTime time.Time, parent *Resource, resolvedResources chan<- *Resource) int {
 	res := make(chan interface{})
 	startTime := time.Now()
 	go func() {
@@ -148,7 +148,7 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, r
 		}
 		totalResources += len(objects)
 		for i := range objects {
-			resource := NewResourceData(&t, parent, objects[i])
+			resource := NewResourceData(&t, parent, fetchTime, objects[i])
 			t.resolveColumns(ctx, meta, resource)
 			if t.PostResourceResolver != nil {
 				meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver started")
@@ -160,7 +160,7 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, r
 			}
 			resolvedResources <- resource
 			for _, rel := range t.Relations {
-				totalResources += rel.Resolve(ctx, meta, resource, resolvedResources)
+				totalResources += rel.Resolve(ctx, meta, fetchTime, resource, resolvedResources)
 			}
 		}
 	}
diff --git a/serve/serve_test.go b/serve/serve_test.go
index ef91314699..f12a4ec829 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -100,7 +100,7 @@ func TestServe(t *testing.T) {
 		"v1.0.0",
 		[]*schema.Table{testTable()},
 		newTestExecutionClient,
-		newTestSourceSpec,
+		plugins.WithSourceExampleConfig(""),
 		plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))),
 	)
 
diff --git a/specs/destination.go b/specs/destination.go
index 047089c9f5..a9d544cd78 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -38,6 +38,17 @@ func (d *Destination) SetDefaults() {
 	}
 }
 
+func (s *Destination) UnmarshalSpec(out interface{}) error {
+	b, err := json.Marshal(s.Spec)
+	if err != nil {
+		return err
+	}
+	dec := json.NewDecoder(nil)
+	dec.UseNumber()
+	dec.DisallowUnknownFields()
+	return json.Unmarshal(b, out)
+}
+
 func (m WriteMode) String() string {
 	return [...]string{"append-only", "overwrite"}[m]
 }
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index 772198a761..4b80e84488 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -6,8 +6,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-
-	"gopkg.in/yaml.v3"
 )
 
 type SpecReader struct {
@@ -32,7 +30,8 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 				return nil, fmt.Errorf("failed to read file %s: %w", file.Name(), err)
 			}
 			var s Spec
-			if err := yaml.Unmarshal(data, &s); err != nil {
+			SpecUnmarshalYamlStrict(data, &s)
+			if err := SpecUnmarshalYamlStrict(data, &s); err != nil {
 				return nil, fmt.Errorf("failed to unmarshal file %s: %w", file.Name(), err)
 			}
 			switch s.Kind {

From 7682bf41f4a5c3d2ad7399df00558e534e351032 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sun, 21 Aug 2022 01:09:13 +0300
Subject: [PATCH 13/25] improvments to codegen

---
 codegen/golang.go         | 19 +++++++++++++++++++
 codegen/table.go          |  1 +
 plugins/source_testing.go |  4 ++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/codegen/golang.go b/codegen/golang.go
index 0f67d10920..11321e9794 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -53,10 +53,25 @@ func WithNameTransformer(transformer func(string) string) TableOptions {
 	}
 }
 
+func WithSkipFields(fields ...string) TableOptions {
+	return func(t *TableDefinition) {
+		t.skipFields = append(t.skipFields, fields...)
+	}
+}
+
 func defaultTransformer(name string) string {
 	return strcase.ToSnake(name)
 }
 
+func sliceContains(arr []string, s string) bool {
+	for _, v := range arr {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
 func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*TableDefinition, error) {
 	t := TableDefinition{
 		Name:            name,
@@ -75,10 +90,14 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 	}
 	for i := 0; i < e.NumField(); i++ {
 		field := e.Type().Field(i)
+		if sliceContains(t.skipFields, field.Name) {
+			continue
+		}
 		columnType, err := valueToSchemaType(field.Type)
 		if err != nil {
 			return nil, err
 		}
+
 		column := ColumnDefinition{
 			Name: t.nameTransformer(field.Name),
 			Type: columnType,
diff --git a/codegen/table.go b/codegen/table.go
index 755da55764..8583d2f6e2 100644
--- a/codegen/table.go
+++ b/codegen/table.go
@@ -21,6 +21,7 @@ type TableDefinition struct {
 	PostResourceResolver string
 	Options              schema.TableCreationOptions
 	nameTransformer      func(string) string
+	skipFields           []string
 }
 
 type ColumnDefinition struct {
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index 2186d4e8ec..b3cad6bc33 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -33,7 +33,7 @@ func init() {
 
 // type
 
-func TestResource(t *testing.T, tc ResourceTestCase) {
+func TestSourcePluginSync(t *testing.T, plugin *SourcePlugin, spec specs.Source) {
 	t.Parallel()
 	t.Helper()
 
@@ -49,7 +49,7 @@ func TestResource(t *testing.T, tc ResourceTestCase) {
 	// tc.Plugin.Logger = zerolog.New(zerolog.NewTestWriter(t))
 	go func() {
 		defer close(resources)
-		fetchErr = tc.Plugin.Sync(context.Background(), tc.Spec, resources)
+		fetchErr = plugin.Sync(context.Background(), spec, resources)
 	}()
 
 	for resource := range resources {

From 7a45296f8905f619b102fed82a2f0ab4efe5058e Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sun, 21 Aug 2022 14:40:26 +0300
Subject: [PATCH 14/25] fix: dont override nil values

---
 schema/table.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/schema/table.go b/schema/table.go
index 3ae7237ca3..3141eb5911 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -179,10 +179,14 @@ func (t Table) resolveColumns(ctx context.Context, meta ClientMeta, resource *Re
 			meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default started")
 			// base use case: try to get column with CamelCase name
 			v := funk.Get(resource.Item, strcase.ToCamel(c.Name), funk.WithAllowZero())
-			if err := resource.Set(c.Name, v); err != nil {
-				meta.Logger().Error().Str("colum_name", c.Name).Str("table_name", t.Name).Err(err).Msg("column resolver default finished with error")
+			if v != nil {
+				if err := resource.Set(c.Name, v); err != nil {
+					meta.Logger().Error().Str("colum_name", c.Name).Str("table_name", t.Name).Err(err).Msg("column resolver default finished with error")
+				}
+				meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully")
+			} else {
+				meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully with nil")
 			}
-			meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully")
 		}
 	}
 }

From b3671bd9a86a4633ebcc174025e119950d25c904 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Sun, 21 Aug 2022 15:35:17 +0300
Subject: [PATCH 15/25] more fixes to base templates

---
 codegen/templates/column.go.tpl | 3 +++
 codegen/templates/table.go.tpl  | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/codegen/templates/column.go.tpl b/codegen/templates/column.go.tpl
index b9e89e26c2..e0c0463a41 100644
--- a/codegen/templates/column.go.tpl
+++ b/codegen/templates/column.go.tpl
@@ -1,4 +1,7 @@
 {
   Name:        "{{.Name}}",
   Type:        schema.{{.Type}},
+  {{- if .Resolver}}
+  Resolver:     {{.Resolver}},
+  {{- end}}
 },
diff --git a/codegen/templates/table.go.tpl b/codegen/templates/table.go.tpl
index cfc52743f8..601fc8e9ce 100644
--- a/codegen/templates/table.go.tpl
+++ b/codegen/templates/table.go.tpl
@@ -8,6 +8,14 @@
     {{- end}}
 		{{- if .IgnoreError}}
     IgnoreError:  {{.IgnoreError}},
+    {{- end}}
+    {{- if .Options}}
+    Options: schema.TableCreationOptions{
+      PrimaryKeys: []string{
+        {{- range .Options.PrimaryKeys}}
+        "{{.}}",{{- end}}
+      },
+    },
     {{- end}}
 		Columns: []schema.Column{
 {{range .Columns}}{{template "column.go.tpl" .}}{{end}}

From 4a19c7eaa34b444d015a8a00c1d709fd819139f5 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Mon, 22 Aug 2022 15:41:34 +0300
Subject: [PATCH 16/25] more codegen improvments

---
 codegen/table.go                |  1 +
 codegen/templates/column.go.tpl |  7 +++++++
 codegen/templates/table.go.tpl  |  8 --------
 schema/column.go                |  5 +++--
 schema/table.go                 | 11 +++++++++++
 5 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/codegen/table.go b/codegen/table.go
index 8583d2f6e2..0de1aa9d20 100644
--- a/codegen/table.go
+++ b/codegen/table.go
@@ -30,4 +30,5 @@ type ColumnDefinition struct {
 	Resolver      string
 	Description   string
 	IgnoreInTests bool
+	Options       schema.ColumnCreationOptions
 }
diff --git a/codegen/templates/column.go.tpl b/codegen/templates/column.go.tpl
index e0c0463a41..50d5f2034f 100644
--- a/codegen/templates/column.go.tpl
+++ b/codegen/templates/column.go.tpl
@@ -4,4 +4,11 @@
   {{- if .Resolver}}
   Resolver:     {{.Resolver}},
   {{- end}}
+  {{- if .Options.PrimaryKey}}
+  CreationOptions: schema.ColumnCreationOptions{
+    {{- if .Options.PrimaryKey}}
+      PrimaryKey: true,
+    {{- end }}
+  },
+  {{- end}}  
 },
diff --git a/codegen/templates/table.go.tpl b/codegen/templates/table.go.tpl
index 601fc8e9ce..cfc52743f8 100644
--- a/codegen/templates/table.go.tpl
+++ b/codegen/templates/table.go.tpl
@@ -8,14 +8,6 @@
     {{- end}}
 		{{- if .IgnoreError}}
     IgnoreError:  {{.IgnoreError}},
-    {{- end}}
-    {{- if .Options}}
-    Options: schema.TableCreationOptions{
-      PrimaryKeys: []string{
-        {{- range .Options.PrimaryKeys}}
-        "{{.}}",{{- end}}
-      },
-    },
     {{- end}}
 		Columns: []schema.Column{
 {{range .Columns}}{{template "column.go.tpl" .}}{{end}}
diff --git a/schema/column.go b/schema/column.go
index 1e70cf743f..69a670c280 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -36,8 +36,9 @@ type ColumnResolver func(ctx context.Context, meta ClientMeta, resource *Resourc
 
 // ColumnCreationOptions allow modification of how column is defined when table is created
 type ColumnCreationOptions struct {
-	Unique  bool `json:"unique,omitempty"`
-	NotNull bool `json:"notnull,omitempty"`
+	PrimaryKey bool `json:"primary_key,omitempty"`
+	Unique     bool `json:"unique,omitempty"`
+	NotNull    bool `json:"notnull,omitempty"`
 }
 
 // Column definition for Table
diff --git a/schema/table.go b/schema/table.go
index 3141eb5911..cdf64e16d3 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -86,6 +86,17 @@ func (t Table) Column(name string) *Column {
 	return nil
 }
 
+func (t Table) PrimaryKeys() []string {
+	var primaryKeys []string
+	for _, c := range t.Columns {
+		if c.CreationOptions.PrimaryKey {
+			primaryKeys = append(primaryKeys, c.Name)
+		}
+	}
+
+	return primaryKeys
+}
+
 func (t Table) ColumnIndex(name string) int {
 	var once sync.Once
 	once.Do(func() {

From 2ac39f777066cfed06ae939c04723438e92d808d Mon Sep 17 00:00:00 2001
From: Herman Schaaf <hermanschaaf@gmail.com>
Date: Wed, 24 Aug 2022 12:20:09 +0200
Subject: [PATCH 17/25] feat: Support timestamp columns (#12)

---
 codegen/golang.go      | 22 +++++++++++++++-------
 codegen/golang_test.go | 15 +++++++++++++--
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/codegen/golang.go b/codegen/golang.go
index 11321e9794..54d589b87e 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -3,18 +3,18 @@ package codegen
 import (
 	"embed"
 	"fmt"
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/iancoleman/strcase"
 	"io"
 	"reflect"
 	"text/template"
-
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/iancoleman/strcase"
+	"time"
 )
 
 //go:embed templates/*.go.tpl
 var TemplatesFS embed.FS
 
-func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
+func valueToSchemaType(v reflect.Type, f interface{}) (schema.ValueType, error) {
 	k := v.Kind()
 	switch k {
 	case reflect.String:
@@ -26,10 +26,17 @@ func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
 		return schema.TypeInt, nil
 	case reflect.Float32, reflect.Float64:
 		return schema.TypeFloat, nil
-	case reflect.Struct, reflect.Map:
+	case reflect.Map:
+		return schema.TypeJSON, nil
+	case reflect.Struct:
+		switch f.(type) {
+		case time.Time:
+			return schema.TypeTimestamp, nil
+		}
 		return schema.TypeJSON, nil
 	case reflect.Pointer:
-		return valueToSchemaType(v.Elem())
+		inf := reflect.ValueOf(v.Elem()).Interface()
+		return valueToSchemaType(v.Elem(), inf)
 	case reflect.Slice:
 		switch v.Elem().Kind() {
 		case reflect.String:
@@ -93,7 +100,8 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 		if sliceContains(t.skipFields, field.Name) {
 			continue
 		}
-		columnType, err := valueToSchemaType(field.Type)
+		v := reflect.Indirect(e).FieldByIndex([]int{i}).Interface()
+		columnType, err := valueToSchemaType(field.Type, v)
 		if err != nil {
 			return nil, err
 		}
diff --git a/codegen/golang_test.go b/codegen/golang_test.go
index 076ebb21ce..2386a30721 100644
--- a/codegen/golang_test.go
+++ b/codegen/golang_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/cloudquery/plugin-sdk/schema"
 )
@@ -17,8 +18,10 @@ type testStruct struct {
 		IntCol    int    `json:"int_col,omitempty"`
 		StringCol string `json:"string_col,omitempty"`
 	}
-	IntArrayCol    []int    `json:"int_array_col,omitempty"`
-	StringArrayCol []string `json:"string_array_col,omitempty"`
+	IntArrayCol    []int      `json:"int_array_col,omitempty"`
+	StringArrayCol []string   `json:"string_array_col,omitempty"`
+	TimeCol        time.Time  `json:"time_col,omitempty"`
+	TimePointerCol *time.Time `json:"time_pointer_col,omitempty"`
 }
 
 var expectedTestTable = TableDefinition{
@@ -52,6 +55,14 @@ var expectedTestTable = TableDefinition{
 			Name: "string_array_col",
 			Type: schema.TypeStringArray,
 		},
+		{
+			Name: "time_col",
+			Type: schema.TypeTimestamp,
+		},
+		{
+			Name: "time_pointer_col",
+			Type: schema.TypeTimestamp,
+		},
 	},
 	nameTransformer: defaultTransformer,
 }

From 575016dfa6624943076bb6b1639d4d16a9eb3aec Mon Sep 17 00:00:00 2001
From: Herman Schaaf <hermanschaaf@gmail.com>
Date: Thu, 25 Aug 2022 08:58:09 +0200
Subject: [PATCH 18/25] Always generate path resolver (#14)

---
 codegen/golang.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/codegen/golang.go b/codegen/golang.go
index 54d589b87e..b582118710 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -106,9 +106,12 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 			return nil, err
 		}
 
+		// generate a PathResolver to use by default
+		pathResolver := fmt.Sprintf("schema.PathResolver(%q)", field.Name)
 		column := ColumnDefinition{
-			Name: t.nameTransformer(field.Name),
-			Type: columnType,
+			Name:     t.nameTransformer(field.Name),
+			Type:     columnType,
+			Resolver: pathResolver,
 		}
 		t.Columns = append(t.Columns, column)
 	}

From 55d453617cb000015c7fc8d0d9b54f4ceddf7f68 Mon Sep 17 00:00:00 2001
From: Herman Schaaf <hermanschaaf@gmail.com>
Date: Thu, 25 Aug 2022 09:24:22 +0200
Subject: [PATCH 19/25] fix: NewTableFromStruct pointer to time.Time (#15)

---
 codegen/golang.go      | 13 ++++------
 codegen/golang_test.go | 56 +++++++++++++++++++++++++-----------------
 2 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/codegen/golang.go b/codegen/golang.go
index b582118710..03afcd0767 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -8,13 +8,12 @@ import (
 	"io"
 	"reflect"
 	"text/template"
-	"time"
 )
 
 //go:embed templates/*.go.tpl
 var TemplatesFS embed.FS
 
-func valueToSchemaType(v reflect.Type, f interface{}) (schema.ValueType, error) {
+func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
 	k := v.Kind()
 	switch k {
 	case reflect.String:
@@ -29,14 +28,13 @@ func valueToSchemaType(v reflect.Type, f interface{}) (schema.ValueType, error)
 	case reflect.Map:
 		return schema.TypeJSON, nil
 	case reflect.Struct:
-		switch f.(type) {
-		case time.Time:
+		t := v.PkgPath() + "." + v.Name()
+		if t == "time.Time" {
 			return schema.TypeTimestamp, nil
 		}
 		return schema.TypeJSON, nil
 	case reflect.Pointer:
-		inf := reflect.ValueOf(v.Elem()).Interface()
-		return valueToSchemaType(v.Elem(), inf)
+		return valueToSchemaType(v.Elem())
 	case reflect.Slice:
 		switch v.Elem().Kind() {
 		case reflect.String:
@@ -100,8 +98,7 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 		if sliceContains(t.skipFields, field.Name) {
 			continue
 		}
-		v := reflect.Indirect(e).FieldByIndex([]int{i}).Interface()
-		columnType, err := valueToSchemaType(field.Type, v)
+		columnType, err := valueToSchemaType(field.Type)
 		if err != nil {
 			return nil, err
 		}
diff --git a/codegen/golang_test.go b/codegen/golang_test.go
index 2386a30721..ed7a87a404 100644
--- a/codegen/golang_test.go
+++ b/codegen/golang_test.go
@@ -3,6 +3,8 @@ package codegen
 import (
 	"bytes"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
@@ -14,7 +16,7 @@ type testStruct struct {
 	StringCol string  `json:"string_col,omitempty"`
 	FloatCol  float64 `json:"float_col,omitempty"`
 	BoolCol   bool    `json:"bool_col,omitempty"`
-	JsonCol   struct {
+	JSONCol   struct {
 		IntCol    int    `json:"int_col,omitempty"`
 		StringCol string `json:"string_col,omitempty"`
 	}
@@ -28,40 +30,49 @@ var expectedTestTable = TableDefinition{
 	Name: "test_struct",
 	Columns: []ColumnDefinition{
 		{
-			Name: "int_col",
-			Type: schema.TypeInt,
+			Name:     "int_col",
+			Type:     schema.TypeInt,
+			Resolver: `schema.PathResolver("IntCol")`,
 		},
 		{
-			Name: "string_col",
-			Type: schema.TypeString,
+			Name:     "string_col",
+			Type:     schema.TypeString,
+			Resolver: `schema.PathResolver("StringCol")`,
 		},
 		{
-			Name: "float_col",
-			Type: schema.TypeFloat,
+			Name:     "float_col",
+			Type:     schema.TypeFloat,
+			Resolver: `schema.PathResolver("FloatCol")`,
 		},
 		{
-			Name: "bool_col",
-			Type: schema.TypeBool,
+			Name:     "bool_col",
+			Type:     schema.TypeBool,
+			Resolver: `schema.PathResolver("BoolCol")`,
 		},
 		{
-			Name: "json_col",
-			Type: schema.TypeJSON,
+			Name:     "json_col",
+			Type:     schema.TypeJSON,
+			Resolver: `schema.PathResolver("JSONCol")`,
 		},
 		{
-			Name: "int_array_col",
-			Type: schema.TypeIntArray,
+			Name:     "int_array_col",
+			Type:     schema.TypeIntArray,
+			Resolver: `schema.PathResolver("IntArrayCol")`,
 		},
 		{
-			Name: "string_array_col",
-			Type: schema.TypeStringArray,
+			Name:     "string_array_col",
+			Type:     schema.TypeStringArray,
+			Resolver: `schema.PathResolver("StringArrayCol")`,
 		},
 		{
-			Name: "time_col",
-			Type: schema.TypeTimestamp,
+			Name:     "time_col",
+			Type:     schema.TypeTimestamp,
+			Resolver: `schema.PathResolver("TimeCol")`,
 		},
 		{
-			Name: "time_pointer_col",
-			Type: schema.TypeTimestamp,
+			Name:     "time_pointer_col",
+			Type:     schema.TypeTimestamp,
+			Resolver: `schema.PathResolver("TimePointerCol")`,
 		},
 	},
 	nameTransformer: defaultTransformer,
@@ -72,9 +83,10 @@ func TestTableFromGoStruct(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	// if !reflect.DeepEqual(table, &expectedTestTable) {
-	// 	t.Fatalf("expected:\n%+v, got:\n%+v", expectedTestTable, table)
-	// }
+	if diff := cmp.Diff(table, &expectedTestTable,
+		cmpopts.IgnoreUnexported(TableDefinition{})); diff != "" {
+		t.Fatalf("table does not match expected. diff (-got, +want): %v", diff)
+	}
 	buf := bytes.NewBufferString("")
 	if err := table.GenerateTemplate(buf); err != nil {
 		t.Fatal(err)

From 0b3b7ef1c33ddd851680a147a02d8a3e4935fdd4 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Mon, 29 Aug 2022 01:52:34 +0300
Subject: [PATCH 20/25] feat: Add WithDescriptionEnabled to NewTableFromStruct

I suggest not to use it as it is just duplicated bloat and
it is super slow
---
 clients/destination.go           |  4 +-
 codegen/golang.go                | 85 +++++++++++++++++++++++++++++---
 codegen/golang_test.go           | 10 +++-
 codegen/table.go                 | 14 ++++++
 codegen/templates/column.go.tpl  |  3 ++
 internal/servers/destinations.go |  2 +-
 plugins/destination.go           |  2 +-
 plugins/source.go                |  6 ++-
 plugins/source_testing.go        |  7 ++-
 schema/column.go                 | 38 +++++++-------
 schema/meta.go                   | 60 ----------------------
 schema/resource.go               | 58 +++++++++++-----------
 schema/table.go                  |  8 +++
 specs/destination.go             |  8 +--
 14 files changed, 178 insertions(+), 127 deletions(-)

diff --git a/clients/destination.go b/clients/destination.go
index 3f98a231db..b4df62f6d0 100644
--- a/clients/destination.go
+++ b/clients/destination.go
@@ -73,7 +73,7 @@ func (c *DestinationClient) Migrate(ctx context.Context, tables []*schema.Table)
 	return nil
 }
 
-func (c *DestinationClient) Write(ctx context.Context, resource *schema.Resource) error {
+func (c *DestinationClient) Write(ctx context.Context, table string, data map[string]interface{}) error {
 	// var saveClient pb.Destination_SaveClient
 	// var err error
 	// if c.pbClient != nil {
@@ -83,7 +83,7 @@ func (c *DestinationClient) Write(ctx context.Context, resource *schema.Resource
 	// 	}
 	// }
 	if c.localClient != nil {
-		if err := c.localClient.Write(ctx, resource); err != nil {
+		if err := c.localClient.Write(ctx, table, data); err != nil {
 			return fmt.Errorf("failed to save resources: %w", err)
 		}
 	}
diff --git a/codegen/golang.go b/codegen/golang.go
index 03afcd0767..173ee54346 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -3,11 +3,15 @@ package codegen
 import (
 	"embed"
 	"fmt"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/iancoleman/strcase"
+	"go/ast"
 	"io"
 	"reflect"
+	"strings"
 	"text/template"
+
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/iancoleman/strcase"
+	"golang.org/x/tools/go/packages"
 )
 
 //go:embed templates/*.go.tpl
@@ -58,9 +62,21 @@ func WithNameTransformer(transformer func(string) string) TableOptions {
 	}
 }
 
-func WithSkipFields(fields ...string) TableOptions {
+func WithSkipFields(fields []string) TableOptions {
 	return func(t *TableDefinition) {
-		t.skipFields = append(t.skipFields, fields...)
+		t.skipFields = fields
+	}
+}
+
+func WithOverrideColumns(columns []ColumnDefinition) TableOptions {
+	return func(t *TableDefinition) {
+		t.overrideColumns = columns
+	}
+}
+
+func WithDescriptionsEnabled() TableOptions {
+	return func(t *TableDefinition) {
+		t.descriptionsEnabled = true
 	}
 }
 
@@ -93,11 +109,25 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 	if e.Kind() != reflect.Struct {
 		return nil, fmt.Errorf("expected struct, got %s", e.Kind())
 	}
+
+	comments := make(map[string]string)
+	if t.descriptionsEnabled {
+		comments = readStructComments(e.Type().PkgPath(), e.Type().Name())
+	}
+
 	for i := 0; i < e.NumField(); i++ {
 		field := e.Type().Field(i)
 		if sliceContains(t.skipFields, field.Name) {
 			continue
 		}
+
+		if t.overrideColumns != nil {
+			if col := t.overrideColumns.GetByName(t.nameTransformer(field.Name)); col != nil {
+				t.Columns = append(t.Columns, *col)
+				continue
+			}
+		}
+
 		columnType, err := valueToSchemaType(field.Type)
 		if err != nil {
 			return nil, err
@@ -106,9 +136,10 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 		// generate a PathResolver to use by default
 		pathResolver := fmt.Sprintf("schema.PathResolver(%q)", field.Name)
 		column := ColumnDefinition{
-			Name:     t.nameTransformer(field.Name),
-			Type:     columnType,
-			Resolver: pathResolver,
+			Name:        t.nameTransformer(field.Name),
+			Type:        columnType,
+			Resolver:    pathResolver,
+			Description: strings.ReplaceAll(comments[field.Name], "`", "'"),
 		}
 		t.Columns = append(t.Columns, column)
 	}
@@ -126,3 +157,43 @@ func (t *TableDefinition) GenerateTemplate(wr io.Writer) error {
 	}
 	return nil
 }
+
+// type commentReader struct {
+// 	pkgPath  string
+// 	//comments is a map of type->comment
+// 	comments map[string]string
+// }
+
+// func newCommentsReader() {
+
+// }
+
+func readStructComments(pkgPath string, structName string) map[string]string {
+	cfg := &packages.Config{Mode: packages.NeedFiles | packages.NeedSyntax}
+	pkgs, err := packages.Load(cfg, pkgPath)
+	if err != nil {
+		panic(err)
+	}
+	comments := make(map[string]string, 0)
+	for _, p := range pkgs {
+		for _, f := range p.Syntax {
+			ast.Inspect(f, func(n ast.Node) bool {
+				switch x := n.(type) {
+				case *ast.TypeSpec:
+					if st, ok := x.Type.(*ast.StructType); ok {
+						if x.Name.Name == structName {
+							for _, field := range st.Fields.List {
+								if len(field.Names) > 0 {
+									comments[field.Names[0].Name] = field.Doc.Text()
+								}
+							}
+						}
+					}
+				}
+				return true
+			})
+		}
+
+	}
+	return comments
+}
diff --git a/codegen/golang_test.go b/codegen/golang_test.go
index ed7a87a404..2d7d431f9c 100644
--- a/codegen/golang_test.go
+++ b/codegen/golang_test.go
@@ -3,15 +3,17 @@ package codegen
 import (
 	"bytes"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/cloudquery/plugin-sdk/schema"
 )
 
 type testStruct struct {
+	// IntCol this is an example documentation comment
 	IntCol    int     `json:"int_col,omitempty"`
 	StringCol string  `json:"string_col,omitempty"`
 	FloatCol  float64 `json:"float_col,omitempty"`
@@ -93,3 +95,7 @@ func TestTableFromGoStruct(t *testing.T) {
 	}
 	fmt.Println(buf.String())
 }
+
+// func TestReadComments(t *testing.T) {
+// 	readComments("github.com/google/go-cmp/cmp")
+// }
diff --git a/codegen/table.go b/codegen/table.go
index 0de1aa9d20..16a1df2608 100644
--- a/codegen/table.go
+++ b/codegen/table.go
@@ -22,9 +22,14 @@ type TableDefinition struct {
 	Options              schema.TableCreationOptions
 	nameTransformer      func(string) string
 	skipFields           []string
+	overrideColumns      ColumnDefinitions
+	descriptionsEnabled  bool
 }
 
+type ColumnDefinitions []ColumnDefinition
+
 type ColumnDefinition struct {
+	// Name name of the column
 	Name          string
 	Type          schema.ValueType
 	Resolver      string
@@ -32,3 +37,12 @@ type ColumnDefinition struct {
 	IgnoreInTests bool
 	Options       schema.ColumnCreationOptions
 }
+
+func (c ColumnDefinitions) GetByName(name string) *ColumnDefinition {
+	for _, col := range c {
+		if col.Name == name {
+			return &col
+		}
+	}
+	return nil
+}
diff --git a/codegen/templates/column.go.tpl b/codegen/templates/column.go.tpl
index 50d5f2034f..5f936eaaaf 100644
--- a/codegen/templates/column.go.tpl
+++ b/codegen/templates/column.go.tpl
@@ -4,6 +4,9 @@
   {{- if .Resolver}}
   Resolver:     {{.Resolver}},
   {{- end}}
+  {{- if .Description}}
+  Description:     `{{.Description}}`,
+  {{- end}}
   {{- if .Options.PrimaryKey}}
   CreationOptions: schema.ColumnCreationOptions{
     {{- if .Options.PrimaryKey}}
diff --git a/internal/servers/destinations.go b/internal/servers/destinations.go
index e62c490415..d9ec38a7d5 100644
--- a/internal/servers/destinations.go
+++ b/internal/servers/destinations.go
@@ -46,7 +46,7 @@ func (s *DestinationServer) Write(msg pb.Destination_WriteServer) error {
 		if err := json.Unmarshal(r.Resource, &resource); err != nil {
 			return status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err)
 		}
-		if err := s.Plugin.Write(context.Background(), resource); err != nil {
+		if err := s.Plugin.Write(context.Background(), resource.TableName, resource.Data); err != nil {
 			return fmt.Errorf("write: failed to write resource: %w", err)
 		}
 	}
diff --git a/plugins/destination.go b/plugins/destination.go
index 9949c25b47..65283aaa41 100644
--- a/plugins/destination.go
+++ b/plugins/destination.go
@@ -34,7 +34,7 @@ type DestinationPlugin interface {
 	ExampleConfig() string
 	Initialize(ctx context.Context, spec specs.Destination) error
 	Migrate(ctx context.Context, tables schema.Tables) error
-	Write(ctx context.Context, resources *schema.Resource) error
+	Write(ctx context.Context, table string, data map[string]interface{}) error
 	SetLogger(logger zerolog.Logger)
 }
 
diff --git a/plugins/source.go b/plugins/source.go
index f2cfcf802f..5a163f07d1 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -67,7 +67,11 @@ func WithClassifyError(ignoreError schema.IgnoreErrorFunc) SourceOption {
 // Add internal columns
 func addInternalColumns(tables []*schema.Table) {
 	for _, table := range tables {
-		table.Columns = append(schema.CqColumns, table.Columns...)
+		cqId := schema.CqIdColumn
+		if len(table.PrimaryKeys()) == 0 {
+			cqId.CreationOptions.PrimaryKey = true
+		}
+		table.Columns = append(table.Columns, cqId, schema.CqFetchTime)
 		addInternalColumns(table.Relations)
 	}
 }
diff --git a/plugins/source_testing.go b/plugins/source_testing.go
index b3cad6bc33..74ba12fd0d 100644
--- a/plugins/source_testing.go
+++ b/plugins/source_testing.go
@@ -36,7 +36,6 @@ func init() {
 func TestSourcePluginSync(t *testing.T, plugin *SourcePlugin, spec specs.Source) {
 	t.Parallel()
 	t.Helper()
-
 	// No need for configuration or db connection, get it out of the way first
 	// testTableIdentifiersForProvider(t, resource.Provider)
 
@@ -51,10 +50,14 @@ func TestSourcePluginSync(t *testing.T, plugin *SourcePlugin, spec specs.Source)
 		defer close(resources)
 		fetchErr = plugin.Sync(context.Background(), spec, resources)
 	}()
-
+	totalResources := 0
 	for resource := range resources {
+		totalResources++
 		validateResource(t, resource)
 	}
+	if totalResources == 0 {
+		t.Fatal("no resources fetched")
+	}
 
 	if fetchErr != nil {
 		t.Fatal(fetchErr)
diff --git a/schema/column.go b/schema/column.go
index 69a670c280..e4b0a27c9a 100644
--- a/schema/column.go
+++ b/schema/column.go
@@ -317,27 +317,27 @@ func SetColumnMeta(c Column, m *ColumnMeta) Column {
 }
 
 // Sift gets a column list and returns a list of provider columns, and another list of internal columns, cqId column being the very last one
-func (c ColumnList) Sift() (providerCols ColumnList, internalCols ColumnList) {
-	providerCols, internalCols = make(ColumnList, 0, len(c)), make(ColumnList, 0, len(c))
-	cqIdColIndex := -1
-	for i := range c {
-		if c[i].internal {
-			if c[i].Name == cqIdColumn.Name {
-				cqIdColIndex = len(internalCols)
-			}
+// func (c ColumnList) Sift() (providerCols ColumnList, internalCols ColumnList) {
+// 	providerCols, internalCols = make(ColumnList, 0, len(c)), make(ColumnList, 0, len(c))
+// 	cqIdColIndex := -1
+// 	for i := range c {
+// 		if c[i].internal {
+// 			if c[i].Name == cqIdColumn.Name {
+// 				cqIdColIndex = len(internalCols)
+// 			}
 
-			internalCols = append(internalCols, c[i])
-		} else {
-			providerCols = append(providerCols, c[i])
-		}
-	}
+// 			internalCols = append(internalCols, c[i])
+// 		} else {
+// 			providerCols = append(providerCols, c[i])
+// 		}
+// 	}
 
-	// resolve cqId last, as it would need other PKs to be resolved, some might be internal (cq_fetch_date)
-	if lastIndex := len(internalCols) - 1; cqIdColIndex > -1 && cqIdColIndex != lastIndex {
-		internalCols[cqIdColIndex], internalCols[lastIndex] = internalCols[lastIndex], internalCols[cqIdColIndex]
-	}
-	return providerCols, internalCols
-}
+// 	// resolve cqId last, as it would need other PKs to be resolved, some might be internal (cq_fetch_date)
+// 	if lastIndex := len(internalCols) - 1; cqIdColIndex > -1 && cqIdColIndex != lastIndex {
+// 		internalCols[cqIdColIndex], internalCols[lastIndex] = internalCols[lastIndex], internalCols[cqIdColIndex]
+// 	}
+// 	return providerCols, internalCols
+// }
 
 func (c ColumnList) Names() []string {
 	ret := make([]string, len(c))
diff --git a/schema/meta.go b/schema/meta.go
index cc91268f8c..8ed69590b0 100644
--- a/schema/meta.go
+++ b/schema/meta.go
@@ -24,63 +24,3 @@ var CqColumns = []Column{
 	CqIdColumn,
 	CqFetchTime,
 }
-
-var (
-	// cqMeta = Column{
-	// 	Name:        "cq_meta",
-	// 	Type:        TypeJSON,
-	// 	Description: "Meta column holds fetch information",
-	// 	Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
-	// 		mi := Meta{
-	// 			LastUpdate: time.Now().UTC(),
-	// 		}
-	// 		if val, ok := resource.GetMeta(FetchIdMetaKey); ok {
-	// 			if s, ok := val.(string); ok {
-	// 				mi.FetchId = s
-	// 			}
-	// 		}
-	// 		b, _ := json.Marshal(mi)
-	// 		return resource.Set(c.Name, b)
-	// 	},
-	// 	internal: true,
-	// }
-	cqIdColumn = Column{
-		Name:        "cq_id",
-		Type:        TypeUUID,
-		Description: "Unique CloudQuery Id added to every resource",
-		// Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
-		// 	if err := resource.GenerateCQId(); err != nil {
-		// 		if resource.Parent == nil {
-		// 			return err
-		// 		}
-
-		// 		meta.Logger().Debug("one of the table pk is nil", "table", resource.table.Name)
-		// 	}
-		// 	return resource.Set(c.Name, resource.Id())
-		// },
-		CreationOptions: ColumnCreationOptions{
-			Unique:  true,
-			NotNull: true,
-		},
-		internal: true,
-	}
-	// cqFetchDateColumn = Column{
-	// 	Name:        "cq_fetch_date",
-	// 	Type:        TypeTimestamp,
-	// 	Description: "Time of fetch for this resource",
-	// 	// Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
-	// 	// 	val, ok := resource.GetMeta("cq_fetch_date")
-	// 	// 	if !ok && !resource.executionStart.IsZero() {
-	// 	// 		val = resource.executionStart
-	// 	// 	}
-	// 	// 	if val == nil {
-	// 	// 		return fmt.Errorf("zero cq_fetch date")
-	// 	// 	}
-	// 	// 	return resource.Set(c.Name, val)
-	// 	// },
-	// 	CreationOptions: ColumnCreationOptions{
-	// 		NotNull: true,
-	// 	},
-	// 	internal: true,
-	// }
-)
diff --git a/schema/resource.go b/schema/resource.go
index bf0857edab..92f7f6df6d 100644
--- a/schema/resource.go
+++ b/schema/resource.go
@@ -1,6 +1,8 @@
 package schema
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -34,32 +36,32 @@ func NewResourceData(t *Table, parent *Resource, fetchTime time.Time, item inter
 	return &r
 }
 
-// func (r *Resource) PrimaryKeyValues() []string {
-// 	tablePrimKeys := r.dialect.PrimaryKeys(r.table)
-// 	if len(tablePrimKeys) == 0 {
-// 		return []string{}
-// 	}
-// 	results := make([]string, 0)
-// 	for _, primKey := range tablePrimKeys {
-// 		data := r.Get(primKey)
-// 		if data == nil {
-// 			continue
-// 		}
-// 		// we can have more types, but PKs are usually either ints, strings or a structure
-// 		// hopefully supporting Stringer interface, otherwise we fallback
-// 		switch v := data.(type) {
-// 		case fmt.Stringer:
-// 			results = append(results, v.String())
-// 		case *string:
-// 			results = append(results, *v)
-// 		case *int:
-// 			results = append(results, fmt.Sprintf("%d", *v))
-// 		default:
-// 			results = append(results, fmt.Sprintf("%v", v))
-// 		}
-// 	}
-// 	return results
-// }
+func (r *Resource) PrimaryKeyValue() string {
+	pks := r.Table.PrimaryKeys()
+	if len(pks) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	for _, primKey := range pks {
+		data := r.Get(primKey)
+		if data == nil {
+			continue
+		}
+		// we can have more types, but PKs are usually either ints, strings or a structure
+		// hopefully supporting Stringer interface, otherwise we fallback
+		switch v := data.(type) {
+		case fmt.Stringer:
+			sb.WriteString(v.String())
+		case *string:
+			sb.WriteString(*v)
+		case *int:
+			sb.WriteString(fmt.Sprintf("%d", *v))
+		default:
+			sb.WriteString(fmt.Sprintf("%d", v))
+		}
+	}
+	return sb.String()
+}
 
 func (r *Resource) Get(key string) interface{} {
 	return r.Data[key]
@@ -71,10 +73,10 @@ func (r *Resource) Set(key string, value interface{}) error {
 }
 
 func (r *Resource) Id() uuid.UUID {
-	if r.Data[cqIdColumn.Name] == nil {
+	if r.Data[CqIdColumn.Name] == nil {
 		return uuid.UUID{}
 	}
-	return r.Data[cqIdColumn.Name].(uuid.UUID)
+	return r.Data[CqIdColumn.Name].(uuid.UUID)
 }
 
 func (r *Resource) Columns() []string {
diff --git a/schema/table.go b/schema/table.go
index cdf64e16d3..a539d15592 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -151,6 +151,9 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, fetchTime time.Time
 		meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Msg("table resolver finished successfully")
 	}()
 	totalResources := 0
+	// we want to check for data integirty
+	// in the future we can do that as an optinoal feature via a flag
+	pks := map[string]bool{}
 	// each result is an array of interface{}
 	for elem := range res {
 		objects := helpers.InterfaceSlice(elem)
@@ -169,6 +172,11 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, fetchTime time.Time
 					meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver finished successfully")
 				}
 			}
+			if pks[resource.PrimaryKeyValue()] {
+				meta.Logger().Error().Str("table_name", t.Name).Str("primary_key", resource.PrimaryKeyValue()).Msg("duplicate primary key found")
+			} else {
+				pks[resource.PrimaryKeyValue()] = true
+			}
 			resolvedResources <- resource
 			for _, rel := range t.Relations {
 				totalResources += rel.Resolve(ctx, meta, fetchTime, resource, resolvedResources)
diff --git a/specs/destination.go b/specs/destination.go
index a9d544cd78..b4d7b4ed53 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -10,7 +10,7 @@ import (
 type WriteMode int
 
 const (
-	WriteModeAppendOnly WriteMode = iota
+	WriteModeAppend WriteMode = iota
 	WriteModeOverwrite
 )
 
@@ -50,7 +50,7 @@ func (s *Destination) UnmarshalSpec(out interface{}) error {
 }
 
 func (m WriteMode) String() string {
-	return [...]string{"append-only", "overwrite"}[m]
+	return [...]string{"append", "overwrite"}[m]
 }
 
 func (m WriteMode) MarshalJSON() ([]byte, error) {
@@ -73,8 +73,8 @@ func (m *WriteMode) UnmarshalJSON(data []byte) (err error) {
 
 func WriteModeFromString(s string) (WriteMode, error) {
 	switch s {
-	case "append-only":
-		return WriteModeAppendOnly, nil
+	case "append":
+		return WriteModeAppend, nil
 	case "overwrite":
 		return WriteModeOverwrite, nil
 	}

From 83afae1c76b28eb314c309455af4a720e026c23b Mon Sep 17 00:00:00 2001
From: Herman Schaaf <hermanschaaf@gmail.com>
Date: Mon, 29 Aug 2022 11:54:51 +0200
Subject: [PATCH 21/25] Add table descriptions (#17)

---
 codegen/templates/table.go.tpl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/codegen/templates/table.go.tpl b/codegen/templates/table.go.tpl
index cfc52743f8..48312412a2 100644
--- a/codegen/templates/table.go.tpl
+++ b/codegen/templates/table.go.tpl
@@ -1,5 +1,8 @@
 {
 		Name:         "{{.Name}}",
+		{{- if .Description}}
+        Description:     "{{.Description}}",
+    {{- end}}
 		{{- if .Resolver}}
     Resolver:     {{.Resolver}},
     {{- end}}

From acd6f2283fa2cfd3919639bfed78fb34eb68c3b1 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 30 Aug 2022 09:53:53 +0300
Subject: [PATCH 22/25] detect duplicate columns on start

---
 codegen/golang.go | 26 ++++++++++++++++----------
 codegen/table.go  |  1 +
 plugins/source.go | 17 ++++++++++++++---
 schema/table.go   | 26 ++++++++++++++++++++++++++
 4 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/codegen/golang.go b/codegen/golang.go
index 173ee54346..01db7e53f4 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -74,6 +74,12 @@ func WithOverrideColumns(columns []ColumnDefinition) TableOptions {
 	}
 }
 
+func WithExtraColumns(columns []ColumnDefinition) TableOptions {
+	return func(t *TableDefinition) {
+		t.extraColumns = columns
+	}
+}
+
 func WithDescriptionsEnabled() TableOptions {
 	return func(t *TableDefinition) {
 		t.descriptionsEnabled = true
@@ -115,6 +121,16 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 		comments = readStructComments(e.Type().PkgPath(), e.Type().Name())
 	}
 
+	for _, c := range t.extraColumns {
+		if t.overrideColumns != nil {
+			if col := t.overrideColumns.GetByName(c.Name); col != nil {
+				t.Columns = append(t.Columns, *col)
+				continue
+			}
+		}
+		t.Columns = append(t.Columns, c)
+	}
+
 	for i := 0; i < e.NumField(); i++ {
 		field := e.Type().Field(i)
 		if sliceContains(t.skipFields, field.Name) {
@@ -158,16 +174,6 @@ func (t *TableDefinition) GenerateTemplate(wr io.Writer) error {
 	return nil
 }
 
-// type commentReader struct {
-// 	pkgPath  string
-// 	//comments is a map of type->comment
-// 	comments map[string]string
-// }
-
-// func newCommentsReader() {
-
-// }
-
 func readStructComments(pkgPath string, structName string) map[string]string {
 	cfg := &packages.Config{Mode: packages.NeedFiles | packages.NeedSyntax}
 	pkgs, err := packages.Load(cfg, pkgPath)
diff --git a/codegen/table.go b/codegen/table.go
index 16a1df2608..9bf185d782 100644
--- a/codegen/table.go
+++ b/codegen/table.go
@@ -23,6 +23,7 @@ type TableDefinition struct {
 	nameTransformer      func(string) string
 	skipFields           []string
 	overrideColumns      ColumnDefinitions
+	extraColumns         ColumnDefinitions
 	descriptionsEnabled  bool
 }
 
diff --git a/plugins/source.go b/plugins/source.go
index 5a163f07d1..87bc7a1e2e 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -83,17 +83,28 @@ func NewSourcePlugin(name string, version string, tables []*schema.Table, newExe
 		tables:             tables,
 		newExecutionClient: newExecutionClient,
 	}
-	if newExecutionClient == nil {
-		panic("newExecutionClient function not defined for source plugin:" + name)
-	}
 	for _, opt := range opts {
 		opt(&p)
 	}
 	addInternalColumns(p.tables)
+	if err := p.validate(); err != nil {
+		panic(err)
+	}
 	// add default columns to tables
 	return &p
 }
 
+func (p *SourcePlugin) validate() error {
+	if p.newExecutionClient == nil {
+		return fmt.Errorf("newExecutionClient function not defined for source plugin:" + p.name)
+	}
+
+	if err := p.tables.ValidateDuplicateColumns(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (p *SourcePlugin) Tables() schema.Tables {
 	return p.tables
 }
diff --git a/schema/table.go b/schema/table.go
index a539d15592..70f1c0dc13 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -2,6 +2,7 @@ package schema
 
 import (
 	"context"
+	"fmt"
 	"runtime/debug"
 	"sync"
 	"time"
@@ -77,6 +78,31 @@ func (tt Tables) TableNames() []string {
 	return ret
 }
 
+func (tt Tables) ValidateDuplicateColumns() error {
+	for _, t := range tt {
+		if err := t.ValidateDuplicateColumns(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t Table) ValidateDuplicateColumns() error {
+	columns := make(map[string]bool, len(t.Columns))
+	for _, c := range t.Columns {
+		if _, ok := columns[c.Name]; ok {
+			return fmt.Errorf("duplicate column %s in table %s", c.Name, t.Name)
+		}
+		columns[c.Name] = true
+	}
+	for _, rel := range t.Relations {
+		if err := rel.ValidateDuplicateColumns(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t Table) Column(name string) *Column {
 	for _, c := range t.Columns {
 		if c.Name == name {

From 36663779eb3860bcc0b1a07e6a3538ea4dd4bb41 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 30 Aug 2022 10:36:37 +0300
Subject: [PATCH 23/25] fixed golang-lint

---
 codegen/golang.go      | 18 ++++++--------
 plugins/source.go      | 10 +++-----
 plugins/source_test.go | 14 +++++------
 plugins/template.go    | 17 -------------
 schema/table.go        |  2 +-
 serve/serve_test.go    | 56 +++++++++++++++++++-----------------------
 specs/destination.go   |  4 +--
 specs/source.go        |  2 +-
 specs/spec.go          | 14 +++++------
 specs/spec_reader.go   |  1 -
 10 files changed, 53 insertions(+), 85 deletions(-)
 delete mode 100644 plugins/template.go

diff --git a/codegen/golang.go b/codegen/golang.go
index 01db7e53f4..1432fd7243 100644
--- a/codegen/golang.go
+++ b/codegen/golang.go
@@ -14,6 +14,8 @@ import (
 	"golang.org/x/tools/go/packages"
 )
 
+type TableOptions func(*TableDefinition)
+
 //go:embed templates/*.go.tpl
 var TemplatesFS embed.FS
 
@@ -54,8 +56,6 @@ func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
 	}
 }
 
-type TableOptions func(*TableDefinition)
-
 func WithNameTransformer(transformer func(string) string) TableOptions {
 	return func(t *TableDefinition) {
 		t.Name = transformer(t.Name)
@@ -184,14 +184,11 @@ func readStructComments(pkgPath string, structName string) map[string]string {
 	for _, p := range pkgs {
 		for _, f := range p.Syntax {
 			ast.Inspect(f, func(n ast.Node) bool {
-				switch x := n.(type) {
-				case *ast.TypeSpec:
-					if st, ok := x.Type.(*ast.StructType); ok {
-						if x.Name.Name == structName {
-							for _, field := range st.Fields.List {
-								if len(field.Names) > 0 {
-									comments[field.Names[0].Name] = field.Doc.Text()
-								}
+				if st, ok := n.(*ast.TypeSpec); ok {
+					if st.Name.Name == structName {
+						for _, field := range st.Type.(*ast.StructType).Fields.List {
+							if len(field.Names) > 0 {
+								comments[field.Names[0].Name] = field.Doc.Text()
 							}
 						}
 					}
@@ -199,7 +196,6 @@ func readStructComments(pkgPath string, structName string) map[string]string {
 				return true
 			})
 		}
-
 	}
 	return comments
 }
diff --git a/plugins/source.go b/plugins/source.go
index 128b1f110b..1be134ad01 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -2,7 +2,6 @@ package plugins
 
 import (
 	"context"
-	_ "embed"
 	"fmt"
 	"sync"
 	"time"
@@ -22,6 +21,8 @@ const ExampleSourceConfig = `
 # skip_tables: []
 `
 
+const minGoRoutines = 5
+
 type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error)
 
 // SourcePlugin is the base structure required to pass to sdk.serve
@@ -98,10 +99,7 @@ func (p *SourcePlugin) validate() error {
 		return fmt.Errorf("newExecutionClient function not defined for source plugin:" + p.name)
 	}
 
-	if err := p.tables.ValidateDuplicateColumns(); err != nil {
-		return err
-	}
-	return nil
+	return p.tables.ValidateDuplicateColumns()
 }
 
 func (p *SourcePlugin) Tables() schema.Tables {
@@ -124,8 +122,6 @@ func (p *SourcePlugin) SetLogger(log zerolog.Logger) {
 	p.logger = log
 }
 
-const minGoRoutines = 5
-
 // Sync data from source to the given channel
 func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error {
 	c, err := p.newExecutionClient(ctx, p, spec)
diff --git a/plugins/source_test.go b/plugins/source_test.go
index 6af9fba097..8fc34c5acf 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -22,14 +22,14 @@ type Account struct {
 	Regions []string `json:"regions,omitempty"`
 }
 
-type testSourceSpec struct {
-	Accounts []Account `json:"accounts,omitempty"`
-	Regions  []string  `json:"regions,omitempty"`
-}
+// type testSourceSpec struct {
+// 	Accounts []Account `json:"accounts,omitempty"`
+// 	Regions  []string  `json:"regions,omitempty"`
+// }
 
-func newTestSourceSpec() interface{} {
-	return &testSourceSpec{}
-}
+// func newTestSourceSpec() interface{} {
+// 	return &testSourceSpec{}
+// }
 
 func testTable() *schema.Table {
 	return &schema.Table{
diff --git a/plugins/template.go b/plugins/template.go
deleted file mode 100644
index abc7829324..0000000000
--- a/plugins/template.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package plugins
-
-import (
-	"strings"
-	"text/template"
-)
-
-func templateFuncMap() template.FuncMap {
-	return template.FuncMap{
-		"indent": indent,
-	}
-}
-
-func indent(spaces int, v string) string {
-	pad := strings.Repeat(" ", spaces)
-	return pad + strings.ReplaceAll(v, "\n", "\n"+pad)
-}
diff --git a/schema/table.go b/schema/table.go
index 70f1c0dc13..a407808644 100644
--- a/schema/table.go
+++ b/schema/table.go
@@ -177,7 +177,7 @@ func (t Table) Resolve(ctx context.Context, meta ClientMeta, fetchTime time.Time
 		meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Msg("table resolver finished successfully")
 	}()
 	totalResources := 0
-	// we want to check for data integirty
+	// we want to check for data integrity
 	// in the future we can do that as an optinoal feature via a flag
 	pks := map[string]bool{}
 	// each result is an array of interface{}
diff --git a/serve/serve_test.go b/serve/serve_test.go
index 4462a0fba0..5cad8a631a 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -19,8 +19,12 @@ import (
 
 var _ schema.ClientMeta = &testExecutionClient{}
 
-type testSourceSpec struct {
-	Accounts []string `json:"accounts,omitempty"`
+type TestSourcePluginSpec struct {
+	Accounts []string `json:"accounts,omitempty" yaml:"accounts,omitempty"`
+}
+
+type testExecutionClient struct {
+	logger zerolog.Logger
 }
 
 var expectedExampleSpecConfig = specs.Spec{
@@ -34,16 +38,6 @@ var expectedExampleSpecConfig = specs.Spec{
 	},
 }
 
-func newTestSourceSpec() interface{} {
-	return &testSourceSpec{
-		Accounts: []string{"all"},
-	}
-}
-
-type testExecutionClient struct {
-	logger zerolog.Logger
-}
-
 func testTable() *schema.Table {
 	return &schema.Table{
 		Name: "test_table",
@@ -62,10 +56,6 @@ func testTable() *schema.Table {
 	}
 }
 
-type TestSourcePluginSpec struct {
-	Accounts []string `json:"accounts,omitempty" yaml:"accounts,omitempty"`
-}
-
 func (c *testExecutionClient) Logger() *zerolog.Logger {
 	return &c.logger
 }
@@ -75,20 +65,20 @@ func newTestExecutionClient(context.Context, *plugins.SourcePlugin, specs.Source
 }
 
 // https://stackoverflow.com/questions/32840687/timeout-for-waitgroup-wait
-func waitTimeout(wg *errgroup.Group, timeout time.Duration) (bool, error) {
-	c := make(chan struct{})
-	var err error
-	go func() {
-		defer close(c)
-		err = wg.Wait()
-	}()
-	select {
-	case <-c:
-		return false, err // completed normally
-	case <-time.After(timeout):
-		return true, err // timed out
-	}
-}
+// func waitTimeout(wg *errgroup.Group, timeout time.Duration) (bool, error) {
+// 	c := make(chan struct{})
+// 	var err error
+// 	go func() {
+// 		defer close(c)
+// 		err = wg.Wait()
+// 	}()
+// 	select {
+// 	case <-c:
+// 		return false, err // completed normally
+// 	case <-time.After(timeout):
+// 		return true, err // timed out
+// 	}
+// }
 
 func bufDialer(context.Context, string) (net.Conn, error) {
 	return testListener.Dial()
@@ -119,8 +109,9 @@ spec:
 	})
 	cmd.SetArgs([]string{"serve", "--network", "test"})
 
+	var serveErr error
 	go func() {
-		cmd.Execute()
+		serveErr = cmd.Execute()
 	}()
 
 	// wait for the server to start
@@ -130,6 +121,9 @@ spec:
 		}
 		t.Log("waiting for grpc server to start")
 		time.Sleep(time.Millisecond * 200)
+		if serveErr != nil {
+			t.Fatal(serveErr)
+		}
 	}
 
 	// https://stackoverflow.com/questions/42102496/testing-a-grpc-service
diff --git a/specs/destination.go b/specs/destination.go
index b4d7b4ed53..664776de71 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -38,8 +38,8 @@ func (d *Destination) SetDefaults() {
 	}
 }
 
-func (s *Destination) UnmarshalSpec(out interface{}) error {
-	b, err := json.Marshal(s.Spec)
+func (d *Destination) UnmarshalSpec(out interface{}) error {
+	b, err := json.Marshal(d.Spec)
 	if err != nil {
 		return err
 	}
diff --git a/specs/source.go b/specs/source.go
index 45c8d958f4..4cd2ba92b4 100644
--- a/specs/source.go
+++ b/specs/source.go
@@ -48,6 +48,6 @@ func (s *Source) UnmarshalSpec(out interface{}) error {
 	return json.Unmarshal(b, out)
 }
 
-func (s *Source) Validate() (*gojsonschema.Result, error) {
+func (*Source) Validate() (*gojsonschema.Result, error) {
 	return nil, nil
 }
diff --git a/specs/spec.go b/specs/spec.go
index 30cc749900..9129cecb77 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -8,13 +8,18 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-type Kind int
-
 const (
 	KindSource Kind = iota
 	KindDestination
 )
 
+type Kind int
+
+type Spec struct {
+	Kind Kind        `json:"kind"`
+	Spec interface{} `json:"spec"`
+}
+
 func (k Kind) String() string {
 	return [...]string{"source", "destination"}[k]
 }
@@ -48,11 +53,6 @@ func KindFromString(s string) (Kind, error) {
 	}
 }
 
-type Spec struct {
-	Kind Kind        `json:"kind"`
-	Spec interface{} `json:"spec"`
-}
-
 func (s *Spec) UnmarshalJSON(data []byte) error {
 	var t struct {
 		Kind Kind        `json:"kind"`
diff --git a/specs/spec_reader.go b/specs/spec_reader.go
index 4b80e84488..40478d0e2f 100644
--- a/specs/spec_reader.go
+++ b/specs/spec_reader.go
@@ -30,7 +30,6 @@ func NewSpecReader(directory string) (*SpecReader, error) {
 				return nil, fmt.Errorf("failed to read file %s: %w", file.Name(), err)
 			}
 			var s Spec
-			SpecUnmarshalYamlStrict(data, &s)
 			if err := SpecUnmarshalYamlStrict(data, &s); err != nil {
 				return nil, fmt.Errorf("failed to unmarshal file %s: %w", file.Name(), err)
 			}

From a8503f21d25f69707d21b7ffc0d6f6dda3893904 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 30 Aug 2022 10:41:07 +0300
Subject: [PATCH 24/25] run gci

---
 codegen/golang_test.go             | 3 +--
 internal/pb/base.pb.go             | 5 +++--
 internal/pb/destination.pb.go      | 5 +++--
 internal/pb/destination_grpc.pb.go | 1 +
 internal/pb/source.pb.go           | 5 +++--
 internal/pb/source_grpc.pb.go      | 1 +
 6 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/codegen/golang_test.go b/codegen/golang_test.go
index 2d7d431f9c..4a728eb7cb 100644
--- a/codegen/golang_test.go
+++ b/codegen/golang_test.go
@@ -6,10 +6,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-
-	"github.com/cloudquery/plugin-sdk/schema"
 )
 
 type testStruct struct {
diff --git a/internal/pb/base.pb.go b/internal/pb/base.pb.go
index a0aa659dd2..f3f0f253f7 100644
--- a/internal/pb/base.pb.go
+++ b/internal/pb/base.pb.go
@@ -7,10 +7,11 @@
 package pb
 
 import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 )
 
 const (
diff --git a/internal/pb/destination.pb.go b/internal/pb/destination.pb.go
index 0c76315f5e..2e004a4744 100644
--- a/internal/pb/destination.pb.go
+++ b/internal/pb/destination.pb.go
@@ -7,10 +7,11 @@
 package pb
 
 import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 )
 
 const (
diff --git a/internal/pb/destination_grpc.pb.go b/internal/pb/destination_grpc.pb.go
index 7e2b94d44e..244fd1ec61 100644
--- a/internal/pb/destination_grpc.pb.go
+++ b/internal/pb/destination_grpc.pb.go
@@ -8,6 +8,7 @@ package pb
 
 import (
 	context "context"
+
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
diff --git a/internal/pb/source.pb.go b/internal/pb/source.pb.go
index fa25af4285..00b71e7fc8 100644
--- a/internal/pb/source.pb.go
+++ b/internal/pb/source.pb.go
@@ -7,10 +7,11 @@
 package pb
 
 import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	reflect "reflect"
 	sync "sync"
+
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 )
 
 const (
diff --git a/internal/pb/source_grpc.pb.go b/internal/pb/source_grpc.pb.go
index 06c3185e59..07d4670883 100644
--- a/internal/pb/source_grpc.pb.go
+++ b/internal/pb/source_grpc.pb.go
@@ -8,6 +8,7 @@ package pb
 
 import (
 	context "context"
+
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"

From 5af7ec6065399ac187ad762276cf4b9a5ca2ddc3 Mon Sep 17 00:00:00 2001
From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com>
Date: Tue, 30 Aug 2022 11:16:57 +0300
Subject: [PATCH 25/25] more linting

---
 go.mod                 |  1 +
 go.sum                 |  2 ++
 plugins/source.go      | 22 +++++++++++-----------
 plugins/source_test.go |  4 ++--
 serve/serve.go         | 12 ++++++------
 serve/serve_test.go    |  4 ++--
 specs/destination.go   | 10 +++++-----
 specs/spec.go          | 10 +++++-----
 8 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/go.mod b/go.mod
index 221ff4861e..02c6e7e0a7 100644
--- a/go.mod
+++ b/go.mod
@@ -53,6 +53,7 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	gitlab.com/bosi/decorder v0.2.3 // indirect
 	golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
diff --git a/go.sum b/go.sum
index 325c7656db..3fcd7b7bac 100644
--- a/go.sum
+++ b/go.sum
@@ -271,6 +271,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+gitlab.com/bosi/decorder v0.2.3 h1:gX4/RgK16ijY8V+BRQHAySfQAb354T7/xQpDB2n10P0=
+gitlab.com/bosi/decorder v0.2.3/go.mod h1:9K1RB5+VPNQYtXtTDAzd2OEftsZb1oV0IrJrzChSdGE=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
diff --git a/plugins/source.go b/plugins/source.go
index 1be134ad01..72c991fe6c 100644
--- a/plugins/source.go
+++ b/plugins/source.go
@@ -12,17 +12,6 @@ import (
 	"github.com/thoas/go-funk"
 )
 
-const ExampleSourceConfig = `
-# max_goroutines to use when fetching. 0 means default and calculated by CloudQuery
-# max_goroutines: 0
-# By default CloudQuery will fetch all tables in the source plugin
-# tables: ["*"]
-# skip_tables specify which tables to skip. especially useful when using "*" for tables
-# skip_tables: []
-`
-
-const minGoRoutines = 5
-
 type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error)
 
 // SourcePlugin is the base structure required to pass to sdk.serve
@@ -46,6 +35,17 @@ type SourcePlugin struct {
 
 type SourceOption func(*SourcePlugin)
 
+const ExampleSourceConfig = `
+# max_goroutines to use when fetching. 0 means default and calculated by CloudQuery
+# max_goroutines: 0
+# By default CloudQuery will fetch all tables in the source plugin
+# tables: ["*"]
+# skip_tables specify which tables to skip. especially useful when using "*" for tables
+# skip_tables: []
+`
+
+const minGoRoutines = 5
+
 func WithSourceExampleConfig(exampleConfig string) SourceOption {
 	return func(p *SourcePlugin) {
 		p.exampleConfig = exampleConfig
diff --git a/plugins/source_test.go b/plugins/source_test.go
index 8fc34c5acf..06b1743348 100644
--- a/plugins/source_test.go
+++ b/plugins/source_test.go
@@ -11,8 +11,6 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var _ schema.ClientMeta = &testExecutionClient{}
-
 type testExecutionClient struct {
 	logger zerolog.Logger
 }
@@ -31,6 +29,8 @@ type Account struct {
 // 	return &testSourceSpec{}
 // }
 
+var _ schema.ClientMeta = &testExecutionClient{}
+
 func testTable() *schema.Table {
 	return &schema.Table{
 		Name: "testTable",
diff --git a/serve/serve.go b/serve/serve.go
index 9611cbdf24..cee4662a6b 100644
--- a/serve/serve.go
+++ b/serve/serve.go
@@ -19,22 +19,22 @@ import (
 	"google.golang.org/grpc/test/bufconn"
 )
 
-// bufSize used for unit testing grpc server and client
-const testBufSize = 1024 * 1024
-
-// lis used for unit testing grpc server and client
-var testListener *bufconn.Listener
-
 type Options struct {
 	// Required: Source or destination plugin to serve.
 	SourcePlugin      *plugins.SourcePlugin
 	DestinationPlugin plugins.DestinationPlugin
 }
 
+// bufSize used for unit testing grpc server and client
+const testBufSize = 1024 * 1024
+
 const (
 	serveShort = `Start plugin server`
 )
 
+// lis used for unit testing grpc server and client
+var testListener *bufconn.Listener
+
 func newCmdServe(opts Options) *cobra.Command {
 	var address string
 	var network string
diff --git a/serve/serve_test.go b/serve/serve_test.go
index 5cad8a631a..b4197ff8a7 100644
--- a/serve/serve_test.go
+++ b/serve/serve_test.go
@@ -17,8 +17,6 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-var _ schema.ClientMeta = &testExecutionClient{}
-
 type TestSourcePluginSpec struct {
 	Accounts []string `json:"accounts,omitempty" yaml:"accounts,omitempty"`
 }
@@ -27,6 +25,8 @@ type testExecutionClient struct {
 	logger zerolog.Logger
 }
 
+var _ schema.ClientMeta = &testExecutionClient{}
+
 var expectedExampleSpecConfig = specs.Spec{
 	Kind: specs.KindSource,
 	Spec: &specs.Source{
diff --git a/specs/destination.go b/specs/destination.go
index 664776de71..8d5abc7d80 100644
--- a/specs/destination.go
+++ b/specs/destination.go
@@ -9,11 +9,6 @@ import (
 
 type WriteMode int
 
-const (
-	WriteModeAppend WriteMode = iota
-	WriteModeOverwrite
-)
-
 type Destination struct {
 	Name      string      `json:"name,omitempty"`
 	Version   string      `json:"version,omitempty"`
@@ -23,6 +18,11 @@ type Destination struct {
 	Spec      interface{} `json:"spec,omitempty"`
 }
 
+const (
+	WriteModeAppend WriteMode = iota
+	WriteModeOverwrite
+)
+
 func (d *Destination) SetDefaults() {
 	if d.Registry.String() == "" {
 		d.Registry = RegistryGithub
diff --git a/specs/spec.go b/specs/spec.go
index 9129cecb77..ec4ccbe40f 100644
--- a/specs/spec.go
+++ b/specs/spec.go
@@ -8,11 +8,6 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-const (
-	KindSource Kind = iota
-	KindDestination
-)
-
 type Kind int
 
 type Spec struct {
@@ -20,6 +15,11 @@ type Spec struct {
 	Spec interface{} `json:"spec"`
 }
 
+const (
+	KindSource Kind = iota
+	KindDestination
+)
+
 func (k Kind) String() string {
 	return [...]string{"source", "destination"}[k]
 }