feat(fastly)!: Migrate to Arrow native SDK (#11018)

Closes [#10751](#10741) BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#11018) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
cloudquery · May 26, 2023 · f558615 · f558615
1 parent 13df940
commit f558615
Show file tree

Hide file tree

Showing 48 changed files with 429 additions and 454 deletions.
diff --git a/plugins/source/fastly/client/client.go b/plugins/source/fastly/client/client.go
@@ -7,8 +7,8 @@ import (
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
 	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/rs/zerolog"
 	"github.com/thoas/go-funk"

diff --git a/plugins/source/fastly/client/multiplexers.go b/plugins/source/fastly/client/multiplexers.go
@@ -1,7 +1,7 @@
 package client
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 )
 
 func ServiceRegionMultiplex(meta schema.ClientMeta) []schema.ClientMeta {

diff --git a/plugins/source/fastly/client/resolvers.go b/plugins/source/fastly/client/resolvers.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/thoas/go-funk"
 )
 

diff --git a/plugins/source/fastly/client/testing.go b/plugins/source/fastly/client/testing.go
@@ -8,8 +8,8 @@ import (
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
 	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/golang/mock/gomock"
 	"github.com/rs/zerolog"

diff --git a/plugins/source/fastly/go.mod b/plugins/source/fastly/go.mod
@@ -3,9 +3,10 @@ module github.com/cloudquery/cloudquery/plugins/source/fastly
 go 1.19
 
 require (
+	github.com/apache/arrow/go/v13 v13.0.0-20230526080908-77a71305090f
 	github.com/cloudquery/codegen v0.2.1
 	github.com/cloudquery/plugin-pb-go v1.0.8
-	github.com/cloudquery/plugin-sdk/v2 v2.7.0
+	github.com/cloudquery/plugin-sdk/v3 v3.6.4
 	github.com/fastly/go-fastly/v7 v7.0.0
 	github.com/golang/mock v1.6.0
 	github.com/rs/zerolog v1.29.0
@@ -17,9 +18,9 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
 	github.com/apache/thrift v0.16.0 // indirect
 	github.com/cloudquery/plugin-sdk v1.45.0 // indirect
+	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.20.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
@@ -43,6 +44,7 @@ require (
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/peterhellberg/link v1.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.6.1 // indirect

diff --git a/plugins/source/fastly/go.sum b/plugins/source/fastly/go.sum
@@ -54,6 +54,8 @@ github.com/cloudquery/plugin-sdk v1.45.0 h1:5vrfQZtaO1dp6ebKt8ouXDmPC7eeLuOB3JMd
 github.com/cloudquery/plugin-sdk v1.45.0/go.mod h1:9KGuuTGjTCKgh9amKwS+7Zrrqq7/M6lormteOyqoKwg=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
+github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
+github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -196,6 +198,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc=
 github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

diff --git a/plugins/source/fastly/main.go b/plugins/source/fastly/main.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/resources/plugin"
-	"github.com/cloudquery/plugin-sdk/v2/serve"
+	"github.com/cloudquery/plugin-sdk/v3/serve"
 )
 
 const sentryDSN = "https://c4e711b87a8846de90e5ba2c785fb901@o1396617.ingest.sentry.io/4504379395604480"

diff --git a/plugins/source/fastly/resources/plugin/plugin.go b/plugins/source/fastly/resources/plugin/plugin.go
@@ -2,7 +2,7 @@ package plugin
 
 import (
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
 )
 
 var (

diff --git a/plugins/source/fastly/resources/plugin/tables.go b/plugins/source/fastly/resources/plugin/tables.go
diff --git a/plugins/source/fastly/resources/services/account/account_events.go b/plugins/source/fastly/resources/services/account/account_events.go
@@ -1,8 +1,9 @@
 package account
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 
@@ -14,16 +15,14 @@ func AccountEvents() *schema.Table {
 		Transform:   transformers.TransformWithStruct(&fastly.Event{}),
 		Columns: []schema.Column{
 			{
-				Name:     "id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("ID"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("ID"),
+				PrimaryKey: true,
 			},
 			{
 				Name:     "ip",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: schema.PathResolver("IP"),
 			},
 		},

diff --git a/plugins/source/fastly/resources/services/account/account_events_fetch.go b/plugins/source/fastly/resources/services/account/account_events_fetch.go
@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 

diff --git a/plugins/source/fastly/resources/services/account/account_events_mock_test.go b/plugins/source/fastly/resources/services/account/account_events_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/golang/mock/gomock"
 )

diff --git a/plugins/source/fastly/resources/services/account/account_users.go b/plugins/source/fastly/resources/services/account/account_users.go
@@ -1,8 +1,9 @@
 package account
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 
@@ -14,12 +15,10 @@ func AccountUsers() *schema.Table {
 		Transform:   transformers.TransformWithStruct(&fastly.User{}),
 		Columns: []schema.Column{
 			{
-				Name:     "id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("ID"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("ID"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/fastly/resources/services/account/account_users_fetch.go b/plugins/source/fastly/resources/services/account/account_users_fetch.go
@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 

diff --git a/plugins/source/fastly/resources/services/account/account_users_mock_test.go b/plugins/source/fastly/resources/services/account/account_users_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/golang/mock/gomock"
 )

diff --git a/plugins/source/fastly/resources/services/auth/auth_tokens.go b/plugins/source/fastly/resources/services/auth/auth_tokens.go
@@ -1,8 +1,9 @@
 package auth
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 
@@ -14,16 +15,14 @@ func AuthTokens() *schema.Table {
 		Transform:   transformers.TransformWithStruct(&fastly.Token{}),
 		Columns: []schema.Column{
 			{
-				Name:     "id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("ID"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("ID"),
+				PrimaryKey: true,
 			},
 			{
 				Name:     "ip",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: schema.PathResolver("IP"),
 			},
 		},

diff --git a/plugins/source/fastly/resources/services/auth/auth_tokens_fetch.go b/plugins/source/fastly/resources/services/auth/auth_tokens_fetch.go
@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 )
 
 func fetchAuthTokens(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {

diff --git a/plugins/source/fastly/resources/services/auth/auth_tokens_mock_test.go b/plugins/source/fastly/resources/services/auth/auth_tokens_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/golang/mock/gomock"
 )

diff --git a/plugins/source/fastly/resources/services/services/service_backends.go b/plugins/source/fastly/resources/services/services/service_backends.go
@@ -1,8 +1,9 @@
 package services
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 
@@ -14,33 +15,27 @@ func ServiceBackends() *schema.Table {
 		Transform:   transformers.TransformWithStruct(&fastly.Backend{}),
 		Columns: []schema.Column{
 			{
-				Name:     "name",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("Name"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("Name"),
+				PrimaryKey: true,
 			},
 			{
 				Name:     "sslca_cert",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: schema.PathResolver("SSLCACert"),
 			},
 			{
-				Name:     "service_id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("ServiceID"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "service_id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("ServiceID"),
+				PrimaryKey: true,
 			},
 			{
-				Name:     "service_version",
-				Type:     schema.TypeInt,
-				Resolver: schema.PathResolver("ServiceVersion"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "service_version",
+				Type:       arrow.PrimitiveTypes.Int64,
+				Resolver:   schema.PathResolver("ServiceVersion"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/fastly/resources/services/services/service_backends_fetch.go b/plugins/source/fastly/resources/services/services/service_backends_fetch.go
@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 

diff --git a/plugins/source/fastly/resources/services/services/service_backends_mock_test.go b/plugins/source/fastly/resources/services/services/service_backends_mock_test.go
@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/fastly/go-fastly/v7/fastly"
 	"github.com/golang/mock/gomock"
 )

diff --git a/plugins/source/fastly/resources/services/services/service_domains.go b/plugins/source/fastly/resources/services/services/service_domains.go
@@ -1,8 +1,9 @@
 package services
 
 import (
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/fastly/go-fastly/v7/fastly"
 )
 
@@ -14,28 +15,22 @@ func ServiceDomains() *schema.Table {
 		Transform:   transformers.TransformWithStruct(&fastly.Domain{}),
 		Columns: []schema.Column{
 			{
-				Name:     "name",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("Name"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("Name"),
+				PrimaryKey: true,
 			},
 			{
-				Name:     "service_id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("ServiceID"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "service_id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("ServiceID"),
+				PrimaryKey: true,
 			},
 			{
-				Name:     "service_version",
-				Type:     schema.TypeInt,
-				Resolver: schema.PathResolver("ServiceVersion"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "service_version",
+				Type:       arrow.PrimitiveTypes.Int64,
+				Resolver:   schema.PathResolver("ServiceVersion"),
+				PrimaryKey: true,
 			},
 		},
 	}