feat(datadog)!: Upgrade to SDK V3 Arrow native (#10976)

Closes #10738 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10976) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
cloudquery · May 26, 2023 · eb380c2 · eb380c2
1 parent 5d8ee53
commit eb380c2
Show file tree

Hide file tree

Showing 57 changed files with 442 additions and 470 deletions.
diff --git a/plugins/source/datadog/CONTRIBUTING.md b/plugins/source/datadog/CONTRIBUTING.md
@@ -26,8 +26,8 @@ import (
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 )
 
 func Bar() *schema.Table {

diff --git a/plugins/source/datadog/client/client.go b/plugins/source/datadog/client/client.go
@@ -6,8 +6,8 @@ import (
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
 	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 )

diff --git a/plugins/source/datadog/client/multiplexers.go b/plugins/source/datadog/client/multiplexers.go
@@ -1,6 +1,6 @@
 package client
 
-import "github.com/cloudquery/plugin-sdk/v2/schema"
+import "github.com/cloudquery/plugin-sdk/v3/schema"
 
 func AccountMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
 	var l = make([]schema.ClientMeta, 0)

diff --git a/plugins/source/datadog/client/resolvers.go b/plugins/source/datadog/client/resolvers.go
@@ -6,7 +6,7 @@ import (
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
 
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/thoas/go-funk"
 )
 

diff --git a/plugins/source/datadog/client/testing.go b/plugins/source/datadog/client/testing.go
@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/golang/mock/gomock"
 	"github.com/rs/zerolog"
 )

diff --git a/plugins/source/datadog/codegen/services/generate.go b/plugins/source/datadog/codegen/services/generate.go
@@ -12,7 +12,7 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/cloudquery/plugin-sdk/v2/caser"
+	"github.com/cloudquery/plugin-sdk/v3/caser"
 )
 
 //go:embed templates/*.go.tpl

diff --git a/plugins/source/datadog/go.mod b/plugins/source/datadog/go.mod
@@ -4,8 +4,9 @@ go 1.19
 
 require (
 	github.com/DataDog/datadog-api-client-go/v2 v2.9.0
+	github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
 	github.com/cloudquery/plugin-pb-go v1.0.8
-	github.com/cloudquery/plugin-sdk/v2 v2.7.0
+	github.com/cloudquery/plugin-sdk/v3 v3.6.4
 	github.com/golang/mock v1.6.0
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.29.0
@@ -18,8 +19,8 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
 require (
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
 	github.com/apache/thrift v0.16.0 // indirect
+	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.20.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
@@ -38,6 +39,7 @@ require (
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.6.1 // indirect

diff --git a/plugins/source/datadog/go.sum b/plugins/source/datadog/go.sum
@@ -54,6 +54,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
 github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
+github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
+github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -180,6 +182,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

diff --git a/plugins/source/datadog/main.go b/plugins/source/datadog/main.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/resources/plugin"
-	"github.com/cloudquery/plugin-sdk/v2/serve"
+	"github.com/cloudquery/plugin-sdk/v3/serve"
 )
 
 const sentryDSN = "https://a0c2570b96264748a6759bb62e8cdef5@o1396617.ingest.sentry.io/4504220208267264"

diff --git a/plugins/source/datadog/resources/plugin/plugin.go b/plugins/source/datadog/resources/plugin/plugin.go
@@ -2,7 +2,7 @@ package plugin
 
 import (
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/plugins/source"
+	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
 )
 
 var (

diff --git a/plugins/source/datadog/resources/plugin/tables.go b/plugins/source/datadog/resources/plugin/tables.go
@@ -11,7 +11,7 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/resources/services/roles"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/resources/services/synthetics"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/resources/services/users"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 )
 
 func Tables() []*schema.Table {

diff --git a/plugins/source/datadog/resources/services/dashboard_lists/dashboard_lists.go b/plugins/source/datadog/resources/services/dashboard_lists/dashboard_lists.go
@@ -4,9 +4,10 @@ import (
 	"context"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 )
 
 func DashboardLists() *schema.Table {
@@ -17,20 +18,16 @@ func DashboardLists() *schema.Table {
 		Transform: transformers.TransformWithStruct(&datadogV1.DashboardList{}),
 		Columns: []schema.Column{
 			{
-				Name:     "account_name",
-				Type:     schema.TypeString,
-				Resolver: client.ResolveAccountName,
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "account_name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   client.ResolveAccountName,
+				PrimaryKey: true,
 			},
 			{
-				Name:     "id",
-				Type:     schema.TypeInt,
-				Resolver: schema.PathResolver("Id"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.PrimitiveTypes.Int64,
+				Resolver:   schema.PathResolver("Id"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/datadog/resources/services/dashboard_lists/dashboard_lists_mock_test.go b/plugins/source/datadog/resources/services/dashboard_lists/dashboard_lists_mock_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client/mocks"
 
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/golang/mock/gomock"
 )
 

diff --git a/plugins/source/datadog/resources/services/dashboards/dashboards.go b/plugins/source/datadog/resources/services/dashboards/dashboards.go
@@ -4,9 +4,10 @@ import (
 	"context"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 )
 
 func Dashboards() *schema.Table {
@@ -17,20 +18,16 @@ func Dashboards() *schema.Table {
 		Transform: transformers.TransformWithStruct(&datadogV1.DashboardSummaryDefinition{}),
 		Columns: []schema.Column{
 			{
-				Name:     "account_name",
-				Type:     schema.TypeString,
-				Resolver: client.ResolveAccountName,
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "account_name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   client.ResolveAccountName,
+				PrimaryKey: true,
 			},
 			{
-				Name:     "id",
-				Type:     schema.TypeString,
-				Resolver: schema.PathResolver("Id"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.PathResolver("Id"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/datadog/resources/services/dashboards/dashboards_mock_test.go b/plugins/source/datadog/resources/services/dashboards/dashboards_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client/mocks"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/golang/mock/gomock"
 )
 

diff --git a/plugins/source/datadog/resources/services/downtimes/downtimes.go b/plugins/source/datadog/resources/services/downtimes/downtimes.go
@@ -4,9 +4,10 @@ import (
 	"context"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 )
 
 func Downtimes() *schema.Table {
@@ -17,20 +18,16 @@ func Downtimes() *schema.Table {
 		Transform: transformers.TransformWithStruct(&datadogV1.Downtime{}),
 		Columns: []schema.Column{
 			{
-				Name:     "account_name",
-				Type:     schema.TypeString,
-				Resolver: client.ResolveAccountName,
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "account_name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   client.ResolveAccountName,
+				PrimaryKey: true,
 			},
 			{
-				Name:     "id",
-				Type:     schema.TypeInt,
-				Resolver: schema.PathResolver("Id"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.PrimitiveTypes.Int64,
+				Resolver:   schema.PathResolver("Id"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/datadog/resources/services/downtimes/downtimes_mock_test.go b/plugins/source/datadog/resources/services/downtimes/downtimes_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client/mocks"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/golang/mock/gomock"
 )
 

diff --git a/plugins/source/datadog/resources/services/hosts/hosts.go b/plugins/source/datadog/resources/services/hosts/hosts.go
@@ -4,9 +4,10 @@ import (
 	"context"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
-	"github.com/cloudquery/plugin-sdk/v2/transformers"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
 )
 
 func Hosts() *schema.Table {
@@ -17,20 +18,16 @@ func Hosts() *schema.Table {
 		Transform: transformers.TransformWithStruct(&datadogV1.Host{}),
 		Columns: []schema.Column{
 			{
-				Name:     "account_name",
-				Type:     schema.TypeString,
-				Resolver: client.ResolveAccountName,
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "account_name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   client.ResolveAccountName,
+				PrimaryKey: true,
 			},
 			{
-				Name:     "id",
-				Type:     schema.TypeInt,
-				Resolver: schema.PathResolver("Id"),
-				CreationOptions: schema.ColumnCreationOptions{
-					PrimaryKey: true,
-				},
+				Name:       "id",
+				Type:       arrow.PrimitiveTypes.Int64,
+				Resolver:   schema.PathResolver("Id"),
+				PrimaryKey: true,
 			},
 		},
 	}

diff --git a/plugins/source/datadog/resources/services/hosts/hosts_mock_test.go b/plugins/source/datadog/resources/services/hosts/hosts_mock_test.go
@@ -6,7 +6,7 @@ import (
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client/mocks"
-	"github.com/cloudquery/plugin-sdk/v2/faker"
+	"github.com/cloudquery/plugin-sdk/v3/faker"
 	"github.com/golang/mock/gomock"
 )
 

diff --git a/plugins/source/datadog/resources/services/incidents/incident_attachments.go b/plugins/source/datadog/resources/services/incidents/incident_attachments.go
@@ -1,13 +1,13 @@
-// Code generated by codegen; DO NOT EDIT.
-
 package incidents
 
 import (
 	"context"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/datadog/client"
-	"github.com/cloudquery/plugin-sdk/v2/schema"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/types"
 )
 
 func IncidentAttachments() *schema.Table {
@@ -17,34 +17,33 @@ func IncidentAttachments() *schema.Table {
 		Columns: []schema.Column{
 			{
 				Name:     "account_name",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: client.ResolveAccountName,
 			},
 			{
 				Name:     "attributes",
-				Type:     schema.TypeJSON,
+				Type:     types.ExtensionTypes.JSON,
 				Resolver: schema.PathResolver("Attributes"),
 			},
 			{
 				Name:     "id",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: schema.PathResolver("Id"),
 			},
 			{
 				Name:     "relationships",
-				Type:     schema.TypeJSON,
+				Type:     types.ExtensionTypes.JSON,
 				Resolver: schema.PathResolver("Relationships"),
 			},
 			{
 				Name:     "type",
-				Type:     schema.TypeString,
+				Type:     arrow.BinaryTypes.String,
 				Resolver: schema.PathResolver("Type"),
 			},
 		},
 	}
 }
 
-
 func fetchIncidentAttachments(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
 	p := parent.Item.(datadogV2.IncidentResponseData)
 	c := meta.(*client.Client)
@@ -56,4 +55,3 @@ func fetchIncidentAttachments(ctx context.Context, meta schema.ClientMeta, paren
 	res <- resp.GetData()
 	return nil
 }
-