Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(fastly)!: Migrate to Arrow native SDK #11018

Merged
merged 2 commits into from
May 26, 2023
Merged

feat(fastly)!: Migrate to Arrow native SDK #11018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afa4116
Migrate to v3
hermanschaaf May 26, 2023
c964648
Merge branch 'main' into migrate-fastly-v3
kodiakhq[bot] May 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions plugins/source/fastly/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (

"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
"github.com/cloudquery/plugin-pb-go/specs"
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/rs/zerolog"
"github.com/thoas/go-funk"
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/client/multiplexers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package client

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
)

func ServiceRegionMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/client/resolvers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"
"time"

"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/thoas/go-funk"
)

Expand Down
4 changes: 2 additions & 2 deletions plugins/source/fastly/client/testing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (

"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
"github.com/cloudquery/plugin-pb-go/specs"
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/golang/mock/gomock"
"github.com/rs/zerolog"
Expand Down
6 changes: 4 additions & 2 deletions plugins/source/fastly/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ module github.com/cloudquery/cloudquery/plugins/source/fastly
go 1.19

require (
github.com/apache/arrow/go/v13 v13.0.0-20230526080908-77a71305090f
github.com/cloudquery/codegen v0.2.1
github.com/cloudquery/plugin-pb-go v1.0.8
github.com/cloudquery/plugin-sdk/v2 v2.7.0
github.com/cloudquery/plugin-sdk/v3 v3.6.4
github.com/fastly/go-fastly/v7 v7.0.0
github.com/golang/mock v1.6.0
github.com/rs/zerolog v1.29.0
Expand All @@ -17,9 +18,9 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13

require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
github.com/apache/thrift v0.16.0 // indirect
github.com/cloudquery/plugin-sdk v1.45.0 // indirect
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/getsentry/sentry-go v0.20.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
Expand All @@ -43,6 +44,7 @@ require (
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/peterhellberg/link v1.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.6.1 // indirect
Expand Down
3 changes: 3 additions & 0 deletions plugins/source/fastly/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ github.com/cloudquery/plugin-sdk v1.45.0 h1:5vrfQZtaO1dp6ebKt8ouXDmPC7eeLuOB3JMd
github.com/cloudquery/plugin-sdk v1.45.0/go.mod h1:9KGuuTGjTCKgh9amKwS+7Zrrqq7/M6lormteOyqoKwg=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
Expand Down Expand Up @@ -196,6 +198,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc=
github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package main

import (
"github.com/cloudquery/cloudquery/plugins/source/fastly/resources/plugin"
"github.com/cloudquery/plugin-sdk/v2/serve"
"github.com/cloudquery/plugin-sdk/v3/serve"
)

const sentryDSN = "https://c4e711b87a8846de90e5ba2c785fb901@o1396617.ingest.sentry.io/4504379395604480"
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/plugin/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package plugin

import (
"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
)

var (
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/plugin/tables.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 8 additions & 9 deletions plugins/source/fastly/resources/services/account/account_events.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package account

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v2/transformers"
"github.com/apache/arrow/go/v13/arrow"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/transformers"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand All @@ -14,16 +15,14 @@ func AccountEvents() *schema.Table {
Transform: transformers.TransformWithStruct(&fastly.Event{}),
Columns: []schema.Column{
{
Name: "id",
Type: schema.TypeString,
Resolver: schema.PathResolver("ID"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "id",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("ID"),
PrimaryKey: true,
},
{
Name: "ip",
Type: schema.TypeString,
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("IP"),
},
},
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/account/account_events_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/account/account_events_mock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
"github.com/cloudquery/plugin-sdk/v2/faker"
"github.com/cloudquery/plugin-sdk/v3/faker"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/golang/mock/gomock"
)
Expand Down
15 changes: 7 additions & 8 deletions plugins/source/fastly/resources/services/account/account_users.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package account

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v2/transformers"
"github.com/apache/arrow/go/v13/arrow"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/transformers"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand All @@ -14,12 +15,10 @@ func AccountUsers() *schema.Table {
Transform: transformers.TransformWithStruct(&fastly.User{}),
Columns: []schema.Column{
{
Name: "id",
Type: schema.TypeString,
Resolver: schema.PathResolver("ID"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "id",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("ID"),
PrimaryKey: true,
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/account/account_users_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/account/account_users_mock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
"github.com/cloudquery/plugin-sdk/v2/faker"
"github.com/cloudquery/plugin-sdk/v3/faker"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/golang/mock/gomock"
)
Expand Down
17 changes: 8 additions & 9 deletions plugins/source/fastly/resources/services/auth/auth_tokens.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package auth

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v2/transformers"
"github.com/apache/arrow/go/v13/arrow"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/transformers"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand All @@ -14,16 +15,14 @@ func AuthTokens() *schema.Table {
Transform: transformers.TransformWithStruct(&fastly.Token{}),
Columns: []schema.Column{
{
Name: "id",
Type: schema.TypeString,
Resolver: schema.PathResolver("ID"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "id",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("ID"),
PrimaryKey: true,
},
{
Name: "ip",
Type: schema.TypeString,
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("IP"),
},
},
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/auth/auth_tokens_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
)

func fetchAuthTokens(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/auth/auth_tokens_mock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
"github.com/cloudquery/plugin-sdk/v2/faker"
"github.com/cloudquery/plugin-sdk/v3/faker"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/golang/mock/gomock"
)
Expand Down
37 changes: 16 additions & 21 deletions plugins/source/fastly/resources/services/services/service_backends.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package services

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v2/transformers"
"github.com/apache/arrow/go/v13/arrow"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/transformers"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand All @@ -14,33 +15,27 @@ func ServiceBackends() *schema.Table {
Transform: transformers.TransformWithStruct(&fastly.Backend{}),
Columns: []schema.Column{
{
Name: "name",
Type: schema.TypeString,
Resolver: schema.PathResolver("Name"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "name",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("Name"),
PrimaryKey: true,
},
{
Name: "sslca_cert",
Type: schema.TypeString,
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("SSLCACert"),
},
{
Name: "service_id",
Type: schema.TypeString,
Resolver: schema.PathResolver("ServiceID"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "service_id",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("ServiceID"),
PrimaryKey: true,
},
{
Name: "service_version",
Type: schema.TypeInt,
Resolver: schema.PathResolver("ServiceVersion"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "service_version",
Type: arrow.PrimitiveTypes.Int64,
Resolver: schema.PathResolver("ServiceVersion"),
PrimaryKey: true,
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/services/service_backends_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand Down
2 changes: 1 addition & 1 deletion plugins/source/fastly/resources/services/services/service_backends_mock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"testing"

"github.com/cloudquery/cloudquery/plugins/source/fastly/client/mocks"
"github.com/cloudquery/plugin-sdk/v2/faker"
"github.com/cloudquery/plugin-sdk/v3/faker"
"github.com/fastly/go-fastly/v7/fastly"
"github.com/golang/mock/gomock"
)
Expand Down
35 changes: 15 additions & 20 deletions plugins/source/fastly/resources/services/services/service_domains.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package services

import (
"github.com/cloudquery/plugin-sdk/v2/schema"
"github.com/cloudquery/plugin-sdk/v2/transformers"
"github.com/apache/arrow/go/v13/arrow"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/transformers"
"github.com/fastly/go-fastly/v7/fastly"
)

Expand All @@ -14,28 +15,22 @@ func ServiceDomains() *schema.Table {
Transform: transformers.TransformWithStruct(&fastly.Domain{}),
Columns: []schema.Column{
{
Name: "name",
Type: schema.TypeString,
Resolver: schema.PathResolver("Name"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "name",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("Name"),
PrimaryKey: true,
},
{
Name: "service_id",
Type: schema.TypeString,
Resolver: schema.PathResolver("ServiceID"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "service_id",
Type: arrow.BinaryTypes.String,
Resolver: schema.PathResolver("ServiceID"),
PrimaryKey: true,
},
{
Name: "service_version",
Type: schema.TypeInt,
Resolver: schema.PathResolver("ServiceVersion"),
CreationOptions: schema.ColumnCreationOptions{
PrimaryKey: true,
},
Name: "service_version",
Type: arrow.PrimitiveTypes.Int64,
Resolver: schema.PathResolver("ServiceVersion"),
PrimaryKey: true,
},
},
}
Expand Down
Loading