Skip to content

Commit

Permalink
fix(deps): Update github.com/cloudquery/plugin-sdk/v3 to v3.10.0 (#…
Browse files Browse the repository at this point in the history 
…11116)
  • Loading branch information
@candiduslynx
candiduslynx committed Jun 2, 2023
1 parent 8a0822e commit bba7c4e
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 66 deletions.
35 changes: 9 additions & 26 deletions plugins/destination/duckdb/client/read.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ func reverseTransformRecord(sc *arrow.Schema, rec arrow.Record) arrow.Record {
}

func reverseTransformArray(dt arrow.DataType, arr arrow.Array) arrow.Array {
if arrow.TypeEqual(dt, transformTypeForWriting(dt)) {
return arr
}

switch dt := dt.(type) {
case *types.UUIDType:
return array.NewExtensionArrayWithStorage(dt, arr.(*array.FixedSizeBinary))
Expand All @@ -114,12 +118,15 @@ func reverseTransformArray(dt arrow.DataType, arr arrow.Array) arrow.Array {
case *arrow.Uint8Type:
return reverseTransformUint8(arr.(*array.Uint32))
case *arrow.TimestampType:
return reverseTransformTimestamp(dt, arr.(*array.Timestamp))
return transformTimestamp(dt, arr.(*array.Timestamp))
case *arrow.MapType:
child := reverseTransformArray(dt.ValueType(), arr.(*array.List).ListValues()).Data()
return array.NewMapData(array.NewData(dt, arr.Len(), arr.Data().Buffers(), []arrow.ArrayData{child}, arr.NullN(), arr.Data().Offset()))
case listLike:
child := reverseTransformArray(dt.Elem(), arr.(array.ListLike).ListValues()).Data()
return array.NewListData(array.NewData(dt, arr.Len(), arr.Data().Buffers(), []arrow.ArrayData{child}, arr.NullN(), arr.Data().Offset()))
default:
return arr
return reverseTransformFromString(dt, arr.(*array.String))
}
}

Expand Down Expand Up @@ -163,27 +170,3 @@ func reverseTransformUint16(arr *array.Uint32) arrow.Array {

return builder.NewArray()
}

func reverseTransformTimestamp(dt *arrow.TimestampType, arr *array.Timestamp) arrow.Array {
builder := array.NewTimestampBuilder(memory.DefaultAllocator, dt)
for i := 0; i < arr.Len(); i++ {
if arr.IsNull(i) {
builder.AppendNull()
continue
}
t := arr.Value(i).ToTime(arr.DataType().(*arrow.TimestampType).Unit)
switch dt.Unit {
case arrow.Second:
builder.Append(arrow.Timestamp(t.Unix()))
case arrow.Millisecond:
builder.Append(arrow.Timestamp(t.UnixMilli()))
case arrow.Microsecond:
builder.Append(arrow.Timestamp(t.UnixMicro()))
case arrow.Nanosecond:
builder.Append(arrow.Timestamp(t.UnixNano()))
default:
panic(fmt.Errorf("unsupported timestamp unit: %s", dt.Unit))
}
}
return builder.NewTimestampArray()
}
33 changes: 19 additions & 14 deletions plugins/destination/duckdb/client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,21 @@ type listLike interface {
Elem() arrow.DataType
}

func transformSchemaForWriting(sc *arrow.Schema) *arrow.Schema {
fields := sc.Fields()
for i := range fields {
fields[i].Type = transformTypeForWriting(fields[i].Type)
}
md := sc.Metadata()
return arrow.NewSchema(fields, &md)
}

func transformTypeForWriting(dt arrow.DataType) arrow.DataType {
if dt, ok := dt.(listLike); ok {
switch dt := dt.(type) {
case listLike:
return arrow.ListOf(transformTypeForWriting(dt.Elem()))
case *arrow.MapType:
return arrow.ListOf(transformTypeForWriting(dt.ValueType()))
}

switch dt := duckDBToArrow(arrowToDuckDB(dt)).(type) {
Expand All @@ -25,19 +37,12 @@ func transformTypeForWriting(dt arrow.DataType) arrow.DataType {
}
}

func transformSchemaForWriting(sc *arrow.Schema) *arrow.Schema {
fields := sc.Fields()
for i := range fields {
fields[i].Type = transformTypeForWriting(fields[i].Type)
}
md := sc.Metadata()
return arrow.NewSchema(fields, &md)
}

func arrowToDuckDB(t arrow.DataType) string {
switch v := t.(type) {
case *arrow.ListType:
return arrowToDuckDB(v.Elem()) + "[]"
func arrowToDuckDB(dt arrow.DataType) string {
switch dt := dt.(type) {
case listLike:
return arrowToDuckDB(dt.Elem()) + "[]"
case *arrow.MapType:
return arrowToDuckDB(arrow.ListOf(dt.ValueType()))
case *arrow.BooleanType:
return "boolean"
case *arrow.Int8Type:
Expand Down
41 changes: 19 additions & 22 deletions plugins/destination/duckdb/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ module github.com/cloudquery/cloudquery/plugins/destination/duckdb
go 1.19

require (
github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
github.com/apache/arrow/go/v13 v13.0.0-20230601164043-3299d12efc91
github.com/cloudquery/plugin-pb-go v1.0.8
github.com/cloudquery/plugin-sdk/v3 v3.6.3
github.com/cloudquery/plugin-sdk/v3 v3.10.0
github.com/google/uuid v1.3.0
github.com/marcboeker/go-duckdb v1.3.0
github.com/rs/zerolog v1.29.0
)
Expand All @@ -19,47 +20,43 @@ require (
github.com/apache/thrift v0.18.1 // indirect
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/getsentry/sentry-go v0.20.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.16.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
github.com/getsentry/sentry-go v0.20.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.3.0
github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.6.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/grpc v1.55.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
8 changes: 4 additions & 4 deletions plugins/destination/duckdb/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v3 v3.6.3 h1:TyljGXffaPICARPBg8geOfKI4biP5sjW9OjSkjMXwig=
github.com/cloudquery/plugin-sdk/v3 v3.6.3/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
github.com/cloudquery/plugin-sdk/v3 v3.10.0 h1:zXbkfw2XwIZoAu7rliWGQ+Ucca/1l/eBI2EeJprIWwE=
github.com/cloudquery/plugin-sdk/v3 v3.10.0/go.mod h1:Ul9Z25RZ1FTrwj4FMC6zQHaQvrHTqtwhh4DYI+/05qc=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
Expand Down Expand Up @@ -464,8 +464,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down

0 comments on commit bba7c4e

Please sign in to comment.