Skip to content

Commit

Permalink
fix(deps): Update module github.com/cloudquery/plugin-sdk/v3 to v3.10…
Browse files Browse the repository at this point in the history 
….3 (#11221)



#### Summary

Extracted from #11150 as BigQuery failed on decimal types and maps.
I added support for decimals as it was not too much work, but skipped maps for now.
See https://github.com/cloudquery/cloudquery/actions/runs/5176150317/jobs/9324607901.

Please note the support for decimals is not a breaking change as before BigQuery panicked on those

<!--
  • Loading branch information
@erezrokah
erezrokah committed Jun 5, 2023
1 parent c0b9f0a commit e3925ab
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 6 deletions.
1 change: 1 addition & 0 deletions plugins/destination/bigquery/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,6 @@ func TestPlugin(t *testing.T) {
MigrateStrategyAppend: migrateStrategy,
},
destination.WithTestIgnoreNullsInLists(),
destination.WithTestSourceSkipMaps(),
)
}
30 changes: 30 additions & 0 deletions plugins/destination/bigquery/client/read.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@ package client
import (
"context"
"fmt"
"math/big"
"strings"
"time"

"cloud.google.com/go/bigquery"
"cloud.google.com/go/civil"
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/array"
"github.com/apache/arrow/go/v13/arrow/decimal128"
"github.com/apache/arrow/go/v13/arrow/decimal256"
"github.com/apache/arrow/go/v13/arrow/memory"
"github.com/cloudquery/plugin-sdk/v3/schema"
"github.com/cloudquery/plugin-sdk/v3/types"
Expand Down Expand Up @@ -58,6 +61,15 @@ func (c *Client) Read(ctx context.Context, table *schema.Table, sourceName strin
return nil
}

func parseRat(v *big.Rat) (str string, precision int32, scale int32) {
str = v.FloatString(10)
parts := strings.Split(str, ".")
if len(parts) == 2 {
return str, int32(len(parts[0]) + len(parts[1])), int32(len(parts[1]))
}
return str, int32(len(parts[0])), 0
}

func appendValue(builder array.Builder, value any) error {
if value == nil {
builder.AppendNull()
Expand Down Expand Up @@ -150,6 +162,24 @@ func appendValue(builder array.Builder, value any) error {
bldr.Append(ts)
return nil
}
case *array.Decimal128Builder:
r := value.(*big.Rat)
str, precision, scale := parseRat(r)
dec, err := decimal128.FromString(str, precision, scale)
if err != nil {
return err
}
bldr.Append(dec)
return nil
case *array.Decimal256Builder:
r := value.(*big.Rat)
str, precision, scale := parseRat(r)
dec, err := decimal256.FromString(str, precision, scale)
if err != nil {
return err
}
bldr.Append(dec)
return nil
case *types.JSONBuilder:
return bldr.AppendValueFromString(value.(string))
default:
Expand Down
4 changes: 4 additions & 0 deletions plugins/destination/bigquery/client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ func (c *Client) DataTypeToBigQueryType(dataType arrow.DataType) bigquery.FieldT
case typeOneOf(dataType,
arrow.FixedWidthTypes.MonthDayNanoInterval):
return bigquery.RecordFieldType
// We don't use `typeOneOf` as `arrow.TypeEqual` checks for equality of precision and scale.
case arrow.IsDecimal(dataType.ID()):
// BigQuery NumericFieldType has a scale limit of 9, so we use BigNumeric for both decimal128 and decimal256.
return bigquery.BigNumericFieldType
default:
panic("unsupported data type: " + dataType.String())
}
Expand Down
4 changes: 2 additions & 2 deletions plugins/destination/bigquery/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
cloud.google.com/go/bigquery v1.50.0
github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
github.com/cloudquery/plugin-pb-go v1.0.8
github.com/cloudquery/plugin-sdk/v3 v3.6.7
github.com/cloudquery/plugin-sdk/v3 v3.10.3
github.com/goccy/go-json v0.9.11
github.com/rs/zerolog v1.29.0
golang.org/x/sync v0.1.0
Expand Down Expand Up @@ -35,7 +35,7 @@ require (
golang.org/x/mod v0.8.0 // indirect
golang.org/x/tools v0.6.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

Expand Down
8 changes: 4 additions & 4 deletions plugins/destination/bigquery/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v3 v3.6.7 h1:QJqZGHs+3uN+CE3y9oBOjkchH/v5XisRnVP2X4aj9Wo=
github.com/cloudquery/plugin-sdk/v3 v3.6.7/go.mod h1:+ta6OETfGfzh6nCpCyZi5Er1rj+zvn7m2QR2wokEvA8=
github.com/cloudquery/plugin-sdk/v3 v3.10.3 h1:aMofD3hHU4Dm+raxNgIOdSg+hGQrkTUTV2KXjxSwtqE=
github.com/cloudquery/plugin-sdk/v3 v3.10.3/go.mod h1:P3zucEOH+IEhdM9FGD5q3WqciXIBOPCKw2kHZT4UrlQ=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
Expand Down Expand Up @@ -494,8 +494,8 @@ google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdL
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e h1:AZX1ra8YbFMSb7+1pI8S9v4rrgRR7jU1FmuFSSjTVcQ=
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down

0 comments on commit e3925ab

Please sign in to comment.