fix(deps): Update github.com/cloudquery/plugin-sdk/v3 to v3.8.1 (#…

…11078)

Extracted from #11043
Depends on:
* cloudquery/plugin-sdk#913
* apache/arrow#35823
* cloudquery/plugin-sdk#921

plugins/destination/clickhouse/client/client_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/resources/plugin"
 	"github.com/cloudquery/plugin-pb-go/specs"
 	"github.com/cloudquery/plugin-sdk/v3/plugins/destination"
@@ -52,5 +53,17 @@ func TestPlugin(t *testing.T) {
 
 			MigrateStrategyAppend: migrateStrategy,
 		},
+		destination.WithTestSourceAllowNull(func(dt arrow.DataType) bool {
+			switch dt.(type) {
+			case *arrow.StructType,
+				*arrow.MapType,
+				*arrow.ListType,
+				*arrow.LargeListType,
+				*arrow.FixedSizeListType:
+				return false
+			default:
+				return true
+			}
+		}),
 	)
 }

plugins/destination/clickhouse/go.mod

@@ -3,23 +3,21 @@ module github.com/cloudquery/cloudquery/plugins/destination/clickhouse
 go 1.19
 
 require (
-	github.com/ClickHouse/clickhouse-go/v2 v2.9.2
+	github.com/ClickHouse/clickhouse-go/v2 v2.10.0
 	github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
 	github.com/cloudquery/plugin-pb-go v1.0.8
-	github.com/cloudquery/plugin-sdk/v3 v3.5.2
+	github.com/cloudquery/plugin-sdk/v3 v3.8.1
 	github.com/google/uuid v1.3.0
 	github.com/rs/zerolog v1.29.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.3
 	golang.org/x/sync v0.2.0
 )
 
-require google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
-
-// TODO: remove once al updates are merged
-replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
+// TODO: remove once all updates are merged
+replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed
 
 require (
-	github.com/ClickHouse/ch-go v0.55.0 // indirect
+	github.com/ClickHouse/ch-go v0.56.0 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/apache/thrift v0.18.1 // indirect
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
@@ -53,15 +51,16 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/thoas/go-funk v0.9.3 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
-	go.opentelemetry.io/otel v1.15.1 // indirect
-	go.opentelemetry.io/otel/trace v1.15.1 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.9.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

plugins/destination/clickhouse/go.sum

@@ -33,10 +33,10 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ClickHouse/ch-go v0.55.0 h1:jw4Tpx887YXrkyL5DfgUome/po8MLz92nz2heOQ6RjQ=
-github.com/ClickHouse/ch-go v0.55.0/go.mod h1:kQT2f+yp2p+sagQA/7kS6G3ukym+GQ5KAu1kuFAFDiU=
-github.com/ClickHouse/clickhouse-go/v2 v2.9.2 h1:P9az39xLJGwdL+Bq04Qrcq0lJspTgGT0VD7ESouFOYg=
-github.com/ClickHouse/clickhouse-go/v2 v2.9.2/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg=
+github.com/ClickHouse/ch-go v0.56.0 h1:Uli/jOQl7+pYyA2E+/VvOSQRX3jHZ4FjESCRReAeFuM=
+github.com/ClickHouse/ch-go v0.56.0/go.mod h1:nKtI66Lz8pRvbO7NZDrgbRaC9ldsD4HMeWKb1hpbpBI=
+github.com/ClickHouse/clickhouse-go/v2 v2.10.0 h1:0w/A50D5MfsRUYBaV6rLKwZ4LXWKLZKJ1u31QXjTIO4=
+github.com/ClickHouse/clickhouse-go/v2 v2.10.0/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg=
 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
@@ -48,14 +48,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
-github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
+github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed h1:FQPwpdGI7TIRUfgn7WgVSx9PCa8/kmelveY6FFqriAM=
+github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
 github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
 github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
-github.com/cloudquery/plugin-sdk/v3 v3.5.2 h1:Ss8m1/T3tY4IFFo4ZRku4fCJEOOXILe33SUGB5sfpvk=
-github.com/cloudquery/plugin-sdk/v3 v3.5.2/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
+github.com/cloudquery/plugin-sdk/v3 v3.8.1 h1:Rj+3zBLmscKSbG+JPLT5bzgv56oPwBHRSMGyJ1DSfyc=
+github.com/cloudquery/plugin-sdk/v3 v3.8.1/go.mod h1:8PHS8cMjWPeXrurnI30dyHwViK4HJUZLA6uys+F2fXQ=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -218,16 +218,12 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
 github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -248,10 +244,10 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8=
-go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc=
-go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY=
-go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8=
+go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
+go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
+go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
+go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -495,8 +491,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

plugins/destination/clickhouse/typeconv/arrow/values/map.go

@@ -40,10 +40,10 @@ func buildMapFromReflect(builder *array.MapBuilder, value reflect.Value) error {
 		return nil
 	}
 
+	builder.Append(true)
 	keyBuilder, itemBuilder := builder.KeyBuilder(), builder.ItemBuilder()
 	it := value.MapRange()
 	for it.Next() {
-		builder.Append(true)
 		if err := buildValue(keyBuilder, it.Key().Interface()); err != nil {
 			return err
 		}

plugins/destination/clickhouse/typeconv/arrow/values/string.go

@@ -10,5 +10,13 @@ func buildFromString(builder array.Builder, value any) error {
 		builder.AppendNull()
 		return nil
 	}
-	return builder.AppendValueFromString(v)
+
+	if len(v) > 0 {
+		return builder.AppendValueFromString(v)
+	}
+
+	// binary types are handled separately, so here we have a builder that most likely can't handle empty string.
+	// having empty string in CH means that this was an empty value
+	builder.AppendEmptyValue()
+	return nil
 }

plugins/destination/clickhouse/typeconv/arrow/values/time.go

@@ -1,7 +1,6 @@
 package values
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/apache/arrow/go/v13/arrow"
@@ -12,29 +11,52 @@ func buildDate32Values(builder primitiveBuilder[arrow.Date32], value any) {
 	v, ok := unwrap[time.Time](value)
 	if !ok {
 		builder.AppendNull()
+		return
 	}
+
+	if v.IsZero() {
+		// work-around for empty values
+		builder.AppendEmptyValue()
+		return
+	}
+
 	builder.Append(arrow.Date32FromTime(v))
 }
 
 func buildDate64Values(builder primitiveBuilder[arrow.Date64], value any) {
 	v, ok := unwrap[time.Time](value)
 	if !ok {
 		builder.AppendNull()
+		return
+	}
+
+	if v.IsZero() {
+		// work-around for empty values
+		builder.AppendEmptyValue()
+		return
 	}
+
 	builder.Append(arrow.Date64FromTime(v))
 }
 
 func buildTimestampValues(builder *array.TimestampBuilder, value any) error {
 	v, ok := unwrap[time.Time](value)
 	if !ok {
 		builder.AppendNull()
+		return nil
 	}
 
 	t, err := timeToTimestamp(v, builder.Type().(*arrow.TimestampType))
 	if err != nil {
 		return err
 	}
 
+	if v.IsZero() {
+		// work-around for empty values
+		builder.AppendEmptyValue()
+		return nil
+	}
+
 	builder.Append(t)
 	return nil
 }
@@ -48,16 +70,5 @@ func timeToTimestamp(value time.Time, tsType *arrow.TimestampType) (arrow.Timest
 		value = value.In(loc)
 	}
 
-	switch tsType.Unit {
-	case arrow.Second:
-		return arrow.Timestamp(value.Unix()), nil
-	case arrow.Millisecond:
-		return arrow.Timestamp(value.Unix()*1e3 + int64(value.Nanosecond())/1e6), nil
-	case arrow.Microsecond:
-		return arrow.Timestamp(value.Unix()*1e6 + int64(value.Nanosecond())/1e3), nil
-	case arrow.Nanosecond:
-		return arrow.Timestamp(value.UnixNano()), nil
-	default:
-		return arrow.Timestamp(0), fmt.Errorf("unsupported Apache Arrow time unit: %s", tsType.Unit.String())
-	}
+	return arrow.TimestampFromTime(value, tsType.Unit)
 }

plugins/destination/clickhouse/typeconv/ch/values/list.go

@@ -5,7 +5,9 @@ import (
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/column"
+	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/apache/arrow/go/v13/arrow/array"
+	"github.com/apache/arrow/go/v13/arrow/memory"
 	"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/typeconv/ch/types"
 )
 
@@ -22,12 +24,14 @@ func listValue(arr array.ListLike) (any, error) {
 	}
 	valueType := col.ScanType()
 
+	sanitized, err := sanitizeNested(arr)
+	if err != nil {
+		return nil, err
+	}
+	arr = sanitized.(array.ListLike)
+
 	elems := make([]any, arr.Len())
 	for i := 0; i < arr.Len(); i++ {
-		if arr.IsNull(i) {
-			continue
-		}
-
 		from, to := arr.ValueOffsets(i)
 		elems[i], err = FromArray(array.NewSlice(arr.ListValues(), from, to))
 		if err != nil {
@@ -37,13 +41,33 @@ func listValue(arr array.ListLike) (any, error) {
 
 	res := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(valueType)), len(elems), len(elems)) // we do []*(type) for nullable assignment
 	for i, elem := range elems {
-		// we need to fill in for the in-depth recursive parsing by ClickHouse SDK
+		// lists aren't nullable themselves
+		// https://clickhouse.com/docs/en/sql-reference/data-types/nullable
 		val := reflect.New(valueType)
-		if elem != nil {
-			val.Elem().Set(reflect.ValueOf(elem))
-		}
+		val.Elem().Set(reflect.ValueOf(elem))
 		res.Index(i).Set(val)
 	}
 
 	return res.Interface(), nil
 }
+
+// sanitizeNested will replace all null entries with empty ones as in CH nested types aren't nullable themselves
+// https://clickhouse.com/docs/en/sql-reference/data-types/nullable
+func sanitizeNested(arr arrow.Array) (arrow.Array, error) {
+	if arr.NullN() == 0 {
+		return arr, nil
+	}
+
+	builder := array.NewBuilder(memory.DefaultAllocator, arr.DataType())
+	for i := 0; i < arr.Len(); i++ {
+		if arr.IsNull(i) {
+			builder.AppendEmptyValue()
+			continue
+		}
+		if err := builder.AppendValueFromString(arr.ValueStr(i)); err != nil {
+			return nil, err
+		}
+	}
+
+	return builder.NewArray(), nil
+}