feat(spanner): add support for Proto Columns (#9315)

* feat(spanner): add support for Proto Columns (#6886)

* feat(spanner): Adding support for Proto Message and Proto Enum

* feat(spanner): Add license header to new files

* code refactoring and additional checks

* nit: code changes

* go.mod update to prevent failing builds

* go.mod update to prevent failing builds

* go.mod and build_samples.sh update to prevent failing builds

* revert back grpc version

* revert back changes

* Using standard Singers example, refactoring redundant code

* code and proto refactoring

* Add proto_type_fqn for Proto Message and Proto Enum

* code refactoring

* go mod tidy: go.mod and go.sum version updates

* add changes to support compatibility between Int64 and Enum & compatibility between Bytes and Proto

* Revert "go mod tidy: go.mod and go.sum version updates"

This reverts commit 484b00c.

* add Integration Tests for Proto Message, Proto Enum, compatibility tests

* code refactoring

* code refactoring

* add unit tests for nil proto types

* Add error when nil proto message or nil enum is passed

* feat(spanner): add support for handling null value in Proto columns (#6954)

* feat(spanner): add support for handling null value in Proto columns

* code refactor: get protoTypeFqn from user defined nil types

* code refactoring

* code refactoring

* code refactoring

* code refactoring

* Add tests for MarshalJSON and UnmarshalJSON methods

* refactoring test file

* code refactoring

* feat(spanner): add support for Array of Proto columns (#7014)

* feat(spanner): add support for array of proto columns

* refactoring comments and added negative test cases while reading array of protos

* change decoding logic of handling array of proto columns

* feat(spanner): add support for handling null values in array of proto columns (#7042)

* feat(spanner): add support for handling null values in Array of Proto Columns

* add comments for code readability

* nit: change in error message

* feat(spanner): Modify configuration for integration test and add license header (#7059)

* feat(spanner): update go-genproto dependency (#7066)

* feat(spanner): support for enum columns as keys, index and integration tests (#7091)

* feat(spanner): support for proto columns as primary key and tests for parameterized queries, primary key and indexes

* feat(spanner): close read-only transaction to prevent session leak

* feat(spanner): update table schema to have GPK on proto field

* feat(spanner): add proto changes to support proto columns feature

* feat(spanner): remove gen-proto dep overwrite for proto column support

* feat(spanner): remove gen-proto dep overwrite from kokoro build

* feat(spanner): reset array of proto, enum when null from database (#7176)

* feat(spanner): reset array of proto, enum when null from database

* feat(spanner): fix license header to fix vet.sh build

* feat(spanner): fix proto generated file to fix vet.sh build

* feat(spanner): organize imports to fix vet.sh build

* feat(spanner): golint changes to fix vet.sh build

* feat(spanner): add support for Proto column DDL (#7292)

* feat(spanner): add proto changes for Proto Columns DDL support

* feat(spanner): add Proto descriptor file and integration tests for Proto DDL feature

* feat(spanner): skip pg tests and code refactoring for proto ddl

* feat(spanner): rename NewDatabaseAdminRESTClient to NewDatabaseAdminClient due to visibility label issue fix

* feat(spanner): remove hardcoded cloud-devel host

* feat(spanner): remove hardcoded project id

* feat(spanner): revert autogenerated code

* feat(spanner): just to validate integration tests running

* feat(spanner): change copyright year

* feat(spanner): set project id for integration tests

* feat(spanner): use jsoniter instead of json for marshal and unmarshal

* feat(spanner): revert presubmit.sh changes

* chore(spanner): run integration tests in presubmit

* chore(spanner): revert presubmit.sh changes

* feat(spanner): fix json

---------

Co-authored-by: rahul2393 <irahul@google.com>

spanner/key.go

@@ -25,6 +25,7 @@ import (
 	"cloud.google.com/go/civil"
 	sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/protobuf/reflect/protoreflect"
 	proto3 "google.golang.org/protobuf/types/known/structpb"
 )
 
@@ -58,6 +59,7 @@ import (
 //   - string and NullString are mapped to Cloud Spanner's STRING type.
 //   - time.Time and NullTime are mapped to Cloud Spanner's TIMESTAMP type.
 //   - civil.Date and NullDate are mapped to Cloud Spanner's DATE type.
+//   - protoreflect.Enum and NullProtoEnum are mapped to Cloud Spanner's ENUM type.
 type Key []interface{}
 
 // errInvdKeyPartType returns error for unsupported key part type.
@@ -83,7 +85,7 @@ func keyPartValue(part interface{}) (pb *proto3.Value, err error) {
 		pb, _, err = encodeValue(int64(v))
 	case uint32:
 		pb, _, err = encodeValue(int64(v))
-	case int64, float64, float32, NullInt64, NullFloat64, NullFloat32, bool, NullBool, []byte, string, NullString, time.Time, civil.Date, NullTime, NullDate, big.Rat, NullNumeric:
+	case int64, float64, float32, NullInt64, NullFloat64, NullFloat32, bool, NullBool, []byte, string, NullString, time.Time, civil.Date, NullTime, NullDate, big.Rat, NullNumeric, protoreflect.Enum, NullProtoEnum:
 		pb, _, err = encodeValue(v)
 	case Encoder:
 		part, err = v.EncodeSpanner()
@@ -138,7 +140,7 @@ func (key Key) String() string {
 
 func (key Key) elemString(b *bytes.Buffer, part interface{}) {
 	switch v := part.(type) {
-	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, float32, float64, bool:
+	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, float32, float64, bool, protoreflect.Enum:
 		// Use %v to print numeric types and bool.
 		fmt.Fprintf(b, "%v", v)
 	case string:
@@ -149,7 +151,7 @@ func (key Key) elemString(b *bytes.Buffer, part interface{}) {
 		} else {
 			fmt.Fprint(b, nullString)
 		}
-	case NullInt64, NullFloat64, NullBool, NullNumeric:
+	case NullInt64, NullFloat64, NullBool, NullNumeric, NullProtoEnum:
 		// The above types implement fmt.Stringer.
 		fmt.Fprintf(b, "%s", v)
 	case NullString, NullDate, NullTime:

spanner/key_test.go

@@ -24,6 +24,7 @@ import (
 
 	"cloud.google.com/go/civil"
 	sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
+	pb "cloud.google.com/go/spanner/testdata/protos"
 	proto3 "google.golang.org/protobuf/types/known/structpb"
 )
 
@@ -244,6 +245,16 @@ func TestKey(t *testing.T) {
 			wantProto: nil,
 			wantStr:   `(error)`,
 		},
+		{
+			k:         Key{pb.Genre_ROCK},
+			wantProto: listValueProto(stringProto("3")),
+			wantStr:   "(ROCK)",
+		},
+		{
+			k:         Key{NullProtoEnum{pb.Genre_FOLK, true}},
+			wantProto: listValueProto(stringProto("2")),
+			wantStr:   "(FOLK)",
+		},
 	} {
 		if got := test.k.String(); got != test.wantStr {
 			t.Errorf("%v.String() = %v, want %v", test.k, got, test.wantStr)

spanner/protoutils.go

@@ -24,6 +24,8 @@ import (
 
 	"cloud.google.com/go/civil"
 	sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
 	proto3 "google.golang.org/protobuf/types/known/structpb"
 )
 
@@ -144,3 +146,20 @@ func structType(fields ...*sppb.StructType_Field) *sppb.Type {
 func nullProto() *proto3.Value {
 	return &proto3.Value{Kind: &proto3.Value_NullValue{NullValue: proto3.NullValue_NULL_VALUE}}
 }
+
+func protoMessageType(fqn string) *sppb.Type {
+	return &sppb.Type{Code: sppb.TypeCode_PROTO, ProtoTypeFqn: fqn}
+}
+
+func protoEnumType(fqn string) *sppb.Type {
+	return &sppb.Type{Code: sppb.TypeCode_ENUM, ProtoTypeFqn: fqn}
+}
+
+func protoMessageProto(m proto.Message) *proto3.Value {
+	var b, _ = proto.Marshal(m)
+	return &proto3.Value{Kind: &proto3.Value_StringValue{StringValue: base64.StdEncoding.EncodeToString(b)}}
+}
+
+func protoEnumProto(e protoreflect.Enum) *proto3.Value {
+	return &proto3.Value{Kind: &proto3.Value_StringValue{StringValue: strconv.FormatInt(int64(e.Number()), 10)}}
+}

spanner/testdata/protos/singer.proto

@@ -0,0 +1,34 @@
+/*
+Copyright 2024 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+syntax = "proto2";
+
+package spanner.examples.music;
+option go_package = "./protos";
+
+message SingerInfo {
+  optional int64 singer_id = 1;
+  optional string birth_date = 2;
+  optional string nationality = 3;
+  optional Genre genre = 4;
+}
+
+enum Genre {
+  POP = 0;
+  JAZZ = 1;
+  FOLK = 2;
+  ROCK = 3;
+}