BigQuery: Storage Write does not support protobuf's oneof.

[jlmille4]

Client

BigQuery Storage API

Environment
macOS Big Sur Intel

Go Environment
$go version go1.17.5 darwin/amd64
$ go env
GO111MODULE="" GOARCH="amd64" GOBIN="" GOCACHE="/Users/jamiller/Library/Caches/go-build" GOENV="/Users/jamiller/Library/Application Support/go/env" GOEXE="" GOEXPERIMENT="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="darwin" GOINSECURE="" GOMODCACHE="/Users/jamiller/go/pkg/mod" GONOPROXY="" GONOSUMDB="" GOOS="darwin" GOPATH="/Users/jamiller/go" GOPRIVATE="" GOPROXY="https://proxy.golang.org,direct" GOROOT="/usr/local/Cellar/go/1.17.5/libexec" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/Cellar/go/1.17.5/libexec/pkg/tool/darwin_amd64" GOVCS="" GOVERSION="go1.17.5" GCCGO="gccgo" AR="ar" CC="clang" CXX="clang++" CGO_ENABLED="1" GOMOD="/Users/jamiller/Work/bq_storage_api_bug/go.mod" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -arch x86_64 -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/55/zfv_sh9n27x4w6455vtxzs4m0000gp/T/go-build3105813587=/tmp/go-build -gno-record-gcc-switches -fno-common"

Code

Proto file

syntax="proto2";

option go_package = "github.com/jlmille4/bqstorage/main;main";

package main;

message AgentParams {
    optional string key = 1;
    oneof value_oneof {
        string string_value = 2;
        int32 int_value = 3;
        double double_value = 4;
        bool bool_value = 5;
    }
  }

package main

import (
	"context"
	"fmt"
	"log"

	storage "cloud.google.com/go/bigquery/storage/apiv1"
	"cloud.google.com/go/bigquery/storage/managedwriter/adapt"
	storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1"
	"google.golang.org/protobuf/proto"
)

const (
	projectId = "{{ projectId }}"
	datasetId = "example"
	tableId   = "agent_params"
)

func main() {
	ctx := context.Background()
	client, err := storage.NewBigQueryWriteClient(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer client.Close()

	message := AgentParams{
		Key: proto.String("ThisIsTheKey"),
		ValueOneof: &AgentParams_StringValue{
			StringValue: "This is a string value",
		},
	}

	descriptor, err := adapt.NormalizeDescriptor(message.ProtoReflect().Descriptor())
	if err != nil {
		log.Fatal("NormalizeDescriptor: ", err)
	}

	var records [][]byte
	bytes, err := proto.Marshal(&message)
	if err != nil {
		log.Fatal(err)
	}

	records = append(records, bytes)

	stream, err := client.AppendRows(ctx)
	if err != nil {
		log.Fatal(err)
	}

	protoSchema := &storagepb.ProtoSchema{
		ProtoDescriptor: descriptor,
	}
	protoRows := storagepb.ProtoRows{
		SerializedRows: records,
	}
	appendRowsRequestProtoData := storagepb.AppendRowsRequest_ProtoData{
		WriterSchema: protoSchema,
		Rows:         &protoRows,
	}
	appendRowsRequestProtoRows := storagepb.AppendRowsRequest_ProtoRows{
		ProtoRows: &appendRowsRequestProtoData,
	}
	appendRowRequest := storagepb.AppendRowsRequest{
		WriteStream: fmt.Sprintf("projects/%s/datasets/%s/tables/%s/_default", projectId, datasetId, tableId),
		TraceId:     "trace-id",
		Rows:        &appendRowsRequestProtoRows,
	}

	if err := stream.Send(&appendRowRequest); err != nil {
		log.Fatal(err)
	}

	resp, err := stream.Recv()
	if err != nil {
		log.Fatal(err)
	}

	log.Println(resp)

}

BQ Table DDL

CREATE TABLE `example.agent_params`
(
  key STRING,
  int_value INT64,
  double_value FLOAT64,
  bool_value BOOL,
  string_value STRING
);

Expected behavior
Record appears in the BQ table

Actual behavior

I get the following rpc errors

rpc error: code = InvalidArgument desc = Invalid proto schema: BqMessage.proto: main_AgentParams.string_value: FieldDescriptorProto.oneof_index 0 is out of range for type "main_AgentParams".
BqMessage.proto: main_AgentParams.int_value: FieldDescriptorProto.oneof_index 0 is out of range for type "main_AgentParams".
BqMessage.proto: main_AgentParams.double_value: FieldDescriptorProto.oneof_index 0 is out of range for type "main_AgentParams".
BqMessage.proto: main_AgentParams.bool_value: FieldDescriptorProto.oneof_index 0 is out of range for type "main_AgentParams". Entity: projects/{{ projectId }}/datasets/example/tables/agent_params/_default
exit status 1
``

[shollyman]

Thanks for the report.

[shollyman]

verifying expected behavior with backend team, internal issue 219956936

[shollyman]

Fix for this is out for review, feel free to leave feedback.