bigquery: adapt.NormalizeDescriptor produces Invalid DescriptorProto with nested enums

[jlmille4]

Client

BigQuery

Environment

Mac OSX

Go Environment
$ go version go1.17.5 darwin/amd64

GO111MODULE=""
GOARCH="amd64"
GOBIN=""
GOCACHE="/Users/jamiller/Library/Caches/go-build"
GOENV="/Users/jamiller/Library/Application Support/go/env"
GOEXE=""
GOEXPERIMENT=""
GOFLAGS=""
GOHOSTARCH="amd64"
GOHOSTOS="darwin"
GOINSECURE=""
GOMODCACHE="/Users/jamiller/go/pkg/mod"
GONOPROXY=""
GONOSUMDB=""
GOOS="darwin"
GOPATH="/Users/jamiller/go"
GOPRIVATE=""
GOPROXY="https://proxy.golang.org,direct"
GOROOT="/usr/local/Cellar/go/1.17.5/libexec"
GOSUMDB="sum.golang.org"
GOTMPDIR=""
GOTOOLDIR="/usr/local/Cellar/go/1.17.5/libexec/pkg/tool/darwin_amd64"
GOVCS=""
GOVERSION="go1.17.5"
GCCGO="gccgo"
AR="ar"
CC="clang"
CXX="clang++"
CGO_ENABLED="1"
GOMOD="/Users/jamiller/code/Work/bq_storage_api_bug/go.mod"
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -arch x86_64 -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/55/zfv_sh9n27x4w6455vtxzs4m0000gp/T/go-build16015807=/tmp/go-build -gno-record-gcc-switches -fno-common"

Code

proto definition

syntax = "proto2";

package bqstorage;

option go_package = "github.com/jlmille4/bqstorage";

message Message {
    optional Am am = 1;
    optional Bm bm = 2;
}

message Am {
    optional Flag active = 1;
}

message Bm {
    optional Flag active = 1;
}

enum Flag {
    false = 0;
    true = 1;
}

Usage

    ctx := context.Background()
	client, err := storage.NewBigQueryWriteClient(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer client.Close()


	message := Message{
		Am: &Am{
			Active: Flag_true.Enum(),
		},
		Bm: &Bm{
			Active: Flag_false.Enum(),
		},
	}

	descriptor, err := adapt.NormalizeDescriptor(message.ProtoReflect().Descriptor())
	if err != nil {
		log.Fatal("NormalizeDescriptor: ", err)
	}

	var records [][]byte
	bytes, err := proto.Marshal(&message)
	if err != nil {
		log.Fatal(err)
	}
	records = append(records, bytes)

	stream, err := client.AppendRows(ctx)
	if err != nil {
		log.Fatal(err)
	}

	protoSchema := &storagepb.ProtoSchema{
		ProtoDescriptor: descriptor,
	}
	protoRows := storagepb.ProtoRows{
		SerializedRows: records,
	}
	appendRowsRequestProtoData := storagepb.AppendRowsRequest_ProtoData{
		WriterSchema: protoSchema,
		Rows:         &protoRows,
	}
	appendRowsRequestProtoRows := storagepb.AppendRowsRequest_ProtoRows{
		ProtoRows: &appendRowsRequestProtoData,
	}
	appendRowRequest := storagepb.AppendRowsRequest{
		WriteStream: fmt.Sprintf("projects/%s/datasets/%s/tables/%s/_default", projectId, datasetId, tableId),
		TraceId:     "trace-id",
		Rows:        &appendRowsRequestProtoRows,
	}

	if err := stream.Send(&appendRowRequest); err != nil {
		log.Fatal(err)
	}

	resp, err := stream.Recv()
	if err != nil {
		log.Fatal(err)
	}

	log.Println(resp)

BigQuery Table Schema

CREATE TABLE `{project}.{dataset}.{table}`
(
  am STRUCT<Active INT64>,
  bm STRUCT<Active INT64>
);

Expected behavior
The proto records are writen to the BigQuery Table

Actual behavior

2022/03/23 17:31:59 rpc error: code = InvalidArgument desc = Invalid proto schema: BqMessage.proto: bqstorage_Message.bqstorage_Bm.active: "bqstorage_Flag_E.Flag" is not defined. Entity: projects/{project}/datasets/{dataset}/tables/{table}/_default

Additional Information
if you look at the descriptor produced by adapt.NormalizeDescriptor(message.ProtoReflect().Descriptor()) The nested type for bqstorage_Flag_E is only defined within the bqstorage_Am type. I think that's the bug. The Enum should be defined at the root level nested type.

{
  "name": "bqstorage_Message",
  "field":
    [
      {
        "name": "am",
        "number": 1,
        "label": 1,
        "type": 11,
        "type_name": "bqstorage_Am",
        "json_name": "am",
      },
      {
        "name": "bm",
        "number": 2,
        "label": 1,
        "type": 11,
        "type_name": "bqstorage_Bm",
        "json_name": "bm",
      },
    ],
  "nested_type":
    [
      {
        "name": "bqstorage_Am",
        "field":
          [
            {
              "name": "active",
              "number": 1,
              "label": 1,
              "type": 14,
              "type_name": "bqstorage_Flag_E.Flag",
              "json_name": "active",
            },
          ],
        "nested_type":
          [
            {
              "name": "bqstorage_Flag_E",
              "enum_type":
                [
                  {
                    "name": "Flag",
                    "value":
                      [
                        { "name": "false", "number": 0 },
                        { "name": "true", "number": 1 },
                      ],
                  },
                ],
            },
          ],
      },
      {
        "name": "bqstorage_Bm",
        "field":
          [
            {
              "name": "active",
              "number": 1,
              "label": 1,
              "type": 14,
              "type_name": "bqstorage_Flag_E.Flag",
              "json_name": "active",
            },
          ],
      },
    ],
}

[shollyman]

Thanks for the clear description. I'll work on getting this addressed.

[shollyman]

internal issue 227352429