fix: refactor codebase to protobuf-go v2 (#1489)

* refactor gengapic protov2
* refactor gencli protov2
* remove last remnants
* update test script and ci
* update bazel deps to proto go v2
* remove unnecessary wkt resolution directives
* update repositories.bzl

.github/workflows/ci.yaml

@@ -66,7 +66,8 @@ jobs:
     - name: Install tools and dependencies
       run: |
         go install golang.org/x/exp/cmd/apidiff@latest
-        go install github.com/golang/protobuf/protoc-gen-go@latest
+        go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+        go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
         curl -sSL https://github.com/googleapis/googleapis/archive/master.zip > googleapis.zip
         unzip googleapis.zip -x "googleapis-master/google/ads/*"
         mv googleapis-master /tmp/googleapis

BUILD

@@ -11,10 +11,3 @@ gazelle(name = "gazelle")
 # gazelle:resolve proto go google/api/annotations.proto  @org_golang_google_genproto//googleapis/api/annotations
 # gazelle:resolve proto proto google/longrunning/operations.proto @com_google_googleapis//google/longrunning:operations_proto
 # gazelle:resolve proto go google/longrunning/operations.proto @com_google_googleapis//google/longrunning:longrunning_go_proto
-
-# Note: Direct gazelle to resolve the protobuf-go v1 module to rules_go well
-# known types. This will be removed when we migrate to protobuf-go v2.
-#
-# gazelle:resolve go go github.com/golang/protobuf/protoc-gen-go/plugin @io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto
-# gazelle:resolve go go github.com/golang/protobuf/protoc-gen-go/descriptor @io_bazel_rules_go//proto/wkt:descriptor_go_proto
-# gazelle:resolve go go github.com/golang/protobuf/ptypes/duration @io_bazel_rules_go//proto/wkt:duration_go_proto

cmd/protoc-gen-go_cli/BUILD.bazel

@@ -8,8 +8,8 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//internal/gencli",
-        "@io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto",
         "@org_golang_google_protobuf//proto",
+        "@org_golang_google_protobuf//types/pluginpb",
     ],
 )
 

cmd/protoc-gen-go_cli/main.go

@@ -19,17 +19,17 @@ import (
 	"log"
 	"os"
 
-	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
 	"github.com/googleapis/gapic-generator-go/internal/gencli"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/pluginpb"
 )
 
 func main() {
 	reqBytes, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatal(err)
 	}
-	var genReq plugin.CodeGeneratorRequest
+	var genReq pluginpb.CodeGeneratorRequest
 	if err := proto.Unmarshal(reqBytes, &genReq); err != nil {
 		log.Fatal(err)
 	}
@@ -38,7 +38,7 @@ func main() {
 	if err != nil {
 		genResp.Error = proto.String(err.Error())
 	}
-	genResp.SupportedFeatures = proto.Uint64(uint64(plugin.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL))
+	genResp.SupportedFeatures = proto.Uint64(uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL))
 
 	outBytes, err := proto.Marshal(genResp)
 	if err != nil {

cmd/protoc-gen-go_gapic/BUILD

@@ -8,8 +8,8 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//internal/gengapic",
-        "@io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto",
         "@org_golang_google_protobuf//proto",
+        "@org_golang_google_protobuf//types/pluginpb",
     ],
 )
 

cmd/protoc-gen-go_gapic/main.go

@@ -19,17 +19,17 @@ import (
 	"log"
 	"os"
 
-	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
 	"github.com/googleapis/gapic-generator-go/internal/gengapic"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/pluginpb"
 )
 
 func main() {
 	reqBytes, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatal(err)
 	}
-	var genReq plugin.CodeGeneratorRequest
+	var genReq pluginpb.CodeGeneratorRequest
 	if err := proto.Unmarshal(reqBytes, &genReq); err != nil {
 		log.Fatal(err)
 	}
@@ -39,7 +39,7 @@ func main() {
 		genResp.Error = proto.String(err.Error())
 	}
 
-	genResp.SupportedFeatures = proto.Uint64(uint64(plugin.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL))
+	genResp.SupportedFeatures = proto.Uint64(uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL))
 
 	outBytes, err := proto.Marshal(genResp)
 	if err != nil {

go.mod

@@ -4,7 +4,6 @@ require (
 	cloud.google.com/go/iam v1.1.7
 	cloud.google.com/go/longrunning v0.5.6
 	github.com/ghodss/yaml v1.0.0
-	github.com/golang/protobuf v1.5.4
 	github.com/google/go-cmp v0.6.0
 	github.com/jhump/protoreflect v1.15.6
 	gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a
@@ -17,6 +16,7 @@ require (
 
 require (
 	github.com/bufbuild/protocompile v0.8.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 // indirect
 	gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 // indirect
 	gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 // indirect

internal/gencli/BUILD.bazel

@@ -18,10 +18,10 @@ go_library(
         "//internal/printer",
         "@com_github_jhump_protoreflect//desc",
         "@com_google_cloud_go_longrunning//autogen/longrunningpb",
-        "@io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto",
-        "@io_bazel_rules_go//proto/wkt:descriptor_go_proto",
         "@org_golang_google_genproto_googleapis_api//annotations",
         "@org_golang_google_protobuf//proto",
+        "@org_golang_google_protobuf//types/descriptorpb",
+        "@org_golang_google_protobuf//types/pluginpb",
         "@org_golang_x_text//cases",
         "@org_golang_x_text//language",
     ],
@@ -42,6 +42,6 @@ go_test(
     deps = [
         "//internal/pbinfo",
         "//internal/txtdiff",
-        "@io_bazel_rules_go//proto/wkt:descriptor_go_proto",
+        "@org_golang_google_protobuf//types/descriptorpb",
     ],
 )

internal/gencli/cmd_file_test.go

@@ -18,9 +18,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/golang/protobuf/protoc-gen-go/descriptor"
 	"github.com/googleapis/gapic-generator-go/internal/pbinfo"
 	"github.com/googleapis/gapic-generator-go/internal/txtdiff"
+	"google.golang.org/protobuf/types/descriptorpb"
 )
 
 func TestCommandFile(t *testing.T) {
@@ -46,7 +46,7 @@ func TestCommandFile(t *testing.T) {
 		Flags: []*Flag{
 			&Flag{
 				Name:      "task",
-				Type:      descriptor.FieldDescriptorProto_TYPE_STRING,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_STRING,
 				Required:  true,
 				FieldName: "Task",
 				VarName:   "CreateTodoInput",
@@ -56,14 +56,14 @@ func TestCommandFile(t *testing.T) {
 				Name:      "done",
 				FieldName: "Done",
 				VarName:   "CreateTodoInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_BOOL,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_BOOL,
 				Usage:     "task completion status",
 				Optional:  true,
 			},
 			&Flag{
 				Name:          "priority",
 				FieldName:     "Priority",
-				Type:          descriptor.FieldDescriptorProto_TYPE_ENUM,
+				Type:          descriptorpb.FieldDescriptorProto_TYPE_ENUM,
 				Usage:         "importance of the task",
 				Message:       "Priority",
 				MessageImport: pbinfo.ImportSpec{Name: "todopb"},
@@ -93,7 +93,7 @@ func TestCommandFile(t *testing.T) {
 				Name:      "id",
 				FieldName: "Id",
 				VarName:   "StartTodoInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_INT32,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_INT32,
 				Required:  true,
 				Usage:     "task to start",
 			},
@@ -134,7 +134,7 @@ func TestCommandFile(t *testing.T) {
 				Name:      "id",
 				FieldName: "Id",
 				VarName:   "WatchTodoInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_INT32,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_INT32,
 				Required:  true,
 				Usage:     "task to watch",
 			},

internal/gencli/flag.go

@@ -20,15 +20,15 @@ import (
 	"unicode"
 	"unicode/utf8"
 
-	"github.com/golang/protobuf/protoc-gen-go/descriptor"
 	"github.com/googleapis/gapic-generator-go/internal/pbinfo"
 	"github.com/jhump/protoreflect/desc"
+	"google.golang.org/protobuf/types/descriptorpb"
 )
 
 // Flag is used to represent fields as flags
 type Flag struct {
 	Name          string
-	Type          descriptor.FieldDescriptorProto_Type
+	Type          descriptorpb.FieldDescriptorProto_Type
 	Message       string
 	Repeated      bool
 	Required      bool
@@ -61,7 +61,7 @@ func (f *Flag) GenFlag() string {
 	fType := toTitle(tStr)
 
 	if f.Repeated {
-		if f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
+		if f.Type == descriptorpb.FieldDescriptorProto_TYPE_MESSAGE {
 			// repeated Messages are entered as JSON strings and unmarshaled into the Message type later
 			return fmt.Sprintf(`StringArrayVar(&%s, "%s", []string{}, "%s")`, f.VarName, f.Name, f.Usage)
 		}
@@ -111,17 +111,17 @@ func (f *Flag) GenFlag() string {
 
 // IsMessage is a template helper that reports if the flag is a message type
 func (f *Flag) IsMessage() bool {
-	return f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE
+	return f.Type == descriptorpb.FieldDescriptorProto_TYPE_MESSAGE
 }
 
 // IsEnum is a template helper that reports if the flag is of an enum type
 func (f *Flag) IsEnum() bool {
-	return f.Type == descriptor.FieldDescriptorProto_TYPE_ENUM
+	return f.Type == descriptorpb.FieldDescriptorProto_TYPE_ENUM
 }
 
 // IsBytes is a helper that reports if the flag is of a type bytes
 func (f *Flag) IsBytes() bool {
-	return f.Type == descriptor.FieldDescriptorProto_TYPE_BYTES
+	return f.Type == descriptorpb.FieldDescriptorProto_TYPE_BYTES
 }
 
 // EnumFieldAccess constructs the input message field accessor for an enum

internal/gencli/flag_test.go

@@ -17,7 +17,7 @@ package gencli
 import (
 	"testing"
 
-	"github.com/golang/protobuf/protoc-gen-go/descriptor"
+	"google.golang.org/protobuf/types/descriptorpb"
 )
 
 func TestGenFlag(t *testing.T) {
@@ -30,7 +30,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_STRING,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_STRING,
 				Usage:     "this is the usage",
 			},
 			want: `StringVar(&ClientInput.Field, "field", "", "this is the usage")`,
@@ -40,7 +40,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_BOOL,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_BOOL,
 				Usage:     "this is the usage",
 			},
 			want: `BoolVar(&ClientInput.Field, "field", false, "this is the usage")`,
@@ -50,7 +50,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_INT32,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_INT32,
 				Usage:     "this is the usage",
 			},
 			want: `Int32Var(&ClientInput.Field, "field", 0, "this is the usage")`,
@@ -60,7 +60,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_FLOAT,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_FLOAT,
 				Usage:     "this is the usage",
 			},
 			want: `Float32Var(&ClientInput.Field, "field", 0.0, "this is the usage")`,
@@ -70,7 +70,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_DOUBLE,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_DOUBLE,
 				Usage:     "this is the usage",
 			},
 			want: `Float64Var(&ClientInput.Field, "field", 0.0, "this is the usage")`,
@@ -80,7 +80,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_BYTES,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_BYTES,
 				Usage:     "this is the usage",
 			},
 			want: `BytesHexVar(&ClientInput.Field, "field", []byte{}, "this is the usage")`,
@@ -90,7 +90,7 @@ func TestGenFlag(t *testing.T) {
 				Name:      "field",
 				FieldName: "Field",
 				VarName:   "ClientInput",
-				Type:      descriptor.FieldDescriptorProto_TYPE_STRING,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_STRING,
 				Usage:     "this is the usage",
 				Repeated:  true,
 			},
@@ -100,7 +100,7 @@ func TestGenFlag(t *testing.T) {
 			f: &Flag{
 				Name:      "field",
 				FieldName: "Field",
-				Type:      descriptor.FieldDescriptorProto_TYPE_MESSAGE,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_MESSAGE,
 				Usage:     "this is the usage",
 				Repeated:  true,
 				VarName:   "ClientInputField",
@@ -111,7 +111,7 @@ func TestGenFlag(t *testing.T) {
 			f: &Flag{
 				Name:      "field",
 				FieldName: "Field",
-				Type:      descriptor.FieldDescriptorProto_TYPE_ENUM,
+				Type:      descriptorpb.FieldDescriptorProto_TYPE_ENUM,
 				Usage:     "this is the usage",
 				VarName:   "ClientInputField",
 			},
@@ -121,7 +121,7 @@ func TestGenFlag(t *testing.T) {
 			f: &Flag{
 				Name:         "oneof.field",
 				FieldName:    "Field",
-				Type:         descriptor.FieldDescriptorProto_TYPE_STRING,
+				Type:         descriptorpb.FieldDescriptorProto_TYPE_STRING,
 				Usage:        "this is the usage",
 				VarName:      "ClientInputOneofField",
 				IsOneOfField: true,
@@ -132,7 +132,7 @@ func TestGenFlag(t *testing.T) {
 			f: &Flag{
 				Name:    "oneof_selector",
 				VarName: "ClientInputOneofSelector",
-				Type:    descriptor.FieldDescriptorProto_TYPE_STRING,
+				Type:    descriptorpb.FieldDescriptorProto_TYPE_STRING,
 				Usage:   "this is the usage",
 				OneOfs:  map[string]*Flag{"test": &Flag{}},
 			},
@@ -151,11 +151,11 @@ func TestIsMessage(t *testing.T) {
 		want bool
 	}{
 		{
-			f:    &Flag{Type: descriptor.FieldDescriptorProto_TYPE_MESSAGE},
+			f:    &Flag{Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE},
 			want: true,
 		},
 		{
-			f:    &Flag{Type: descriptor.FieldDescriptorProto_TYPE_STRING},
+			f:    &Flag{Type: descriptorpb.FieldDescriptorProto_TYPE_STRING},
 			want: false,
 		},
 	} {
@@ -171,11 +171,11 @@ func TestIsEnum(t *testing.T) {
 		want bool
 	}{
 		{
-			f:    &Flag{Type: descriptor.FieldDescriptorProto_TYPE_ENUM},
+			f:    &Flag{Type: descriptorpb.FieldDescriptorProto_TYPE_ENUM},
 			want: true,
 		},
 		{
-			f:    &Flag{Type: descriptor.FieldDescriptorProto_TYPE_STRING},
+			f:    &Flag{Type: descriptorpb.FieldDescriptorProto_TYPE_STRING},
 			want: false,
 		},
 	} {