Manually vendor the required .proto files from googleapis (#2765)

This resolves some weird references to GOPATH, that while it worked, was weird (#2763). It also simplifies new user onboarding (one less instruction to follow), makes CI more lightweight (one less big repo to clone), and avoids mismatched versions of these protos from causing havoc (because they were less hermetic before).

CHANGELOG.md

@@ -12,6 +12,13 @@
 * Updated golangci-lint to v1.46.1 (developers should update to this version)
 * Removed dependency on certificate-transparency-go
 
+### Developer updates
+
+* #2765 copies the required protos from `googleapis` into `third_party` in this
+  repository. This simplifies the preconditions in order to compile the proto
+  definitions, and removes a big dependency on `$GOPATH/src` which was archaic;
+  `$GOPATH/src/github.com/googleapis/googleapis` is no longer required.
+
 ## v1.4.1
 
 * `countFromInformationSchema` function to add support for MySQL 8.

README.md

@@ -179,12 +179,6 @@ the original files; if you do, you'll need to install the prerequisites:
     go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc; \
     go install golang.org/x/tools/cmd/stringer
     ```
-  - protocol buffer definitions for standard Google APIs:
-    TODO(#2763): this doesn't need to be in `GOPATH/src`.
-
-    ```bash
-    git clone https://github.com/googleapis/googleapis.git $(go env GOPATH)/src/github.com/googleapis/googleapis
-    ```
 
 and run the following:
 

gen.go

@@ -14,7 +14,7 @@
 
 package trillian
 
-//go:generate protoc -I=. -I=$GOPATH/src/github.com/googleapis/googleapis --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. --go-grpc_opt=require_unimplemented_servers=false trillian_log_api.proto trillian_admin_api.proto trillian.proto --doc_out=markdown,api.md:./docs/
+//go:generate protoc -I=. -I=third_party/googleapis --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. --go-grpc_opt=require_unimplemented_servers=false trillian_log_api.proto trillian_admin_api.proto trillian.proto --doc_out=markdown,api.md:./docs/
 //go:generate protoc -I=. --go_out=paths=source_relative:. crypto/keyspb/keyspb.proto
 
 //go:generate mockgen -package tmock -destination testonly/tmock/mock_log_server.go  github.com/google/trillian TrillianLogServer

integration/cloudbuild/prepare.sh

@@ -5,13 +5,6 @@ set -ex
 # TODO(pavelkalinnikov): This script can be made a definitive "how to" for
 # setting up dev environment. Eliminate duplicating these steps in many places.
 
-# Install Google API definitions. Some APIs are used by the protoc tool when
-# [re-]generating Trillian API packages from protobufs.
-#
-# TODO(pavelkalinnikov): It doesn't have to be within $GOPATH. There is no Go
-# code/module in this repository, and we use it only for API proto definitions.
-git clone --depth=1 https://github.com/googleapis/googleapis.git "$GOPATH/src/github.com/googleapis/googleapis"
-
 # Install the tooling used for auto-generating code. Specifically, these are the
 # tools mentioned in //go:generate comments throughout this repository, and used
 # by the "go generate" command. In CI this is used for ensuring that developers

quota/etcd/quotapb/gen.go

@@ -14,4 +14,4 @@
 
 package quotapb
 
-//go:generate protoc -I=. -I=$GOPATH/src/github.com/googleapis/googleapis --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. --go-grpc_opt=require_unimplemented_servers=false quotapb.proto
+//go:generate protoc -I=. -I=../../../third_party/googleapis --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. --go-grpc_opt=require_unimplemented_servers=false quotapb.proto

storage/cloudspanner/spannerpb/gen.go

@@ -14,4 +14,4 @@
 
 package spannerpb
 
-//go:generate protoc -I=. -I=$GOPATH/src -I=$GOPATH/src/github.com/googleapis/googleapis/ --go_out=paths=source_relative:. spanner.proto
+//go:generate protoc -I=. -I=../../../third_party/googleapis --go_out=paths=source_relative:. spanner.proto

storage/storagepb/gen.go

@@ -24,4 +24,4 @@
 //  b) data structures are defined in the application layer.
 package storagepb
 
-//go:generate protoc -I=. -I=$GOPATH/src --go_out=paths=source_relative:. storage.proto
+//go:generate protoc -I=. --go_out=paths=source_relative:. storage.proto

third_party/googleapis/README.md

@@ -0,0 +1,26 @@
+## Purpose
+
+This directory is manually vendored in order to take the required proto definitions from `googleapis`.
+The principle is that the dependency should be hermetic and lightweight.
+
+Previously, the instructions were to manually clone the whole googleapis repository at head, into
+a known location.
+
+## Updating
+
+The required files were determined by manual inspection of the codebase and closing the
+transitive dependencies by checking the imports of the files.
+Should the upstream repository be significantly refactored then it is possible that further files
+may need to be imported if an update is performed.
+
+The files here were cloned at commit `c81bb70`.
+The workflow is simple and documented here for future maintainers:
+```sh
+export GA_VERSION=c81bb701eb53991d6faf74b2656eaf539261a122
+mkdir -p google/api
+wget https://raw.githubusercontent.com/googleapis/googleapis/$GA_VERSION/google/api/annotations.proto -O google/api/annotations.proto
+wget https://raw.githubusercontent.com/googleapis/googleapis/$GA_VERSION/google/api/http.proto -O google/api/http.proto
+mkdir -p google/rpc
+wget https://raw.githubusercontent.com/googleapis/googleapis/$GA_VERSION/google/rpc/code.proto -O google/rpc/code.proto
+wget https://raw.githubusercontent.com/googleapis/googleapis/$GA_VERSION/google/rpc/status.proto -O google/rpc/status.proto
+```

third_party/googleapis/google/api/annotations.proto

@@ -0,0 +1,31 @@
+// Copyright 2015 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.api;
+
+import "google/api/http.proto";
+import "google/protobuf/descriptor.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
+option java_multiple_files = true;
+option java_outer_classname = "AnnotationsProto";
+option java_package = "com.google.api";
+option objc_class_prefix = "GAPI";
+
+extend google.protobuf.MethodOptions {
+  // See `HttpRule`.
+  HttpRule http = 72295728;
+}