proto: update the mechanism for generating protos from spec repo (#7269)

This pull request updates the `protocgen.sh` script to insert the `go_package` option to all of the downloaded proto files. A related pull request into the spec repo removes this options from the .proto files: tendermint/spec#358

This pull requests, along with the related spec PR, aim to move the creation of the `tendermintdev/docker-build-proto` container into the spec repo. This change also relies on several fixes to that container that are made in the PR into the spec repo.

Makefile

@@ -15,7 +15,7 @@ endif
 LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
-DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -85,24 +85,24 @@ proto-all: proto-gen proto-lint proto-check-breaking
 proto-gen:
 	@docker pull -q tendermintdev/docker-build-proto
 	@echo "Generating Protobuf files"
-	@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
+	@$(DOCKER_PROTO_BUILDER) sh ./scripts/protocgen.sh
 .PHONY: proto-gen
 
 proto-lint:
-	@$(DOCKER_BUF) lint --error-format=json
+	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json
 .PHONY: proto-lint
 
 proto-format:
 	@echo "Formatting Protobuf files"
-	docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
+	@$(DOCKER_PROTO_BUILDER) find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
 .PHONY: proto-format
 
 proto-check-breaking:
-	@$(DOCKER_BUF) breaking --against .git#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git#branch=master
 .PHONY: proto-check-breaking
 
 proto-check-breaking-ci:
-	@$(DOCKER_BUF) breaking --against $(HTTPS_GIT)#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT)#branch=master
 .PHONY: proto-check-breaking-ci
 
 ###############################################################################

scripts/protocgen.sh

@@ -1,39 +1,44 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
-set -eo pipefail
+# By default, this script runs against the latest commit to the master branch
+# in the Tendermint spec repository. To use this script with a different version
+# of the spec repository, run it with the $VERS environment variable set to the
+# desired branch name or commit hash from the spec repo.
 
 : ${VERS:=master}
-URL_PATH=archive/
-if [[ VERS -ne master ]]; then
-    URL_PATH=archive/refs/tags/v
-fi
 
-# Edit this line to clone your branch, if you are modifying protobuf files
-curl -qL "https://github.com/tendermint/spec/${URL_PATH}${VERS}.tar.gz" | tar -xjf - spec-"$VERS"/proto/
+echo "fetching proto files"
 
-cp -r ./spec-"$VERS"/proto/tendermint/** ./proto/tendermint
+# Get shortened ref of commit
+REF=$(curl -H "Accept: application/vnd.github.v3.sha" -qL \
+  "https://api.github.com/repos/tendermint/spec/commits/${VERS}" \
+  | cut -c -7)
 
-buf generate --path proto/tendermint
+readonly OUTDIR="tendermint-spec-${REF}"
+curl -qL "https://api.github.com/repos/tendermint/spec/tarball/${REF}" | tar -xzf - ${OUTDIR}/
+
+cp -r ${OUTDIR}/proto/tendermint/* ./proto/tendermint
+cp -r ${OUTDIR}/third_party/** ./third_party
+
+MODNAME="$(go list -m)"
+find ./proto/tendermint -name '*.proto' -not -path "./proto/tendermint/abci/types.proto" \
+	-exec sh ./scripts/protopackage.sh {} "$MODNAME" ';'
+
+# For historical compatibility, the abci file needs to get a slightly different import name
+# so that it can be moved into the ./abci/types directory.
+sh ./scripts/protopackage.sh ./proto/tendermint/abci/types.proto $MODNAME "abci/types"
+
+buf generate --path proto/tendermint --template ./${OUTDIR}/buf.gen.yaml --config ./${OUTDIR}/buf.yaml
 
 mv ./proto/tendermint/abci/types.pb.go ./abci/types
 
-echo "proto files have been generated"
+echo "proto files have been compiled"
 
 echo "removing copied files"
 
-rm -rf ./proto/tendermint/abci
-rm -rf ./proto/tendermint/blocksync/types.proto
-rm -rf ./proto/tendermint/consensus/types.proto
-rm -rf ./proto/tendermint/mempool/types.proto
-rm -rf ./proto/tendermint/p2p/types.proto
-rm -rf ./proto/tendermint/p2p/conn.proto
-rm -rf ./proto/tendermint/p2p/pex.proto
-rm -rf ./proto/tendermint/statesync/types.proto
-rm -rf ./proto/tendermint/types/block.proto
-rm -rf ./proto/tendermint/types/evidence.proto
-rm -rf ./proto/tendermint/types/params.proto
-rm -rf ./proto/tendermint/types/types.proto
-rm -rf ./proto/tendermint/types/validator.proto
-rm -rf ./proto/tendermint/version/types.proto
-
-rm -rf ./spec-"$VERS"
+find ${OUTDIR}/proto/tendermint/ -name *.proto \
+	| sed "s/$OUTDIR\/\(.*\)/\1/g" \
+	| xargs -I {} rm {}
+
+rm -rf ${OUTDIR}

scripts/protopackage.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/sh
+set -eo pipefail
+
+# This script appends the "option go_package" proto option to the file located at $FNAME.
+# This option specifies what the package will be named when imported by other packages.
+# This option is not directly included in the proto files to allow the files to more easily
+# be hosted in github.com/tendermint/spec and shared between other repos.
+# If the option is already specified in the file, it will be replaced using the
+# arguments passed to this script.
+
+FNAME="${1:?missing required .proto filename}"
+MODNAME=$(echo $2| sed 's/\//\\\//g')
+PACKAGE="$(dirname $FNAME | sed 's/^\.\/\(.*\)/\1/g' | sed 's/\//\\\//g')"
+if [[ ! -z "$3" ]]; then
+	PACKAGE="$(echo $3 | sed 's/\//\\\//g')"
+fi
+
+
+if ! grep -q 'option\s\+go_package\s\+=\s\+.*;' $FNAME; then 
+	sed -i "s/\(package tendermint.*\)/\1\n\noption go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+else
+	sed -i "s/option\s\+go_package\s\+=\s\+.*;/option go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+fi