Merge branch 'master' into workflow-auth

.gitattributes

@@ -0,0 +1,11 @@
+# vendor is marked as linguist-vendored by default, but this
+# only excluses the code from stats.
+#
+# Forcing it to be linguist-generated will also exclude it from diffs.
+#
+# See https://github.com/github-linguist/linguist/blob/master/docs/overrides.md#summary
+# for more info.
+*.pb.go linguist-generated
+*.mock.go linguist-generated
+internal/protojson/** linguist-vendored
+internal/strcase/** linguist-vendored

.github/workflows/ci.yml

@@ -14,6 +14,6 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.18"
+          go-version: "1.20"
       - name: Check
         run: make check test

.github/workflows/update-proto.yml

@@ -33,11 +33,11 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: "1.20"
 
       - uses: arduino/setup-protoc@v1
         with:
-          version: '3.x'
+          version: "3.x"
 
       - name: Re-build proto
         run: |
@@ -52,4 +52,9 @@ jobs:
         run: |
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
           git add .
-          git commit -m "${{ github.event.inputs.commit_message }}" && git push || echo "No changes to commit"
+          git commit -m "${{ github.event.inputs.commit_message }}"
+          if [ $? -eq 0 ]; then
+            git push
+          else
+            echo "No changes to commit"
+          fi

.gitignore

@@ -1,2 +1,3 @@
 .idea
-.gobincache
+.gobincache
+.go-helpers-installed

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "proto/api"]
 	path = proto/api
 	url = https://github.com/temporalio/api
+[submodule "proto/api-cloud"]
+	path = proto/api-cloud
+	url = https://github.com/temporalio/api-cloud

Makefile

@@ -1,10 +1,11 @@
 $(VERBOSE).SILENT:
+all: install test
 ############################# Main targets #############################
 # Install everything, update submodule, and compile proto files.
 install: grpc-install mockgen-install goimports-install update-proto
 
 # Compile proto files.
-proto: grpc goimports proxy grpc-mock copyright
+proto: http-api-docs grpc goimports proxy grpc-mock copyright
 
 # Update submodule and compile proto files.
 update-proto: update-proto-submodule proto update-dependencies gomodtidy
@@ -22,48 +23,66 @@ COLOR := "\e[1;36m%s\e[0m\n"
 
 PINNED_DEPENDENCIES := \
 
-
 PROTO_ROOT := proto/api
-PROTO_FILES = $(shell find $(PROTO_ROOT) -name "*.proto")
-PROTO_DIRS = $(sort $(dir $(PROTO_FILES)))
+PROTO_CLOUD_ROOT := proto/api-cloud
 PROTO_OUT := .
 PROTO_IMPORTS = \
-	-I=$(PROTO_ROOT) \
-	-I=$(shell go list -modfile build/go.mod -m -f '{{.Dir}}' github.com/temporalio/gogo-protobuf)/protobuf \
-	-I=$(shell go list -m -f '{{.Dir}}' github.com/grpc-ecosystem/grpc-gateway)/third_party/googleapis
+	-I=$(PROTO_ROOT)
+PROTO_PATHS = paths=source_relative:$(PROTO_OUT)
+
+OAPI_ROOT := $(PROTO_ROOT)/openapi
+OAPI_OUT := temporalproto/openapi
 
 $(PROTO_OUT):
 	mkdir $(PROTO_OUT)
 
 ##### git submodule for proto files #####
 update-proto-submodule:
 	printf $(COLOR) "Update proto-submodule..."
-	git submodule update --init --force --remote $(PROTO_ROOT)
+	git -c protocol.file.allow=always submodule update --init --force --remote $(PROTO_ROOT)
 
 ##### Compile proto files for go #####
-grpc: gogo-grpc fix-path
+grpc: http-api-docs go-grpc copy-helpers
+
+# Only install helper when its source has changed
+HELPER_FILES = $(shell find ./cmd/protoc-gen-go-helpers)
+.go-helpers-installed: $(HELPER_FILES)
+	printf $(COLOR) "Installing protoc plugin"
+	@go install ./cmd/protoc-gen-go-helpers
+	@touch $@
+
+go-grpc: clean .go-helpers-installed $(PROTO_OUT)
+	printf $(COLOR) "Compile for go-gRPC..."
+	go run ./cmd/protogen \
+		--root=$(PROTO_ROOT) \
+		--root=$(PROTO_CLOUD_ROOT) \
+		--output=$(PROTO_OUT) \
+		--exclude=internal \
+		--exclude=proto/api/google \
+		-p go-grpc_out=$(PROTO_PATHS) \
+		-p grpc-gateway_out=allow_patch_feature=false,$(PROTO_PATHS) \
+		-p go-helpers_out=$(PROTO_PATHS)
 
-gogo-grpc: clean $(PROTO_OUT)
-	printf $(COLOR) "Compiling for gogo-gRPC..."
-	$(foreach PROTO_DIR,$(PROTO_DIRS),\
-		protoc --fatal_warnings $(PROTO_IMPORTS) \
-			--gogoslick_out=Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/wrappers.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,plugins=grpc,paths=source_relative:$(PROTO_OUT) \
-			--grpc-gateway_out=allow_patch_feature=false,paths=source_relative:$(PROTO_OUT) \
-		$(PROTO_DIR)*.proto;)
-
-fix-path:
 	mv -f $(PROTO_OUT)/temporal/api/* $(PROTO_OUT) && rm -rf $(PROTO_OUT)/temporal
-	# Also copy the payload JSON helper
-	cp $(PROTO_OUT)/internal/temporalcommonv1/payload_json.go $(PROTO_OUT)/common/v1/
 
-# All generated service files pathes relative to PROTO_OUT.
-PROTO_GRPC_SERVICES = $(patsubst $(PROTO_OUT)/%,%,$(shell find $(PROTO_OUT) -name "service.pb.go"))
-service_name = $(firstword $(subst /, ,$(1)))
-mock_file_name = $(call service_name,$(1))mock/$(subst $(call service_name,$(1))/,,$(1:go=mock.go))
+http-api-docs: go-grpc
+	go run cmd/encode-openapi-spec/main.go \
+		-v2=$(OAPI_ROOT)/openapiv2.json \
+		-v2-out=$(OAPI_OUT)/openapiv2.go \
+		-v3=$(OAPI_ROOT)/openapiv3.yaml \
+		-v3-out=$(OAPI_OUT)/openapiv3.go
+
+# Copy the payload helpers
+copy-helpers:
+	chmod +w $(PROTO_OUT)/common/v1/payload_json.go 2>/dev/null || true
+	cp $(PROTO_OUT)/internal/temporalcommonv1/payload_json.go $(PROTO_OUT)/common/v1/
+	chmod -w $(PROTO_OUT)/common/v1/payload_json.go
 
 grpc-mock:
 	printf $(COLOR) "Generate gRPC mocks..."
-	$(foreach PROTO_GRPC_SERVICE,$(PROTO_GRPC_SERVICES),cd $(PROTO_OUT) && mockgen -package $(call service_name,$(PROTO_GRPC_SERVICE))mock -source $(PROTO_GRPC_SERVICE) -destination $(call mock_file_name,$(PROTO_GRPC_SERVICE))$(NEWLINE) )
+	mockgen -package operatorservicemock -source operatorservice/v1/service_grpc.pb.go -destination operatorservicemock/v1/service_grpc.pb.mock.go
+	mockgen -package workflowservicemock -source workflowservice/v1/service_grpc.pb.go -destination workflowservicemock/v1/service_grpc.pb.mock.go
+	mockgen -package cloudservicemock -source cloud/cloudservice/v1/service_grpc.pb.go -destination cloud/cloudservicemock/v1/service_grpc.pb.mock.go
 
 .PHONY: proxy
 proxy:
@@ -75,21 +94,15 @@ goimports:
 	goimports -w $(PROTO_OUT)
 
 ##### Plugins & tools #####
-grpc-install: gogo-protobuf-install
-	printf $(COLOR) "Install/update gRPC plugins..."
-	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
-
-gogo-protobuf-install: go-protobuf-install
-	go install github.com/temporalio/gogo-protobuf/protoc-gen-gogoslick@latest
-	go install -modfile build/go.mod github.com/temporalio/gogo-protobuf/protoc-gen-gogoslick
-	go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@latest
-
-go-protobuf-install:
-	go install github.com/golang/protobuf/protoc-gen-go@latest
+grpc-install:
+	@printf $(COLOR) "Install/update grpc and plugins..."
+	@go install google.golang.org/protobuf/cmd/protoc-gen-go@latest 
+	@go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+	@go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@latest
 
 mockgen-install:
 	printf $(COLOR) "Install/update mockgen..."
-	go install github.com/golang/mock/mockgen@v1.6.0
+	go install -modfile=build/go.mod github.com/golang/mock/mockgen
 
 goimports-install:
 	printf $(COLOR) "Install/update goimports..."
@@ -111,19 +124,22 @@ gomodtidy:
 
 ##### Test #####
 
-test:
-	go test ./proxy ./serviceerror
+# We need to ensure protos are up to date to test our UTF-8 post-processing
+test: copy-helpers
+	go test -tags protolegacy ./...
 
 ##### Check #####
 
 generatorcheck:
 	printf $(COLOR) "Check generated code is not stale..."
-	(cd ./cmd/proxygenerator && go mod tidy && go run ./ -verifyOnly)
+	#(cd ./cmd/proxygenerator && go mod tidy && go run ./ -verifyOnly)
 
 check: generatorcheck
 
 ##### Clean #####
 clean:
 	printf $(COLOR) "Deleting generated go files..."
 	# Delete all directories with *.pb.go and *.mock.go files from $(PROTO_OUT)
-	find $(PROTO_OUT) \( -name "*.pb.go" -o -name "*.mock.go" \) | xargs -I{} dirname {} | sort -u | xargs rm -rf
+	find $(PROTO_OUT) \( -name "*.pb.go" -o -name "*.mock.go" -o -name "*.go-helpers.go" \) | xargs -I{} dirname {} | egrep -v 'testprotos' | sort -u | xargs rm -rf
+	# Delete entire cloud dir
+	rm -rf cloud

README.md

@@ -2,18 +2,21 @@
 
 Generated Go files from Temporal [api](https://github.com/temporalio/api) repository.
 
+_Note that any changes merged to [api](https://github.com/temporalio/api) will automatically trigger a [GitHub workflow](https://github.com/temporalio/api-go/blob/master/.github/workflows/update-proto.yml) that recompiles the proto files and commits the results to master._
+
 ## How to use
 
 To install in your project run:
 ```
-go get -u go.temporal.io/api
+go get go.temporal.io/api
 ```
 
 ## Rebuild
 
 Run `make` once to install all plugins and tools (`protoc` and `go` must be installed manually).
 
-Run `make update-proto` to update submodule and recompile proto files.
+Run `make update-proto` to update the `proto/api` submodule and recompile proto files. The `proto/api-cloud` submodule
+must be updated manually.
 
 ## License