java/protobuf: add support for grpc-java codegen (#16350)

Add support for generating gRPC stubs for Protobuf and Java using https://github.com/grpc/grpc-java.

Note: The gRPC Java plugin is a native binary which is distributed through Maven Central as a non-jar artifact.

Fixes #15691.

[ci skip-rust]

[ci skip-build-wheels]

build-support/bin/_generate_all_lockfiles_helper.py

@@ -12,6 +12,7 @@
 
 from pants.backend.cc.lint.clangformat.subsystem import ClangFormat
 from pants.backend.codegen.avro.java.subsystem import AvroSubsystem
+from pants.backend.codegen.protobuf.java.subsystem import JavaProtobufGrpcSubsystem
 from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import PythonProtobufMypyPlugin
 from pants.backend.codegen.protobuf.scala.subsystem import ScalaPBSubsystem
 from pants.backend.codegen.thrift.scrooge.subsystem import ScroogeSubsystem
@@ -126,6 +127,9 @@ def jvm(cls, tool: type[JvmToolBase], *, backend: str | None = None) -> DefaultT
     DefaultTool.jvm(JUnit),
     DefaultTool.jvm(KtlintSubsystem, backend="pants.backend.experimental.kotlin.lint.ktlint"),
     DefaultTool.jvm(ScalaPBSubsystem, backend="pants.backend.experimental.codegen.protobuf.scala"),
+    DefaultTool.jvm(
+        JavaProtobufGrpcSubsystem, backend="pants.backend.experimental.codegen.protobuf.java"
+    ),
     DefaultTool.jvm(ScalafmtSubsystem),
     DefaultTool.jvm(Scalatest),
     DefaultTool.jvm(

src/python/pants/backend/codegen/protobuf/java/BUILD

@@ -1,7 +1,8 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_sources()
+python_sources(dependencies=[":lockfile"])
+resource(name="lockfile", source="grpc-java.default.lockfile.txt")
 
 python_tests(name="tests", dependencies=[":test_resources"])
 resources(name="test_resources", sources=["*.test.lock"])

src/python/pants/backend/codegen/protobuf/java/dependency_inference.py

@@ -1,9 +1,14 @@
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
 from dataclasses import dataclass
 from typing import FrozenSet
 
-from pants.backend.codegen.protobuf.target_types import ProtobufDependenciesField
+from pants.backend.codegen.protobuf.target_types import (
+    ProtobufDependenciesField,
+    ProtobufGrpcToggleField,
+)
 from pants.build_graph.address import Address
 from pants.engine.internals.selectors import Get
 from pants.engine.rules import collect_rules, rule
@@ -23,18 +28,20 @@
 
 
 @dataclass(frozen=True)
-class ProtobufJavaRuntimeDependencyInferenceFiedSet(FieldSet):
+class ProtobufJavaRuntimeDependencyInferenceFieldSet(FieldSet):
     required_fields = (
         ProtobufDependenciesField,
         JvmResolveField,
+        ProtobufGrpcToggleField,
     )
 
     dependencies: ProtobufDependenciesField
     resolve: JvmResolveField
+    grpc: ProtobufGrpcToggleField
 
 
 class InferProtobufJavaRuntimeDependencyRequest(InferDependenciesRequest):
-    infer_from = ProtobufJavaRuntimeDependencyInferenceFiedSet
+    infer_from = ProtobufJavaRuntimeDependencyInferenceFieldSet
 
 
 @dataclass(frozen=True)
@@ -69,6 +76,54 @@ async def resolve_protobuf_java_runtime_for_resolve(
     return ProtobufJavaRuntimeForResolve(addresses)
 
 
+@dataclass(frozen=True)
+class ProtobufJavaGrpcRuntimeForResolveRequest:
+    resolve_name: str
+
+
+@dataclass(frozen=True)
+class ProtobufJavaGrpcRuntimeForResolve:
+    addresses: FrozenSet[Address]
+
+
+@rule
+async def resolve_protobuf_java_grpc_runtime_for_resolve(
+    jvm_artifact_targets: AllJvmArtifactTargets,
+    jvm: JvmSubsystem,
+    request: ProtobufJavaGrpcRuntimeForResolveRequest,
+) -> ProtobufJavaGrpcRuntimeForResolve:
+    addresses = find_jvm_artifacts_or_raise(
+        required_coordinates=[
+            # For non-Android uses:
+            # TODO: Maybe support Android jars? See https://github.com/grpc/grpc-java#download for
+            # the differences in required jars.
+            UnversionedCoordinate(
+                group="io.grpc",
+                artifact="grpc-netty-shaded",
+            ),
+            UnversionedCoordinate(
+                group="io.grpc",
+                artifact="grpc-protobuf",
+            ),
+            UnversionedCoordinate(
+                group="io.grpc",
+                artifact="grpc-stub",
+            ),
+            # TODO: This is only required for JDK 9+ according to https://github.com/grpc/grpc-java#download.
+            UnversionedCoordinate(
+                group="org.apache.tomcat",
+                artifact="annotations-api",
+            ),
+        ],
+        resolve=request.resolve_name,
+        jvm_artifact_targets=jvm_artifact_targets,
+        jvm=jvm,
+        subsystem="the Protobuf Java gRPC runtime",
+        target_type="protobuf_sources",
+    )
+    return ProtobufJavaGrpcRuntimeForResolve(addresses)
+
+
 @rule
 async def infer_protobuf_java_runtime_dependency(
     request: InferProtobufJavaRuntimeDependencyRequest,
@@ -79,8 +134,15 @@ async def infer_protobuf_java_runtime_dependency(
     protobuf_java_runtime_target_info = await Get(
         ProtobufJavaRuntimeForResolve, ProtobufJavaRuntimeForResolveRequest(resolve)
     )
+    addresses: set[Address] = set(protobuf_java_runtime_target_info.addresses)
+
+    if request.field_set.grpc.value:
+        grpc_runtime_info = await Get(
+            ProtobufJavaGrpcRuntimeForResolve, ProtobufJavaGrpcRuntimeForResolveRequest(resolve)
+        )
+        addresses.update(grpc_runtime_info.addresses)
 
-    return InferredDependencies(protobuf_java_runtime_target_info.addresses)
+    return InferredDependencies(frozenset(addresses))
 
 
 def rules():

src/python/pants/backend/codegen/protobuf/java/grpc-java.default.lockfile.txt

@@ -0,0 +1,72 @@
+# This lockfile was autogenerated by Pants. To regenerate, run:
+#
+#    ./pants generate-lockfiles
+#
+# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
+# {
+#   "version": 1,
+#   "generated_with_requirements": [
+#     "io.grpc:protoc-gen-grpc-java:1.48.0,url=not_provided,jar=not_provided,type=exe,classifier=linux-aarch_64",
+#     "io.grpc:protoc-gen-grpc-java:1.48.0,url=not_provided,jar=not_provided,type=exe,classifier=linux-x86_64",
+#     "io.grpc:protoc-gen-grpc-java:1.48.0,url=not_provided,jar=not_provided,type=exe,classifier=osx-aarch_64",
+#     "io.grpc:protoc-gen-grpc-java:1.48.0,url=not_provided,jar=not_provided,type=exe,classifier=osx-x86_64"
+#   ]
+# }
+# --- END PANTS LOCKFILE METADATA ---
+
+[[entries]]
+directDependencies = []
+dependencies = []
+file_name = "io.grpc_protoc-gen-grpc-java_exe_linux-aarch_64_1.48.0.exe"
+
+[entries.coord]
+group = "io.grpc"
+artifact = "protoc-gen-grpc-java"
+version = "1.48.0"
+packaging = "exe"
+classifier = "linux-aarch_64"
+[entries.file_digest]
+fingerprint = "9a2e8b0440778e66d8db2d1464bda94524bec187bff8c0d9c7af476a359458e9"
+serialized_bytes_length = 2302136
+[[entries]]
+directDependencies = []
+dependencies = []
+file_name = "io.grpc_protoc-gen-grpc-java_exe_linux-x86_64_1.48.0.exe"
+
+[entries.coord]
+group = "io.grpc"
+artifact = "protoc-gen-grpc-java"
+version = "1.48.0"
+packaging = "exe"
+classifier = "linux-x86_64"
+[entries.file_digest]
+fingerprint = "c189d17495903bfe9d09595b52836988c531b2eb3c83d3423bfea311bf0cb7f3"
+serialized_bytes_length = 3040480
+[[entries]]
+directDependencies = []
+dependencies = []
+file_name = "io.grpc_protoc-gen-grpc-java_exe_osx-aarch_64_1.48.0.exe"
+
+[entries.coord]
+group = "io.grpc"
+artifact = "protoc-gen-grpc-java"
+version = "1.48.0"
+packaging = "exe"
+classifier = "osx-aarch_64"
+[entries.file_digest]
+fingerprint = "351a301cb9eeac6f09bd72b5df617855e2ffa8b8b672b9241832cb13838530cf"
+serialized_bytes_length = 6334176
+[[entries]]
+directDependencies = []
+dependencies = []
+file_name = "io.grpc_protoc-gen-grpc-java_exe_osx-x86_64_1.48.0.exe"
+
+[entries.coord]
+group = "io.grpc"
+artifact = "protoc-gen-grpc-java"
+version = "1.48.0"
+packaging = "exe"
+classifier = "osx-x86_64"
+[entries.file_digest]
+fingerprint = "351a301cb9eeac6f09bd72b5df617855e2ffa8b8b672b9241832cb13838530cf"
+serialized_bytes_length = 6334176