[Core] Upgrade grpc from 1.46.6 to 1.57.0

.bazelrc

@@ -8,6 +8,11 @@ build:windows --action_env=PATH
 # For --compilation_mode=dbg, consider enabling checks in the standard library as well (below).
 build --compilation_mode=opt
 # Using C++ 17 on all platforms.
+build:linux --host_cxxopt="-std=c++17"
+build:macos --host_cxxopt="-std=c++17"
+build:clang-cl --host_cxxopt="-std=c++17"
+build:msvc-cl --host_cxxopt="/std:c++17"
+build:windows --host_cxxopt="/std:c++17"
 build:linux --cxxopt="-std=c++17"
 build:macos --cxxopt="-std=c++17"
 build:clang-cl --cxxopt="-std=c++17"
@@ -187,12 +192,12 @@ build:debug --strip="never"
 
 # Undefined Behavior Sanitizer
 build:ubsan --strip=never
-build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -fno-sanitize=vptr
 build:ubsan --copt -fno-sanitize-recover=all
 build:ubsan --copt -g
 build:ubsan --linkopt -fsanitize=undefined
 build:ubsan --linkopt -fno-sanitize-recover=all
+build:ubsan --per_file_copt="-external/com_github_grpc_grpc/.*@-fsanitize=undefined"
 
 # Import local specific llvm config options, which can be generated by
 # ci/env/install-llvm-dependencies.sh

.buildkite/lint.rayci.yml

@@ -72,3 +72,9 @@ steps:
     commands:
       - ./ci/lint/check-documentation-style.sh
     depends_on: forge
+
+  - label: ":lint-roller: lint: python proto"
+    commands:
+      - pip install -r ci/compile_py_proto/requirements_compile_py_proto.txt
+      - ./ci/lint/check-python-proto.sh
+    depends_on: forge

.gitattributes

@@ -0,0 +1,3 @@
+/python/ray/core/generated/**/* linguist-generated=true
+/python/ray/serve/generated/**/* linguist-generated=true
+

.gitignore

@@ -1,7 +1,7 @@
 # The build output should clearly not be checked in
 *test-output.xml
 /bazel-*
-/python/ray/core
+/python/ray/core/**/*
 /python/ray/pickle5_files/
 /python/ray/thirdparty_files/
 /python/ray/_private/runtime_env/agent/thirdparty_files/
@@ -98,6 +98,10 @@ python/ray/_raylet.pyd
 *_pb2.py
 *.pb.h
 *.pb.cc
+!/python/ray/core/generated/
+!/python/ray/core/generated/*.py
+!/python/ray/serve/generated/
+!/python/ray/serve/generated/*.py
 
 # Ray cluster configuration
 scripts/nodes.txt

BUILD.bazel

@@ -2517,45 +2517,11 @@ copy_to_workspace(
     dstdir = "python/ray/core/",
 )
 
-
-genrule(
-    name = "install_py_proto",
-    srcs = [
-        ":cp_all_py_proto",
-        ":cp_serve_py_proto",
-    ],
-    outs = ["install_py_proto.out"],
-    cmd = """
-        # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
-        # command to change the import path in the generated file.
-        # shellcheck disable=SC2006
-        files=(
-            `ls python/ray/core/generated/*_pb2*.py` \
-            `ls python/ray/serve/generated/*_pb2*.py` \
-            )
-        sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
-        # Help the generated serve files to have the correct module
-        serve_files=(`ls python/ray/serve/generated/*_pb2*.py`)
-        sed -i -E 's/'"'"'src.ray.protobuf./'"'"'ray.serve.generated./' "$${serve_files[@]}"
-        # TODO(sang): Build our own proto instead of creating a new proto for opencensus separately.
-        # https://github.com/ray-project/ray/issues/31358
-        sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
-        sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
-        # TODO(rickyx): Remove this experimental path rewrite once we move autoscaler proto out
-        # of experimental.
-        autoscale_files=(`ls python/ray/core/generated/autoscaler_pb2*.py`)
-        sed -i -E 's/from ..experimental/from ./' "$${autoscale_files[@]}"
-        echo "$${PWD}" > $@
-    """,
-    local = 1,
-)
-
 genrule(
     name = "ray_pkg",
     srcs = [
         ":cp_raylet_so",
         ":python_sources",
-        ":install_py_proto",
         ":cp_redis",
         ":cp_raylet",
         ":cp_gcs_server",
@@ -2574,4 +2540,3 @@ genrule(
     """,
     local = 1,
 )
-

WORKSPACE

@@ -64,13 +64,22 @@ pip_parse(
     requirements_lock = "//release:requirements_buildkite.txt",
 )
 
+pip_parse(
+    name = "py_deps_compile_py_proto",
+    python_interpreter_target = python39,
+    requirements_lock = "//ci/compile_py_proto:requirements_compile_py_proto.txt",
+)
+
 load("@py_deps_buildkite//:requirements.bzl", install_py_deps_buildkite = "install_deps")
 load("@py_deps_ray_ci//:requirements.bzl", install_py_deps_ray_ci = "install_deps")
+load("@py_deps_compile_py_proto//:requirements.bzl", install_py_deps_compile_py_proto = "install_deps")
 
 install_py_deps_buildkite()
 
 install_py_deps_ray_ci()
 
+install_py_deps_compile_py_proto()
+
 register_toolchains("//:python_toolchain")
 
 register_execution_platforms(

bazel/ray_deps_setup.bzl

@@ -86,11 +86,19 @@ def auto_http_archive(
 def ray_deps_setup():
     # Explicitly bring in protobuf dependency to work around
     # https://github.com/ray-project/ray/issues/14117
+    # This is copied from grpc's bazel/grpc_deps.bzl
     http_archive(
         name = "com_google_protobuf",
-        strip_prefix = "protobuf-3.19.4",
-        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.4.tar.gz"],
-        sha256 = "3bd7828aa5af4b13b99c191e8b1e884ebfa9ad371b0ce264605d347f135d2568",
+        sha256 = "76a33e2136f23971ce46c72fd697cd94dc9f73d56ab23b753c3e16854c90ddfd",
+        strip_prefix = "protobuf-2c5fa078d8e86e5f4bd34e6f4c9ea9e8d7d4d44a",
+        urls = [
+            # https://github.com/protocolbuffers/protobuf/commits/v23.4
+            "https://github.com/protocolbuffers/protobuf/archive/2c5fa078d8e86e5f4bd34e6f4c9ea9e8d7d4d44a.tar.gz",
+        ],
+        patches = [
+            "@com_github_grpc_grpc//third_party:protobuf.patch",
+        ],
+        patch_args = ["-p1"],
     )
 
     # NOTE(lingxuan.zlx): 3rd party dependencies could be accessed, so it suggests
@@ -205,15 +213,18 @@ def ray_deps_setup():
             "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-harvest-interval.patch",
             "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-shutdown-api.patch",
         ],
-        patch_args = ["-p1"],        
+        patch_args = ["-p1"],
     )
 
     # OpenCensus depends on Abseil so we have to explicitly pull it in.
     # This is how diamond dependencies are prevented.
     auto_http_archive(
         name = "com_google_absl",
-        url = "https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz",
-        sha256 = "91ac87d30cc6d79f9ab974c51874a704de9c2647c40f6932597329a282217ba8",
+        sha256 = "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36",
+        strip_prefix = "abseil-cpp-20230125.3",
+        urls = [
+            "https://github.com/abseil/abseil-cpp/archive/20230125.3.tar.gz",
+        ],
     )
 
     # OpenCensus depends on jupp0r/prometheus-cpp
@@ -232,14 +243,13 @@ def ray_deps_setup():
     auto_http_archive(
         name = "com_github_grpc_grpc",
         # NOTE: If you update this, also update @boringssl's hash.
-        url = "https://github.com/grpc/grpc/archive/refs/tags/v1.46.6.tar.gz",
-        sha256 = "6514b3e6eab9e9c7017304512d4420387a47b1a9c5caa986643692977ed44e8a",
+        url = "https://github.com/grpc/grpc/archive/refs/tags/v1.57.0.tar.gz",
+        sha256 = "8393767af531b2d0549a4c26cf8ba1f665b16c16fb6c9238a7755e45444881dd",
         patches = [
             "@com_github_ray_project_ray//thirdparty/patches:grpc-cython-copts.patch",
-            "@com_github_ray_project_ray//thirdparty/patches:grpc-python.patch",
         ],
     )
-    
+
     http_archive(
         name = "openssl",
         strip_prefix = "openssl-1.1.1f",
@@ -249,7 +259,7 @@ def ray_deps_setup():
         ],
         build_file = "@rules_foreign_cc_thirdparty//openssl:BUILD.openssl.bazel",
     )
-    
+
     http_archive(
         name = "rules_foreign_cc",
         sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
@@ -279,11 +289,11 @@ def ray_deps_setup():
         # https://github.com/grpc/grpc/blob/1ff1feaa83e071d87c07827b0a317ffac673794f/bazel/grpc_deps.bzl#L189
         # Ensure this rule matches the rule used by grpc's bazel/grpc_deps.bzl
         name = "boringssl",
-        sha256 = "534fa658bd845fd974b50b10f444d392dfd0d93768c4a51b61263fd37d851c40",
-        strip_prefix = "boringssl-b9232f9e27e5668bc0414879dcdedb2a59ea75f2",
+        sha256 = "0675a4f86ce5e959703425d6f9063eaadf6b61b7f3399e77a154c0e85bad46b1",
+        strip_prefix = "boringssl-342e805bc1f5dfdd650e3f031686d6c939b095d9",
         urls = [
-            "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/b9232f9e27e5668bc0414879dcdedb2a59ea75f2.tar.gz",
-            "https://github.com/google/boringssl/archive/b9232f9e27e5668bc0414879dcdedb2a59ea75f2.tar.gz",
+            "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/342e805bc1f5dfdd650e3f031686d6c939b095d9.tar.gz",
+            "https://github.com/google/boringssl/archive/342e805bc1f5dfdd650e3f031686d6c939b095d9.tar.gz",
         ],
     )
 

ci/ci.sh

@@ -344,7 +344,6 @@ build_dashboard_front_end() {
 }
 
 build_sphinx_docs() {
-  _bazel_build_protobuf
   install_ray
 
   (
@@ -394,10 +393,6 @@ _bazel_build_before_install() {
 }
 
 
-_bazel_build_protobuf() {
-  bazel build "//:install_py_proto"
-}
-
 install_ray() {
   # TODO(mehrdadn): This function should be unified with the one in python/build-wheel-windows.sh.
   (

ci/compile_py_proto/BUILD.bazel

@@ -0,0 +1,106 @@
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@py_deps_compile_py_proto//:requirements.bzl", "requirement")
+
+compile_pip_requirements(
+    name = "requirements_compile_py_proto",
+    requirements_in = "requirements_compile_py_proto.in",
+    requirements_txt = "requirements_compile_py_proto.txt",
+    extra_args = ["--allow-unsafe"],
+    tags = ["team:ci"],
+    visibility = ["//visibility:public"],
+)
+
+py_binary(
+    name = "grpc_tools_protoc",
+    srcs = ["grpc_tools_protoc.py"],
+    deps = [
+        requirement("grpcio-tools"),
+    ],
+    exec_compatible_with = ["//:hermetic_python"],
+    tags = ["team:ci"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "compile_py_proto",
+    srcs = [
+        "@io_opencensus_proto//opencensus/proto/metrics/v1:metrics.proto",
+        "@io_opencensus_proto//opencensus/proto/resource/v1:resource.proto",
+        "//src/ray/protobuf:core_protos",
+        "//src/ray/protobuf:serve_protos",
+    ],
+    cmd = """
+        TMP=$$(mktemp -d)
+
+        mkdir -p $${TMP}/core
+        $(location :grpc_tools_protoc) --python_out=$${TMP}/core --grpc_python_out=$${TMP}/core $(locations //src/ray/protobuf:core_protos) --proto_path=. --proto_path=$$(dirname $(locations @io_opencensus_proto//opencensus/proto/metrics/v1:metrics.proto))/../../../..
+        mkdir -p opencensus/proto/metrics/v1
+        mkdir -p opencensus/proto/resource/v1
+        cp $(locations @io_opencensus_proto//opencensus/proto/metrics/v1:metrics.proto) opencensus/proto/metrics/v1/metrics.proto
+        cp $(locations @io_opencensus_proto//opencensus/proto/resource/v1:resource.proto) opencensus/proto/resource/v1/resource.proto
+        $(location :grpc_tools_protoc) --python_out=$${TMP}/core --grpc_python_out=$${TMP}/core opencensus/proto/metrics/v1/metrics.proto opencensus/proto/resource/v1/resource.proto --proto_path=.
+        mv $${TMP}/core/src/ray/protobuf/*.py $${TMP}/core/
+        mv $${TMP}/core/src/ray/protobuf/**/*.py $${TMP}/core/
+        mv $${TMP}/core/opencensus/proto/**/v1/*.py $${TMP}/core/
+
+        mkdir -p $${TMP}/serve
+        $(location :grpc_tools_protoc) --python_out=$${TMP}/serve --grpc_python_out=$${TMP}/serve $(locations //src/ray/protobuf:serve_protos) --proto_path=.
+        mv $${TMP}/serve/src/ray/protobuf/*.py $${TMP}/serve/
+
+        # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
+        # command to change the import path in the generated file.
+        # shellcheck disable=SC2006
+        files=(
+            `ls $${TMP}/*/*_pb2*.py`
+        )
+        sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
+        # TODO(sang): Build our own proto instead of creating a new proto for opencensus separately.
+        # https://github.com/ray-project/ray/issues/31358
+        sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
+        sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
+        # TODO(rickyx): Remove this experimental path rewrite once we move instance manager proto out
+        # of experimental.
+        autoscale_files=(`ls $${TMP}/core/instance_manager_pb2*.py`)
+        sed -i -E 's/from ..experimental/from ./' "$${autoscale_files[@]}"
+
+        # Help the generated serve files to have the correct module
+        serve_files=(`ls $${TMP}/serve/*_pb2*.py`)
+        sed -i -E 's/'"'"'src.ray.protobuf./'"'"'ray.serve.generated./' "$${serve_files[@]}"
+
+        pushd $${TMP}/core
+        tar -cvf compile_py_proto_core.tar *.py
+        popd
+
+        pushd $${TMP}/serve
+        tar -cvf compile_py_proto_serve.tar *.py
+        popd
+
+        mv $${TMP}/core/compile_py_proto_core.tar $(location :compile_py_proto_core.tar)
+        mv $${TMP}/serve/compile_py_proto_serve.tar $(location :compile_py_proto_serve.tar)
+        rm -r $${TMP}
+    """,
+    tools = [
+        ":grpc_tools_protoc",
+    ],
+    outs = [
+        "compile_py_proto_core.tar",
+        "compile_py_proto_serve.tar"
+    ],
+    visibility = ["//visibility:public"],
+)
+
+sh_binary(
+    name = "install_py_proto",
+    srcs = [
+        ":install_py_proto.sh",
+    ],
+    data = [
+        ":compile_py_proto_core.tar",
+        ":compile_py_proto_serve.tar",
+    ],
+    args = [
+        "$(location :compile_py_proto_core.tar)",
+        "$(location :compile_py_proto_serve.tar)",
+    ],
+)

ci/compile_py_proto/grpc_tools_protoc.py

@@ -0,0 +1,7 @@
+import subprocess
+import sys
+
+if __name__ == "__main__":
+    args = [sys.executable, "-m", "grpc_tools.protoc"]
+    args.extend(sys.argv[1:])
+    subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr)

ci/compile_py_proto/install_py_proto.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+mkdir -p "${BUILD_WORKSPACE_DIRECTORY}"/python/ray/core/generated
+tar -xvf "$1" -C "${BUILD_WORKSPACE_DIRECTORY}"/python/ray/core/generated
+
+mkdir -p "${BUILD_WORKSPACE_DIRECTORY}"/python/ray/serve/generated
+tar -xvf "$2" -C "${BUILD_WORKSPACE_DIRECTORY}"/python/ray/serve/generated

ci/compile_py_proto/requirements_compile_py_proto.in

@@ -0,0 +1,4 @@
+# This uses protoc v19 and
+# is compatible with python protobuf 3.19
+grpcio-tools==1.46.5
+setuptools