Merge branch 'ray-project:master' into patch-1

ray-project · Jan 25, 2023 · d21ab3f · d21ab3f
2 parents 0f3a887 + 477910b
commit d21ab3f
Show file tree

Hide file tree

Showing 674 changed files with 23,050 additions and 9,139 deletions.
diff --git a/.buildkite/pipeline.arm64.yml b/.buildkite/pipeline.arm64.yml
@@ -0,0 +1,56 @@
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Linux wheels"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    # Build the wheels
+    - UPLOAD_WHEELS_AS_ARTIFACTS=1 LINUX_WHEELS=1 ./ci/ci.sh build
+    # Upload the wheels
+    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    # Upload to branch directory.
+    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+    # Upload to latest directory.
+    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Post-wheels tests"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/env_info.sh
+    - bazel test --config=ci $(./ci/run/bazel_export_options)
+      --test_tag_filters=post_wheel_build
+      --test_env=CONDA_EXE
+      --test_env=CONDA_PYTHON_EXE
+      --test_env=CONDA_SHLVL
+      --test_env=CONDA_PREFIX
+      --test_env=CONDA_DEFAULT_ENV
+      --test_env=CI
+      --test_env=RAY_CI_POST_WHEEL_TESTS=True
+      python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... doc/...
+
+
+# This currently takes ~3 hours and times out often.
+# Enable this when ARM debug wheels are actually needed (and look into speeding this up).
+
+#- label: ":mechanical_arm: :ferris_wheel: ARM64 Debug Wheels"
+#  conditions:
+#    [
+#        "RAY_CI_LINUX_WHEELS_AFFECTED",
+#    ]
+#  instance_size: arm64-medium
+#  commands:
+#    # Build the debug wheels
+#    - RAY_DEBUG_BUILD=debug LINUX_WHEELS=1 ./ci/ci.sh build
+#    # Upload the wheels.
+#    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+#    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+#    - pip install -q docker aws_requests_auth boto3
+#    - ./ci/env/env_info.sh
+#    # Upload to branch directory.
+#    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+#    # Upload to latest directory.
+#    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -200,7 +200,9 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
+    # Unset CC CXX vars set in Dockerfile. Clang currently runs into problems with ubsan builds, this will revert to
+    # using GCC instead.
+    - unset CC CXX && bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
       --build_tests_only
       --jobs=2
       -- //:all -//:core_worker_test -//:logging_test -//:ray_syncer_test
@@ -337,6 +339,14 @@
     - ./ci/ci.sh test_minimal 3.10
     - ./ci/ci.sh test_latest_core_dependencies 3.10
 
+- label: ":python: Minimal install 3.11"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  instance_size: medium
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/ci.sh test_minimal 3.11
+    - ./ci/ci.sh test_latest_core_dependencies 3.11
+
 - label: ":python: Default install"
   conditions: ["RAY_CI_PYTHON_AFFECTED"]
   instance_size: small
@@ -563,6 +573,7 @@
     - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files
     - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd
     - bazel clean --expunge
+    - export WANDB_MODE=offline
     - ./ci/ci.sh build
 
 - label: ":octopus: Tune multinode tests"
@@ -613,7 +624,7 @@
     - pip install -U typing-extensions
     - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod
     - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=compat_py36 
+    - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=compat_py36
       python/ray/tests/horovod/...
       python/ray/tests/lightgbm/...
       python/ray/tests/ml_py36_compat/...

diff --git a/.gitignore b/.gitignore
@@ -189,6 +189,7 @@ venv
 tags
 tags.lock
 tags.temp
+*.vim
 
 # Emacs
 .#*
@@ -220,3 +221,6 @@ workflow_data/
 /compile_commands.json
 # Directory where clangd puts its indexing work
 /.cache/
+
+# Auto-generated tag mapping
+tag-mapping.json
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1651,9 +1651,21 @@ cc_test(
     ],
     copts = COPTS,
     tags = ["team:core"],
-    target_compatible_with = [
-        "@platforms//os:linux",
+    deps = [
+        ":ray_common",
+        ":raylet_lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_test(
+    name = "worker_killing_policy_group_by_owner_test",
+    size = "small",
+    srcs = [
+        "src/ray/raylet/worker_killing_policy_group_by_owner_test.cc",
     ],
+    copts = COPTS,
+    tags = ["team:core"],
     deps = [
         ":ray_common",
         ":raylet_lib",
@@ -2893,6 +2905,8 @@ genrule(
         # shellcheck disable=SC2006
         files=(`ls python/ray/core/generated/*_pb2*.py` `ls python/ray/serve/generated/*_pb2*.py`)
         sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
+        # TODO(sang): Build our own proto instead of creating a new proto for opencensus separately.
+        # https://github.com/ray-project/ray/issues/31358
         sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
         sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
         echo "$${PWD}" > $@

diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl
@@ -194,8 +194,8 @@ def ray_deps_setup():
     auto_http_archive(
         name = "cython",
         build_file = True,
-        url = "https://github.com/cython/cython/archive/3028e8c7ac296bc848d996e397c3354b3dbbd431.tar.gz",
-        sha256 = "31ea23c2231ddee8572a2a5effd54952e16a1b44e9a4cb3eb645418f8accf20d",
+        url = "https://github.com/cython/cython/archive/c48361d0a0969206e227ec016f654c9d941c2b69.tar.gz",
+        sha256 = "37c466fea398da9785bc37fe16f1455d2645d21a72e402103991d9e2fa1c6ff3",
     )
 
     auto_http_archive(

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
@@ -43,8 +43,9 @@
     "cu112": "nvidia/cuda:11.2.0-cudnn8-devel-ubuntu20.04",
     "cu111": "nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04",
     "cu110": "nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04",
-    "cu102": "nvidia/cuda:10.2-cudnn8-devel-ubuntu20.04",
-    "cu101": "nvidia/cuda:10.1-cudnn8-devel-ubuntu20.04",
+    # there is no ubuntu20.04 image for cuda 10.2 and 10.1
+    "cu102": "nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04",
+    "cu101": "nvidia/cuda:10.1-cudnn8-devel-ubuntu18.04",
     "cpu": "ubuntu:focal",
 }
 
@@ -619,7 +620,7 @@ def push_readmes(merge_build: bool):
         default="py37",
         nargs="*",
         help="Which python versions to build. "
-        "Must be in (py36, py37, py38, py39, py310)",
+        "Must be in (py36, py37, py38, py39, py310, py311)",
     )
     parser.add_argument(
         "--device-types",

diff --git a/ci/build/test-wheels.sh b/ci/build/test-wheels.sh
@@ -100,7 +100,8 @@ elif [[ "$platform" == "macosx" ]]; then
           "3.7"
           "3.8"
           "3.9"
-          "3.10")
+          "3.10"
+          )
 
   for ((i=0; i<${#PY_MMS[@]}; ++i)); do
     PY_MM="${PY_MMS[i]}"

diff --git a/ci/ci.sh b/ci/ci.sh
@@ -414,8 +414,7 @@ validate_wheels_commit_str() {
       continue
     fi
 
-    folder=${basename%%-cp*}
-    WHL_COMMIT=$(unzip -p "$whl" "${folder}.data/purelib/ray/__init__.py" | grep "__commit__" | awk -F'"' '{print $2}')
+    WHL_COMMIT=$(unzip -p "$whl" | grep "^__commit__" | awk -F'"' '{print $2}')
 
     if [ "${WHL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then
       echo "Error: Observed wheel commit (${WHL_COMMIT}) is not expected commit (${EXPECTED_COMMIT}). Aborting."
@@ -455,11 +454,14 @@ build_wheels() {
         -e "RAY_DEBUG_BUILD=${RAY_DEBUG_BUILD:-}"
       )
 
+      IMAGE_NAME="quay.io/pypa/manylinux2014_${HOSTTYPE}"
+      IMAGE_TAG="2022-12-20-b4884d9"
+
       if [ -z "${BUILDKITE-}" ]; then
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the "${MOUNT_BAZEL_CACHE[@]}" part.
         docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+        "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
       else
         rm -rf /ray-mount/*
         rm -rf /ray-mount/.whl || true
@@ -469,7 +471,7 @@ build_wheels() {
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted
         docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \
-          quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+          "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
         cp -rT /ray-mount /ray # copy new files back here
         find . | grep whl # testing
 

diff --git a/ci/docker/base.gpu.Dockerfile b/ci/docker/base.gpu.Dockerfile
@@ -27,9 +27,6 @@ ENV DOCKER_CERT_PATH=/certs/client
 ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_BAZEL_CACHE_URL=${REMOTE_CACHE_URL}
 
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
-
 RUN apt-get update -qq && apt-get upgrade -qq
 RUN apt-get install -y -qq \
     curl python-is-python3 git build-essential \

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:focal
+ARG DOCKER_IMAGE_BASE_UBUNTU=ubuntu:focal
+FROM $DOCKER_IMAGE_BASE_UBUNTU
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
@@ -33,6 +34,7 @@ RUN apt-get install -y -qq \
     sudo unzip unrar apt-utils dialog tzdata wget rsync \
     language-pack-en tmux cmake gdb vim htop \
     libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
+    liblz4-dev libunwind-dev libncurses5 \
     clang-format-12 jq \
     clang-tidy-12 clang-12
 # Make using GCC 9 explicit.

diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile
@@ -9,6 +9,9 @@ ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
 ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
+# Set compiler here to build Ray with CLANG/LLVM
+ENV CC=clang
+ENV CXX=clang++-12
 
 # Move out of working dir /ray
 # Delete stale data
@@ -25,9 +28,8 @@ RUN env
 
 # init also calls install-dependencies.sh
 RUN BUILD=1 bash --login -i ./ci/ci.sh init
-RUN bash --login -i ./ci/ci.sh build
 
-RUN export CC=clang CXX=clang++-12
+RUN bash --login -i ./ci/ci.sh build
 
 # Run determine test to run
 RUN bash --login -i -c "python ./ci/pipeline/determine_tests_to_run.py --output=json > affected_set.json"

diff --git a/ci/env/install-bazel.sh b/ci/env/install-bazel.sh
@@ -5,7 +5,7 @@ ROOT_DIR=$(cd "$(dirname "$0")/$(dirname "$(test -L "$0" && readlink "$0" || ech
 
 arg1="${1-}"
 
-achitecture="${HOSTTYPE}"
+
 platform="unknown"
 case "${OSTYPE}" in
   msys)
@@ -26,6 +26,9 @@ case "${OSTYPE}" in
     exit 1
 esac
 
+architecture="${HOSTTYPE}"
+echo "Architecture is $architecture"
+
 if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   # Sanity check: Verify we have symlinks where we expect them, or Bazel can produce weird "missing input file" errors.
   # This is most likely to occur on Windows, where symlinks are sometimes disabled by default.
@@ -50,26 +53,57 @@ if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   if [ "${OSTYPE}" = "msys" ]; then
     target="${MINGW_DIR-/usr}/bin/bazel.exe"
     mkdir -p "${target%/*}"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe"
+    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${architecture}.exe"
   else
-    target="./install.sh"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${achitecture}.sh"
-    chmod +x "${target}"
+
+    # Buildkite mac instances
     if [[ -n "${BUILDKITE-}" ]] && [ "${platform}" = "darwin" ]; then
-      "${target}" --user
+      mkdir -p "$HOME/bin"
       # Add bazel to the path.
       # shellcheck disable=SC2016
       printf '\nexport PATH="$HOME/bin:$PATH"\n' >> ~/.zshenv
       # shellcheck disable=SC1090
       source ~/.zshenv
+      INSTALL_USER=1
+    # Buildkite linux instance
     elif [ "${CI-}" = true ] || [ "${arg1-}" = "--system" ]; then
-      "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      INSTALL_USER=0
+    # User
     else
-      "${target}" --user > /dev/null
+      mkdir -p "$HOME/bin"
+      INSTALL_USER=1
       export PATH=$PATH:"$HOME/bin"
     fi
-    which bazel
-    rm -f "${target}"
+
+    if [ "${architecture}" = "aarch64" ]; then
+      # architecture is "aarch64", but the bazel tag is "arm64"
+      url="https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-arm64"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        target="$HOME/bin/bazel"
+        curl -f -s -L -R -o "${target}" "${url}"
+        chmod +x "${target}"
+      else
+        target="/bin/bazel"
+        sudo curl -f -s -L -R -o "${target}" "${url}"
+        sudo chmod +x "${target}"
+      fi
+
+      which bazel
+
+    else
+      target="./install.sh"
+      curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${architecture}.sh"
+      chmod +x "${target}"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        "${target}" --user
+      else
+        "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      fi
+      which bazel
+      rm -f "${target}"
+    fi
   fi
 fi
 

diff --git a/ci/env/install-core-prerelease-dependencies.sh b/ci/env/install-core-prerelease-dependencies.sh
@@ -3,7 +3,8 @@
 set -e
 
 # install all unbounded dependencies in setup.py for ray core
-for dependency in attrs jsonschema aiosignal frozenlist requests grpcio protobuf
+# TOOD(scv119) reenable grpcio once https://github.com/grpc/grpc/issues/31885 is fixed.
+for dependency in attrs jsonschema aiosignal frozenlist requests protobuf
 do
     python -m pip install -U --pre --upgrade-strategy=eager $dependency
 done
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
@@ -287,10 +287,15 @@ download_mnist() {
 install_pip_packages() {
 
   # Install modules needed in all jobs.
+  # shellcheck disable=SC2262
   alias pip="python -m pip"
 
   if [ "${MINIMAL_INSTALL-}" != 1 ]; then
+    # Some architectures will build dm-tree from source.
+    # Move bazelrc to a different location temporarily to disable --config=ci settings
+    mv "$HOME/.bazelrc" "$HOME/._brc" || true
     pip install --no-clean dm-tree==0.1.5  # --no-clean is due to: https://github.com/deepmind/tree/issues/5
+    mv "$HOME/._brc" "$HOME/.bazelrc" || true
   fi
 
   if { [ -n "${PYTHON-}" ] || [ "${DL-}" = "1" ]; } && [ "${MINIMAL_INSTALL-}" != 1 ]; then