Merge branch 'master' into cap

.buildkite/core.rayci.yml

@@ -71,6 +71,7 @@ steps:
         --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual
 
   - label: ":ray: core: :windows: python tests"
+    if: pipeline.id == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("RAYCI_CONTINUOUS_BUILD") == "1"
     tags: python
     job_env: WINDOWS
     mount_windows_artifacts: true
@@ -110,7 +111,6 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
         --only-tags=tmpfs --tmp-filesystem=tmpfs
-    job_env: corebuild
 
   - label: ":ray: core: out of disk redis tests"
     tags:
@@ -121,7 +121,6 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
         --test-env=TEST_EXTERNAL_REDIS=1
         --only-tags=tmpfs --tmp-filesystem=tmpfs
-    job_env: corebuild
 
   - label: ":ray: core: workflow tests"
     tags: 
@@ -307,6 +306,7 @@ steps:
         --parallelism-per-worker 2
 
   - label: ":ray: core: :windows: cpp tests"
+    if: pipeline.id == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("RAYCI_CONTINUOUS_BUILD") == "1"
     tags: core_cpp
     job_env: WINDOWS
     mount_windows_artifacts: true
@@ -343,7 +343,7 @@ steps:
 
   - label: ":ray: core: cpp worker tests"
     tags: core_cpp
-    instance_type: small
+    instance_type: medium
     commands:
       - if [[ "$${BUILDKITE_PIPELINE_ID}" == "0189942e-0876-4b8f-80a4-617f988ec59b" ]]; then
           echo "build --remote_upload_local_results=false" >> ~/.bazelrc;

.buildkite/data.rayci.yml

@@ -165,4 +165,18 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
         --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1 --parallelism-per-worker 3
         --build-name data14build
+        --except-tags gpu_only,gpu
     depends_on: data14build
+
+  - label: ":database: data: flaky gpu tests"
+    tags: 
+      - python
+      - data
+      - skip-on-premerge
+    instance_type: gpu-large
+    soft_fail: true
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
+        --build-name docgpubuild
+        --only-tags gpu,gpu_only
+    depends_on: docgpubuild

.buildkite/ml.rayci.yml

@@ -17,6 +17,16 @@ steps:
       IMAGE_TO: mlbuild
       RAYCI_IS_GPU_BUILD: "false"
 
+  - name: mlbuild-multipy
+    wanda: ci/docker/ml.build.wanda.yaml
+    depends_on: oss-ci-base_ml-multipy
+    env:
+      IMAGE_FROM: cr.ray.io/rayproject/oss-ci-base_ml-py{{matrix}}
+      IMAGE_TO: mlbuild-py{{matrix}}
+      RAYCI_IS_GPU_BUILD: "false"
+    matrix:
+      - "3.8"
+
   - name: mllightning2gpubuild
     wanda: ci/docker/mllightning2gpu.build.wanda.yaml
     depends_on: oss-ci-base_gpu
@@ -29,6 +39,16 @@ steps:
       IMAGE_TO: mlgpubuild
       RAYCI_IS_GPU_BUILD: "true"
 
+  - name: mlgpubuild-multipy
+    wanda: ci/docker/ml.build.wanda.yaml
+    depends_on: oss-ci-base_gpu-multipy
+    env:
+      IMAGE_FROM: cr.ray.io/rayproject/oss-ci-base_gpu-py{{matrix}}
+      IMAGE_TO: mlgpubuild-py{{matrix}}
+      RAYCI_IS_GPU_BUILD: "true"
+    matrix:
+      - "3.8"
+
   # tests
   - label: ":train: ml: train tests"
     tags: train
@@ -40,6 +60,21 @@ steps:
         --except-tags gpu_only,gpu,minimal,tune,doctest,needs_credentials 
     depends_on: [ "mlbuild", "forge" ]
 
+  - label: ":train: ml: train python {{matrix.python}} tests ({{matrix.worker_id}})"
+    if: build.env("RAYCI_CONTINUOUS_BUILD") == "1" || build.pull_request.labels includes "continuous-build"
+    tags: train
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/train/... ml 
+        --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 3
+        --except-tags gpu_only,gpu,minimal,tune,doctest,needs_credentials 
+        --python-version {{matrix.python}}
+    depends_on: [ "mlbuild-multipy", "forge" ]
+    matrix:
+      setup:
+        python: ["3.8"]
+        worker_id: ["0", "1"]
+
   - label: ":train: ml: train gpu tests"
     tags: 
       - train
@@ -53,6 +88,24 @@ steps:
         --only-tags gpu,gpu_only
     depends_on: [ "mlgpubuild", "forge" ]
 
+  - label: ":train: ml: train gpu python {{matrix.python}} tests ({{matrix.worker_id}})"
+    if: build.env("RAYCI_CONTINUOUS_BUILD") == "1" || build.pull_request.labels includes "continuous-build"
+    tags: 
+      - train
+      - gpu
+    instance_type: gpu-large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/train/... //python/ray/air/... //doc/... ml
+        --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 2
+        --build-name mlgpubuild-py{{matrix.python}}
+        --only-tags gpu,gpu_only
+        --python-version {{matrix.python}}
+    depends_on: [ "mlgpubuild-multipy", "forge" ]
+    matrix:
+      setup:
+        python: ["3.8"]
+        worker_id: ["0", "1"]
+
   - label: ":train: ml: train authentication tests"
     tags:
       - train
@@ -78,6 +131,18 @@ steps:
         --except-tags soft_imports,gpu_only,rllib,multinode
     depends_on: [ "mlbuild", "forge" ]
 
+  - label: ":train: ml: tune python {{matrix}} tests"
+    if: build.env("RAYCI_CONTINUOUS_BUILD") == "1" || build.pull_request.labels includes "continuous-build"
+    tags: tune
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tune/... ml 
+        --parallelism-per-worker 3
+        --except-tags soft_imports,gpu_only,rllib,multinode
+        --python-version {{matrix}}
+    depends_on: [ "mlbuild-multipy", "forge" ]
+    matrix: ["3.8"]
+
   - label: ":train: ml: tune new output tests"
     tags: tune
     instance_type: large
@@ -110,6 +175,23 @@ steps:
         --skip-ray-installation
     depends_on: [ "mlbuild", "forge" ]
 
+  - label: ":train: ml: air python {{matrix}} tests"
+    if: build.env("RAYCI_CONTINUOUS_BUILD") == "1" || build.pull_request.labels includes "continuous-build"
+    tags: ml
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/air/... ml 
+        --parallelism-per-worker 3
+        --except-tags gpu
+        --python-version {{matrix}}
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... ml 
+        --parallelism-per-worker 3
+        --only-tags ray_air
+        --python-version {{matrix}}
+        --skip-ray-installation
+    depends_on: [ "mlbuild-multipy", "forge" ]
+    matrix: ["3.8"]
+
   - label: ":train: ml: train+tune tests"
     tags: train
     instance_type: medium
@@ -214,6 +296,7 @@ steps:
     depends_on: [ "mllightning2gpubuild", "forge" ]
 
   - label: ":train: ml: :windows: tests"
+    if: pipeline.id == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("RAYCI_CONTINUOUS_BUILD") == "1"
     tags:
       - train
     job_env: WINDOWS

.buildkite/others.rayci.yml

@@ -87,3 +87,10 @@ steps:
     instance_type: small
     commands:
       - bazel run //ci/ray_ci/automation:state_machine_bot -- --production
+
+  - label: ":robot_face: CI weekly green metric"
+    tags: skip-on-premerge
+    if: build.branch == "master" && build.env("RAYCI_CONTINUOUS_BUILD") == "1"
+    instance_type: small
+    commands:
+      - bazel run //ci/ray_ci/automation:weekly_green_metric -- --production

.buildkite/serve.rayci.yml

@@ -77,6 +77,7 @@ steps:
         worker_id: ["0", "1"]
 
   - label: ":ray-serve: serve: :windows: tests"
+    if: pipeline.id == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("RAYCI_CONTINUOUS_BUILD") == "1"
     tags: serve
     job_env: WINDOWS
     mount_windows_artifacts: true

.github/CODEOWNERS

@@ -11,7 +11,6 @@
 # NOTE: Add @ray-project/ray-docs to all following docs subdirs.
 /doc/ @ray-project/ray-docs
 /doc/source/use-cases.rst @ericl @pcmoritz
-/doc/source/templates @justinvyu @matthewdeng @sofianhnaide
 
 # ==== Ray core ====
 
@@ -111,17 +110,16 @@
 # ==== Build and CI ====
 
 # Bazel.
-#/BUILD.bazel @ray-project/ray-core @ray-project/ray-ci
-#/WORKSPACE @ray-project/ray-core @ray-project/ray-ci
-#/bazel/ @ray-project/ray-core @ray-project/ray-ci
+#/BUILD.bazel @ray-project/ray-core @ray-project/ray-ci-reviewers
+#/WORKSPACE @ray-project/ray-core @ray-project/ray-ci-reviewers
+#/bazel/ @ray-project/ray-core @ray-project/ray-ci-reviewers
 
 # CI scripts.
-#/.travis.yml @ray-project/ray-core @ray-project/ray-ci
-#/ci/ @ray-project/ray-core @ray-project/ray-ci
+#/ci/ @ray-project/ray-core @ray-project/ray-ci-reviewers
 
 # Buildkite pipeline management
-.buildkite/hooks @ray-project/ray-ci
+.buildkite/hooks @ray-project/ray-ci-reviewers
 
-/release/ray_release @ray-project/ray-ci
+/release/ray_release @ray-project/ray-ci-reviewers
 
 /.github/ISSUE_TEMPLATE/ @ericl @stephanie-wang @scv119 @pcmoritz

.vale/styles/Vocab/Data/accept.txt

@@ -1,23 +1,24 @@
+autoscaler
 Dask
 Data('s)?
-Modin
-Predibase('s)?
-Spotify('s)?
-TFRecord(s)?
-UDF(s)?
 [Dd]atasource
 [Dd]iscretizer(s)?
+dtype
 [Gg]roupby
 [Ii]ndexable
 [Ii]ngest
 [Ll]ookup(s)?
+Modin
 [Mm]ultiget(s)?
+ndarray(s)?
+Predibase('s)?
 [Pp]refetch
 [Pp]refetching
 [Pp]reprocess
 [Pp]reprocessor(s)?
 [Pp]ushdown
 [Ss]calers
-dtype
-ndarray(s)?
-autoscaler
+Spotify('s)?
+TFRecord(s)?
+UDF(s)?
+XGBoost

bazel/ray_deps_setup.bzl

@@ -243,10 +243,10 @@ def ray_deps_setup():
 
     http_archive(
         name = "openssl",
-        strip_prefix = "openssl-1.1.1f",
-        sha256 = "186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35",
+        strip_prefix = "openssl-1.1.1w",
+        sha256 = "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8",
         urls = [
-            "https://www.openssl.org/source/openssl-1.1.1f.tar.gz",
+            "https://www.openssl.org/source/openssl-1.1.1w.tar.gz",
         ],
         build_file = "@rules_foreign_cc_thirdparty//openssl:BUILD.openssl.bazel",
     )

ci/build/get_build_info.py

@@ -11,7 +11,7 @@
 """
 
 import os
-import sys
+import platform
 import json
 
 
@@ -50,11 +50,7 @@ def get_build_env():
             "TRAVIS_JOB_WEB_URL": (
                 os.environ["BUILDKITE_BUILD_URL"] + "#" + os.environ["BUILDKITE_JOB_ID"]
             ),
-            "TRAVIS_OS_NAME": {  # The map is used to stay consistent with Travis
-                "linux": "linux",
-                "darwin": "osx",
-                "win32": "windows",
-            }[sys.platform],
+            "TRAVIS_OS_NAME": platform.system().lower(),
         }
 
     keys = [

ci/ray_ci/automation/BUILD.bazel

@@ -5,3 +5,15 @@ py_binary(
     srcs = ["state_machine_bot.py"],
     deps = ["//ci/ray_ci:ray_ci_lib"],
 )
+
+py_binary(
+    name = "test_db_bot",
+    srcs = ["test_db_bot.py"],
+    deps = ["//ci/ray_ci:ray_ci_lib"],
+)
+
+py_binary(
+    name = "weekly_green_metric",
+    srcs = ["weekly_green_metric.py"],
+    deps = ["//ci/ray_ci:ray_ci_lib"],
+)

ci/ray_ci/automation/state_machine_bot.py

@@ -8,7 +8,7 @@
 from ray_release.bazel import bazel_runfile
 
 
-ALL_TEST_PREFIXES = ["linux:", "windows:", "macos:"]
+ALL_TEST_PREFIXES = ["linux:", "windows:", "darwin:"]
 
 
 @click.command()

ci/ray_ci/automation/test_db_bot.py

@@ -0,0 +1,28 @@
+import os
+
+import click
+from ci.ray_ci.utils import logger
+from ci.ray_ci.tester_container import TesterContainer, PIPELINE_POSTMERGE
+from ray_release.configs.global_config import init_global_config
+from ray_release.bazel import bazel_runfile
+
+
+@click.command()
+@click.argument("team", required=True, type=str)
+@click.argument("bazel_log_dir", required=True, type=str)
+def main(team: str, bazel_log_dir: str) -> None:
+    init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+    if os.environ.get("BUILDKITE_BRANCH") != "master":
+        logger.info("Skip upload test results. We only upload on master branch.")
+        return
+
+    if os.environ.get("BUILDKITE_PIPELINE_ID") != PIPELINE_POSTMERGE:
+        logger.info("Skip upload test results. We only upload on postmerge pipeline.")
+        return
+
+    TesterContainer.upload_test_results(team, bazel_log_dir)
+
+
+if __name__ == "__main__":
+    main()