run data6

Signed-off-by: can <can@anyscale.com>

.buildkite/_forge.rayci.yml

@@ -21,10 +21,18 @@ steps:
     wanda: ci/docker/core.build.wanda.yaml
     depends_on: oss-ci-base_build
 
+  - name: data6build
+    wanda: ci/docker/data6.build.wanda.yaml
+    depends_on: oss-ci-base_ml
+
   - name: data12build
     wanda: ci/docker/data12.build.wanda.yaml
     depends_on: oss-ci-base_ml
 
+  - name: datanbuild
+    wanda: ci/docker/datan.build.wanda.yaml
+    depends_on: oss-ci-base_ml
+
   - name: servebuild
     wanda: ci/docker/serve.build.wanda.yaml
     depends_on: oss-ci-base_build

.buildkite/data.rayci.yml

@@ -1,5 +1,18 @@
 group: data tests
 steps:
+  - label: ":database: data: arrrow 6 tests"
+    instance_type: medium
+    parallelism: 3
+    commands:
+      - bazel run //ci/ray_ci -- //python/ray/data/... //python/ray/air/... data 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --build-name data6build
+        --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1
+        --except-tags data_integration,doctest
+    depends_on: data6build
+    job_env: data6build
+
   - label: ":database: data: arrrow 12 tests"
     instance_type: medium
     parallelism: 3
@@ -13,6 +26,28 @@ steps:
     depends_on: data12build
     job_env: data12build
 
+  - label: ":database: data: arrrow nightly tests"
+    instance_type: medium
+    parallelism: 3
+    commands:
+      - bazel run //ci/ray_ci -- //python/ray/data/... //python/ray/air/... data 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --build-name datanbuild
+        --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1
+        --except-tags data_integration,doctest
+    depends_on: datanbuild
+    job_env: datanbuild
+
+  - label: ":database: data: arrorw 6 flaky tests"
+    instance_type: medium
+    soft_fail: true
+    commands:
+      - bazel run //ci/ray_ci -- //... data --run-flaky-tests --parallelism-per-worker 3
+        --build-name data6build
+    depends_on: data6build
+    job_env: data6build
+
   - label: ":database: data: arrorw 12 flaky tests"
     instance_type: medium
     soft_fail: true
@@ -21,3 +56,12 @@ steps:
         --build-name data12build
     depends_on: data12build
     job_env: data12build
+
+  - label: ":database: data: arrorw nightly flaky tests"
+    instance_type: medium
+    soft_fail: true
+    commands:
+      - bazel run //ci/ray_ci -- //... data --run-flaky-tests --parallelism-per-worker 3
+        --build-name datanbuild
+    depends_on: datanbuild
+    job_env: datanbuild

.buildkite/pipeline.ml.yml

@@ -457,28 +457,6 @@
     - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
     - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
 
-- label: "Data tests (Arrow nightly)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"]
-  instance_size: medium
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
-
-- label: "Data tests (Arrow 6)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"]
-  instance_size: medium
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
-
 - label: "Workflow tests"
   conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_WORKFLOW_AFFECTED"]
   instance_size: medium

ci/docker/data6.build.wanda.yaml

@@ -0,0 +1,15 @@
+name: "data6build"
+froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
+dockerfile: ci/docker/data.build.Dockerfile
+srcs:
+  - ci/env/install-dependencies.sh
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/ml/dl-cpu-requirements.txt
+  - python/requirements/ml/data-requirements.txt
+  - python/requirements/ml/data-test-requirements.txt
+build_args:
+  - ARROW_VERSION=6.*
+tags:
+  - cr.ray.io/rayproject/data6build

ci/docker/datan.build.wanda.yaml

@@ -0,0 +1,15 @@
+name: "datanbuild"
+froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
+dockerfile: ci/docker/data.build.Dockerfile
+srcs:
+  - ci/env/install-dependencies.sh
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/ml/dl-cpu-requirements.txt
+  - python/requirements/ml/data-requirements.txt
+  - python/requirements/ml/data-test-requirements.txt
+build_args:
+  - ARROW_VERSION=nightly
+tags:
+  - cr.ray.io/rayproject/datanbuild