the rest of data

Signed-off-by: can <can@anyscale.com>
ray-project · Sep 6, 2023 · 1178312 · 1178312
1 parent 30084af
commit 1178312
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 45 deletions.
diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml
@@ -21,10 +21,22 @@ steps:
     wanda: ci/docker/core.build.wanda.yaml
     depends_on: oss-ci-base_build
 
+  - name: data6build
+    wanda: ci/docker/data6.build.wanda.yaml
+    depends_on: oss-ci-base_ml
+
   - name: data12build
     wanda: ci/docker/data12.build.wanda.yaml
     depends_on: oss-ci-base_ml
 
+  - name: datanbuild
+    wanda: ci/docker/datan.build.wanda.yaml
+    depends_on: oss-ci-base_ml
+
+  - name: databbuild
+    wanda: ci/docker/datab.build.wanda.yaml
+    depends_on: oss-ci-base_ml
+
   - name: servebuild
     wanda: ci/docker/serve.build.wanda.yaml
     depends_on: oss-ci-base_build

diff --git a/.buildkite/data.rayci.yml b/.buildkite/data.rayci.yml
@@ -1,5 +1,18 @@
 group: data tests
 steps:
+  - label: ":database: data: arrow 6 tests"
+    instance_type: medium
+    parallelism: 3
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --build-name data6build
+        --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1
+        --except-tags data_integration,doctest
+    depends_on: data6build
+    job_env: data6build
+
   - label: ":database: data: arrow 12 tests"
     instance_type: medium
     parallelism: 3
@@ -13,11 +26,28 @@ steps:
     depends_on: data12build
     job_env: data12build
 
-  - label: ":database: data: arrow 12 flaky tests"
+  - label: ":database: data: arrow nightly tests"
     instance_type: medium
-    soft_fail: true
+    parallelism: 3
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests --parallelism-per-worker 3
-        --build-name data12build
-    depends_on: data12build
-    job_env: data12build
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --build-name datanbuild
+        --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1
+        --except-tags data_integration,doctest
+    depends_on: datanbuild
+    job_env: datanbuild
+
+  - label: ":database: data: bulk executor tests"
+    instance_type: medium
+    parallelism: 3
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data 
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --build-name databbuild
+        --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1
+        --except-tags data_integration,doctest
+    depends_on: databbuild
+    job_env: databbuild
diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
@@ -390,45 +390,6 @@
     - sudo service mongodb stop
     - sudo apt-get purge -y mongodb*
 
-- label: "Data tests (bulk executor)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_DATA_AFFECTED"]
-  instance_size: medium
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DATA_PROCESSING_TESTING=1 ARROW_VERSION=12.* ./ci/env/install-dependencies.sh
-    # TODO(scottjlee): Move datasets to train/data-test-requirements.txt 
-    # (see https://github.com/ray-project/ray/pull/38432/)
-    - pip install "datasets==2.14.0"
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
-
-- label: "Data tests (Arrow nightly)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"]
-  instance_size: medium
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh
-    # TODO(scottjlee): Move datasets to train/data-test-requirements.txt 
-    # (see https://github.com/ray-project/ray/pull/38432/)
-    - pip install "datasets==2.14.0"
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
-
-- label: "Data tests (Arrow 6)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"]
-  instance_size: medium
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/...
-    - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/...
-
 - label: "Workflow tests"
   conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_WORKFLOW_AFFECTED"]
   instance_size: medium

diff --git a/ci/docker/data6.build.wanda.yaml b/ci/docker/data6.build.wanda.yaml
@@ -0,0 +1,15 @@
+name: "data6build"
+froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
+dockerfile: ci/docker/data.build.Dockerfile
+srcs:
+  - ci/env/install-dependencies.sh
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/ml/dl-cpu-requirements.txt
+  - python/requirements/ml/data-requirements.txt
+  - python/requirements/ml/data-test-requirements.txt
+build_args:
+  - ARROW_VERSION=6.*
+tags:
+  - cr.ray.io/rayproject/data6build
diff --git a/ci/docker/datab.build.wanda.yaml b/ci/docker/datab.build.wanda.yaml
@@ -0,0 +1,15 @@
+name: "databbuild"
+froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
+dockerfile: ci/docker/data.build.Dockerfile
+srcs:
+  - ci/env/install-dependencies.sh
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/ml/dl-cpu-requirements.txt
+  - python/requirements/ml/data-requirements.txt
+  - python/requirements/ml/data-test-requirements.txt
+build_args:
+  - ARROW_VERSION=12.*
+tags:
+  - cr.ray.io/rayproject/databbuild
diff --git a/ci/docker/datan.build.wanda.yaml b/ci/docker/datan.build.wanda.yaml
@@ -0,0 +1,15 @@
+name: "datanbuild"
+froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
+dockerfile: ci/docker/data.build.Dockerfile
+srcs:
+  - ci/env/install-dependencies.sh
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/ml/dl-cpu-requirements.txt
+  - python/requirements/ml/data-requirements.txt
+  - python/requirements/ml/data-test-requirements.txt
+build_args:
+  - ARROW_VERSION=nightly
+tags:
+  - cr.ray.io/rayproject/datanbuild