From 1178312ebe1547a1c2a6a693b5b8a6f6c7aa6329 Mon Sep 17 00:00:00 2001 From: can Date: Wed, 6 Sep 2023 07:20:17 +0000 Subject: [PATCH] the rest of data Signed-off-by: can --- .buildkite/_forge.rayci.yml | 12 +++++++++ .buildkite/data.rayci.yml | 42 +++++++++++++++++++++++++++----- .buildkite/pipeline.ml.yml | 39 ----------------------------- ci/docker/data6.build.wanda.yaml | 15 ++++++++++++ ci/docker/datab.build.wanda.yaml | 15 ++++++++++++ ci/docker/datan.build.wanda.yaml | 15 ++++++++++++ 6 files changed, 93 insertions(+), 45 deletions(-) create mode 100644 ci/docker/data6.build.wanda.yaml create mode 100644 ci/docker/datab.build.wanda.yaml create mode 100644 ci/docker/datan.build.wanda.yaml diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 33e22c8c2289c5..f2946c079b80c8 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -21,10 +21,22 @@ steps: wanda: ci/docker/core.build.wanda.yaml depends_on: oss-ci-base_build + - name: data6build + wanda: ci/docker/data6.build.wanda.yaml + depends_on: oss-ci-base_ml + - name: data12build wanda: ci/docker/data12.build.wanda.yaml depends_on: oss-ci-base_ml + - name: datanbuild + wanda: ci/docker/datan.build.wanda.yaml + depends_on: oss-ci-base_ml + + - name: databbuild + wanda: ci/docker/datab.build.wanda.yaml + depends_on: oss-ci-base_ml + - name: servebuild wanda: ci/docker/serve.build.wanda.yaml depends_on: oss-ci-base_build diff --git a/.buildkite/data.rayci.yml b/.buildkite/data.rayci.yml index d535bce3046674..f1b65fbe67c6b8 100644 --- a/.buildkite/data.rayci.yml +++ b/.buildkite/data.rayci.yml @@ -1,5 +1,18 @@ group: data tests steps: + - label: ":database: data: arrow 6 tests" + instance_type: medium + parallelism: 3 + commands: + - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data + --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" + --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 + --build-name data6build + --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1 + --except-tags data_integration,doctest + depends_on: data6build + job_env: data6build + - label: ":database: data: arrow 12 tests" instance_type: medium parallelism: 3 @@ -13,11 +26,28 @@ steps: depends_on: data12build job_env: data12build - - label: ":database: data: arrow 12 flaky tests" + - label: ":database: data: arrow nightly tests" instance_type: medium - soft_fail: true + parallelism: 3 commands: - - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests --parallelism-per-worker 3 - --build-name data12build - depends_on: data12build - job_env: data12build + - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data + --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" + --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 + --build-name datanbuild + --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1 + --except-tags data_integration,doctest + depends_on: datanbuild + job_env: datanbuild + + - label: ":database: data: bulk executor tests" + instance_type: medium + parallelism: 3 + commands: + - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data + --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" + --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 + --build-name databbuild + --test-env RAY_DATA_USE_STREAMING_EXECUTOR=1 + --except-tags data_integration,doctest + depends_on: databbuild + job_env: databbuild diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 7b404ece9a0539..1f2408d2b5ef4f 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -390,45 +390,6 @@ - sudo service mongodb stop - sudo apt-get purge -y mongodb* -- label: "Data tests (bulk executor)" - conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_DATA_AFFECTED"] - instance_size: medium - parallelism: 3 - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - DATA_PROCESSING_TESTING=1 ARROW_VERSION=12.* ./ci/env/install-dependencies.sh - # TODO(scottjlee): Move datasets to train/data-test-requirements.txt - # (see https://github.com/ray-project/ray/pull/38432/) - - pip install "datasets==2.14.0" - - ./ci/env/env_info.sh - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/... - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATA_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/... - -- label: "Data tests (Arrow nightly)" - conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"] - instance_size: medium - parallelism: 3 - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh - # TODO(scottjlee): Move datasets to train/data-test-requirements.txt - # (see https://github.com/ray-project/ray/pull/38432/) - - pip install "datasets==2.14.0" - - ./ci/env/env_info.sh - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/... - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/... - -- label: "Data tests (Arrow 6)" - conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"] - instance_size: medium - parallelism: 3 - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh - - ./ci/env/env_info.sh - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-data_integration,-doctest python/ray/data/... - - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_data,-doctest python/ray/air/... - - label: "Workflow tests" conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_WORKFLOW_AFFECTED"] instance_size: medium diff --git a/ci/docker/data6.build.wanda.yaml b/ci/docker/data6.build.wanda.yaml new file mode 100644 index 00000000000000..39255fce54ea9e --- /dev/null +++ b/ci/docker/data6.build.wanda.yaml @@ -0,0 +1,15 @@ +name: "data6build" +froms: ["cr.ray.io/rayproject/oss-ci-base_ml"] +dockerfile: ci/docker/data.build.Dockerfile +srcs: + - ci/env/install-dependencies.sh + - python/requirements.txt + - python/requirements_compiled.txt + - python/requirements/test-requirements.txt + - python/requirements/ml/dl-cpu-requirements.txt + - python/requirements/ml/data-requirements.txt + - python/requirements/ml/data-test-requirements.txt +build_args: + - ARROW_VERSION=6.* +tags: + - cr.ray.io/rayproject/data6build diff --git a/ci/docker/datab.build.wanda.yaml b/ci/docker/datab.build.wanda.yaml new file mode 100644 index 00000000000000..d9a9e73330b71f --- /dev/null +++ b/ci/docker/datab.build.wanda.yaml @@ -0,0 +1,15 @@ +name: "databbuild" +froms: ["cr.ray.io/rayproject/oss-ci-base_ml"] +dockerfile: ci/docker/data.build.Dockerfile +srcs: + - ci/env/install-dependencies.sh + - python/requirements.txt + - python/requirements_compiled.txt + - python/requirements/test-requirements.txt + - python/requirements/ml/dl-cpu-requirements.txt + - python/requirements/ml/data-requirements.txt + - python/requirements/ml/data-test-requirements.txt +build_args: + - ARROW_VERSION=12.* +tags: + - cr.ray.io/rayproject/databbuild diff --git a/ci/docker/datan.build.wanda.yaml b/ci/docker/datan.build.wanda.yaml new file mode 100644 index 00000000000000..632822e31211f4 --- /dev/null +++ b/ci/docker/datan.build.wanda.yaml @@ -0,0 +1,15 @@ +name: "datanbuild" +froms: ["cr.ray.io/rayproject/oss-ci-base_ml"] +dockerfile: ci/docker/data.build.Dockerfile +srcs: + - ci/env/install-dependencies.sh + - python/requirements.txt + - python/requirements_compiled.txt + - python/requirements/test-requirements.txt + - python/requirements/ml/dl-cpu-requirements.txt + - python/requirements/ml/data-requirements.txt + - python/requirements/ml/data-test-requirements.txt +build_args: + - ARROW_VERSION=nightly +tags: + - cr.ray.io/rayproject/datanbuild