diff --git a/.buildkite/base.rayci.yml b/.buildkite/base.rayci.yml
index d2a823dbaa38..2c43a2367cb5 100644
--- a/.buildkite/base.rayci.yml
+++ b/.buildkite/base.rayci.yml
@@ -64,6 +64,7 @@ steps:
     label: "wanda: oss-ci-base_gpu-py{{matrix}}"
     wanda: ci/docker/base.gpu.wanda.yaml
     matrix:
+      - "3.10"
       - "3.11"
       - "3.12"
     env:
@@ -93,6 +94,9 @@ steps:
     tags: cibase
 
   - name: docgpubuild
+    label: "wanda: docgpubuild-py3.10"
     wanda: ci/docker/docgpu.build.wanda.yaml
-    depends_on: oss-ci-base_gpu
+    depends_on: oss-ci-base_gpu-multipy
+    env:
+      PYTHON: "3.10"
     tags: cibase
diff --git a/.buildkite/data.rayci.yml b/.buildkite/data.rayci.yml
index f43d5c9d691a..ea69da8ef5a2 100644
--- a/.buildkite/data.rayci.yml
+++ b/.buildkite/data.rayci.yml
@@ -217,15 +217,17 @@ steps:
     commands:
       # doc tests
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --only-tags doctest
         --except-tags cpu
+        --python-version 3.10
       # doc examples
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --except-tags doctest
         --only-tags gpu
         --skip-ray-installation
+        --python-version 3.10
     depends_on: docgpubuild
 
   - label: ":database: data: integration tests"
@@ -279,8 +281,9 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --only-tags gpu,gpu_only
+        --python-version 3.10
     depends_on: docgpubuild
   - label: ":data: postmerge authenticated tests"
     key: data_postmerge_authenticated_tests
diff --git a/.buildkite/ml.rayci.yml b/.buildkite/ml.rayci.yml
index 5297a34aa290..981fa4f1db1b 100644
--- a/.buildkite/ml.rayci.yml
+++ b/.buildkite/ml.rayci.yml
@@ -6,11 +6,14 @@ depends_on:
 steps:
   # builds
   - name: minbuild-ml
-    label: "wanda: minbuild-ml-py39"
+    label: "wanda: minbuild-ml-py{{matrix}}"
     wanda: ci/docker/min.build.wanda.yaml
     depends_on: oss-ci-base_build
+    matrix:
+      - "3.9"
+      - "3.10"
     env:
-      PYTHON_VERSION: "3.9"
+      PYTHON_VERSION: "{{matrix}}"
       EXTRA_DEPENDENCY: ml
     tags: cibase
 
diff --git a/.buildkite/serve.rayci.yml b/.buildkite/serve.rayci.yml
index a8cf0b8fd3d8..1dac170ad03f 100644
--- a/.buildkite/serve.rayci.yml
+++ b/.buildkite/serve.rayci.yml
@@ -66,7 +66,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
         --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --build-name servepydantic1build --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=EXPECTED_PYDANTIC_VERSION=1.10.12
+        --build-name servepydantic1build --test-env=EXPECTED_PYTHON_VERSION=3.10 --test-env=EXPECTED_PYDANTIC_VERSION=1.10.12
     depends_on: servepydantic1build
 
   - label: ":ray-serve: serve: same event loop tests"
@@ -81,8 +81,8 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
         --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --build-name servebuild --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=0
-    depends_on: "servebuild"
+        --build-name servebuild-py3.10 --test-env=EXPECTED_PYTHON_VERSION=3.10 --test-env=RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=0
+    depends_on: "servebuild-multipy"
 
   - label: ":ray-serve: serve: python {{matrix.python}} tests ({{matrix.worker_id}})"
     if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
@@ -214,8 +214,8 @@ steps:
     instance_type: gpu
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... serve
-        --build-name docgpubuild
-        --only-tags gpu
+        --build-name docgpubuild-py3.10
+        --only-tags gpu --python-version 3.10
     depends_on: docgpubuild
 
   - label: ":ray-serve: serve: flaky tests"
diff --git a/ci/docker/docgpu.build.Dockerfile b/ci/docker/docgpu.build.Dockerfile
index 8707124ff214..8fa86e415719 100644
--- a/ci/docker/docgpu.build.Dockerfile
+++ b/ci/docker/docgpu.build.Dockerfile
@@ -1,8 +1,7 @@
 # syntax=docker/dockerfile:1.3-labs
 
-ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu
+ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py3.10
 FROM $DOCKER_IMAGE_BASE_BUILD
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .
diff --git a/ci/docker/docgpu.build.wanda.yaml b/ci/docker/docgpu.build.wanda.yaml
index 01d9d91713d4..f783dbbcc53b 100644
--- a/ci/docker/docgpu.build.wanda.yaml
+++ b/ci/docker/docgpu.build.wanda.yaml
@@ -1,5 +1,5 @@
-name: "docgpubuild"
-froms: ["cr.ray.io/rayproject/oss-ci-base_gpu"]
+name: "docgpubuild-py$PYTHON"
+froms: ["cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON"]
 dockerfile: ci/docker/docgpu.build.Dockerfile
 srcs:
   - ci/env/install-dependencies.sh
@@ -17,5 +17,7 @@ srcs:
   - python/requirements/ml/data-test-requirements.txt
   - python/requirements/ml/rllib-requirements.txt
   - python/requirements/ml/rllib-test-requirements.txt
+build_args:
+  - DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON
 tags:
-  - cr.ray.io/rayproject/docgpubuild
+  - cr.ray.io/rayproject/docgpubuild-py$PYTHON