diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d0cfe7533a32..19d05a94db9a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -594,6 +594,7 @@ jobs:
         run: |
           ./build_tools/github_actions/docker_run.sh \
             --env "BUILD_PRESET=test" \
+            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
             ${CONTAINER} \
             ./build_tools/cmake/build_runtime.sh \
             "${BUILD_DIR}"
diff --git a/build_tools/cmake/build_runtime.sh b/build_tools/cmake/build_runtime.sh
index 4fb41fff0c5e..8ed4c0ea5ff4 100755
--- a/build_tools/cmake/build_runtime.sh
+++ b/build_tools/cmake/build_runtime.sh
@@ -17,6 +17,8 @@ set -xeuo pipefail
 
 BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-runtime}}"
 BUILD_PRESET="${BUILD_PRESET:-test}"
+CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-RelWithDebInfo}"
+IREE_BUILD_PYTHON_BINDINGS="${IREE_BUILD_PYTHON_BINDINGS:-ON}"
 
 source build_tools/cmake/setup_build.sh
 # Note: not using ccache since the runtime build should be fast already.
@@ -25,12 +27,16 @@ declare -a args
 args=(
   "-G" "Ninja"
   "-B" "${BUILD_DIR}"
-  "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
-  "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
-  "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+
+  "-DIREE_BUILD_COMPILER=OFF"
+  "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+
   "-DIREE_RUNTIME_OPTIMIZATION_PROFILE=lto"
   "-DIREE_FORCE_LTO_COMPAT_BINUTILS_ON_LINUX=ON"
-  "-DIREE_BUILD_COMPILER=OFF"
+
+  "-DIREE_BUILD_PYTHON_BINDINGS=${IREE_BUILD_PYTHON_BINDINGS}"
+  "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
+  "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
 )
 
 case "${BUILD_PRESET}" in
diff --git a/compiler/bindings/python/test/tools/CMakeLists.txt b/compiler/bindings/python/test/tools/CMakeLists.txt
index fee8f3464868..ec96eeabf64f 100644
--- a/compiler/bindings/python/test/tools/CMakeLists.txt
+++ b/compiler/bindings/python/test/tools/CMakeLists.txt
@@ -7,13 +7,13 @@
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
 if(IREE_BUILD_BUNDLED_LLVM)
-iree_py_test(
-  NAME
-    compiler_core_test
-  SRCS
-    "compiler_core_test.py"
-)
-endif()  # IREE_BUILD_BUNDLED_LLVM
+  iree_py_test(
+    NAME
+      compiler_core_test
+    SRCS
+      "compiler_core_test.py"
+  )
+endif()
 
 if(IREE_INPUT_TORCH)
   iree_py_test(
diff --git a/runtime/bindings/python/CMakeLists.txt b/runtime/bindings/python/CMakeLists.txt
index 9f3151bdb53a..29a3e5559f9e 100644
--- a/runtime/bindings/python/CMakeLists.txt
+++ b/runtime/bindings/python/CMakeLists.txt
@@ -256,9 +256,11 @@ iree_py_test(
     "tests/system_setup_test.py"
 )
 
+# These tests use compiler APIs as well as runtime APIs.
+#
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
-if(IREE_BUILD_BUNDLED_LLVM)
+if(IREE_BUILD_COMPILER AND IREE_BUILD_BUNDLED_LLVM)
   iree_py_test(
     NAME
       benchmark_test
diff --git a/samples/py_custom_module/CMakeLists.txt b/samples/py_custom_module/CMakeLists.txt
index 5e1053dc0a3f..85c82bc9c566 100644
--- a/samples/py_custom_module/CMakeLists.txt
+++ b/samples/py_custom_module/CMakeLists.txt
@@ -8,9 +8,8 @@ if(NOT IREE_BUILD_PYTHON_BINDINGS)
   return()
 endif()
 
-# These tests perform linking via the Compiler API, which is only supported
-# in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM)
+# These tests use compiler APIs as well as runtime APIs.
+if(NOT IREE_BUILD_COMPILER)
   return()
 endif()
 
diff --git a/tests/e2e/collectives/CMakeLists.txt b/tests/e2e/collectives/CMakeLists.txt
index dfa3472cf630..d33e393a3934 100644
--- a/tests/e2e/collectives/CMakeLists.txt
+++ b/tests/e2e/collectives/CMakeLists.txt
@@ -4,9 +4,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# These tests perform linking via the Compiler API, which is only supported
-# in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM OR NOT IREE_ENABLE_COLLECTIVE_RUNTIME_TESTS)
+if(NOT IREE_BUILD_COMPILER OR NOT IREE_ENABLE_COLLECTIVE_RUNTIME_TESTS)
   return()
 endif()
 
diff --git a/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt b/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
index a2ff8eff5ee0..7ae5613b9a1d 100644
--- a/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
+++ b/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
@@ -4,9 +4,11 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# These tests use compiler APIs as well as runtime APIs.
+#
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM)
+if(NOT IREE_BUILD_COMPILER OR NOT IREE_BUILD_BUNDLED_LLVM)
   return()
 endif()