diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index e929c02d11c1d..5349ddd499c22 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -160,7 +160,11 @@ jobs:
           'generic-no-unicode',
           'generic-no-wide-characters',
           'generic-static',
-          'generic-with_llvm_unwinder'
+          'generic-with_llvm_unwinder',
+          # TODO Find a better place for the benchmark and bootstrapping builds to live. They're either very expensive
+          # or don't provide much value since the benchmark run results are too noise on the bots.
+          'benchmarks',
+          'bootstrapping-build'
         ]
         machine: [ 'libcxx-runners-8' ]
         std_modules: [ 'OFF' ]
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index 06b0c55e6f3ca..51b9999026247 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -43,10 +43,6 @@ definitions:
 
 # Define agents using YAML anchors to reduce duplication
 agents_definitions:
-  _linux_agent: &linux_agent
-    agents:
-      queue: libcxx-builders
-      os: linux
   _windows_agent: &windows_agent
     agents:
       queue: windows
@@ -101,20 +97,6 @@ environment_definitions:
 
 
 steps:
-- label: Bootstrapping build
-  command: libcxx/utils/ci/run-buildbot bootstrapping-build
-  env:
-    <<: *common_env
-  <<: *linux_agent
-  <<: *common
-
-- label: Benchmarks
-  command: libcxx/utils/ci/run-buildbot benchmarks
-  env:
-    <<: *common_env
-  <<: *linux_agent
-  <<: *common
-
 - group: ':windows: Windows'
   steps:
   - label: Clang-cl (DLL)