diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index dd5abbe1b4b1b..e539fadb859fe 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -21,6 +21,7 @@ on:
   push:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
@@ -35,6 +36,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt
index 52e456eaab0cc..f28a24cac8d2d 100644
--- a/ci/conda_env_cpp.txt
+++ b/ci/conda_env_cpp.txt
@@ -21,7 +21,7 @@ azure-identity-cpp>=1.6.0
 azure-storage-blobs-cpp>=12.10.0
 azure-storage-common-cpp>=12.5.0
 azure-storage-files-datalake-cpp>=12.9.0
-benchmark>=1.6.0
+benchmark>=1.6.0,!=1.8.4
 boost-cpp>=1.68.0
 brotli
 bzip2