diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index a6b5db382421a..1857873b8cccd 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -5072,6 +5072,8 @@ endif()
 function(build_azure_sdk)
   message(STATUS "Building Azure SDK for C++ from source")
   fetchcontent_declare(azure_sdk
+                       # EXCLUDE_FROM_ALL is available since CMake 3.28
+                       EXCLUDE_FROM_ALL TRUE
                        URL ${ARROW_AZURE_SDK_URL}
                        URL_HASH "SHA256=${ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM}")
   prepare_fetchcontent()
@@ -5084,6 +5086,9 @@ function(build_azure_sdk)
   set(ENV{AZURE_SDK_DISABLE_AUTO_VCPKG} TRUE)
   set(WARNINGS_AS_ERRORS FALSE)
   fetchcontent_makeavailable(azure_sdk)
+  if(CMAKE_VERSION VERSION_LESS 3.28)
+    set_property(DIRECTORY ${azure_sdk_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE)
+  endif()
   set(AZURE_SDK_VENDORED
       TRUE
       PARENT_SCOPE)