diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index b534552c3c024..9f68c5604720b 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -451,7 +451,9 @@ if(ARROW_CPU_FLAG STREQUAL "armv8")
   endif()
   set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${ARROW_ARMV8_ARCH_FLAG}")
 
-  add_definitions(-DARROW_HAVE_NEON)
+  if(NOT ARROW_SIMD_LEVEL STREQUAL "NONE")
+    add_definitions(-DARROW_HAVE_NEON)
+  endif()
 
   if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
      AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4")