[libc++] Remove special handling of the native C++ library in benchmarks

[ldionne]

There were some ad-hoc settings that allowed running the benchmarks against the native C++ Standard Library. While this ability is very useful, it was done before the test suite was quite independent of libc++ itself. Instead, it is better to streamline running the benchmarks on the native standard library by using a custom Lit configuration like we do with the test suite.

A follow-up patch will rework the integration of benchmarks with the Lit configuration used for the test suite so that we can reuse the same mechanism for both, making it easy to benchmark the native standard library.

It will also make benchmarks way more user-friendly to run since we will be able to run them like we run individual tests, which is a pain point right now.

[llvmbot]

@llvm/pr-subscribers-libcxx

Author: Louis Dionne (ldionne)

Changes

There were some ad-hoc settings that allowed running the benchmarks against the native C++ Standard Library. While this ability is very useful, it was done before the test suite was quite independent of libc++ itself. Instead, it is better to streamline running the benchmarks on the native standard library by using a custom Lit configuration like we do with the test suite.

A follow-up patch will rework the integration of benchmarks with the Lit configuration used for the test suite so that we can reuse the same mechanism for both, making it easy to benchmark the native standard library.

It will also make benchmarks way more user-friendly to run since we will be able to run them like we run individual tests, which is a pain point right now.

---

Full diff: https://github.com/llvm/llvm-project/pull/98529.diff

6 Files Affected:

• (modified) libcxx/CMakeLists.txt (-14)
• (modified) libcxx/benchmarks/CMakeLists.txt (+19-81)
• (modified) libcxx/benchmarks/lit.cfg.py (+1-1)
• (modified) libcxx/docs/BuildingLibcxx.rst (-16)
• (modified) libcxx/docs/ReleaseNotes/19.rst (+4)
• (modified) libcxx/docs/TestingLibcxx.rst (+4-11)

diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index e098bd574eec7..6e37174240ae9 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -159,20 +159,6 @@ set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01)
 set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING
     "Arguments to pass when running the benchmarks using check-cxx-benchmarks")
 
-set(LIBCXX_BENCHMARK_NATIVE_STDLIB "" CACHE STRING
-        "Build the benchmarks against the specified native STL.
-         The value must be one of libc++/libstdc++")
-set(LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN "" CACHE STRING
-    "Use alternate GCC toolchain when building the native benchmarks")
-
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-  if (NOT (LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libc++"
-        OR LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libstdc++"))
-    message(FATAL_ERROR "Invalid value for LIBCXX_BENCHMARK_NATIVE_STDLIB: "
-            "'${LIBCXX_BENCHMARK_NATIVE_STDLIB}'")
-  endif()
-endif()
-
 option(LIBCXX_INCLUDE_DOCS "Build the libc++ documentation." ${LLVM_INCLUDE_DOCS})
 set(LIBCXX_LIBDIR_SUFFIX "${LLVM_LIBDIR_SUFFIX}" CACHE STRING
     "Define suffix of library directory name (32/64)")
diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 2101f9c71788c..fa55e77d0825a 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -2,12 +2,12 @@ include(ExternalProject)
 include(CheckCXXCompilerFlag)
 
 #==============================================================================
-# Build Google Benchmark for libc++
+# Build Google Benchmark
 #==============================================================================
 
 set(CMAKE_FOLDER "${CMAKE_FOLDER}/Benchmarks")
 
-set(BENCHMARK_LIBCXX_COMPILE_FLAGS
+set(BENCHMARK_COMPILE_FLAGS
     -Wno-unused-command-line-argument
     -nostdinc++
     -isystem "${LIBCXX_GENERATED_INCLUDE_DIR}"
@@ -16,64 +16,37 @@ set(BENCHMARK_LIBCXX_COMPILE_FLAGS
     ${SANITIZER_FLAGS}
     )
 if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
-  list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS
+  list(APPEND BENCHMARK_COMPILE_FLAGS
     -isystem "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}")
 endif()
 if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH)
-  list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS
+  list(APPEND BENCHMARK_COMPILE_FLAGS
           -L${LIBCXX_CXX_ABI_LIBRARY_PATH}
           -Wl,-rpath,${LIBCXX_CXX_ABI_LIBRARY_PATH})
 endif()
-split_list(BENCHMARK_LIBCXX_COMPILE_FLAGS)
+split_list(BENCHMARK_COMPILE_FLAGS)
 
-ExternalProject_Add(google-benchmark-libcxx
+ExternalProject_Add(google-benchmark
         EXCLUDE_FROM_ALL ON
         DEPENDS cxx cxx-headers
-        PREFIX benchmark-libcxx
+        PREFIX google-benchmark
         SOURCE_DIR ${LLVM_THIRD_PARTY_DIR}/benchmark
-        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/benchmark-libcxx
+        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark
         CMAKE_CACHE_ARGS
           -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
           -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
           -DCMAKE_BUILD_TYPE:STRING=RELEASE
           -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_LIBCXX_COMPILE_FLAGS}
+          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_COMPILE_FLAGS}
           -DBENCHMARK_USE_LIBCXX:BOOL=ON
           -DBENCHMARK_ENABLE_TESTING:BOOL=OFF)
 
-#==============================================================================
-# Build Google Benchmark for the native stdlib
-#==============================================================================
-set(BENCHMARK_NATIVE_TARGET_FLAGS)
-if (LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN)
-  set(BENCHMARK_NATIVE_TARGET_FLAGS
-      --gcc-toolchain=${LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN})
-endif()
-split_list(BENCHMARK_NATIVE_TARGET_FLAGS)
-
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-  ExternalProject_Add(google-benchmark-native
-        EXCLUDE_FROM_ALL ON
-        PREFIX benchmark-native
-        SOURCE_DIR ${LLVM_THIRD_PARTY_DIR}/benchmark
-        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/benchmark-native
-        CMAKE_CACHE_ARGS
-          -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
-          -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
-          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_NATIVE_TARGET_FLAGS}
-          -DCMAKE_BUILD_TYPE:STRING=RELEASE
-          -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-          -DBENCHMARK_ENABLE_TESTING:BOOL=OFF)
-endif()
-
-
 #==============================================================================
 # Benchmark tests configuration
 #==============================================================================
 add_custom_target(cxx-benchmarks)
 set(BENCHMARK_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(BENCHMARK_LIBCXX_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/benchmark-libcxx)
-set(BENCHMARK_NATIVE_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/benchmark-native)
+set(BENCHMARK_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark)
 
 add_library(               cxx-benchmarks-flags INTERFACE)
 
@@ -97,32 +70,14 @@ else()
   target_compile_features( cxx-benchmarks-flags INTERFACE cxx_std_23)
 endif()
 
-target_compile_options(    cxx-benchmarks-flags INTERFACE -fsized-deallocation -nostdinc++)
+target_compile_options(cxx-benchmarks-flags INTERFACE -fsized-deallocation -nostdinc++
+                                                      ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
 target_include_directories(cxx-benchmarks-flags INTERFACE "${LIBCXX_GENERATED_INCLUDE_DIR}"
-                                                INTERFACE "${BENCHMARK_LIBCXX_INSTALL}/include"
+                                                INTERFACE "${BENCHMARK_INSTALL_DIR}/include"
                                                 INTERFACE "${LIBCXX_SOURCE_DIR}/test/support")
-
-add_library(           cxx-benchmarks-flags-native INTERFACE)
-target_link_libraries( cxx-benchmarks-flags-native INTERFACE cxx-benchmarks-flags)
-target_compile_options(cxx-benchmarks-flags-native INTERFACE ${BENCHMARK_NATIVE_TARGET_FLAGS})
-target_link_options(   cxx-benchmarks-flags-native INTERFACE ${BENCHMARK_NATIVE_TARGET_FLAGS} "-L${BENCHMARK_NATIVE_INSTALL}/lib")
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libstdc++")
-  find_library(LIBSTDCXX_FILESYSTEM_TEST stdc++fs
-        PATHS ${LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN}
-        PATH_SUFFIXES lib lib64
-        DOC "The libstdc++ filesystem library used by the benchmarks"
-    )
-  if (LIBSTDCXX_FILESYSTEM_TEST)
-    target_link_libraries(cxx-benchmarks-flags-native INTERFACE -lstdc++fs)
-  endif()
-else()
-  target_link_libraries(cxx-benchmarks-flags-native INTERFACE -lc++fs -lc++experimental)
-endif()
-
-add_library(           cxx-benchmarks-flags-libcxx INTERFACE)
-target_link_libraries( cxx-benchmarks-flags-libcxx INTERFACE cxx-benchmarks-flags)
-target_compile_options(cxx-benchmarks-flags-libcxx INTERFACE ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
-target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -lm -nostdlib++ "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
+target_link_options(cxx-benchmarks-flags INTERFACE -lm -nostdlib++
+                                                   "-L${BENCHMARK_INSTALL_DIR}/lib" "-L${BENCHMARK_INSTALL_DIR}/lib64"
+                                                   ${SANITIZER_FLAGS})
 
 set(libcxx_benchmark_targets)
 
@@ -130,8 +85,8 @@ function(add_benchmark_test name source_file)
   set(libcxx_target ${name}_libcxx)
   list(APPEND libcxx_benchmark_targets ${libcxx_target})
   add_executable(${libcxx_target} EXCLUDE_FROM_ALL ${source_file})
-  target_link_libraries(${libcxx_target} PRIVATE cxx-benchmarks-flags-libcxx)
-  add_dependencies(${libcxx_target} cxx google-benchmark-libcxx)
+  target_link_libraries(${libcxx_target} PRIVATE cxx-benchmarks-flags)
+  add_dependencies(${libcxx_target} cxx google-benchmark)
   add_dependencies(cxx-benchmarks ${libcxx_target})
   if (LIBCXX_ENABLE_SHARED)
     target_link_libraries(${libcxx_target} PRIVATE cxx_shared)
@@ -144,27 +99,10 @@ function(add_benchmark_test name source_file)
   endif()
   set_target_properties(${libcxx_target}
     PROPERTIES
-          OUTPUT_NAME "${name}.libcxx.out"
+          OUTPUT_NAME "${name}.out"
           RUNTIME_OUTPUT_DIRECTORY "${BENCHMARK_OUTPUT_DIR}"
           CXX_EXTENSIONS NO)
   cxx_link_system_libraries(${libcxx_target})
-  if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-    set(native_target ${name}_native)
-    add_executable(${native_target} EXCLUDE_FROM_ALL ${source_file})
-    target_link_libraries(${native_target} PRIVATE cxx-benchmarks-flags-native)
-    add_dependencies(${native_target} google-benchmark-native
-                                      google-benchmark-libcxx)
-    target_link_libraries(${native_target} PRIVATE -lbenchmark)
-    if (LIBCXX_HAS_PTHREAD_LIB)
-      target_link_libraries(${native_target} PRIVATE -pthread)
-    endif()
-    add_dependencies(cxx-benchmarks ${native_target})
-    set_target_properties(${native_target}
-      PROPERTIES
-          OUTPUT_NAME "${name}.native.out"
-          RUNTIME_OUTPUT_DIRECTORY "${BENCHMARK_OUTPUT_DIR}"
-          CXX_EXTENSIONS NO)
-  endif()
 endfunction()
 
 
diff --git a/libcxx/benchmarks/lit.cfg.py b/libcxx/benchmarks/lit.cfg.py
index 7d222ddf9284e..a8209a90110cc 100644
--- a/libcxx/benchmarks/lit.cfg.py
+++ b/libcxx/benchmarks/lit.cfg.py
@@ -19,5 +19,5 @@
 config.test_source_root = config.test_exec_root
 
 config.test_format = GoogleBenchmark(
-    test_sub_dirs=".", test_suffix=".libcxx.out", benchmark_args=config.benchmark_args
+    test_sub_dirs=".", test_suffix=".out", benchmark_args=config.benchmark_args
 )
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index e425b9dadfe7d..66bb19bb5b2cd 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -399,22 +399,6 @@ libc++ Feature Options
   since the primary use of ``check-cxx-benchmarks`` is to get test and sanitizer coverage, not to
   get accurate measurements.
 
-.. option:: LIBCXX_BENCHMARK_NATIVE_STDLIB:STRING
-
-  **Default**:: ``""``
-
-  **Values**:: ``libc++``, ``libstdc++``
-
-  Build the libc++ benchmark tests and Google Benchmark library against the
-  specified standard library on the platform. On Linux this can be used to
-  compare libc++ to libstdc++ by building the benchmark tests against both
-  standard libraries.
-
-.. option:: LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN:STRING
-
-  Use the specified GCC toolchain and standard library when building the native
-  stdlib benchmark tests.
-
 .. option:: LIBCXX_ASSERTION_HANDLER_FILE:PATH
 
   **Default**:: ``"${CMAKE_CURRENT_SOURCE_DIR}/vendor/llvm/default_assertion_handler.in"``
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index b9c6bc84892c5..47f792066e925 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -169,3 +169,7 @@ Build System Changes
   to automatically detect the presence of ``clang-tidy`` and the required ``Clang`` libraries.
 
 - The CMake options ``LIBCXX_INSTALL_MODULES`` now defaults to ``ON``.
+
+- The CMake options ``LIBCXX_BENCHMARK_NATIVE_STDLIB`` and ``LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN`` have
+  been removed. To benchmark the native standard library, configure the test suite against the native
+  standard library directly instead.
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index d9f4fe467fe36..071ed388e0a91 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -351,7 +351,7 @@ Test Filenames`_ when determining the names for new test files.
      - Same as ``FOO.pass.cpp``, but for Objective-C++.
 
    * - ``FOO.compile.pass.cpp``
-     - Checks whether the C++ code in the file compiles successfully. In general, prefer ``compile`` tests over ``verify`` tests, 
+     - Checks whether the C++ code in the file compiles successfully. In general, prefer ``compile`` tests over ``verify`` tests,
        subject to the specific recommendations, below, for when to write ``verify`` tests.
    * - ``FOO.compile.pass.mm``
      - Same as ``FOO.compile.pass.cpp``, but for Objective-C++.
@@ -447,19 +447,12 @@ An example build would look like:
 
 .. code-block:: bash
 
-  $ cd build
-  $ ninja cxx-benchmarks
+  $ ninja -C build cxx-benchmarks
 
 This will build all of the benchmarks under ``<libcxx-src>/benchmarks`` to be
 built against the just-built libc++. The compiled tests are output into
 ``build/projects/libcxx/benchmarks``.
 
-The benchmarks can also be built against the platforms native standard library
-using the ``-DLIBCXX_BUILD_BENCHMARKS_NATIVE_STDLIB=ON`` CMake option. This
-is useful for comparing the performance of libc++ to other standard libraries.
-The compiled benchmarks are named ``<test>.libcxx.out`` if they test libc++ and
-``<test>.native.out`` otherwise.
-
 Also See:
 
   * :ref:`Building Libc++ <build instructions>`
@@ -476,8 +469,8 @@ For example:
 .. code-block:: bash
 
   $ cd build/projects/libcxx/benchmarks
-  $ ./algorithms.libcxx.out # Runs all the benchmarks
-  $ ./algorithms.libcxx.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks
+  $ ./algorithms.out # Runs all the benchmarks
+  $ ./algorithms.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks
 
 For more information about running benchmarks see `Google Benchmark`_.