Refactor test and benchmark infrastructure (#87)

* Work-in-progress for test and benchmarking refactor.

* Support testing point-to-point operations in test_ops.

* Fix CommWrapper.

* Reorganize some code for better reuse.

* Support test_exchange again.

* Support RMA tests again.

* Drop test_multi_nballreduces.

* Drop transfer tests.

* Fix gathering vectors for printing when they differ in sizes.

* Add help message.

* New benchmarking infra.

* Remove now obsolete / unneeded benchmarks.

* Simplify stats printing.

* Fix up older benchmarks.

* Support toggling tests/benchmarks from CMake.

* Ensure non-participating ranks still participate in barriers.

* Support testing different algorithms.

* Support benchmarking a specific algorithm.

* Apply suggestions from code review

Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com>

Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/cxxopts"]
+	path = third_party/cxxopts
+	url = https://github.com/jarro2783/cxxopts.git

CMakeLists.txt

@@ -99,6 +99,13 @@ option(ALUMINUM_ENABLE_TRACE
   "Enable runtime tracing."
   OFF)
 
+option(ALUMINUM_ENABLE_TESTS
+  "Build tests."
+  OFF)
+option(ALUMINUM_ENABLE_BENCHMAKRS
+  "Build benchmarks."
+  OFF)
+
 if (ALUMINUM_HAS_GPU
     AND NOT ALUMINUM_ENABLE_NCCL
     AND NOT ALUMINUM_ENABLE_MPI_CUDA
@@ -114,6 +121,7 @@ endif ()
 string(TOUPPER "${CMAKE_BUILD_TYPE}" AL_BUILD_TYPE_UPPER)
 if (AL_BUILD_TYPE_UPPER MATCHES "DEBUG")
   set(AL_DEBUG ON)
+  set(AL_ENABLE_TESTS ON)
 endif ()
 if (ALUMINUM_DEBUG_HANG_CHECK)
   set(AL_DEBUG_HANG_CHECK ON)
@@ -127,6 +135,12 @@ endif ()
 if (ALUMINUM_ENABLE_TRACE)
   set(AL_TRACE ON)
 endif ()
+if (ALUMINUM_ENABLE_TESTS)
+  set(AL_ENABLE_TESTS ON)
+endif ()
+if (ALUMINUM_ENABLE_BENCHMARKS)
+  set(AL_ENABLE_BENCHMARKS ON)
+endif ()
 
 # Setup CXX requirements
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g")
@@ -349,7 +363,9 @@ add_subdirectory(test)
 
 # The benchmarks depend on some test utility headers, so it must come
 # after the test/ directory.
-add_subdirectory(benchmark)
+if (AL_ENABLE_BENCHMARKS)
+  add_subdirectory(benchmark)
+endif ()
 
 #
 # Install target

benchmark/CMakeLists.txt

@@ -1,17 +1,12 @@
 set_source_path(BENCHMARK_HDRS
+  benchmark_utils.hpp
+  benchmark_utils_mpi.hpp
+  benchmark_utils_nccl.hpp
+  benchmark_utils_ht.hpp
   wait.hpp)
 
 set_source_path(BENCHMARK_SRCS
-  benchmark_allgather.cpp
-  benchmark_allreduce.cpp
-  benchmark_alltoall.cpp
-  benchmark_bcast.cpp
-  benchmark_gather.cpp
-  benchmark_nballreduces.cpp
-  benchmark_reduce.cpp
-  benchmark_reduce_scatter.cpp
-  benchmark_scatter.cpp
-  benchmark_segallreduces.cpp)
+  benchmark_ops.cpp)
 
 if (AL_HAS_ROCM)
   hipify_source_files(AL_BENCHMARK_HEADERS ${BENCHMARK_HDRS})
@@ -35,6 +30,8 @@ foreach(src ${AL_BENCHMARK_SOURCES})
   # Get the test headers
   target_link_libraries(${_benchmark_exe_name}
     PRIVATE Al aluminum_test_headers)
+  target_include_directories(
+    ${_benchmark_exe_name} SYSTEM PRIVATE ${CMAKE_SOURCE_DIR}/third_party/cxxopts/include)
 
   # Use more caution now that "AL_HAS_CUDA" is overloaded.
   if (AL_HAS_CUDA AND NOT AL_HAS_ROCM)
@@ -43,9 +40,7 @@ foreach(src ${AL_BENCHMARK_SOURCES})
 endforeach()
 
 # Do it all again for "special" benchmarks...
-set_source_path(SPECIAL_BENCHMARK_BASE_SRCS
-  benchmark_overlap.cpp
-  benchmark_pingpong.cpp)
+set_source_path(SPECIAL_BENCHMARK_BASE_SRCS)
 if (AL_HAS_CUDA OR AL_HAS_ROCM)
   set_source_path(SPECIAL_BENCHMARK_WAIT_SRCS wait.cu)
   set_source_path(SPECIAL_BENCHMARK_WAIT_HDRS wait.hpp)
@@ -91,6 +86,8 @@ foreach(src ${AL_SPECIAL_BENCHMARK_SOURCES})
 
   target_link_libraries(${_benchmark_exe_name}
     PRIVATE Al aluminum_test_headers)
+  target_include_directories(
+    ${_benchmark_exe_name} SYSTEM PRIVATE ${CMAKE_SOURCE_DIR}/third_party/cxxopts/include)
 
   if (AL_HAS_CUDA AND NOT AL_HAS_ROCM)
     target_link_libraries(${_benchmark_exe_name} PRIVATE cuda)