Add performance testing framework: preparatory refactoring

cmake/macros/macro_deal_ii_add_test.cmake

@@ -69,14 +69,30 @@
 #     - Complete path to the numdiff binary.
 #
 #   TEST_TIME_LIMIT
-#     - specifying the maximal wall clock time in seconds a test is allowed
-#       to run
+#     - Specifies the maximal wall clock time in seconds a test is allowed
+#       to run.
+#
+#   TEST_MPI_RANK_LIMIT
+#     - Specifies the maximal number of MPI ranks that can be used. If a
+#       test variant configures a larger number of MPI ranks (via
+#       .mpirun=N. in the output file) than this limit the test will be
+#       dropped. The special value 0 enforces no limit. Defaults to 0.
+#
+#   TEST_THREAD_LIMIT
+#     - Specifies the maximal number of worker threads that can should be
+#       used by the threading backend. If a test variant configures a
+#       larger number of threads (via .threads=N. in the output file) than
+#       this limit the test will be dropped. Note that individual tests
+#       might exceed this limit by calling
+#       MultithreadInfo::set_thread_limit(), or by manually creating
+#       additional threads. The special value 0 enforces no limit. Defaults
+#       to 0.
 #
 # Usage:
 #     DEAL_II_ADD_TEST(category test_name comparison_file)
 #
 
-MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
+FUNCTION(DEAL_II_ADD_TEST _category _test_name _comparison_file)
 
   IF(NOT DEAL_II_PROJECT_CONFIG_INCLUDED)
     MESSAGE(FATAL_ERROR
@@ -101,11 +117,65 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
   #
   # Determine whether the test should be run with mpirun:
   #
-  STRING(REGEX MATCH "mpirun=([0-9]*)" _n_cpu ${_file})
+  STRING(REGEX MATCH "mpirun=([0-9]+|max)" _n_cpu ${_file})
   IF("${_n_cpu}" STREQUAL "")
     SET(_n_cpu 0) # 0 indicates that no mpirun should be used
   ELSE()
-    STRING(REGEX REPLACE "^mpirun=([0-9]*)$" "\\1" _n_cpu ${_n_cpu})
+    STRING(REGEX REPLACE "^mpirun=([0-9]+|max)$" "\\1" _n_cpu ${_n_cpu})
+  ENDIF()
+
+  #
+  # If we encounter the special string "mpirun=max" set the number of MPI
+  # ranks used for the test to the maximum number of allowed ranks. If no
+  # limit has been specified, i.e., TEST_MPI_RANK_LIMIT is 0, skip defining
+  # the test.
+  #
+  IF("${_n_cpu}" STREQUAL "max")
+    IF(TEST_MPI_RANK_LIMIT EQUAL 0)
+      RETURN()
+    ENDIF()
+    SET(_n_cpu "${TEST_MPI_RANK_LIMIT}")
+  ENDIF()
+
+  #
+  # If the number of MPI ranks specified for the test via .mpirun=N.
+  # exceeds the limit ${TEST_MPI_RANK_LIMIT}, skip defining the test
+  #
+  IF(TEST_MPI_RANK_LIMIT GREATER 0 AND _n_cpu GREATER TEST_MPI_RANK_LIMIT)
+    RETURN()
+  ENDIF()
+
+  #
+  # Determine whether the test declaration specifies a thread pool size via
+  # threads=N:
+  #
+  STRING(REGEX MATCH "threads=([0-9]+|max)" _n_threads ${_file})
+  IF("${_n_threads}" STREQUAL "")
+    SET(_n_threads 0) # 0 indicates that the default thread pool size
+                      # should be used (currently set to 3 in tests.h)
+  ELSE()
+    STRING(REGEX REPLACE "^threads=([0-9]+|max)$" "\\1" _n_threads ${_n_threads})
+  ENDIF()
+
+  #
+  # If we encounter the special string "threads=max" set the number of
+  # threads of the threading pool to the maximum number of allowed threads.
+  # If no limit has been specified, i.e., TEST_THREAD_LIMIT is 0, skip
+  # defining the test.
+  #
+  IF("${_n_threads}" STREQUAL "max")
+    IF(TEST_THREAD_LIMIT EQUAL 0)
+      RETURN()
+    ENDIF()
+    SET(_n_threads "${TEST_THREAD_LIMIT}")
+  ENDIF()
+
+  #
+  # If the number of threads specified for the test via .threads=N. exceeds
+  # the limit ${TEST_THREAD_LIMIT}, skip defining the test
+  #
+  IF(TEST_THREAD_LIMIT GREATER 0 AND _n_threads GREATER TEST_THREAD_LIMIT)
+    RETURN()
   ENDIF()
 
   #
@@ -309,7 +379,8 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
       #
 
       ADD_CUSTOM_COMMAND(OUTPUT ${_test_directory}/output
-        COMMAND sh ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh
+        COMMAND TEST_N_THREADS=${_n_threads}
+          sh ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh
           run "${_test_full}" ${_run_args}
         COMMAND ${PERL_EXECUTABLE}
           -pi ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/normalize.pl
@@ -391,9 +462,9 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
             DEPENDS ${TEST_DEPENDENCIES_${_target}}
             )
         ENDIF()
-        SET(TEST_DEPENDENCIES_${_target} ${_test_full})
+        SET(TEST_DEPENDENCIES_${_target} ${_test_full} PARENT_SCOPE)
       ENDIF()
 
     ENDIF()
   ENDFOREACH()
-ENDMACRO()
+ENDFUNCTION()

cmake/macros/macro_deal_ii_pickup_tests.cmake

@@ -27,15 +27,30 @@
 #     TEST_LIBRARIES
 #     TEST_LIBRARIES_DEBUG
 #     TEST_LIBRARIES_RELEASE
-#       - specifying additional libraries (and targets) to link against.
+#       - Specify additional libraries (and targets) to link against.
 #
 #     TEST_TARGET or
 #     TEST_TARGET_DEBUG and TEST_TARGET_RELEASE
-#       - specifying a test target to be executed for a parameter run.
+#       - Specifies a test target to be executed for a parameter run.
 #
 #     TEST_TIME_LIMIT
 #       - Specifies the maximal wall clock time in seconds a test is
 #         allowed to run. Defaults to 600.
+#     TEST_MPI_RANK_LIMIT
+#       - Specifies the maximal number of MPI ranks that can be used. If a
+#         test variant configures a larger number of MPI ranks (via
+#         .mpirun=N. in the output file) than this limit the test will be
+#         dropped. The special value 0 enforces no limit. Defaults to 0.
+#     TEST_THREAD_LIMIT
+#       - Specifies the maximal number of worker threads that can should be
+#         used by the threading backend. If a test variant configures a
+#         larger number of threads (via .threads=N. in the output file)
+#         than this limit the test will be dropped. Note that individual
+#         tests might exceed this limit by calling
+#         MultithreadInfo::set_thread_limit(), or by manually creating
+#         additional threads. The special value 0 enforces no limit.
+#         Defaults to 0.
+#
 #     TEST_PICKUP_REGEX
 #       - A regular expression to select only a subset of tests during setup.
 #         An empty string is interpreted as a catchall (this is the default).
@@ -47,9 +62,6 @@
 #     DEAL_II_PICKUP_TESTS()
 #
 
-# We use CONTINUE(), which is new in cmake 3.2
-CMAKE_MINIMUM_REQUIRED(VERSION 3.2.0)
-
 #
 # Two very small macros that are used below:
 #
@@ -172,12 +184,18 @@ MACRO(DEAL_II_PICKUP_TESTS)
   ENDIF()
 
   #
-  # Set time limit:
+  # Set various limits:
   #
 
   SET_IF_EMPTY(TEST_TIME_LIMIT "$ENV{TEST_TIME_LIMIT}")
   SET_IF_EMPTY(TEST_TIME_LIMIT 600)
 
+  SET_IF_EMPTY(TEST_MPI_RANK_LIMIT "$ENV{TEST_MPI_RANK_LIMIT}")
+  SET_IF_EMPTY(TEST_MPI_RANK_LIMIT 0)
+
+  SET_IF_EMPTY(TEST_THREAD_LIMIT "$ENV{TEST_THREAD_LIMIT}")
+  SET_IF_EMPTY(TEST_THREAD_LIMIT 0)
+
   #
   # ... and finally pick up tests:
   #

cmake/scripts/run_test.sh

@@ -31,10 +31,6 @@ shift 2
 
 # Ensure uniform sorting for pathname expansion
 export LC_ALL=C
-# Prevent OpenMP from creating additional threads
-export OMP_NUM_THREADS=2
-# Allow oversubscription for MPI (needed for Openmpi@3.0)
-export OMPI_MCA_rmaps_base_oversubscribe=1
 
 case $STAGE in
   run)
@@ -47,6 +43,27 @@ case $STAGE in
     #     failing_output
     ##
 
+    #
+    # If TEST_N_THREADS is not equal to zero then:
+    #  - Export the environment variable DEAL_II_NUM_THREADS set to
+    #    $TEST_N_THREADS. This will enforce an upper bound of
+    #    DEAL_II_NUM_THREADS during thread initialization of the threading
+    #    pool in deal.II.
+    #  - Export TEST_N_THREADS which is internally used in the deal.II
+    #    testsuite to explicitly set the number of threads in the header
+    #    file tests.h.
+    #
+    if [ "${TEST_N_THREADS+0}" -ne 0 ]; then
+      export DEAL_II_NUM_THREADS="${TEST_N_THREADS}"
+      export TEST_N_THREADS
+    fi
+
+    # Limit the OpenMP pool to two threads.
+    export OMP_NUM_THREADS="2"
+
+    # Allow oversubscription for MPI (needed for Openmpi@3.0)
+    export OMPI_MCA_rmaps_base_oversubscribe=1
+
     rm -f failing_output
     rm -f output
     rm -f stdout

doc/developers/testsuite.html

@@ -110,6 +110,21 @@ <h3>For a build directory</h3>
   - The time limit (in seconds) a single test is allowed to take. Defaults
     to 180 seconds
 
+TEST_MPI_RANK_LIMIT
+  - Specifies the maximal number of MPI ranks that can be used. If a
+    test variant configures a larger number of MPI ranks (via
+    .mpirun=N. in the output file) than this limit the test will be
+    dropped. The special value 0 enforces no limit. Defaults to 0.
+
+TEST_THREAD_LIMIT
+  - Specifies the maximal number of worker threads that can should be
+    used by the threading backend. If a test variant configures a larger
+    number of threads (via .threads=N. in the output file) than this limit
+    the test will be dropped. Note that individual tests might exceed this
+    limit by calling MultithreadInfo::set_thread_limit(), or by manually
+    creating additional threads. The special value 0 enforces no limit.
+    Defaults to 0.
+
 TEST_PICKUP_REGEX
   - A regular expression to select only a subset of tests during setup.
     An empty string is interpreted as a catchall (this is the default).
@@ -460,7 +475,7 @@ <h3>Restricting tests to build configurations</h3>
       just <code>category/test.output</code>. In pseudo code:
 <pre>
 category/test.[with_&lt;string&gt;(&lt;=|&gt;=|=|&lt;|&gt;)&lt;on|off|version&gt;.]*
-              [mpirun=&lt;x&gt;.][expect=&lt;y&gt;.][binary.][&lt;debug|release&gt;.]output
+              [mpirun=&lt;N|all&gt;.][threads=&lt;N|all&gt;.][expect=&lt;y&gt;.][binary.][&lt;debug|release&gt;.]output
 </pre>
       Normally, a test will be set up so that it runs twice, once in debug and
       once in release configuration.
@@ -841,7 +856,8 @@ <h2>Submitting test results</h2>
   - Additional options that will be passed directly to make (or ninja).
 </pre>
 
-      Furthermore, the variables TEST_TIME_LIMIT and TEST_PICKUP_REGEX (as
+      Furthermore, the variables TEST_TIME_LIMIT, TEST_MPI_RANK_LIMIT,
+      TEST_THREAD_LIMIT and TEST_PICKUP_REGEX (as
       described <a href="#setupbuild">above</a>), DIFF_DIR, and NUMDIFF_DIR
       can also be set and will be handed automatically down
       to <code>cmake</code>. For more details on the different tracks, see

doc/users/testsuite.html

@@ -312,8 +312,23 @@ <h3>Advanced configuration</h3>
   - pointing to a valid "numdiff" executable
 
 TEST_TIME_LIMIT
-  - specifying the maximal wall clock time in seconds a test is allowed
-    to run</pre>
+  - Specifies the maximal wall clock time in seconds a test is allowed
+    to run.
+
+TEST_MPI_RANK_LIMIT
+  - Specifies the maximal number of MPI ranks that can be used. If a
+    test variant configures a larger number of MPI ranks (via
+    .mpirun=N. in the output file) than this limit the test will be
+    dropped. The special value 0 enforces no limit. Defaults to 0.
+
+TEST_THREAD_LIMIT
+  - Specifies the maximal number of worker threads that can should be
+    used by the threading backend. If a test variant configures a larger
+    number of threads (via .threads=N. in the output file) than this limit
+    the test will be dropped. Note that individual tests might exceed this
+    limit by calling MultithreadInfo::set_thread_limit(), or by manually
+    creating additional threads. The special value 0 enforces no limit.
+    Defaults to 0.</pre>
     <hr />
     <div class="right">
       <a href="http://validator.w3.org/check?uri=referer" target="_top">

tests/CMakeLists.txt

@@ -100,7 +100,7 @@ FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/CTestTestfile.cmake "")
 #
 # Always undefine the following variables in the setup_tests target:
 #
-FOREACH(_var DIFF_DIR NUMDIFF_DIR TEST_PICKUP_REGEX TEST_TIME_LIMIT)
+FOREACH(_var DIFF_DIR NUMDIFF_DIR TEST_PICKUP_REGEX TEST_TIME_LIMIT TEST_MPI_RANK_LIMIT TEST_THREAD_LIMIT)
   LIST(APPEND _options "-U${_var}")
   IF(NOT "${${_var}}" STREQUAL "")
     LIST(APPEND _options "-D${_var}=${${_var}}")

tests/performance/CMakeLists.txt

@@ -0,0 +1,4 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0)
+INCLUDE(../setup_testsubproject.cmake)
+PROJECT(testsuite CXX)
+DEAL_II_PICKUP_TESTS()

tests/setup_testsubproject.cmake

@@ -3,7 +3,7 @@ FIND_PACKAGE(deal.II 9.2.0 REQUIRED HINTS ${DEAL_II_DIR})
 SET(CMAKE_BUILD_TYPE ${DEAL_II_BUILD_TYPE} CACHE STRING "" FORCE)
 DEAL_II_INITIALIZE_CACHED_VARIABLES()
 
-FOREACH(_var DIFF_DIR NUMDIFF_DIR TEST_PICKUP_REGEX TEST_TIME_LIMIT)
+FOREACH(_var DIFF_DIR NUMDIFF_DIR TEST_PICKUP_REGEX TEST_TIME_LIMIT TEST_MPI_RANK_LIMIT TEST_THREAD_LIMIT)
   SET_IF_EMPTY(${_var} "$ENV{${_var}}")
   SET(${_var} "${${_var}}" CACHE STRING "" FORCE)
 ENDFOREACH()

tests/tests.h

@@ -418,15 +418,29 @@ filter_out_small_numbers(const Number number, const double tolerance)
 
 
 /*
- * If we run 64 tests at the same time on a 64-core system, and
- * each of them runs 64 threads, then we get astronomical loads.
- * Limit concurrency to a fixed (small) number of threads, independent
- * of the core count.
+ * If we run 64 tests at the same time on a 64-core system, and each of
+ * them runs 64 threads, then we get astronomical loads. Limit concurrency
+ * to a fixed (small) number of threads, independent of the core count. The
+ * limit defaults to 3 and can be overriden by the environment variable
+ * TEST_N_THREADS.
  */
 inline unsigned int
 testing_max_num_threads()
 {
-  return 3;
+  const int default_n_threads = 3;
+
+  if (const char *penv = std::getenv("TEST_N_THREADS"))
+    try
+      {
+        const int n_threads = Utilities::string_to_int(std::string(penv));
+        return n_threads > 0 ? n_threads : default_n_threads;
+      }
+    catch (...)
+      {
+        return default_n_threads;
+      }
+  else
+    return default_n_threads;
 }
 
 struct LimitConcurrency