Pass the tmp dir to use as argument to the test runner. (#9865)

  - Pass the intended base tmp dir to the testrunner script.
  - The test runner will create the directory before running the tests.

  - The repetition of tests is moved out of the testrunner script and is
    now handled by the script that handles the whole testsuite. This is
    done to allow running a test just once without repetition.

OMEdit/Testsuite/CMakeLists.txt

@@ -29,7 +29,8 @@ macro(add_omedit_test test_name test_source_files)
   target_link_libraries(${test_name} PUBLIC omedit::testsuite::util)
 
   add_test(NAME ${test_name}
-         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/RunOMEditTest.sh $<TARGET_FILE:${test_name}>)
+         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/RunOMEditTest.sh $<TARGET_FILE:${test_name}>
+                                                              ${CMAKE_CURRENT_BINARY_DIR}/OMEditTestResult/)
 
   # Label the tests (does not matter just now but it will of more types of tests are added.).
   set_tests_properties(${TEST_FILE_NAME} PROPERTIES LABELS "OMEditTestsuite")
@@ -53,6 +54,9 @@ add_omedit_test(Transformation "${OMEDIT_TEST_TRANSFORMATION_SOURCES}")
 set(OMEDIT_TEST_HOMOTOPY_SOURCES Homotopy/HomotopyTest.cpp
                                  Homotopy/HomotopyTest.h)
 add_omedit_test(Homotopy "${OMEDIT_TEST_HOMOTOPY_SOURCES}")
+# Set a 5 min timeout for the Homotopy test. This should be investigated
+# to figure out why it takes so long in the first place.
+# set_tests_properties(Homotopy PROPERTIES TIMEOUT 300)
 
 set(OMEDIT_TEST_EXPRESSION_SOURCES Expression/ExpressionTest.cpp
                                    Expression/ExpressionTest.h)

OMEdit/Testsuite/RunOMEditTest.sh

@@ -1,25 +1,37 @@
 #!/bin/bash
 set -e
 
-# If number of arguments less then 1; print usage and exit
-if [ $# -lt 1 ]; then
-    printf "Usage: $0 <test_file>\n"
+# If number of arguments less then 2; print usage and exit
+if [ $# -lt 2 ]; then
+    printf "Usage: $0 <test_file> <BaseTmpDir>\n"
     exit 1
 fi
 
-testexe="$1" # The test executable name.
+test_exe_path="$1" # The path to test executable.
+tmp_dir_for_testsuite="$2" # The base tmp directory for the test.
 
-echo "Running testcase "$testexe
+test_name=$(basename "$test_exe_path")
+tmp_dir_for_test="$tmp_dir_for_testsuite/$test_name"
+
+# Create the tmp dir for the test. If this does not exist OMEdit will try to
+# create it. However, if it fails to create it does not report anything and
+# that can be confusing.
+mkdir -p "$tmp_dir_for_test"
+
+# Save the current env tmp dirs
 ORIGINAL_TEMP=$TEMP
 ORIGINAL_TMP=$TMP
 ORIGINAL_TMPDIR=$TMPDIR
-NEW_TEMP=$OMEditTestResults/$testexe
-NEW_TMP=$OMEditTestResults/$testexe
-NEW_TMPDIR=$OMEditTestResults/$testexe
-export TEMP=$NEW_TEMP
-export TMP=$NEW_TMP
-export TMPDIR=$NEW_TMPDIR
-$testexe || $testexe || $testexe || $testexe || $testexe
+
+# Export the new tmp dirs for this test.
+export TEMP="$tmp_dir_for_test"
+export TMP="$tmp_dir_for_test"
+export TMPDIR="$tmp_dir_for_test"
+
+printf "Running testcase '%s' with tmp dir '%s'\n" "$test_exe_path" "$tmp_dir_for_test"
+$test_exe_path
+
+# Resotore the old env tmp dirs
 export TEMP=$ORIGINAL_TEMP
 export TMP=$ORIGINAL_TMP
 export TMPDIR=$ORIGINAL_TMPDIR

OMEdit/Testsuite/RunOMEditTestsuite.sh

@@ -6,6 +6,11 @@ OMEditTestResults="$PWD/OMEditTestResult"
 
 for testcase in "${testcases[@]}"
 do
-  ./RunOMEditTest.sh ./$testcase
+  # Try 5 times if it does not pass
+  ./RunOMEditTest.sh ./$testcase $OMEditTestResults \
+  || ./RunOMEditTest.sh ./$testcase $OMEditTestResults \
+  || ./RunOMEditTest.sh ./$testcase $OMEditTestResults \
+  || ./RunOMEditTest.sh ./$testcase $OMEditTestResults \
+  || ./RunOMEditTest.sh ./$testcase $OMEditTestResults
 done
 rm -rf $OMEditTestResults