testsuite overhaul -- all temp files end up in build, runtest.py call…

…ed directly and the run.py files shrink down to the bare minimum. Similar to the recent changes in OIIO testsuite.
AcademySoftwareFoundation · Mar 10, 2012 · fc7b1a5 · fc7b1a5
1 parent 12f821f
commit fc7b1a5
Show file tree

Hide file tree

Showing 102 changed files with 338 additions and 2,347 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,14 +1,6 @@
 dist
 build
-*.aux
-*.adx
-*.ind
-*.log
-*.out
-*.toc
-*.idx
-*.ilg
 *~
 \#*\#
 *.pyc
-testsuite/*/*.oso
+*.log
diff --git a/Makefile b/Makefile
@@ -40,6 +40,7 @@ endif
 
 ifneq (${VERBOSE},)
 MY_MAKE_FLAGS += VERBOSE=${VERBOSE}
+TEST_FLAGS += -V
 endif
 
 ifneq (${USE_TBB},)
@@ -69,6 +70,10 @@ ifneq (${MYCXX},)
 MY_CMAKE_FLAGS += -DCMAKE_CXX_COMPILER:STRING=${MYCXX}
 endif
 
+ifneq (${TEST},)
+TEST_FLAGS += -R ${TEST}
+endif
+
 #$(info MY_CMAKE_FLAGS = ${MY_CMAKE_FLAGS})
 #$(info MY_MAKE_FLAGS = ${MY_MAKE_FLAGS})
 
@@ -127,8 +132,15 @@ cmakeinstall: cmake
 dist : cmakeinstall
 
 # 'make test' does a full build and then runs all tests
-test: cmakeinstall
-	( cd ${build_dir} ; ${MAKE} ${MY_MAKE_FLAGS} test )
+test: cmake
+	cmake -E cmake_echo_color --switch=$(COLOR) --cyan "Running tests ${TEST_FLAGS}..."
+	( cd ${build_dir} ; ctest --force-new-ctest-process ${TEST_FLAGS} -E broken )
+
+# 'make testall' does a full build and then runs all tests (even the ones
+# that are expected to fail on some platforms)
+testall: cmake
+	cmake -E cmake_echo_color --switch=$(COLOR) --cyan "Running all tests ${TEST_FLAGS}..."
+	( cd ${build_dir} ; ctest --force-new-ctest-process ${TEST_FLAGS} )
 
 # 'make package' builds everything and then makes an installable package 
 # (platform dependent -- may be .tar.gz, .sh, .dmg, .rpm, .deb. .exe)
@@ -157,14 +169,6 @@ nuke:
 doxygen:
 	doxygen src/doc/Doxyfile
 
-#testclean : ${all_tests}
-#	@ for f in ${all_tests} ; do \
-#	    ( cd $$f ; \
-#	      echo "Cleaning test $$f " ; \
-#	      ./run.py -c ; \
-#	    ) \
-#	done
-
 #########################################################################
 
 
@@ -181,7 +185,8 @@ help:
 	@echo "  make clean        Remove the temporary files in ${build_dir}"
 	@echo "  make realclean    Remove both ${build_dir} AND ${dist_dir}"
 	@echo "  make nuke         Remove ALL of build and dist (not just ${platform})"
-	@echo "  make test         Run all tests"
+	@echo "  make test         Run tests"
+	@echo "  make testall      Run all tests, even broken ones"
 	@echo "  make doxygen      Build the Doxygen docs in ${top_build_dir}/doxygen"
 	@echo ""
 	@echo "Helpful modifiers:"

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -89,9 +89,6 @@ include_directories (
   )
 
 
-# We want CTest for testing
-include (CTest)
-
 include_directories ("include")
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -143,13 +140,46 @@ add_subdirectory (doc)
 #########################################################################
 # Testing
 
+# We want CTest for testing
+include (CTest)
+
+# Make a copy of the testsuite into the build area
+if (DEFINED CMAKE_VERSION AND NOT CMAKE_VERSION VERSION_LESS 2.8)
+    file (COPY ${PROJECT_SOURCE_DIR}/../testsuite
+          DESTINATION ${CMAKE_BINARY_DIR})
+endif()
+
 macro ( TESTSUITE )
-    foreach (_testname ${ARGN})
+    parse_arguments (_ats "LABEL" "" ${ARGN})
+    # If there was a FOUNDVAR param specified and that variable name is
+    # not defined, mark the test as broken.
+    if (_ats_FOUNDVAR AND NOT ${_ats_FOUNDVAR})
+        set (_ats_LABEL "broken")
+    endif ()
+    # Add the tests if all is well.
+    foreach (_testname ${_ats_DEFAULT_ARGS})
+        set (_testdir "${CMAKE_BINARY_DIR}/testsuite/${_testname}")
+        if (_ats_LABEL MATCHES "broken")
+            set (_testname "${_testname}-broken")
+        endif ()
+        set (_runtest python ${CMAKE_BINARY_DIR}/testsuite/runtest.py
+                             ${_testdir} ${_extra_test_args})
+        if (MSVC_IDE)
+            set (_runtest ${_runtest} --devenv-config $<CONFIGURATION>
+                                      --solution-path "${CMAKE_BINARY_DIR}" )
+        endif ()
+
+        message (STATUS "TEST ${_testname}: ${CMAKE_BINARY_DIR}/testsuite/runtest.py ${_testdir} ${_extra_test_args}")
+
         # Run the test unoptimized
-        add_test ( ${_testname} env OPENIMAGEIOHOME=${OPENIMAGEIOHOME} TESTSHADE_OPT=0 python ${PROJECT_SOURCE_DIR}/../testsuite/${_testname}/run.py ${PROJECT_SOURCE_DIR}/../testsuite/${_testname} ${CMAKE_BINARY_DIR} )
+        set (_env TESTSHADE_OPT=0 OPENIMAGEIOHOME=${OPENIMAGEIOHOME})
+        add_test ( NAME ${_testname}
+                   COMMAND env ${_env} ${_runtest} )
         # Run the same test again with aggressive -O2 runtime
         # optimization, triggered by setting TESTSHADE_OPT env variable
-        add_test ( ${_testname}.opt env OPENIMAGEIOHOME=${OPENIMAGEIOHOME} TESTSHADE_OPT=2 python ${PROJECT_SOURCE_DIR}/../testsuite/${_testname}/run.py ${PROJECT_SOURCE_DIR}/../testsuite/${_testname} ${CMAKE_BINARY_DIR} )
+        set (_env TESTSHADE_OPT=2 OPENIMAGEIOHOME=${OPENIMAGEIOHOME})
+        add_test ( NAME ${_testname}.opt
+                   COMMAND env ${_env} ${_runtest} )
     endforeach ()
 endmacro ()
 

diff --git a/src/cmake/CMakeUserUseBison.cmake b/src/cmake/CMakeUserUseBison.cmake
diff --git a/src/cmake/CMakeUserUseFlex.cmake b/src/cmake/CMakeUserUseFlex.cmake
diff --git a/src/cmake/oiio.cmake b/src/cmake/oiio.cmake
@@ -21,13 +21,6 @@ IF (OPENIMAGEIO_INCLUDES AND OPENIMAGEIO_LIBRARY )
     SET ( OPENIMAGEIO_FOUND TRUE )
     MESSAGE ( STATUS "OpenImageIO includes = ${OPENIMAGEIO_INCLUDES}" )
     MESSAGE ( STATUS "OpenImageIO library = ${OPENIMAGEIO_LIBRARY}" )
-    MESSAGE ( STATUS "OpenImageIO namespace = ${OPENIMAGEIO_NAMESPACE}" )
-    if (OPENIMAGEIO_NAMESPACE)
-        add_definitions ("-DOPENIMAGEIO_NAMESPACE=${OPENIMAGEIO_NAMESPACE}")
-    endif ()
-# N.B. -- once we're confident that we only build against OIIO >= 0.9.x,
-# specifically versions after we did the big namespace change, then we
-# can completely eliminate the 'OPENIMAGEIO_NAMESPACE' parts above.
 ELSE ()
     MESSAGE ( STATUS "OpenImageIO not found" )
 ENDIF ()

diff --git a/src/doc/.gitignore b/src/doc/.gitignore
@@ -0,0 +1,8 @@
+*.aux
+*.adx
+*.ind
+*.log
+*.out
+*.toc
+*.idx
+*.ilg
diff --git a/testsuite/arithmetic/run.py b/testsuite/arithmetic/run.py
@@ -1,27 +1,3 @@
 #!/usr/bin/python 
 
-import os
-import sys
-
-path = ""
-command = ""
-if len(sys.argv) > 2 :
-    os.chdir (sys.argv[1])
-    path = sys.argv[2] + "/"
-
-# A command to run
-command = path + "oslc/oslc test.osl > out.txt"
-command = command + "; " + path + "testshade/testshade -g 2 2 test >> out.txt"
-
-# Outputs to check against references
-outputs = [ "out.txt" ]
-
-# Files that need to be cleaned up, IN ADDITION to outputs
-cleanfiles = [ ]
-
-
-# boilerplate
-sys.path = [".."] + sys.path
-import runtest
-ret = runtest.runtest (command, outputs, cleanfiles)
-sys.exit (ret)
+command = testshade("-g 2 2 test")
diff --git a/testsuite/array-derivs/run.py b/testsuite/array-derivs/run.py
@@ -1,27 +1,3 @@
 #!/usr/bin/python 
 
-import os
-import sys
-
-path = ""
-command = ""
-if len(sys.argv) > 2 :
-    os.chdir (sys.argv[1])
-    path = sys.argv[2] + "/"
-
-# A command to run
-command = path + "oslc/oslc test.osl > out.txt"
-command = command + "; " + path + "testshade/testshade -g 2 2 test >> out.txt"
-
-# Outputs to check against references
-outputs = [ "out.txt" ]
-
-# Files that need to be cleaned up, IN ADDITION to outputs
-cleanfiles = [ ]
-
-
-# boilerplate
-sys.path = [".."] + sys.path
-import runtest
-ret = runtest.runtest (command, outputs, cleanfiles)
-sys.exit (ret)
+command = testshade("-g 2 2 test")
diff --git a/testsuite/array-range/run.py b/testsuite/array-range/run.py
@@ -1,28 +1,3 @@
 #!/usr/bin/python 
 
-import os
-import sys
-
-path = ""
-command = ""
-if len(sys.argv) > 2 :
-    os.chdir (sys.argv[1])
-    path = sys.argv[2] + "/"
-
-# A command to run
-command = path + "oslc/oslc test.osl > out1.txt"
-command = command + "; " + path + "testshade/testshade test >& out2.txt"
-command = command + "; cat out1.txt out2.txt > out.txt"
-
-# Outputs to check against references
-outputs = [ "out.txt" ]
-
-# Files that need to be cleaned up, IN ADDITION to outputs
-cleanfiles = [ ]
-
-
-# boilerplate
-sys.path = [".."] + sys.path
-import runtest
-ret = runtest.runtest (command, outputs, cleanfiles)
-sys.exit (ret)
+command = testshade("test")
diff --git a/testsuite/array/run.py b/testsuite/array/run.py
@@ -1,27 +1,3 @@
 #!/usr/bin/python 
 
-import os
-import sys
-
-path = ""
-command = ""
-if len(sys.argv) > 2 :
-    os.chdir (sys.argv[1])
-    path = sys.argv[2] + "/"
-
-# A command to run
-command = path + "oslc/oslc test.osl > out.txt"
-command = command + "; " + path + "testshade/testshade -g 2 2 test >> out.txt"
-
-# Outputs to check against references
-outputs = [ "out.txt" ]
-
-# Files that need to be cleaned up, IN ADDITION to outputs
-cleanfiles = [ ]
-
-
-# boilerplate
-sys.path = [".."] + sys.path
-import runtest
-ret = runtest.runtest (command, outputs, cleanfiles)
-sys.exit (ret)
+command = testshade("-g 2 2 test")