From 569e423a1d79ea0f9b781c239c84daf90c3e36aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Troiti=C3=B1o?= Date: Wed, 25 Sep 2019 17:44:41 -0700 Subject: [PATCH] [test] Split TestFoundation in smaller tests for CTest reporting. At the moment, only one test was created for CTest, which would have run all the test in TestFoundation, and would have reported the final result as the test result, but without details of which test have failed. Following the similar case of GTest and gtest_discover_tests, XCTest can be used to list all the tests in the suite, and execute them as invidual CTest. They will report as different tests in the output, and individual failures will be shown. It will be also a little bit more resilient against crashes, which will only affect one test, and not the full suite. At first I was thinking in doing the parsing in some scripting language, but in order to be platform independent, and because the parsing is easy enough, the parsing of the XCTest output is done in CMake. --- CMakeLists.txt | 1 + TestFoundation/CMakeLists.txt | 12 ++-- cmake/modules/XCTest.cmake | 100 +++++++++++++++++++++++++++++ cmake/modules/XCTestAddTests.cmake | 90 ++++++++++++++++++++++++++ 4 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 cmake/modules/XCTest.cmake create mode 100644 cmake/modules/XCTestAddTests.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bfda92134..6b019fafcc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ find_package(dispatch CONFIG REQUIRED) include(SwiftSupport) include(GNUInstallDirs) +include(XCTest) set(CF_DEPLOYMENT_SWIFT YES CACHE BOOL "Build for Swift" FORCE) diff --git a/TestFoundation/CMakeLists.txt b/TestFoundation/CMakeLists.txt index 42beb11814..e61c5b074e 100644 --- a/TestFoundation/CMakeLists.txt +++ b/TestFoundation/CMakeLists.txt @@ -168,9 +168,11 @@ add_custom_command(TARGET TestFoundation POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_BINARY_DIR}/TestFoundation.app) -add_test(NAME TestFoundation - COMMAND ${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app) -set_tests_properties(TestFoundation PROPERTIES - ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$:$:$:$) +xctest_discover_tests(TestFoundation + COMMAND + ${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation${CMAKE_EXECUTABLE_SUFFIX} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app + PROPERTIES + ENVIRONMENT + LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$:$:$:$) diff --git a/cmake/modules/XCTest.cmake b/cmake/modules/XCTest.cmake new file mode 100644 index 0000000000..c15355a829 --- /dev/null +++ b/cmake/modules/XCTest.cmake @@ -0,0 +1,100 @@ +cmake_policy(PUSH) +cmake_policy(SET CMP0057 NEW) + +# Automatically add tests with CTest by querying the compiled test executable +# for available tests. +# +# xctest_discover_tests(target +# [COMMAND command] +# [WORKING_DIRECTORY dir] +# [PROPERTIES name1 value1...] +# [DISCOVERY_TIMEOUT seconds] +# ) +# +# `xctest_discover_tests` sets up a post-build command on the test executable +# that generates the list of tests by parsing the output from running the test +# with the `--list-tests` argument. +# +# The options are: +# +# `target` +# Specifies the XCTest executable, which must be a known CMake target. CMake +# will substitute the location of the built executable when running the test. +# +# `COMMAND command` +# Override the command used for the test executable. If you executable is not +# created with CMake add_executable, you will have to provide a command path. +# If this option is not provided, the target file of the target is used. +# +# `WORKING_DIRECTORY dir` +# Specifies the directory in which to run the discovered test cases. If this +# option is not provided, the current binary directory is used. +# +# `PROPERTIES name1 value1...` +# Specifies additional properties to be set on all tests discovered by this +# invocation of `xctest_discover_tests`. +# +# `DISCOVERY_TIMEOUT seconds` +# Specifies how long (in seconds) CMake will wait for the test to enumerate +# available tests. If the test takes longer than this, discovery (and your +# build) will fail. The default is 5 seconds. +# +# The inspiration for this is CMake `gtest_discover_tests`. The official +# documentation might be useful for using this function. Many details of that +# function has been dropped in the name of simplicity, and others have been +# improved. +function(xctest_discover_tests TARGET) + cmake_parse_arguments( + "" + "" + "COMMAND;WORKING_DIRECTORY;DISCOVERY_TIMEOUT" + "PROPERTIES" + ${ARGN} + ) + + if(NOT _COMMAND) + set(_COMMAND "$") + endif() + if(NOT _WORKING_DIRECTORY) + set(_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() + if(NOT _DISCOVERY_TIMEOUT) + set(_DISCOVERY_TIMEOUT 5) + endif() + + set(ctest_file_base ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}) + set(ctest_include_file "${ctest_file_base}_include.cmake") + set(ctest_tests_file "${ctest_file_base}_tests.cmake") + + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_EXECUTABLE=${_COMMAND}" + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" + -D "TEST_PROPERTIES=${_PROPERTIES}" + -D "CTEST_FILE=${ctest_tests_file}" + -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" + -P "${_XCTEST_DISCOVER_TESTS_SCRIPT}" + VERBATIM + ) + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" + "endif()\n" + ) + + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) +endfunction() + +set(_XCTEST_DISCOVER_TESTS_SCRIPT + ${CMAKE_CURRENT_LIST_DIR}/XCTestAddTests.cmake +) + +cmake_policy(POP) diff --git a/cmake/modules/XCTestAddTests.cmake b/cmake/modules/XCTestAddTests.cmake new file mode 100644 index 0000000000..b836c96554 --- /dev/null +++ b/cmake/modules/XCTestAddTests.cmake @@ -0,0 +1,90 @@ +set(properties ${TEST_PROPERTIES}) +set(script) +set(tests) + +function(add_command NAME) + set(_args "") + foreach(_arg ${ARGN}) + if(_arg MATCHES "[^-./:a-zA-Z0-9_]") + set(_args "${_args} [==[${_arg}]==]") + else() + set(_args "${_args} ${_arg}") + endif() + endforeach() + set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) +endfunction() + +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable does not exist.\n" + " Path: '${TEST_EXECUTABLE}'" + ) +endif() +# We need to figure out if some environment is needed to run the test listing. +cmake_parse_arguments("_properties" "" "ENVIRONMENT" "" ${properties}) +if(_properties_ENVIRONMENT) + foreach(_env ${_properties_ENVIRONMENT}) + string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\1" _key "${_env}") + string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\2" _value "${_env}") + if(NOT "${_key}" STREQUAL "") + set(ENV{${_key}} "${_value}") + endif() + endforeach() +endif() +execute_process( + COMMAND "${TEST_EXECUTABLE}" --list-tests + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + TIMEOUT ${TEST_DISCOVERY_TIMEOUT} + OUTPUT_VARIABLE output + ERROR_VARIABLE error_output + RESULT_VARIABLE result +) +if(NOT ${result} EQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + string(REPLACE "\n" "\n " error_output "${error_output}") + message(FATAL_ERROR + "Error running test executable.\n" + " Path: '${TEST_EXECUTABLE}'\n" + " Result: ${result}\n" + " Output:\n" + " ${output}\n" + " Error:\n" + " ${error_output}\n" + ) +endif() + +string(REPLACE "\n" ";" output "${output}") + +foreach(line ${output}) + if(line MATCHES "^[ \t]*$") + continue() + elseif(line MATCHES "^Listing [0-9]+ tests? in .+:$") + continue() + elseif(line MATCHES "^.+\\..+/.+$") + # TODO: remove non-ASCII characters from module, class and method names + set(pretty_target "${line}") + string(REGEX REPLACE "/" "-" pretty_target "${pretty_target}") + add_command(add_test + "${pretty_target}" + "${TEST_EXECUTABLE}" + "${line}" + ) + add_command(set_tests_properties + "${pretty_target}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${properties} + ) + list(APPEND tests "${pretty_target}") + else() + message(FATAL_ERROR + "Error parsing test executable output.\n" + " Path: '${TEST_EXECUTABLE}'\n" + " Line: '${line}'" + ) + endif() +endforeach() + +add_command(set "${TARGET}_TESTS" ${tests}) + +file(WRITE "${CTEST_FILE}" "${script}")