From 06f5045fddd83edc5db2282db91dd1ab0c803997 Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Mon, 11 Feb 2019 15:21:12 -0800 Subject: [PATCH] This adds fine grained execution of Google Test test programs First, the Google Test test is executed with `--gtest_list_tests`. Next, based on the output from `--gtest_list_tests`, the testcases are run individually. The output from each unique testcase is based on the standard output it provides, per the test output protocol defined in the GoogleTest docs on github [1], [2], and instrumented via various demo programs I created, which can be found on GitHub [here](https://github.com/ngie-eign/scratch/tree/master/programming/c%2B%2B/gtest). This support is a very rough cut to provide an initial working integration effort. There're additional improvements that can be made by leveraging either the JSON or XML structured output format, instead of scraping standard output using beginning and ending sentinels to search for regular expressions. In order to do that though without reinventing the wheel, Kyua would need to rely on an external JSON or XML library. This test interface supports fine grained execution of test programs like the ATF test interface, but matches the plain/TAP interfaces by instead supporting metadata passing via `TEST_ENV` prefixed environment variables. This support adds additional tests for verifying pass, fail, skip (will be available in version 1.9.0 and is available in FreeBSD base's version of googletest, and disabled result determination, using mock output and a mock test program (`engine/googletest_helpers`), for parity with other test formats (ATF, plain, TAP). 1. https://github.com/google/googletest/blob/master/googletest/docs/primer.md 2. https://github.com/google/googletest/blob/master/googletest/docs/advanced.md Signed-off-by: Enji Cooper --- NEWS.md | 2 + cli/main.cpp | 4 + doc/kyuafile.5.in | 16 +- engine/Kyuafile | 3 + engine/Makefile.am.inc | 27 ++ engine/googletest.cpp | 221 +++++++++++++ engine/googletest.hpp | 67 ++++ engine/googletest_helpers.cpp | 344 ++++++++++++++++++++ engine/googletest_list.cpp | 122 +++++++ engine/googletest_list.hpp | 50 +++ engine/googletest_list_test.cpp | 212 ++++++++++++ engine/googletest_result.cpp | 523 ++++++++++++++++++++++++++++++ engine/googletest_result.hpp | 112 +++++++ engine/googletest_result_fwd.hpp | 43 +++ engine/googletest_result_test.cpp | 431 ++++++++++++++++++++++++ engine/googletest_test.cpp | 361 +++++++++++++++++++++ 16 files changed, 2537 insertions(+), 1 deletion(-) create mode 100644 engine/googletest.cpp create mode 100644 engine/googletest.hpp create mode 100644 engine/googletest_helpers.cpp create mode 100644 engine/googletest_list.cpp create mode 100644 engine/googletest_list.hpp create mode 100644 engine/googletest_list_test.cpp create mode 100644 engine/googletest_result.cpp create mode 100644 engine/googletest_result.hpp create mode 100644 engine/googletest_result_fwd.hpp create mode 100644 engine/googletest_result_test.cpp create mode 100644 engine/googletest_test.cpp diff --git a/NEWS.md b/NEWS.md index 304cfe94..119b5a02 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,8 @@ Changes in version 0.14 **NOT RELEASED YET; STILL UNDER DEVELOPMENT.** +* GoogleTest test program support was added to kyua. + * Explicitly require C++11 language features when compiling Kyua. diff --git a/cli/main.cpp b/cli/main.cpp index 531c252b..d8ef9277 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -56,6 +56,7 @@ extern "C" { #include "cli/common.ipp" #include "cli/config.hpp" #include "engine/atf.hpp" +#include "engine/googletest.hpp" #include "engine/plain.hpp" #include "engine/scheduler.hpp" #include "engine/tap.hpp" @@ -102,6 +103,9 @@ register_scheduler_interfaces(void) scheduler::register_interface( "atf", std::shared_ptr< scheduler::interface >( new engine::atf_interface())); + scheduler::register_interface( + "googletest", std::shared_ptr< scheduler::interface >( + new engine::googletest_interface())); scheduler::register_interface( "plain", std::shared_ptr< scheduler::interface >( new engine::plain_interface())); diff --git a/doc/kyuafile.5.in b/doc/kyuafile.5.in index 06cb2dbc..b5e87800 100644 --- a/doc/kyuafile.5.in +++ b/doc/kyuafile.5.in @@ -25,7 +25,7 @@ .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd July 3, 2015 +.Dd April 2, 2019 .Dt KYUAFILE 5 .Os .Sh NAME @@ -40,6 +40,7 @@ .Fn fs.files "string path" .Fn fs.is_absolute "string path" .Fn fs.join "string path" "string path" +.Fn googletest_test_program "string name" "[string metadata]" .Fn include "string path" .Fn plain_test_program "string name" "[string metadata]" .Fn syntax "int version" @@ -123,6 +124,16 @@ the test cases in the test program. Any metadata properties defined by the test cases themselves override the metadata values defined here. .Pp +.Em Google Test test programs +are those that use the Google Test test infrastructure. +They can be registered with the +.Fn googletest_test_program +table constructor. +This function takes the +.Fa name +of the test program, and a collection of optional metadata settings for the +test program. +.Pp .Em Plain test programs are those that return 0 on success and non-0 on failure; in general, most test programs (even those that use fancy unit-testing libraries) behave this way and @@ -336,6 +347,9 @@ syntax(2) test_suite('second') +googletest_test_program{name='dont_run_as_root_test', + required_user='unprivileged'} + plain_test_program{name='legacy_test', allowed_architectures='amd64 i386', required_files='/bin/ls', diff --git a/engine/Kyuafile b/engine/Kyuafile index 1baa63bc..5337dc19 100644 --- a/engine/Kyuafile +++ b/engine/Kyuafile @@ -8,6 +8,9 @@ atf_test_program{name="atf_result_test"} atf_test_program{name="config_test"} atf_test_program{name="exceptions_test"} atf_test_program{name="filters_test"} +atf_test_program{name="googletest_test"} +atf_test_program{name="googletest_list_test"} +atf_test_program{name="googletest_result_test"} atf_test_program{name="kyuafile_test"} atf_test_program{name="plain_test"} atf_test_program{name="requirements_test"} diff --git a/engine/Makefile.am.inc b/engine/Makefile.am.inc index baa7fe0b..dc972d4f 100644 --- a/engine/Makefile.am.inc +++ b/engine/Makefile.am.inc @@ -46,6 +46,13 @@ libengine_a_SOURCES += engine/exceptions.hpp libengine_a_SOURCES += engine/filters.cpp libengine_a_SOURCES += engine/filters.hpp libengine_a_SOURCES += engine/filters_fwd.hpp +libengine_a_SOURCES += engine/googletest.cpp +libengine_a_SOURCES += engine/googletest.hpp +libengine_a_SOURCES += engine/googletest_list.cpp +libengine_a_SOURCES += engine/googletest_list.hpp +libengine_a_SOURCES += engine/googletest_result.cpp +libengine_a_SOURCES += engine/googletest_result.hpp +libengine_a_SOURCES += engine/googletest_result_fwd.hpp libengine_a_SOURCES += engine/kyuafile.cpp libengine_a_SOURCES += engine/kyuafile.hpp libengine_a_SOURCES += engine/kyuafile_fwd.hpp @@ -106,6 +113,26 @@ engine_filters_test_SOURCES = engine/filters_test.cpp engine_filters_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) engine_filters_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) +tests_engine_PROGRAMS += engine/googletest_helpers +engine_googletest_helpers_SOURCES = engine/googletest_helpers.cpp +engine_googletest_helpers_CXXFLAGS = $(UTILS_CFLAGS) +engine_googletest_helpers_LDADD = $(UTILS_LIBS) + +tests_engine_PROGRAMS += engine/googletest_test +engine_googletest_test_SOURCES = engine/googletest_test.cpp +engine_googletest_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_googletest_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/googletest_list_test +engine_googletest_list_test_SOURCES = engine/googletest_list_test.cpp +engine_googletest_list_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_googletest_list_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/googletest_result_test +engine_googletest_result_test_SOURCES = engine/googletest_result_test.cpp +engine_googletest_result_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_googletest_result_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + tests_engine_PROGRAMS += engine/kyuafile_test engine_kyuafile_test_SOURCES = engine/kyuafile_test.cpp engine_kyuafile_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) diff --git a/engine/googletest.cpp b/engine/googletest.cpp new file mode 100644 index 00000000..255793b9 --- /dev/null +++ b/engine/googletest.cpp @@ -0,0 +1,221 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest.hpp" + +extern "C" { +#include +} + +#include +#include +#include + +#include "engine/googletest_list.hpp" +#include "engine/googletest_result.hpp" +#include "engine/exceptions.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/logging/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/exceptions.hpp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" +#include "utils/stream.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::optional; + + +namespace { + + +/// Basename of the file containing the result written by the googletest +/// testcase. +/// +/// TODO: use more structured output format someday, like the googletest's JSON +/// or XML format. +/// +/// Using either format will require pulling in a third party library and +/// understanding the schema of the format, as it stands in 1.9.0, +/// googletest doesn't document this expectation very well and instead seems +/// to rely on third-party solutions for doing structured output via the +/// listener interfaces. +/// +///static const char* result_name = "result.googletest"; + + +/// Magic numbers returned by exec_list when exec(2) fails. +enum list_exit_code { + exit_eacces = 90, + exit_enoent, + exit_enoexec, +}; + + +} // anonymous namespace + + +/// Executes a test program's list operation. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +void +engine::googletest_interface::exec_list(const model::test_program& test_program, + const config::properties_map& /*vars*/) const +{ + process::args_vector args; + + args.push_back("--gtest_color=no"); + args.push_back("--gtest_list_tests"); + try { + process::exec_unsafe(test_program.absolute_path(), args); + } catch (const process::system_error& e) { + if (e.original_errno() == EACCES) + ::_exit(exit_eacces); + else if (e.original_errno() == ENOENT) + ::_exit(exit_enoent); + else if (e.original_errno() == ENOEXEC) + ::_exit(exit_enoexec); + throw; + } +} + + +/// Computes the test cases list of a test program. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the test. +/// +/// \return A list of test cases. +/// +/// \throw error If there is a problem parsing the test case list. +model::test_cases_map +engine::googletest_interface::parse_list( + const optional< process::status >& status, + const fs::path& stdout_path, + const fs::path& /* stderr_path */) const +{ + if (!status) + throw engine::error("Test case list timed out"); + if (status.get().exited()) { + const int exitstatus = status.get().exitstatus(); + if (exitstatus == EXIT_SUCCESS) { + // Nothing to do; fall through. + } else if (exitstatus == exit_eacces) { + throw engine::error("Permission denied to run test program"); + } else if (exitstatus == exit_enoent) { + throw engine::error("Cannot find test program"); + } else if (exitstatus == exit_enoexec) { + throw engine::error("Invalid test program format"); + } else { + throw engine::error("Test program did not exit cleanly; exited " + "with status: " + std::to_string(exitstatus)); + } + } else { + throw engine::error("Test program received signal"); + } + + std::ifstream input(stdout_path.c_str()); + if (!input) + throw engine::load_error(stdout_path, "Cannot open file for read"); + const model::test_cases_map test_cases = parse_googletest_list(input); + + return test_cases; +} + + +/// Executes a test case of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +void +engine::googletest_interface::exec_test(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& /* control_directory */) const +{ + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second); + } + + process::args_vector args; + args.push_back("--gtest_color=no"); + /// TODO: use more structured output format someday, like the googletest's + /// JSON or XML format. + /// + /// Using either format will require pulling in a third party library and + /// understanding the schema of the format, as it stands in 1.9.0, + /// googletest doesn't document this expectation very well and instead seems + /// to rely on third-party solutions for doing structured output via the + /// listener interfaces. + args.push_back(F("--gtest_filter=%s") % (test_case_name)); + process::exec(test_program.absolute_path(), args); +} + + +/// Computes the result of a test case based on its termination status. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the +/// test. +/// +/// \return A test result. +model::test_result +engine::googletest_interface::compute_result( + const optional< process::status >& status, + const fs::path& /* control_directory */, + const fs::path& stdout_path, + const fs::path& /* stderr_path */) const +{ + if (!status) { + return model::test_result(model::test_result_broken, + "Test case body timed out"); + } + + return calculate_googletest_result(status, stdout_path); +} diff --git a/engine/googletest.hpp b/engine/googletest.hpp new file mode 100644 index 00000000..85f55a4f --- /dev/null +++ b/engine/googletest.hpp @@ -0,0 +1,67 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest.hpp +/// Execution engine for test programs that implement the googletest interface. + +#if !defined(ENGINE_GOOGLETEST_HPP) +#define ENGINE_GOOGLETEST_HPP + +#include "engine/scheduler.hpp" + +namespace engine { + + +/// Implementation of the scheduler interface for googletest test programs. +class googletest_interface : public engine::scheduler::interface { +public: + void exec_list(const model::test_program&, + const utils::config::properties_map&) const UTILS_NORETURN; + + model::test_cases_map parse_list( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&) const; + + void exec_test(const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const + UTILS_NORETURN; + + model::test_result compute_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&, + const utils::fs::path&) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_GOOGLETEST_HPP) diff --git a/engine/googletest_helpers.cpp b/engine/googletest_helpers.cpp new file mode 100644 index 00000000..b05960cb --- /dev/null +++ b/engine/googletest_helpers.cpp @@ -0,0 +1,344 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include + +#include +#include +#include + +extern char** environ; +} + +#include +#include +#include +#include + +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" + +namespace fs = utils::fs; + + +namespace { + + +/// Prefix for all testcases. +const std::string test_suite = "Suite."; + + +/// Logs an error message and exits the test with an error code. +/// +/// \param str The error message to log. +static void +fail(const std::string& str) +{ + std::cerr << str << '\n'; + std::exit(EXIT_FAILURE); +} + +/// An example fail message for test_check_configuration_variables() when it +/// fails. +const char fail_message1[] = ( +"Note: Google Test filter = Suite.Fails\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] Suite.check_configuration_variables\n" +"pass_fail_demo.cc:12: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +"[ FAILED ] Suite.check_configuration_variables (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] Suite.check_configuration_variables\n" +"\n" +" 1 FAILED TEST\n" +); + +/// An example pass message for test_check_configuration_variables() when it +/// passes. +const char pass_message1[] = ( +"Note: Google Test filter = Suite.check_configuration_variables\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.check_configuration_variables\n" +"[ OK ] Suite.check_configuration_variables (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (1 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +/// A test scenario that validates the TEST_ENV_* variables. +static void +test_check_configuration_variables(void) +{ + std::set< std::string > vars; + char** iter; + for (iter = environ; *iter != NULL; ++iter) { + if (std::strstr(*iter, "TEST_ENV_") == *iter) { + vars.insert(*iter); + } + } + + std::set< std::string > exp_vars; + exp_vars.insert("TEST_ENV_first=some value"); + exp_vars.insert("TEST_ENV_second=some other value"); + if (vars == exp_vars) { + std::cout << pass_message1; + } else { + std::cout << fail_message1 + << F(" Expected: %s\nFound: %s\n") % exp_vars % vars; + //std::exit(EXIT_FAILURE); + } +} + + +/// A test scenario that triggers a crash via abort in order to generate a +/// core dump. +static void +test_crash(void) +{ + std::abort(); +} + +/// An example failure message for `test_fail()`. +const char fail_message2[] = ( +"Note: Google Test filter = Suite.fail\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.fail\n" +"gtest_macros_demo.cc:4: Failure\n" +"Failed\n" +"with a reason\n" +"[ FAILED ] Suite.fail (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] Suite.fail\n" +"\n" +" 1 FAILED TEST\n" +); + +/// A test scenario that reports some tests as failed. +static void +test_fail(void) +{ + std::cout << fail_message2; + std::exit(EXIT_FAILURE); +} + + +/// An example pass message for `test_pass()`. +const char pass_message2[] = ( +"Note: Google Test filter = Suite.pass\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.pass\n" +"[ OK ] Suite.pass (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +/// A test scenario that passes. +static void +test_pass(void) +{ + std::cout << pass_message2; +} + + +/// An example pass message for `test_pass_but_exit_failure()`. +const char pass_message3[] = ( +"Note: Google Test filter = Suite.pass_but_exit_failure\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.pass_but_exit_failure\n" +"[ OK ] Suite.pass_but_exit_failure (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +/// A test scenario that passes but then exits with non-zero. +static void +test_pass_but_exit_failure(void) +{ + std::cout << pass_message3; + std::exit(70); +} + + +/// An example incomplete output for `test_timeout`. +const char incomplete_test[] = ( +"Note: Google Test filter = Suite.incomplete\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.incomplete\n" +); + +/// A test scenario that times out. +/// +/// Note that the timeout is defined in the Kyuafile, as the TAP interface has +/// no means for test programs to specify this by themselves. +static void +test_timeout(void) +{ + std::cout << incomplete_test; + + ::sleep(10); + const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get()); + std::ofstream file((control_dir / "cookie").c_str()); + if (!file) + fail("Failed to create the control cookie"); + file.close(); +} + + +} // anonymous namespace + + +/// Prints out program usage and exits with a non-zero exit code. +static void +usage(const char* argv0) { + std::cout << "usage: " << argv0 << " " + << "[--gtest_color=*] " + << "[--gtest_filter=POSITIVE_PATTERNS] " + << "[--gtest_list_tests]" + << "\n\n" + << "This program mocks a googletest test program." + << "\n"; + std::exit(EXIT_FAILURE); +} + +/// Entry point to the test program. +/// +/// The caller can select which test scenario to run by modifying the program's +/// basename on disk (either by a copy or by a hard link). +/// +/// \todo It may be worth to split this binary into separate, smaller binaries, +/// one for every "test scenario". We use this program as a dispatcher for +/// different "main"s, the only reason being to keep the amount of helper test +/// programs to a minimum. However, putting this each function in its own +/// binary could simplify many other things. +/// +/// \param argc The number of CLI arguments. +/// \param argv The CLI arguments themselves. These are not used because +/// Kyua will not pass any arguments to the plain test program. +int +main(int argc, char** argv) +{ + using scenario_fn_t = void (*)(void); + + std::map scenarios; + std::string testcase; + + const char *gtest_color_flag = "--gtest_color="; + const char *gtest_filter_flag = "--gtest_filter="; + char *argv0 = argv[0]; + + bool list_tests = false; + + scenarios["check_configuration_variables"] = + test_check_configuration_variables; + scenarios["crash"] = test_crash; + scenarios["fail"] = test_fail; + scenarios["pass"] = test_pass; + scenarios["pass_but_exit_failure"] = test_pass_but_exit_failure; + scenarios["timeout"] = test_timeout; + + /// NB: this avoids getopt_long*, because its behavior isn't portable. + int i; + for (i = 1; i < argc; i++) { + if (strcmp("--gtest_list_tests", argv[i]) == 0) { + list_tests = true; + } + /// Ignore `--gtest_color` arguments. + else if (strncmp(gtest_color_flag, argv[i], + strlen(gtest_color_flag)) == 0) { + continue; + } + /// Try looking for the test name with `--gtest_filter`. + else { + std::string filter_flag_match = + std::string(gtest_filter_flag) + test_suite; + std::string arg = std::string(argv[i]); + if (arg.find_first_of(filter_flag_match) == arg.npos) { + usage(argv0); + } + testcase = arg.erase(0, filter_flag_match.length()); + } + } + + if (i < argc) { + usage(argv0); + } + + if (list_tests) { + std::cout << test_suite << "\n"; + for (auto it = scenarios.begin(); it != scenarios.end(); it++) { + std::cout << " " << it->first << "\n"; + } + return EXIT_SUCCESS; + } + + auto scenario = scenarios.find(testcase); + if (scenario == scenarios.end()) { + usage(argv0); + } + + scenario->second(); + + return EXIT_SUCCESS; +} diff --git a/engine/googletest_list.cpp b/engine/googletest_list.cpp new file mode 100644 index 00000000..e7bb52d5 --- /dev/null +++ b/engine/googletest_list.cpp @@ -0,0 +1,122 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_list.hpp" + +#include +#include +#include +#include + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "utils/config/exceptions.hpp" +#include "utils/format/macros.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; + + +namespace { + +/// A regular expression that should match either a test suite or a test case. +const std::string name_expr = "([[:alpha:][:digit:]_]+[[:alpha:][:digit:]_/]*)"; + +/// The separator between a test suite and a test case. +const std::string testsuite_testcase_separator = "."; + +/// A complete regular expression representing a line with a test suite +/// definition. +/// +/// e.g., +/// "TestSuite." +/// or +/// "TestSuite/Prefix." +/// or +/// "TestSuite/Prefix. # TypeParam = .+" +const std::string testsuite_expr = + name_expr + "\\.([[:space:]]+# TypeParam = .+)?"; + +/// A complete regular expression representing a line with a test case +/// definition. +/// +/// e.g., +/// " TestCase" +/// or +/// " TestCase/0" +/// or +/// " TestCase/0 # GetParam() = 4" +const std::string testcase_expr = + " " + name_expr + "([[:space:]]+# GetParam\\(\\) = .+)?"; + +} // anonymous namespace + + +/// Parses the googletest list of test cases from an open stream. +/// +/// \param input The stream to read from. +/// +/// \return The collection of parsed test cases. +/// +/// \throw format_error If there is any problem in the input data. +model::test_cases_map +engine::parse_googletest_list(std::istream& input) +{ + std::regex testcase_re(testcase_expr), testsuite_re(testsuite_expr); + std::smatch match; + std::string line; + std::string testcase_name, test_suite; + + model::test_cases_map_builder test_cases_builder; + while (std::getline(input, line).good()) { + if (std::regex_match(line, match, testcase_re)) { + std::string test_case; + + if (test_suite.empty()) { + throw format_error("Invalid testcase definition: not preceded " + "by a test suite definition"); + } + test_case = std::string(match[1]); + test_cases_builder.add(test_suite + test_case); + } else if (std::regex_match(line, match, testsuite_re)) { + test_suite = std::string(match[1]) + + testsuite_testcase_separator; + } + // else, ignore the line; something might have used output a diagnostic + // message to stdout, e.g., gtest_main. + } + const model::test_cases_map test_cases = test_cases_builder.build(); + if (test_cases.empty()) { + // The scheduler interface also checks for the presence of at least one + // test case. However, because the atf format itself requires one test + // case to be always present, we check for this condition here as well. + throw format_error("No test cases"); + } + return test_cases; +} diff --git a/engine/googletest_list.hpp b/engine/googletest_list.hpp new file mode 100644 index 00000000..3efb648a --- /dev/null +++ b/engine/googletest_list.hpp @@ -0,0 +1,50 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf_list.hpp +/// Parser of googletest test case lists. + +#if !defined(ENGINE_GOOGLETEST_LIST_HPP) +#define ENGINE_GOOGLETEST_LIST_HPP + +#include + +#include "model/metadata_fwd.hpp" +#include "model/test_case_fwd.hpp" +#include "model/types.hpp" +#include "utils/fs/path_fwd.hpp" + +namespace engine { + + +model::test_cases_map parse_googletest_list(std::istream&); + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_LIST_HPP) diff --git a/engine/googletest_list_test.cpp b/engine/googletest_list_test.cpp new file mode 100644 index 00000000..7a2085c1 --- /dev/null +++ b/engine/googletest_list_test.cpp @@ -0,0 +1,212 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_list.hpp" + +#include +#include + +#include + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/types.hpp" +#include "utils/datetime.hpp" +#include "utils/format/containers.ipp" +#include "utils/fs/path.hpp" +#include "utils/units.hpp" + +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace units = utils::units; + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__invalid_testcase_definition); +ATF_TEST_CASE_BODY(parse_googletest_list__invalid_testcase_definition) +{ + const std::string text1 = + " \n"; + const std::string text2 = + " TestcaseWithoutSuite\n"; + std::istringstream input1(text1); + std::istringstream input2(text2); + + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input1)); + + ATF_REQUIRE_THROW_RE(engine::format_error, + "Invalid testcase definition: not preceded by a test suite definition", + engine::parse_googletest_list(input2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__invalid_testsuite_definition); +ATF_TEST_CASE_BODY(parse_googletest_list__invalid_testsuite_definition) +{ + const std::string text1 = + "\n"; + const std::string text2 = + "TestSuiteWithoutSeparator\n"; + std::istringstream input1(text1); + std::istringstream input2(text2); + + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input1)); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__no_test_cases); +ATF_TEST_CASE_BODY(parse_googletest_list__no_test_cases) +{ + const std::string text = ""; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_test_case); +ATF_TEST_CASE_BODY(parse_googletest_list__one_test_case) +{ + const std::string text = + "TestSuite.\n" + " TestCase\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite.TestCase").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_case); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_case) +{ + const std::string text = + "TestSuite.\n" + " TestCase/0 # GetParam() = 'c'\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite.TestCase/0").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_suite); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_suite) +{ + const std::string text = + "TestSuite/0. # TypeParam = int\n" + " TestCase\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite/0.TestCase").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_case_and_test_suite); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_case_and_test_suite) +{ + const std::string text = + "TestSuite/0. # TypeParam = int\n" + " TestCase/0 # GetParam() = \"herp\"\n" + " TestCase/1 # GetParam() = \"derp\"\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite/0.TestCase/0") + .add("TestSuite/0.TestCase/1") + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__many_test_cases); +ATF_TEST_CASE_BODY(parse_googletest_list__many_test_cases) +{ + const std::string text = + "FirstTestSuite.\n" + " ATestCase\n" + "SecondTestSuite.\n" + " AnotherTestCase\n" + "ThirdTestSuite.\n" + " _\n" + "FourthTestSuite/0. # TypeParam = std::list\n" + " TestCase\n" + "FourthTestSuite/1. # TypeParam = std::list\n" + " TestCase\n" + "FifthTestSuite.\n" + " TestCase/0 # GetParam() = 0\n" + " TestCase/1 # GetParam() = (1, 2, 3)\n" + " TestCase/2 # GetParam() = \"developers. developers\"\n" + "SixthTestSuite/0. # TypeParam = std::map\n" + " TestCase/0 # GetParam() = 0\n" + " TestCase/1 # GetParam() = (1, 2, 3)\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("FirstTestSuite.ATestCase", model::metadata_builder() + .build()) + .add("SecondTestSuite.AnotherTestCase", model::metadata_builder() + .build()) + .add("ThirdTestSuite._") + .add("FourthTestSuite/0.TestCase") + .add("FourthTestSuite/1.TestCase") + .add("FifthTestSuite.TestCase/0") + .add("FifthTestSuite.TestCase/1") + .add("FifthTestSuite.TestCase/2") + .add("SixthTestSuite/0.TestCase/0") + .add("SixthTestSuite/0.TestCase/1") + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__invalid_testcase_definition); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__invalid_testsuite_definition); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__no_test_cases); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_test_case); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_parameterized_test_case); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_parameterized_test_suite); + ATF_ADD_TEST_CASE(tcs, + parse_googletest_list__one_parameterized_test_case_and_test_suite); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__many_test_cases); +} diff --git a/engine/googletest_result.cpp b/engine/googletest_result.cpp new file mode 100644 index 00000000..e3ab3fee --- /dev/null +++ b/engine/googletest_result.cpp @@ -0,0 +1,523 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_result.hpp" + +#include +#include +#include +#include + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/fs/path.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace fs = utils::fs; +namespace process = utils::process; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + +/// Parses a test result that does not accept a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be "successful". +static engine::googletest_result +parse_without_reason(const std::string& status, const std::string& rest) +{ + if (!rest.empty()) + throw engine::format_error(F("%s cannot have a reason") % status); + + if (status == "skipped") + return engine::googletest_result(engine::googletest_result::skipped); + else { + INV(status == "successful"); + return engine::googletest_result(engine::googletest_result::successful); + } +} + + +/// Parses a test result that needs a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be one of "broken", "disabled", "failed", or "skipped". +static engine::googletest_result +parse_with_reason(const std::string& status, const std::string& rest) +{ + using engine::googletest_result; + + INV(!rest.empty()); + + if (status == "broken") + return googletest_result(googletest_result::broken, rest); + else if (status == "disabled") + return googletest_result(googletest_result::disabled, rest); + else if (status == "failed") + return googletest_result(googletest_result::failed, rest); + else if (status == "skipped") + return googletest_result(googletest_result::skipped, rest); + else + PRE_MSG(false, "Unexpected status"); +} + + +/// Formats the termination status of a process to be used with validate_result. +/// +/// \param status The status to format. +/// +/// \return A string describing the status. +static std::string +format_status(const process::status& status) +{ + if (status.exited()) + return F("exited with code %s") % status.exitstatus(); + else if (status.signaled()) + return F("received signal %s%s") % status.termsig() % + (status.coredump() ? " (core dumped)" : ""); + else + return F("terminated in an unknown manner"); +} + + +} // anonymous namespace + + +/// Constructs a raw result with a type. +/// +/// The reason and the argument are left uninitialized. +/// +/// \param type_ The type of the result. +engine::googletest_result::googletest_result(const types type_) : + _type(type_) +{ +} + + +/// Constructs a raw result with a type and a reason. +/// +/// The argument is left uninitialized. +/// +/// \param type_ The type of the result. +/// \param reason_ The reason for the result. +engine::googletest_result::googletest_result(const types type_, + const std::string& reason_) : + _type(type_), _reason(reason_) +{ +} + + +/// Constructs a raw result with a type, an optional argument and a reason. +/// +/// \param type_ The type of the result. +/// \param argument_ The optional argument for the result. +/// \param reason_ The reason for the result. +engine::googletest_result::googletest_result(const types type_, + const utils::optional< int >& argument_, + const std::string& reason_) : + _type(type_), _argument(argument_), _reason(reason_) +{ +} + + +const std::string invalid_output_message = "invalid output"; +const std::regex disabled_re(R"RE((YOU HAVE [[:digit:]]+ DISABLED TESTS?))RE"); +const std::regex starting_sentinel_re( + R"(\[[[:space:]]+RUN[[:space:]]+\][[:space:]]+[A-Za-z0-9]+\.[A-Za-z0-9]+)"); +const std::regex ending_sentinel_re( + R"RE(\[[[:space:]]+(FAILED|OK|SKIPPED)[[:space:]]+\])RE"); + + +/// Parses an input stream to extract a test result. +/// +/// If the parsing fails for any reason, the test result is 'broken' and it +/// contains the reason for the parsing failure. Test cases that report results +/// in an inconsistent state cannot be trusted (e.g. the test program code may +/// have a bug), and thus why they are reported as broken instead of just failed +/// (which is a legitimate result for a test case). +/// +/// \param input The stream to read from. +/// +/// \return A generic representation of the result of the test case. +/// +/// \throw format_error If the input is invalid. +engine::googletest_result +engine::googletest_result::parse(std::istream& input) +{ + std::vector lines; + std::smatch matches; + std::string context, googletest_res, status; + bool capture_context, valid_output; + + do { + std::string line; + std::getline(input, line, '\n'); + if (!input.eof()) + line.push_back('\n'); + lines.push_back(line); + } while (input.good()); + + valid_output = false; + for (auto& line: lines) { + if (regex_search(line, matches, disabled_re)) { + context = matches[1]; + status = "disabled"; + valid_output = true; + break; + } + if (regex_search(line, matches, starting_sentinel_re)) { + capture_context = true; + context = ""; + continue; + } + if (regex_search(line, matches, ending_sentinel_re)) { + googletest_res = matches[1]; + if (googletest_res == "OK") { + context = ""; + status = "successful"; + } else if (googletest_res == "FAILED") { + status = "failed"; + } else { + INV(googletest_res == "SKIPPED"); + status = "skipped"; + } + capture_context = false; + valid_output = true; + } + if (capture_context) { + context += line; + } + } + if (!valid_output) { + context = invalid_output_message; + status = "broken"; + } + + if (context == "") + return parse_without_reason(status, context); + else + return parse_with_reason(status, context); +} + + +/// Loads a test case result from a file. +/// +/// \param file The file to parse. +/// +/// \return The parsed test case result if all goes well. +/// +/// \throw std::runtime_error If the file does not exist. +/// \throw engine::format_error If the contents of the file are bogus. +engine::googletest_result +engine::googletest_result::load(const fs::path& file) +{ + std::ifstream input(file.c_str()); + if (!input) + throw std::runtime_error("Cannot open results file"); + else + return parse(input); +} + + +/// Gets the type of the result. +/// +/// \return A result type. +engine::googletest_result::types +engine::googletest_result::type(void) const +{ + return _type; +} + + +/// Gets the optional argument of the result. +/// +/// \return The argument of the result if present; none otherwise. +const optional< int >& +engine::googletest_result::argument(void) const +{ + return _argument; +} + + +/// Gets the optional reason of the result. +/// +/// \return The reason of the result if present; none otherwise. +const optional< std::string >& +engine::googletest_result::reason(void) const +{ + return _reason; +} + + +/// Checks whether the result should be reported as good or not. +/// +/// \return True if the result can be considered "good", false otherwise. +bool +engine::googletest_result::good(void) const +{ + switch (_type) { + case googletest_result::disabled: + case googletest_result::skipped: + case googletest_result::successful: + return true; + + case googletest_result::broken: + case googletest_result::failed: + return false; + + default: + UNREACHABLE; + } +} + + +/// Reinterprets a raw result based on the termination status of the test case. +/// +/// This reinterpretation ensures that the termination conditions of the program +/// match what is expected of the paticular result reported by the test program. +/// If such conditions do not match, the test program is considered bogus and is +/// thus reported as broken. +/// +/// This is just a helper function for calculate_result(); the real result of +/// the test case cannot be inferred from apply() only. +/// +/// \param status The exit status of the test program, or none if the test +/// program timed out. +/// +/// \result The adjusted result. The original result is transformed into broken +/// if the exit status of the program does not match our expectations. +engine::googletest_result +engine::googletest_result::apply(const optional< process::status >& status) + const +{ + if (!status) { + return *this; + } + + INV(status); + switch (_type) { + case googletest_result::broken: + return *this; + + case googletest_result::failed: + if (status.get().exited() && status.get().exitstatus() == EXIT_FAILURE) + return *this; + return googletest_result(googletest_result::broken, + "Failed test case should have reported failure but " + + format_status(status.get())); + + case googletest_result::disabled: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + return googletest_result(googletest_result::broken, + "Disabled test case should have reported success but " + + format_status(status.get())); + + case googletest_result::skipped: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + return googletest_result(googletest_result::broken, + "Skipped test case should have reported success but " + + format_status(status.get())); + + case googletest_result::successful: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + return googletest_result(googletest_result::broken, + "Passed test case should have reported success but " + + format_status(status.get())); + + } + + UNREACHABLE; +} + + +/// Converts an internal result to the interface-agnostic representation. +/// +/// \return A generic result instance representing this result. +model::test_result +engine::googletest_result::externalize(void) const +{ + switch (_type) { + case googletest_result::broken: + return model::test_result(model::test_result_broken, _reason.get()); + + case googletest_result::failed: + return model::test_result(model::test_result_failed, _reason.get()); + + case googletest_result::disabled: + return model::test_result(model::test_result_skipped, _reason.get()); + case googletest_result::skipped: + if (_reason) { + return model::test_result(model::test_result_skipped, + _reason.get()); + } else { + return model::test_result(model::test_result_skipped); + } + case googletest_result::successful: + return model::test_result(model::test_result_passed); + + default: + UNREACHABLE; + } +} + + +/// Compares two raw results for equality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are equal; false otherwise. +bool +engine::googletest_result::operator==(const googletest_result& other) const +{ + return _type == other._type && _argument == other._argument && + _reason == other._reason; +} + + +/// Compares two raw results for inequality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are different; false otherwise. +bool +engine::googletest_result::operator!=(const googletest_result& other) const +{ + return !(*this == other); +} + + +/// Injects the object into a stream. +/// +/// \param output The stream into which to inject the object. +/// \param object The object to format. +/// +/// \return The output stream. +std::ostream& +engine::operator<<(std::ostream& output, const googletest_result& object) +{ + std::string result_name; + switch (object.type()) { + case googletest_result::broken: result_name = "broken"; break; + case googletest_result::disabled: result_name = "disabled"; break; + case googletest_result::failed: result_name = "failed"; break; + case googletest_result::skipped: result_name = "skipped"; break; + case googletest_result::successful: result_name = "successful"; break; + } + + const optional< int >& argument = object.argument(); + + const optional< std::string >& reason = object.reason(); + + output << F("model::test_result{type=%s, argument=%s, reason=%s}") + % text::quote(result_name, '\'') + % (argument ? (F("%s") % argument.get()).str() : "none") + % (reason ? text::quote(reason.get(), '\'') : "none"); + + return output; +} + + +/// Calculates the user-visible result of a test case. +/// +/// This function needs to perform magic to ensure that what the test case +/// reports as its result is what the user should really see: i.e. it adjusts +/// the reported status of the test to the exit conditions of its body and +/// cleanup parts. +/// +/// \param body_status The termination status of the process that executed +/// the body of the test. None if the body timed out. +/// \param results_file The path to the results file that the test case body is +/// supposed to have created. +/// +/// \return The calculated test case result. +model::test_result +engine::calculate_googletest_result( + const optional< process::status >& body_status, + const fs::path& results_file) +{ + using engine::googletest_result; + + googletest_result result(googletest_result::broken, "Unknown result"); + try { + result = googletest_result::load(results_file); + /// atf_result.cpp handles this by side-effect by setting a broken + /// result with "", which results in a std::runtime_error throw from + /// atf_result::parse(..). googletest_result::parse doesn't do that and + /// instead returns a broken result with a reason of + /// `invalid_output_message`, so that case needs to be tested for after + /// the fact to make sure the reason why `invalid_output_message` was + /// because of a program that crashed or failed. + if (result.type() == googletest_result::broken && body_status) { + const optional< std::string >& reason = result.reason(); + if (reason && reason.get() == invalid_output_message) { + result = googletest_result( + googletest_result::broken, + F("Premature exit; test case %s") % + format_status(body_status.get())); + } + } + } catch (const engine::format_error& error) { + result = googletest_result(googletest_result::broken, error.what()); + } catch (const std::runtime_error& error) { + if (body_status) + result = googletest_result( + googletest_result::broken, F("Premature exit; test case %s") % + format_status(body_status.get())); + else { + // The test case timed out. apply() handles this case later. + } + } + + result = result.apply(body_status); + + return result.externalize(); +} diff --git a/engine/googletest_result.hpp b/engine/googletest_result.hpp new file mode 100644 index 00000000..84b9d921 --- /dev/null +++ b/engine/googletest_result.hpp @@ -0,0 +1,112 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest_result.hpp +/// Functions and types to process the results of googletest-based test cases. + +#if !defined(ENGINE_GOOGLETEST_RESULT_HPP) +#define ENGINE_GOOGLETEST_RESULT_HPP + +#include "engine/googletest_result_fwd.hpp" + +#include +#include + +#include "model/test_result_fwd.hpp" +#include "utils/optional.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/process/status_fwd.hpp" + +namespace engine { + + +/// Internal representation of the raw result files of googletest-based tests. +/// +/// This class is used exclusively to represent the transient result files read +/// from test cases before generating the "public" version of the result. This +/// class should actually not be exposed in the header files, but it is for +/// testing purposes only. +class googletest_result { +public: + /// List of possible types for the test case result. + enum types { + broken, + successful, + skipped, + failed, + disabled + }; + +private: + /// The test case result. + types _type; + + /// The optional integral argument that may accompany the result. + /// + /// Should only be present if the type is expected_exit or expected_signal. + utils::optional< int > _argument; + + /// A description of the test case result. + /// + /// Should always be present except for the passed type. + utils::optional< std::string > _reason; + +public: + googletest_result(const types); + googletest_result(const types, const std::string&); + googletest_result(const types, const utils::optional< int >&, + const std::string&); + + static googletest_result parse(std::istream&); + static googletest_result load(const utils::fs::path&); + + types type(void) const; + const utils::optional< int >& argument(void) const; + const utils::optional< std::string >& reason(void) const; + + bool good(void) const; + googletest_result apply( + const utils::optional< utils::process::status >&) const; + model::test_result externalize(void) const; + + bool operator==(const googletest_result&) const; + bool operator!=(const googletest_result&) const; +}; + + +std::ostream& operator<<(std::ostream&, const googletest_result&); + + +model::test_result calculate_googletest_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&); + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_IFACE_RESULTS_HPP) diff --git a/engine/googletest_result_fwd.hpp b/engine/googletest_result_fwd.hpp new file mode 100644 index 00000000..6ecd751c --- /dev/null +++ b/engine/googletest_result_fwd.hpp @@ -0,0 +1,43 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest_result_fwd.hpp +/// Forward declarations for engine/googletest_result.hpp + +#if !defined(ENGINE_GOOGLETEST_RESULT_FWD_HPP) +#define ENGINE_GOOGLETEST_RESULT_FWD_HPP + +namespace engine { + + +class googletest_result; + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_RESULT_FWD_HPP) diff --git a/engine/googletest_result_test.cpp b/engine/googletest_result_test.cpp new file mode 100644 index 00000000..02851433 --- /dev/null +++ b/engine/googletest_result_test.cpp @@ -0,0 +1,431 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_result.hpp" + +extern "C" { +#include +} + +#include +#include +#include +#include + +#include + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/process/status.hpp" + +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Performs a test for results::parse() that should succeed. +/// +/// \param exp_type The expected type of the result. +/// \param exp_argument The expected argument in the result, if any. +/// \param exp_reason The expected reason describing the result, if any. +/// \param text The literal input to parse; can include multiple lines. +static void +parse_ok_test(const engine::googletest_result::types& exp_type, + const optional< int >& exp_argument, + const char* exp_reason, const char* text) +{ + std::istringstream input(text); + const engine::googletest_result actual = + engine::googletest_result::parse(input); + ATF_REQUIRE_EQ(exp_type, actual.type()); + ATF_REQUIRE_EQ(exp_argument, actual.argument()); + if (exp_reason != NULL) { + ATF_REQUIRE(actual.reason()); + ATF_REQUIRE_EQ(exp_reason, actual.reason().get()); + } else { + ATF_REQUIRE(!actual.reason()); + } +} + + +/// Wrapper around parse_ok_test to define a test case. +/// +/// \param name The name of the test case; will be prefixed with +/// "googletest_result__parse__". +/// \param exp_type The expected type of the result. +/// \param exp_argument The expected argument in the result, if any. +/// \param exp_reason The expected reason describing the result, if any. +/// \param input The literal input to parse. +#define PARSE(name, exp_type, exp_argument, exp_reason, input) \ + ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__parse__ ## name); \ + ATF_TEST_CASE_BODY(googletest_result__parse__ ## name) \ + { \ + parse_ok_test(exp_type, exp_argument, exp_reason, input); \ + } + + +} // anonymous namespace + + +PARSE(broken, + engine::googletest_result::broken, none, "invalid output", + "invalid input"); + +const char disabled_context[] = ( +"YOU HAVE 1 DISABLED TEST" +); + +const char disabled_message[] = ( +"[==========] Running 0 tests from 0 test cases.\n" +"[==========] 0 tests from 0 test cases ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"\n" +" YOU HAVE 1 DISABLED TEST\n" +"\n" +"\n" +); + +PARSE(disabled, + engine::googletest_result::disabled, none, disabled_context, + disabled_message); + +const char failed_context[] = ( +"pass_fail_demo.cc:8: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +); + +const char failed_message[] = ( +"Note: Google Test filter = PassFailTest.Fails\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.Fails\n" +"pass_fail_demo.cc:8: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +"[ FAILED ] PassFailTest.Fails (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] PassFailTest.Fails\n" +"\n" +" 1 FAILED TEST\n" +); + +PARSE(failed, + engine::googletest_result::failed, none, failed_context, + failed_message); + +const char skipped_message[] = ( +"Note: Google Test filter = SkipTest.DoesSkip\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from SkipTest\n" +"[ RUN ] SkipTest.DoesSkip\n" +"[ SKIPPED ] SkipTest.DoesSkip (0 ms)\n" +"[----------] 1 test from SkipTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ SKIPPED ] 1 test, listed below:\n" +"[ SKIPPED ] SkipTest.DoesSkip\n" +); + +PARSE(skipped, + engine::googletest_result::skipped, none, NULL, + skipped_message +); + +const char skipped_with_reason_context[] = ( +"This is a reason\n" +); + +const char skipped_with_reason_message[] = ( +"Note: Google Test filter = SkipTest.SkipWithReason\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from SkipTest\n" +"[ RUN ] SkipTest.SkipWithReason\n" +"This is a reason\n" +"[ SKIPPED ] SkipTest.SkipWithReason (0 ms)\n" +"[----------] 1 test from SkipTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ SKIPPED ] 1 test, listed below:\n" +"[ SKIPPED ] SkipTest.SkipWithReason\n" +); + +PARSE(skipped_with_reason, + engine::googletest_result::skipped, none, skipped_with_reason_context, + skipped_with_reason_message); + +const char successful_message[] = ( +"Note: Google Test filter = PassFailTest.Passes\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.Passes\n" +"[ OK ] PassFailTest.Passes (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +PARSE(successful, + engine::googletest_result::successful, none, NULL, + successful_message); + +const char successful_message2[] = ( +"Note: Google Test filter = ValuesTest.ValuesWorks\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from ValuesTest\n" +"[ RUN ] ValuesTest.ValuesWorks\n" +"[ OK ] ValuesTest.ValuesWorks (0 ms)\n" +"[----------] 1 test from ValuesTest (0 ms total)\n" +" \n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +PARSE(successful2, + engine::googletest_result::successful, none, NULL, + successful_message2); + +const char successful_parameterized_message[] = ( +"Note: Google Test filter = RangeZeroToFive/ParamDerivedTest/0\n" +"[==========] Running 5 tests from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 5 tests from RangeZeroToFive/ParamDerivedTest\n" +"[ RUN ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0\n" +"[ OK ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0 (0 ms)\n" +"[----------] 1 test from RangeZeroToFive/ParamDerivedTest/0 (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 tests from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 tests.\n" +); + +PARSE(successful_parameterized, + engine::googletest_result::successful, none, NULL, + successful_parameterized_message); + +const char successful_message_with_reason[] = ( +"Note: Google Test filter = PassFailTest.PassesWithReason\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.PassesWithReason\n" +"This is a reason\n" +"[ OK ] PassFailTest.PassesWithReason (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 tests.\n" +); + +PARSE(successful_with_reason, + engine::googletest_result::successful, none, NULL, + successful_message_with_reason); + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__ok); +ATF_TEST_CASE_BODY(googletest_result__load__ok) +{ + std::ofstream output("result.txt"); + ATF_REQUIRE(output); + output << skipped_with_reason_message; + output.close(); + + const engine::googletest_result result = engine::googletest_result::load( + utils::fs::path("result.txt")); + ATF_REQUIRE_EQ(engine::googletest_result::skipped, result.type()); + ATF_REQUIRE(!result.argument()); + ATF_REQUIRE(result.reason()); + ATF_REQUIRE_EQ(skipped_with_reason_context, result.reason().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__missing_file); +ATF_TEST_CASE_BODY(googletest_result__load__missing_file) +{ + ATF_REQUIRE_THROW_RE( + std::runtime_error, "Cannot open", + engine::googletest_result::load(utils::fs::path("result.txt"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__broken); +ATF_TEST_CASE_BODY(googletest_result__apply__broken) +{ + const process::status status = process::status::fake_exited(EXIT_FAILURE); + const engine::googletest_result broken(engine::googletest_result::broken, + "The reason"); + ATF_REQUIRE_EQ(broken, broken.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__disabled); +ATF_TEST_CASE_BODY(googletest_result__apply__disabled) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result disabled( + engine::googletest_result::disabled, "The reason"); + ATF_REQUIRE_EQ(disabled, disabled.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__failed); +ATF_TEST_CASE_BODY(googletest_result__apply__failed) +{ + const process::status status = process::status::fake_exited(EXIT_FAILURE); + const engine::googletest_result failed(engine::googletest_result::failed, + "The reason"); + ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__skipped); +ATF_TEST_CASE_BODY(googletest_result__apply__skipped) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result skipped(engine::googletest_result::skipped, + "The reason"); + ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__successful); +ATF_TEST_CASE_BODY(googletest_result__apply__successful) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result + successful(engine::googletest_result::successful); + ATF_REQUIRE_EQ(successful, successful.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__broken); +ATF_TEST_CASE_BODY(googletest_result__externalize__broken) +{ + const engine::googletest_result raw(engine::googletest_result::broken, + "The reason"); + const model::test_result expected(model::test_result_broken, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__disabled); +ATF_TEST_CASE_BODY(googletest_result__externalize__disabled) +{ + const engine::googletest_result raw(engine::googletest_result::disabled, + "The reason"); + const model::test_result expected(model::test_result_skipped, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__failed); +ATF_TEST_CASE_BODY(googletest_result__externalize__failed) +{ + const engine::googletest_result raw(engine::googletest_result::failed, + "The reason"); + const model::test_result expected(model::test_result_failed, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__skipped); +ATF_TEST_CASE_BODY(googletest_result__externalize__skipped) +{ + const engine::googletest_result raw(engine::googletest_result::skipped, + "The reason"); + const model::test_result expected(model::test_result_skipped, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__successful); +ATF_TEST_CASE_BODY(googletest_result__externalize__successful) +{ + const engine::googletest_result raw(engine::googletest_result::successful); + const model::test_result expected(model::test_result_passed); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped_with_reason); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful2); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_parameterized); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_with_reason); + + ATF_ADD_TEST_CASE(tcs, googletest_result__load__ok); + ATF_ADD_TEST_CASE(tcs, googletest_result__load__missing_file); + + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__successful); + + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__successful); +} diff --git a/engine/googletest_test.cpp b/engine/googletest_test.cpp new file mode 100644 index 00000000..84eb6131 --- /dev/null +++ b/engine/googletest_test.cpp @@ -0,0 +1,361 @@ +// Copyright 2019 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest.hpp" + +extern "C" { +#include + +#include +} + +#include + +#include "engine/config.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program_fwd.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/stacktrace.hpp" +#include "utils/test_utils.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +namespace { + + +/// Prefix for all testcases. +const std::string test_suite = "Suite."; + + +/// Runs one googletest test program and checks its result. +/// +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +/// \param user_config User-provided configuration. +/// +/// \return The list of loaded test cases. +static model::test_cases_map +list(const char* program_name, + const fs::path& root, + config::tree user_config = engine::empty_config()) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::lazy_test_program program( + "googletest", fs::path(program_name), root, "the-suite", + model::metadata_builder().build(), user_config, handle); + + const model::test_cases_map test_cases = handle.list_tests( + &program, user_config); + + handle.cleanup(); + + return test_cases; +} + + +/// Runs a bogus test program and checks the error result. +/// +/// \param exp_error Expected error string to find. +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +static void +check_list_fail(const char* exp_error, + const char* program_name, + const fs::path& root) +{ + const model::test_cases_map test_cases = list( + program_name, root); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_MATCH(exp_error, + test_case.fake_result().get().reason()); +} + + +/// Runs one googletest test program and checks its result. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param test_case_name Name of the "test case" to select from the helper +/// program. +/// \param exp_result The expected result. +/// \param metadata The test case metadata. +/// \param user_config User-provided configuration. +static void +run_one(const atf::tests::tc* tc, const char* test_case_name, + const model::test_result& exp_result, + const model::metadata& metadata = model::metadata_builder().build(), + config::tree user_config = engine::empty_config()) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const std::string test_name = test_suite + std::string(test_case_name); + + const model::test_program_ptr program(new scheduler::lazy_test_program( + "googletest", fs::path("googletest_helpers"), + fs::path(tc->get_config_var("srcdir")), + "the-suite", metadata, user_config, handle)); + + (void)handle.spawn_test(program, test_name.c_str(), user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); + atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(list__ok); +ATF_TEST_CASE_BODY(list__ok) +{ + const model::test_cases_map test_cases = list( + "googletest_helpers", fs::path(get_config_var("srcdir"))); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory); +ATF_TEST_CASE_BODY(list__current_directory) +{ + const fs::path helpers = + fs::path(get_config_var("srcdir")) / "googletest_helpers"; + ATF_REQUIRE(::symlink(helpers.c_str(), "googletest_helpers") != -1); + const model::test_cases_map test_cases = list( + "googletest_helpers", fs::path(".")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path); +ATF_TEST_CASE_BODY(list__relative_path) +{ + const fs::path helpers = + fs::path(get_config_var("srcdir")) / "googletest_helpers"; + ATF_REQUIRE(::mkdir("dir1", 0755) != -1); + ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1); + ATF_REQUIRE_EQ(::symlink(helpers.c_str(), + "dir1/dir2/googletest_helpers"), 0); + const model::test_cases_map test_cases = list( + "dir2/googletest_helpers", fs::path("dir1")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program); +ATF_TEST_CASE_BODY(list__missing_test_program) +{ + check_list_fail("Cannot find test program", "non-existent", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program); +ATF_TEST_CASE_BODY(list__not_a_test_program) +{ + atf::utils::create_file("not-valid", "garbage\n"); + ATF_REQUIRE_EQ(::chmod("not-valid", 0755), 0); + check_list_fail("Invalid test program format", "not-valid", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions); +ATF_TEST_CASE_BODY(list__no_permissions) +{ + atf::utils::create_file("not-executable", "garbage\n"); + check_list_fail("Permission denied to run test program", + "not-executable", fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__abort); +ATF_TEST_CASE_BODY(list__abort) +{ + atf::utils::create_file("kills_self", "#!/bin/sh\nkill -2 $$\n"); + ATF_REQUIRE_EQ(::chmod("kills_self", 0755), 0); + check_list_fail("Test program received signal", "kills_self", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__empty); +ATF_TEST_CASE_BODY(list__empty) +{ + atf::utils::create_file("empty_list", "#!/bin/sh\n"); + ATF_REQUIRE_EQ(::chmod("empty_list", 0755), 0); + check_list_fail("No test cases", "empty_list", fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes); +ATF_TEST_CASE_BODY(test__body_only__passes) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__fails); +ATF_TEST_CASE_BODY(test__body_only__fails) +{ + const std::string failure_message = ( +"gtest_macros_demo.cc:4: Failure\n" +"Failed\n" +"with a reason\n" +); + const model::test_result exp_result(model::test_result_failed, + failure_message); + run_one(this, "fail", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes); +ATF_TEST_CASE_BODY(test__body_only__crashes) +{ + utils::prepare_coredump_test(this); + + const model::test_result exp_result( + model::test_result_broken, + F("Premature exit; test case received signal %s (core dumped)") % + SIGABRT); + run_one(this, "crash", exp_result); +} + + +ATF_TEST_CASE(test__body_only__times_out); +ATF_TEST_CASE_HEAD(test__body_only__times_out) +{ + set_md_var("timeout", "60"); +} +ATF_TEST_CASE_BODY(test__body_only__times_out) +{ + config::tree user_config = engine::empty_config(); + utils::setenv("CONTROL_DIR", fs::current_path().str()); + const model::metadata metadata = model::metadata_builder() + .set_timeout(datetime::delta(1, 0)).build(); + const model::test_result exp_result( + model::test_result_broken, "Test case body timed out"); + run_one(this, "timeout", exp_result, metadata, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables); +ATF_TEST_CASE_BODY(test__body_only__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.first", "some value"); + user_config.set_string("test_suites.the-suite.second", "some other value"); + + const model::test_result exp_result(model::test_result_passed); + run_one(this, "check_configuration_variables", exp_result, + model::metadata_builder().build(), user_config); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "googletest", std::shared_ptr< scheduler::interface >( + new engine::googletest_interface())); + + ATF_ADD_TEST_CASE(tcs, list__ok); + ATF_ADD_TEST_CASE(tcs, list__current_directory); + ATF_ADD_TEST_CASE(tcs, list__relative_path); + ATF_ADD_TEST_CASE(tcs, list__missing_test_program); + ATF_ADD_TEST_CASE(tcs, list__not_a_test_program); + ATF_ADD_TEST_CASE(tcs, list__no_permissions); + ATF_ADD_TEST_CASE(tcs, list__abort); + ATF_ADD_TEST_CASE(tcs, list__empty); + + ATF_ADD_TEST_CASE(tcs, test__body_only__passes); + ATF_ADD_TEST_CASE(tcs, test__body_only__fails); + ATF_ADD_TEST_CASE(tcs, test__body_only__crashes); + ATF_ADD_TEST_CASE(tcs, test__body_only__times_out); + ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables); +}