diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 9923b5693f..c4fa6e0c66 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -7,7 +7,7 @@ import os ; import ../../predef/check/predef - : check + : check : predef-check ; project boost/test @@ -25,17 +25,10 @@ project boost/test all # on - # adding a dependency on boost/timer as the header are needed, and the junction needs - # to be there in order to build the library. - /boost/timer//boost_timer : usage-requirements BOOST_TEST_NO_AUTO_LINK=1 # Disable Warning about boost::noncopyable not being exported shared,msvc:-wd4275 - - # Adding a dependency on boost/timer as the headers need to be there in case of the - # header-only usage variant - /boost/timer//boost_timer ; PRG_EXEC_MON_SOURCES = diff --git a/doc/closing_chapters/change_log.qbk b/doc/closing_chapters/change_log.qbk index 6e99153bcb..3bc26e34ec 100644 --- a/doc/closing_chapters/change_log.qbk +++ b/doc/closing_chapters/change_log.qbk @@ -11,6 +11,13 @@ [h5 New features] +* Boost.Test does not depend on Boost.Timer any more (which was pulling also Boost.Chrono + and Boost.System as transitive dependencies). +* Now Boost.Test raises an exception when the test case times-out on Windows. Prior to this release, + times-out on Windows were not failing the test cases. Note that signaling is not available on Windows, + and it is not possible to interrupt a test even in case of time out. +* Time-out now applies to test-suites as well: a test-suite is marked as timed-out if it exceeds the allocated + time. The test units that were not executed at the time-point of the time-out are skipped. * It is now possible to pass several values for the same context via the tool __BOOST_TEST_CONTEXT__. * A new macro __BOOST_TEST_INFO_SCOPE__ let define a context for the current scope in a sticky way. @@ -65,6 +72,7 @@ # [github_issue 181] `doc_example22` (and `23`) are expected to fail, but do not on clang with release variant builds # [github_issue 194] `master_test_suite` declared twice # [github_issue 199] Runtime `type_mismatch` after upgrade to `1.69` +# [github_issue 202] `boost/timer.hpp` is deprecated # [github_issue 203] Test cases with datasets and fixtures don't support flexible fixture interface # [github_issue 204] Feature Request: Allow specifying timeouts for test cases with datasests. @@ -76,6 +84,8 @@ # [pull_request 197] Feature Request: `BOOST_TEST_CONTEXT` that doesn't require introducing a new scope with braces # [pull_request 205] Fix MinGW `vsnprintf` compile errors and warnings +# [ticket 7397] Boost.Test, since boost `1.48` is using the deprecated `Boost.Timer` class (solved via [github_issue 202]) +# [ticket 9434] error: `namespace boost::timer {}` redeclared as different kind of symbol (solved via [github_issue 202]) # [ticket 13106] `libs/test/tools/console_test_runner` does not compile diff --git a/doc/testing_tools/testing_tools_reference.qbk b/doc/testing_tools/testing_tools_reference.qbk index 0d20bde240..f2ff061b74 100644 --- a/doc/testing_tools/testing_tools_reference.qbk +++ b/doc/testing_tools/testing_tools_reference.qbk @@ -86,12 +86,26 @@ See [link boost_test.testing_tools.expected_failures here] for more details. [section:decorator_timeout timeout (decorator)] `` -timeout(unsigned seconds); +timeout(unsigned int seconds); `` -Specifies a time-out for a *test-case*, above which the test-case is forced to stop and reported as failing. +Specifies a time-out for a *test-case* or a *test-suite*, in wall-clock time. + +If a test-case lasts longer than the timeout, the test is flagged as failed. On some systems (see below), +the test-case is forced to stop. + +For test-suites, the mechanism is similar: every test-unit under the test-suite is allocated a maximum +duration time that is the remainder of the timeout after the previous tests have been executed. If a timeout occurs +during the execution of the suite, the suite is flagged as timed-out and the remaining test-units are skipped. + See [link boost_test.testing_tools.timeout here] for more details. +[note The macro + `BOOST_SIGACTION_BASED_SIGNAL_HANDLING` is defined + if Boost.Test is able to force the test-case to stop.] + +[note The support for test suites has been added in [link ref_CHANGE_LOG_3_10 Boost 1.70 / __UTF__ v3.10]] + [endsect] [/ section timeout] diff --git a/doc/testing_tools/timeout.qbk b/doc/testing_tools/timeout.qbk index b97ed1d135..30e0d23954 100644 --- a/doc/testing_tools/timeout.qbk +++ b/doc/testing_tools/timeout.qbk @@ -8,15 +8,31 @@ [section:timeout Time-out for test cases] -The __UTF__ provides the decorator __decorator_timeout__ that specifies a time-out for a specific *test case*, -The argument time (in seconds) sets the maximum allowed duration of a test case. If this time is -exceeded the test case is forced to stop and is reported as failure. +The __UTF__ provides the decorator __decorator_timeout__ that specifies a time-out for a specific *test unit*. +The argument time is always expressed in *seconds ans wall-clock* time. -[bt_example decorator_11..decorator timeout..run-fail] +For test-cases, the time-out value sets the maximum allowed duration for the test. If this time is +exceeded, the test case is reported as failed. On some systems, the __UTF__ is able to force the test-case +to stop through a `SIGALRM` signal (see below). + +For test-suites, the time-out value sets the maximum allowed duration for the entire suite to complete. This duration +is the accumulated time of all the test-cases contained in the sub-tree rooted on the test-suite, plus some extra +execution time needed by the __UTF__. For each test-units under a test-suite with time-out, the maximum allowed duration +is set as being the test-suite time out minus the accumulated execution time before the execution of the test-unit. +If this test-unit is a test-case, it is equivalent to setting the decorator __decorator_timeout__ to the test-case +with a time-out value expressed as before. -[note Applied at test suite level, this decorator has no effect.] +In case the test-suite times out, the +suite is flagged as `timed-out` and `failed`, and all the test units (suites and cases) that have not been executed +up to the time-out point are all skipped. + +[bt_example decorator_11..decorator timeout..run-fail] -[caution Decorator `timeout` has no effect on Windows build. This feature is not implemented on Windows yet.] +[note The macro + `BOOST_SIGACTION_BASED_SIGNAL_HANDLING` is defined + if Boost.Test is able to force the test-case to stop. This feature is for instance not supported on Windows. + The __UTF__ will still be able to report the test-case as failed (once the test-case finishes).] +[note The support of test suite level time-out has been added in [link ref_CHANGE_LOG_3_10 Boost 1.70 / __UTF__ v3.10] ] [endsect] diff --git a/include/boost/test/execution_monitor.hpp b/include/boost/test/execution_monitor.hpp index bda732b980..25e0129edd 100644 --- a/include/boost/test/execution_monitor.hpp +++ b/include/boost/test/execution_monitor.hpp @@ -111,7 +111,8 @@ namespace boost { /// /// @section DesignRationale Design Rationale /// -/// The Execution Monitor design assumes that it can be used when no (or almost no) memory available. Also the Execution Monitor is intended to be portable to as many platforms as possible. +/// The Execution Monitor design assumes that it can be used when no (or almost no) memory available. Also the Execution Monitor +/// is intended to be portable to as many platforms as possible. /// /// @section UserGuide User's guide /// The Execution Monitor is designed to solve the problem of executing potentially dangerous function that may result in any number of error conditions, @@ -337,9 +338,9 @@ class BOOST_TEST_DECL execution_monitor { /// Specifies the seconds that elapse before a timer_error occurs. /// - /// The @em p_timeout property is an integer timeout (in seconds) for monitored function execution. Use this parameter to monitor code with possible deadlocks - /// or indefinite loops. This feature is only available for some operating systems (not yet Microsoft Windows). - unit_test::readwrite_property p_timeout; + /// The @em p_timeout property is an integer timeout (in microseconds) for monitored function execution. Use this parameter to monitor code with possible deadlocks + /// or infinite loops. This feature is only available for some operating systems (not yet Microsoft Windows). + unit_test::readwrite_property p_timeout; /// Should monitor use alternative stack for the signal catching. /// diff --git a/include/boost/test/impl/execution_monitor.ipp b/include/boost/test/impl/execution_monitor.ipp index a1b8388850..6d7547eaa7 100644 --- a/include/boost/test/impl/execution_monitor.ipp +++ b/include/boost/test/impl/execution_monitor.ipp @@ -50,11 +50,12 @@ #include #include // for varargs #include +#include // for ceil #include // for varargs #ifdef BOOST_NO_STDC_NAMESPACE -namespace std { using ::strerror; using ::strlen; using ::strncat; } +namespace std { using ::strerror; using ::strlen; using ::strncat; using ::ceil; } #endif // to use vsnprintf @@ -100,6 +101,10 @@ using std::va_list; # define BOOST_TEST_CRT_SET_HOOK(H) (void*)(H) # endif +# if defined(_MSC_VER) && (_WIN32_WINNT >= 0x0501) /* WinXP */ +# define BOOST_TEST_WIN32_WAITABLE_TIMERS +# endif + # if (!BOOST_WORKAROUND(_MSC_VER, >= 1400 ) && \ !defined(BOOST_COMO)) || defined(UNDER_CE) @@ -691,7 +696,7 @@ signal_action::~signal_action() class signal_handler { public: // Constructor - explicit signal_handler( bool catch_system_errors, bool detect_fpe, unsigned timeout, bool attach_dbg, char* alt_stack ); + explicit signal_handler( bool catch_system_errors, bool detect_fpe, unsigned timeout_microseconds, bool attach_dbg, char* alt_stack ); // Destructor ~signal_handler(); @@ -714,7 +719,7 @@ public: private: // Data members signal_handler* m_prev_handler; - unsigned m_timeout; + unsigned m_timeout_microseconds; // Note: We intentionality do not catch SIGCHLD. Users have to deal with it themselves signal_action m_ILL_action; @@ -738,24 +743,24 @@ signal_handler* signal_handler::s_active_handler = signal_handler_ptr(); //____________________________________________________________________________// -signal_handler::signal_handler( bool catch_system_errors, bool detect_fpe, unsigned timeout, bool attach_dbg, char* alt_stack ) +signal_handler::signal_handler( bool catch_system_errors, bool detect_fpe, unsigned timeout_microseconds, bool attach_dbg, char* alt_stack ) : m_prev_handler( s_active_handler ) -, m_timeout( timeout ) -, m_ILL_action ( SIGILL , catch_system_errors, attach_dbg, alt_stack ) -, m_FPE_action ( SIGFPE , detect_fpe , attach_dbg, alt_stack ) -, m_SEGV_action( SIGSEGV, catch_system_errors, attach_dbg, alt_stack ) -, m_BUS_action ( SIGBUS , catch_system_errors, attach_dbg, alt_stack ) +, m_timeout_microseconds( timeout_microseconds ) +, m_ILL_action ( SIGILL , catch_system_errors, attach_dbg, alt_stack ) +, m_FPE_action ( SIGFPE , detect_fpe , attach_dbg, alt_stack ) +, m_SEGV_action( SIGSEGV, catch_system_errors, attach_dbg, alt_stack ) +, m_BUS_action ( SIGBUS , catch_system_errors, attach_dbg, alt_stack ) #ifdef BOOST_TEST_CATCH_SIGPOLL -, m_POLL_action( SIGPOLL, catch_system_errors, attach_dbg, alt_stack ) +, m_POLL_action( SIGPOLL, catch_system_errors, attach_dbg, alt_stack ) #endif -, m_ABRT_action( SIGABRT, catch_system_errors, attach_dbg, alt_stack ) -, m_ALRM_action( SIGALRM, timeout > 0 , attach_dbg, alt_stack ) +, m_ABRT_action( SIGABRT, catch_system_errors, attach_dbg, alt_stack ) +, m_ALRM_action( SIGALRM, timeout_microseconds > 0, attach_dbg, alt_stack ) { s_active_handler = this; - if( m_timeout > 0 ) { + if( m_timeout_microseconds > 0 ) { ::alarm( 0 ); - ::alarm( timeout ); + ::alarm( static_cast(std::ceil(timeout_microseconds / 1E6) )); // alarm has a precision to the seconds } #ifdef BOOST_TEST_USE_ALT_STACK @@ -781,7 +786,7 @@ signal_handler::~signal_handler() { assert( s_active_handler == this ); - if( m_timeout > 0 ) + if( m_timeout_microseconds > 0 ) ::alarm( 0 ); #ifdef BOOST_TEST_USE_ALT_STACK @@ -897,8 +902,10 @@ public: , m_se_id( 0 ) , m_fault_address( 0 ) , m_dir( false ) + , m_timeout( false ) {} + void set_timed_out(); void report() const; int operator()( unsigned id, _EXCEPTION_POINTERS* exps ); @@ -909,6 +916,7 @@ private: unsigned m_se_id; void* m_fault_address; bool m_dir; + bool m_timeout; }; //____________________________________________________________________________// @@ -923,6 +931,14 @@ seh_catch_preventer( unsigned /* id */, _EXCEPTION_POINTERS* /* exps */ ) //____________________________________________________________________________// +void +system_signal_exception::set_timed_out() +{ + m_timeout = true; +} + +//____________________________________________________________________________// + int system_signal_exception::operator()( unsigned id, _EXCEPTION_POINTERS* exps ) { @@ -1074,7 +1090,12 @@ system_signal_exception::report() const break; default: - detail::report_error( execution_exception::system_error, "unrecognized exception. Id: 0x%08lx", m_se_id ); + if( m_timeout ) { + detail::report_error(execution_exception::timeout_error, "timeout while executing function"); + } + else { + detail::report_error( execution_exception::system_error, "unrecognized exception. Id: 0x%08lx", m_se_id ); + } break; } } @@ -1133,6 +1154,31 @@ execution_monitor::catch_signals( boost::function const& F ) #endif } +#if defined(BOOST_TEST_WIN32_WAITABLE_TIMERS) + HANDLE htimer = INVALID_HANDLE_VALUE; + BOOL bTimerSuccess = FALSE; + + if( p_timeout ) { + htimer = ::CreateWaitableTimer( + NULL, + TRUE, + TEXT("Boost.Test timer")); + + if( htimer != INVALID_HANDLE_VALUE ) { + LARGE_INTEGER liDueTime; + liDueTime.QuadPart = - static_cast(p_timeout) * 10; // resolution of 100 ns + + bTimerSuccess = ::SetWaitableTimer( + htimer, + &liDueTime, + 0, + 0, + 0, + FALSE); // Do not restore a suspended system + } + } +#endif + detail::system_signal_exception SSE( this ); int ret_val = 0; @@ -1146,8 +1192,26 @@ execution_monitor::catch_signals( boost::function const& F ) __except( SSE( GetExceptionCode(), GetExceptionInformation() ) ) { throw SSE; } + + // we check for time outs: we do not have any signaling facility on Win32 + // however, we signal a timeout as a hard error as for the other operating systems + // and throw the signal error handler + if( bTimerSuccess && htimer != INVALID_HANDLE_VALUE) { + if (::WaitForSingleObject(htimer, 0) == WAIT_OBJECT_0) { + SSE.set_timed_out(); + throw SSE; + } + } + } __finally { + +#if defined(BOOST_TEST_WIN32_WAITABLE_TIMERS) + if( htimer != INVALID_HANDLE_VALUE ) { + ::CloseHandle(htimer); + } +#endif + if( l_catch_system_errors ) { BOOST_TEST_CRT_SET_HOOK( old_crt_hook ); @@ -1250,15 +1314,8 @@ execution_monitor::execute( boost::function const& F ) #endif CATCH_AND_REPORT_STD_EXCEPTION( std::bad_alloc ) - -#if BOOST_WORKAROUND(__BORLANDC__, <= 0x0551) - CATCH_AND_REPORT_STD_EXCEPTION( std::bad_cast ) - CATCH_AND_REPORT_STD_EXCEPTION( std::bad_typeid ) -#else CATCH_AND_REPORT_STD_EXCEPTION( std::bad_cast ) CATCH_AND_REPORT_STD_EXCEPTION( std::bad_typeid ) -#endif - CATCH_AND_REPORT_STD_EXCEPTION( std::bad_exception ) CATCH_AND_REPORT_STD_EXCEPTION( std::domain_error ) CATCH_AND_REPORT_STD_EXCEPTION( std::invalid_argument ) diff --git a/include/boost/test/impl/framework.ipp b/include/boost/test/impl/framework.ipp index 9d5d5849f1..c35469eeb3 100644 --- a/include/boost/test/impl/framework.ipp +++ b/include/boost/test/impl/framework.ipp @@ -48,7 +48,7 @@ #include // Boost -#include +#include #include // STL @@ -58,6 +58,7 @@ #include #include #include +#include #ifdef BOOST_NO_CXX98_RANDOM_SHUFFLE #include #endif @@ -486,7 +487,7 @@ private: // ************** framework::state ************** // // ************************************************************************** // -unsigned const TIMEOUT_EXCEEDED = static_cast( -1 ); +unsigned long int const TIMEOUT_EXCEEDED = static_cast( -1 ); class state { public: @@ -665,7 +666,7 @@ public: // Executes the test tree with the root at specified test unit execution_result execute_test_tree( test_unit_id tu_id, - unsigned timeout = 0, + unsigned long int timeout_microseconds = 0, random_generator_helper const * const p_random_generator = 0) { test_unit const& tu = framework::get( tu_id, TUT_ANY ); @@ -677,15 +678,16 @@ public: // 10. Check preconditions, including zero time left for execution and // successful execution of all dependencies - if( timeout == TIMEOUT_EXCEEDED ) { + if( timeout_microseconds == TIMEOUT_EXCEEDED ) { // notify all observers about skipped test unit BOOST_TEST_FOREACH( test_observer*, to, m_observers ) to->test_unit_skipped( tu, "timeout for the test unit is exceeded" ); return unit_test_monitor_t::os_timeout; } - else if( timeout == 0 || timeout > tu.p_timeout ) // deduce timeout for this test unit - timeout = tu.p_timeout; + else if( timeout_microseconds == 0 || (tu.p_timeout > 0 && timeout_microseconds > (tu.p_timeout * 1000000) ) ) // deduce timeout for this test unit + timeout_microseconds = tu.p_timeout * 1000000; + test_tools::assertion_result const precondition_res = tu.check_preconditions(); if( !precondition_res ) { @@ -716,12 +718,13 @@ public: } } - // This is the time we are going to spend executing the test unit - unsigned long elapsed = 0; + // This is the time we are going to spend executing the test unit (in microseconds + // as expected by test_observer::test_unit_finish) + unsigned long elapsed_microseconds = 0; if( result == unit_test_monitor_t::test_ok ) { // 40. We are going to time the execution - boost::timer tu_timer; + boost::unit_test::timer::timer tu_timer; // we pass the random generator const random_generator_helper& rand_gen = p_random_generator ? *p_random_generator : random_generator_helper(); @@ -733,12 +736,29 @@ public: typedef std::pair value_type; BOOST_TEST_FOREACH( value_type, chld, ts.m_ranked_children ) { - unsigned chld_timeout = child_timeout( timeout, tu_timer.elapsed() ); + // tu_timer.elapsed() returns nanosec, timeout and child_timeout in microsec + unsigned long int chld_timeout = child_timeout( + timeout_microseconds, + static_cast( microsecond_wall_time(tu_timer.elapsed()) )); result = (std::min)( result, execute_test_tree( chld.second, chld_timeout, &rand_gen ) ); if( unit_test_monitor.is_critical_error( result ) ) break; + + // we check for the time elapsed. If this is too high, we fail the current suite and return from here + elapsed_microseconds = static_cast( microsecond_wall_time(tu_timer.elapsed()) ); + + if( (timeout_microseconds > 0) && (elapsed_microseconds > timeout_microseconds) && (timeout_microseconds != TIMEOUT_EXCEEDED ) ) { + BOOST_TEST_FOREACH( test_observer*, to, m_observers ) { + to->test_unit_timed_out(tu); + } + result = (std::min)( result, unit_test_monitor_t::os_timeout ); + timeout_microseconds = TIMEOUT_EXCEEDED; + //break; + // we continue to explore the children, such that we can at least update their + // status to skipped + } } } else { @@ -765,17 +785,30 @@ public: #endif BOOST_TEST_FOREACH( test_unit_id, chld, children_with_the_same_rank ) { - unsigned chld_timeout = child_timeout( timeout, tu_timer.elapsed() ); + unsigned long int chld_timeout = child_timeout( + timeout_microseconds, + static_cast(microsecond_wall_time(tu_timer.elapsed())) ); result = (std::min)( result, execute_test_tree( chld, chld_timeout, &rand_gen ) ); if( unit_test_monitor.is_critical_error( result ) ) break; + + // we check for the time elapsed. If this is too high, we fail the current suite and return from here + elapsed_microseconds = static_cast( microsecond_wall_time(tu_timer.elapsed()) ); + if( (timeout_microseconds > 0) && (elapsed_microseconds > timeout_microseconds) && (timeout_microseconds != TIMEOUT_EXCEEDED ) ) { + BOOST_TEST_FOREACH( test_observer*, to, m_observers ) { + to->test_unit_timed_out(tu); + } + result = (std::min)( result, unit_test_monitor_t::os_timeout ); + timeout_microseconds = TIMEOUT_EXCEEDED; + //break; + // we continue to explore the children, such that we can at least update their + // status to skipped + } } } } - - elapsed = static_cast( tu_timer.elapsed() * 1e6 ); } else { // TUT_CASE test_case const& tc = static_cast( tu ); @@ -786,9 +819,9 @@ public: // setup current test case ut_detail::test_unit_id_restore restore_current_test_unit(m_curr_test_unit, tc.p_id); - // execute the test case body - result = unit_test_monitor.execute_and_translate( tc.p_test_func, timeout ); - elapsed = static_cast( tu_timer.elapsed() * 1e6 ); + // execute the test case body, transforms the time out to seconds + result = unit_test_monitor.execute_and_translate( tc.p_test_func, timeout_microseconds ); + elapsed_microseconds = static_cast( microsecond_wall_time(tu_timer.elapsed()) ); // cleanup leftover context m_context.clear(); @@ -817,21 +850,21 @@ public: // notify all observers about completion BOOST_TEST_REVERSE_FOREACH( test_observer*, to, m_observers ) - to->test_unit_finish( tu, elapsed ); + to->test_unit_finish( tu, elapsed_microseconds ); return result; } ////////////////////////////////////////////////////////////////// - unsigned child_timeout( unsigned tu_timeout, double elapsed ) + unsigned long int child_timeout( unsigned long tu_timeout_microseconds, unsigned long elpsed_microsec ) { - if( tu_timeout == 0U ) - return 0U; - - unsigned elpsed_sec = static_cast(elapsed); // rounding to number of whole seconds + if( tu_timeout_microseconds == 0UL || tu_timeout_microseconds == TIMEOUT_EXCEEDED) + return tu_timeout_microseconds; - return tu_timeout > elpsed_sec ? tu_timeout - elpsed_sec : TIMEOUT_EXCEEDED; + return tu_timeout_microseconds > elpsed_microsec ? + tu_timeout_microseconds - elpsed_microsec + : TIMEOUT_EXCEEDED; } struct priority_order { @@ -1096,7 +1129,7 @@ setup_loggers() boost::ref(std::cout) ); if( ++current_format_specs != utils::string_token_iterator() && current_format_specs->size() ) { - stream_logger.setup( *current_format_specs, + stream_logger.setup( *current_format_specs, log_cleaner ); } else { @@ -1154,7 +1187,7 @@ init( init_unit_test_func init_func, int argc, char* argv[] ) s_frk_state().m_report_sink.setup( runtime_config::get( runtime_config::btrt_report_sink ), report_cleaner ); } - + results_reporter::set_stream( s_frk_state().m_report_sink.ref() ); // 40. Register default test observers @@ -1200,14 +1233,14 @@ finalize_setup_phase( test_unit_id master_tu_id ) class apply_decorators : public test_tree_visitor { private: // test_tree_visitor interface - + virtual bool test_suite_start( test_suite const& ts) { const_cast(ts).generate(); const_cast(ts).check_for_duplicate_test_cases(); return test_tree_visitor::test_suite_start(ts); } - + virtual bool visit( test_unit const& tu ) { BOOST_TEST_FOREACH( decorator::base_ptr, d, tu.p_decorators.get() ) diff --git a/include/boost/test/impl/junit_log_formatter.ipp b/include/boost/test/impl/junit_log_formatter.ipp index 5cffb2fa05..477915fc8a 100644 --- a/include/boost/test/impl/junit_log_formatter.ipp +++ b/include/boost/test/impl/junit_log_formatter.ipp @@ -242,7 +242,10 @@ public: } if( tu.p_type == TUT_SUITE ) { - name += "-setup-teardown"; + if(tr->p_timed_out) + name += "-timed-execution"; + else + name += "-setup-teardown"; } m_stream << "p_assertions_passed + tr->p_assertions_failed; + nb_assertions = static_cast(tr->p_assertions_passed + tr->p_assertions_failed); } return nb_assertions; @@ -423,7 +426,7 @@ public: << " tests" << utils::attr_value() << tr.p_test_cases_passed << " skipped" << utils::attr_value() << tr.p_test_cases_skipped << " errors" << utils::attr_value() << tr.p_test_cases_aborted - << " failures" << utils::attr_value() << tr.p_test_cases_failed + << " failures" << utils::attr_value() << tr.p_test_cases_failed + tr.p_test_suites_timed_out + tr.p_test_cases_timed_out << " id" << utils::attr_value() << m_id++ << " name" << utils::attr_value() << tu_name_normalize(ts.p_name) << " time" << utils::attr_value() << (tr.p_duration_microseconds * 1E-6) @@ -546,6 +549,25 @@ junit_log_formatter::test_unit_aborted( std::ostream& /*ostr*/, test_unit const& //____________________________________________________________________________// +void +junit_log_formatter::test_unit_timed_out( std::ostream& /*os*/, test_unit const& tu) +{ + if(tu.p_type == TUT_SUITE) + { + // if we reach this call, it means that the test has already started and + // test_unit_start has already been called on the tu. + junit_impl::junit_log_helper& last_entry = get_current_log_entry(); + junit_impl::junit_log_helper::assertion_entry entry; + entry.logentry_message = "test-suite time out"; + entry.logentry_type = "execution timeout"; + entry.log_entry = junit_impl::junit_log_helper::assertion_entry::log_entry_error; + entry.output = "the current suite exceeded the allocated execution time"; + last_entry.assertion_entries.push_back(entry); + } +} + +//____________________________________________________________________________// + void junit_log_formatter::test_unit_skipped( std::ostream& /*ostr*/, test_unit const& tu, const_string reason ) { diff --git a/include/boost/test/impl/plain_report_formatter.ipp b/include/boost/test/impl/plain_report_formatter.ipp index cbf5a4c029..c69b895a8f 100644 --- a/include/boost/test/impl/plain_report_formatter.ipp +++ b/include/boost/test/impl/plain_report_formatter.ipp @@ -108,6 +108,8 @@ plain_report_formatter::test_unit_report_start( test_unit const& tu, std::ostrea descr = "has passed"; else if( tr.p_skipped ) descr = "was skipped"; + else if( tr.p_timed_out ) + descr = "has timed out"; else if( tr.p_aborted ) descr = "was aborted"; else @@ -122,8 +124,9 @@ plain_report_formatter::test_unit_report_start( test_unit const& tu, std::ostrea return; } + // aborted test case within failed ones, timed-out TC exclusive with failed/aborted counter_t total_assertions = tr.p_assertions_passed + tr.p_assertions_failed; - counter_t total_tc = tr.p_test_cases_passed + tr.p_test_cases_warned + tr.p_test_cases_failed + tr.p_test_cases_skipped; + counter_t total_tc = tr.p_test_cases_passed + tr.p_test_cases_warned + tr.p_test_cases_failed + tr.p_test_cases_skipped + tr.p_test_cases_timed_out; if( total_assertions > 0 || total_tc > 0 || tr.p_warnings_failed > 0) ostr << " with:"; @@ -134,6 +137,8 @@ plain_report_formatter::test_unit_report_start( test_unit const& tu, std::ostrea print_stat_value( ostr, tr.p_test_cases_passed , m_indent, total_tc , "test case", "passed" ); print_stat_value( ostr, tr.p_test_cases_warned , m_indent, total_tc , "test case", "passed with warnings" ); print_stat_value( ostr, tr.p_test_cases_failed , m_indent, total_tc , "test case", "failed" ); + print_stat_value( ostr, tr.p_test_cases_timed_out, m_indent, total_tc , "test case", "timed-out" ); + print_stat_value( ostr, tr.p_test_suites_timed_out, m_indent, tr.p_test_suites, "test suite", "timed-out" ); print_stat_value( ostr, tr.p_test_cases_skipped, m_indent, total_tc , "test case", "skipped" ); print_stat_value( ostr, tr.p_test_cases_aborted, m_indent, total_tc , "test case", "aborted" ); print_stat_value( ostr, tr.p_assertions_passed , m_indent, total_assertions, "assertion", "passed" ); @@ -174,6 +179,12 @@ plain_report_formatter::do_confirmation_report( test_unit const& tu, std::ostrea return; } + if( tr.p_timed_out ) { + ostr << "*** The test " << tu.p_type_name << ' ' << quote() << tu.full_name() << " has timed out" + << "; see standard output for details\n"; + return; + } + if( tr.p_aborted ) { ostr << "*** The test " << tu.p_type_name << ' ' << quote() << tu.full_name() << " was aborted" << "; see standard output for details\n"; diff --git a/include/boost/test/impl/results_collector.ipp b/include/boost/test/impl/results_collector.ipp index cfc34cf793..bda11f2d96 100644 --- a/include/boost/test/impl/results_collector.ipp +++ b/include/boost/test/impl/results_collector.ipp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -56,7 +57,9 @@ test_results::passed() const p_test_cases_failed == 0 && p_assertions_failed <= p_expected_failures && // p_test_cases_skipped == 0 && - !p_aborted; + !p_timed_out && + p_test_cases_timed_out == 0 && + !aborted(); } //____________________________________________________________________________// @@ -81,7 +84,7 @@ int test_results::result_code() const { return passed() ? exit_success - : ( (p_assertions_failed > p_expected_failures || p_skipped ) + : ( (p_assertions_failed > p_expected_failures || p_skipped || p_timed_out || p_test_cases_timed_out ) ? exit_test_failure : exit_exception_failure ); } @@ -91,6 +94,7 @@ test_results::result_code() const void test_results::operator+=( test_results const& tr ) { + p_test_suites.value += tr.p_test_suites; p_assertions_passed.value += tr.p_assertions_passed; p_assertions_failed.value += tr.p_assertions_failed; p_warnings_failed.value += tr.p_warnings_failed; @@ -99,6 +103,8 @@ test_results::operator+=( test_results const& tr ) p_test_cases_failed.value += tr.p_test_cases_failed; p_test_cases_skipped.value += tr.p_test_cases_skipped; p_test_cases_aborted.value += tr.p_test_cases_aborted; + p_test_cases_timed_out.value += tr.p_test_cases_timed_out; + p_test_suites_timed_out.value += tr.p_test_suites_timed_out; p_duration_microseconds.value += tr.p_duration_microseconds; } @@ -107,6 +113,7 @@ test_results::operator+=( test_results const& tr ) void test_results::clear() { + p_test_suites.value = 0; p_assertions_passed.value = 0; p_assertions_failed.value = 0; p_warnings_failed.value = 0; @@ -116,9 +123,12 @@ test_results::clear() p_test_cases_failed.value = 0; p_test_cases_skipped.value = 0; p_test_cases_aborted.value = 0; + p_test_cases_timed_out.value = 0; + p_test_suites_timed_out.value = 0; p_duration_microseconds.value= 0; p_aborted.value = false; p_skipped.value = false; + p_timed_out.value = false; } //____________________________________________________________________________// @@ -179,6 +189,8 @@ public: else m_tr.p_test_cases_passed.value++; } + else if( tr.p_timed_out ) + m_tr.p_test_cases_timed_out.value++; else if( tr.p_skipped ) m_tr.p_test_cases_skipped.value++; else { @@ -194,6 +206,10 @@ public: return true; m_tr += results_collector.results( ts.p_id ); + m_tr.p_test_suites.value++; + + if( results_collector.results( ts.p_id ).p_timed_out ) + m_tr.p_test_suites_timed_out.value++; return false; } @@ -212,6 +228,8 @@ results_collector_t::test_unit_finish( test_unit const& tu, unsigned long elapse results_collect_helper ch( s_rc_impl().m_results_store[tu.p_id], tu ); traverse_test_tree( tu, ch ); + + s_rc_impl().m_results_store[tu.p_id].p_duration_microseconds.value = elapsed_in_microseconds; } else { test_results & tr = s_rc_impl().m_results_store[tu.p_id]; @@ -233,7 +251,6 @@ void results_collector_t::test_unit_skipped( test_unit const& tu, const_string /*reason*/ ) { test_results& tr = s_rc_impl().m_results_store[tu.p_id]; - tr.clear(); tr.p_skipped.value = true; @@ -248,6 +265,15 @@ results_collector_t::test_unit_skipped( test_unit const& tu, const_string /*reas //____________________________________________________________________________// +void +results_collector_t::test_unit_timed_out(test_unit const& tu) +{ + test_results& tr = s_rc_impl().m_results_store[tu.p_id]; + tr.p_timed_out.value = true; +} + +//____________________________________________________________________________// + void results_collector_t::assertion_result( unit_test::assertion_result ar ) { @@ -266,11 +292,14 @@ results_collector_t::assertion_result( unit_test::assertion_result ar ) //____________________________________________________________________________// void -results_collector_t::exception_caught( execution_exception const& ) +results_collector_t::exception_caught( execution_exception const& ex) { test_results& tr = s_rc_impl().m_results_store[framework::current_test_case_id()]; tr.p_assertions_failed.value++; + if( ex.code() == execution_exception::timeout_error ) { + tr.p_timed_out.value = true; + } } //____________________________________________________________________________// diff --git a/include/boost/test/impl/test_tree.ipp b/include/boost/test/impl/test_tree.ipp index db58e2569e..d050735ad7 100644 --- a/include/boost/test/impl/test_tree.ipp +++ b/include/boost/test/impl/test_tree.ipp @@ -30,9 +30,6 @@ #include -// Boost -#include - // STL #include #include diff --git a/include/boost/test/impl/unit_test_log.ipp b/include/boost/test/impl/unit_test_log.ipp index 40b54be223..f44bd0909a 100644 --- a/include/boost/test/impl/unit_test_log.ipp +++ b/include/boost/test/impl/unit_test_log.ipp @@ -267,6 +267,20 @@ unit_test_log_t::test_unit_aborted( test_unit const& tu ) } } +void +unit_test_log_t::test_unit_timed_out( test_unit const& tu ) +{ + if( s_log_impl().has_entry_in_progress() ) + *this << log::end(); + + BOOST_TEST_FOREACH( unit_test_log_data_helper_impl&, current_logger_data, s_log_impl().m_log_formatter_data ) { + if( !current_logger_data.m_enabled || current_logger_data.get_log_level() > log_test_units ) + continue; + + current_logger_data.m_log_formatter->test_unit_timed_out(current_logger_data.stream(), tu ); + } +} + //____________________________________________________________________________// void diff --git a/include/boost/test/impl/unit_test_monitor.ipp b/include/boost/test/impl/unit_test_monitor.ipp index 70e78513b6..63a04c8f4f 100644 --- a/include/boost/test/impl/unit_test_monitor.ipp +++ b/include/boost/test/impl/unit_test_monitor.ipp @@ -37,11 +37,11 @@ BOOST_TEST_SINGLETON_CONS_IMPL(unit_test_monitor_t) // ************************************************************************** // unit_test_monitor_t::error_level -unit_test_monitor_t::execute_and_translate( boost::function const& func, unsigned timeout ) +unit_test_monitor_t::execute_and_translate( boost::function const& func, unsigned long int timeout_microseconds ) { BOOST_TEST_I_TRY { p_catch_system_errors.value = runtime_config::get( runtime_config::btrt_catch_sys_errors ); - p_timeout.value = timeout; + p_timeout.value = timeout_microseconds; p_auto_start_dbg.value = runtime_config::get( runtime_config::btrt_auto_start_dbg ); p_use_alt_stack.value = runtime_config::get( runtime_config::btrt_use_alt_stack ); p_detect_fp_exceptions.value = runtime_config::get( runtime_config::btrt_detect_fp_except ); diff --git a/include/boost/test/impl/xml_report_formatter.ipp b/include/boost/test/impl/xml_report_formatter.ipp index 424ef4ba44..2718895b80 100644 --- a/include/boost/test/impl/xml_report_formatter.ipp +++ b/include/boost/test/impl/xml_report_formatter.ipp @@ -59,6 +59,8 @@ xml_report_formatter::test_unit_report_start( test_unit const& tu, std::ostream& descr = "passed"; else if( tr.p_skipped ) descr = "skipped"; + else if( tr.p_timed_out ) + descr = "timed-out"; else if( tr.p_aborted ) descr = "aborted"; else @@ -70,14 +72,18 @@ xml_report_formatter::test_unit_report_start( test_unit const& tu, std::ostream& << " assertions_passed" << utils::attr_value() << tr.p_assertions_passed << " assertions_failed" << utils::attr_value() << tr.p_assertions_failed << " warnings_failed" << utils::attr_value() << tr.p_warnings_failed - << " expected_failures" << utils::attr_value() << tr.p_expected_failures; + << " expected_failures" << utils::attr_value() << tr.p_expected_failures + ; if( tu.p_type == TUT_SUITE ) { ostr << " test_cases_passed" << utils::attr_value() << tr.p_test_cases_passed << " test_cases_passed_with_warnings" << utils::attr_value() << tr.p_test_cases_warned << " test_cases_failed" << utils::attr_value() << tr.p_test_cases_failed << " test_cases_skipped" << utils::attr_value() << tr.p_test_cases_skipped - << " test_cases_aborted" << utils::attr_value() << tr.p_test_cases_aborted; + << " test_cases_aborted" << utils::attr_value() << tr.p_test_cases_aborted + << " test_cases_timed_out" << utils::attr_value() << tr.p_test_cases_timed_out + << " test_suites_timed_out"<< utils::attr_value() << tr.p_test_suites_timed_out + ; } ostr << '>'; diff --git a/include/boost/test/output/junit_log_formatter.hpp b/include/boost/test/output/junit_log_formatter.hpp index 713d3b016c..ee37fa5792 100644 --- a/include/boost/test/output/junit_log_formatter.hpp +++ b/include/boost/test/output/junit_log_formatter.hpp @@ -107,6 +107,7 @@ class junit_log_formatter : public unit_test_log_formatter { void test_unit_finish( std::ostream&, test_unit const& tu, unsigned long elapsed ); void test_unit_skipped( std::ostream&, test_unit const& tu, const_string reason ); void test_unit_aborted( std::ostream& os, test_unit const& tu ); + void test_unit_timed_out( std::ostream& os, test_unit const& tu); void log_exception_start( std::ostream&, log_checkpoint_data const&, execution_exception const& ex ); void log_exception_finish( std::ostream& ); diff --git a/include/boost/test/results_collector.hpp b/include/boost/test/results_collector.hpp index 8e8a6202ed..1c26a74a2a 100644 --- a/include/boost/test/results_collector.hpp +++ b/include/boost/test/results_collector.hpp @@ -62,6 +62,7 @@ class BOOST_TEST_DECL test_results { (test_results) (results_collect_helper) ) bool_prop; + counter_prop p_test_suites; //!< Number of test suites counter_prop p_assertions_passed; //!< Number of successful assertions counter_prop p_assertions_failed; //!< Number of failing assertions counter_prop p_warnings_failed; //!< Number of warnings @@ -71,9 +72,12 @@ class BOOST_TEST_DECL test_results { counter_prop p_test_cases_failed; //!< Number of failing test cases counter_prop p_test_cases_skipped; //!< Number of skipped test cases counter_prop p_test_cases_aborted; //!< Number of aborted test cases + counter_prop p_test_cases_timed_out; //!< Number of timed out test cases + counter_prop p_test_suites_timed_out; //!< Number of timed out test suites counter_prop p_duration_microseconds; //!< Duration of the test in microseconds bool_prop p_aborted; //!< Indicates that the test unit execution has been aborted bool_prop p_skipped; //!< Indicates that the test unit execution has been skipped + bool_prop p_timed_out; //!< Indicates that the test unit has timed out /// Returns true if test unit passed bool passed() const; @@ -123,6 +127,7 @@ class BOOST_TEST_DECL results_collector_t : public test_observer { virtual void test_unit_finish( test_unit const&, unsigned long ); virtual void test_unit_skipped( test_unit const&, const_string ); virtual void test_unit_aborted( test_unit const& ); + virtual void test_unit_timed_out( test_unit const& ); virtual void assertion_result( unit_test::assertion_result ); virtual void exception_caught( execution_exception const& ); diff --git a/include/boost/test/tree/observer.hpp b/include/boost/test/tree/observer.hpp index bd6fc9bff5..1c5805dd80 100644 --- a/include/boost/test/tree/observer.hpp +++ b/include/boost/test/tree/observer.hpp @@ -72,6 +72,12 @@ class BOOST_TEST_DECL test_observer { virtual void test_unit_skipped( test_unit const& tu, const_string ) { test_unit_skipped( tu ); } virtual void test_unit_skipped( test_unit const& ) {} ///< backward compatibility + //! Called when the test timed out + //! + //! This function is called to signal that a test unit (case or suite) timed out. + //! A valid test unit is available through boost::unit_test::framework::current_test_unit + virtual void test_unit_timed_out( test_unit const& ) {} + //! Called when a test unit indicates a fatal error. //! //! A fatal error happens when @@ -81,12 +87,14 @@ class BOOST_TEST_DECL test_observer { virtual void assertion_result( unit_test::assertion_result ar ) { + /* switch( ar ) { case AR_PASSED: assertion_result( true ); break; case AR_FAILED: assertion_result( false ); break; case AR_TRIGGERED: break; default: break; } + */ } //! Called when an exception is intercepted @@ -101,8 +109,6 @@ class BOOST_TEST_DECL test_observer { virtual int priority() { return 0; } protected: - //! Deprecated - virtual void assertion_result( bool /* passed */ ) {} BOOST_TEST_PROTECTED_VIRTUAL ~test_observer() {} }; diff --git a/include/boost/test/unit_test_log.hpp b/include/boost/test/unit_test_log.hpp index d65a728747..5de9f43340 100644 --- a/include/boost/test/unit_test_log.hpp +++ b/include/boost/test/unit_test_log.hpp @@ -119,6 +119,7 @@ class BOOST_TEST_DECL unit_test_log_t : public test_observer { virtual void test_unit_finish( test_unit const&, unsigned long elapsed ); virtual void test_unit_skipped( test_unit const&, const_string ); virtual void test_unit_aborted( test_unit const& ); + virtual void test_unit_timed_out( test_unit const& ); virtual void exception_caught( execution_exception const& ex ); diff --git a/include/boost/test/unit_test_log_formatter.hpp b/include/boost/test/unit_test_log_formatter.hpp index 79b74e0849..dc664fa85f 100644 --- a/include/boost/test/unit_test_log_formatter.hpp +++ b/include/boost/test/unit_test_log_formatter.hpp @@ -176,6 +176,10 @@ class BOOST_TEST_DECL unit_test_log_formatter { /// Invoked when a test unit is aborted virtual void test_unit_aborted( std::ostream& /* os */, test_unit const& /* tu */) {} + /// Invoked when a test unit times-out + virtual void test_unit_timed_out( std::ostream& /* os */, test_unit const& /* tu */) {} + + // @} // @name Uncaught exception report diff --git a/include/boost/test/unit_test_monitor.hpp b/include/boost/test/unit_test_monitor.hpp index b056051caf..3e7919a5aa 100644 --- a/include/boost/test/unit_test_monitor.hpp +++ b/include/boost/test/unit_test_monitor.hpp @@ -45,7 +45,8 @@ class BOOST_TEST_DECL unit_test_monitor_t :public execution_monitor { static bool is_critical_error( error_level e ) { return e <= fatal_error; } // monitor method - error_level execute_and_translate( boost::function const& func, unsigned timeout = 0 ); + // timeout is expressed in seconds + error_level execute_and_translate( boost::function const& func, unsigned long int timeout_microseconds = 0 ); // singleton pattern BOOST_TEST_SINGLETON_CONS( unit_test_monitor_t ) diff --git a/include/boost/test/utils/timer.hpp b/include/boost/test/utils/timer.hpp new file mode 100644 index 0000000000..8aed78e73c --- /dev/null +++ b/include/boost/test/utils/timer.hpp @@ -0,0 +1,164 @@ +// (C) Copyright Raffi Enficiaud 2019. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/test for the library home page. +// +// Description : timer and elapsed types +// *************************************************************************** + +#ifndef BOOST_TEST_UTILS_TIMER_HPP +#define BOOST_TEST_UTILS_TIMER_HPP + +#include +#include +#include +#include + +# if defined(_WIN32) || defined(__CYGWIN__) +# define BOOST_TEST_TIMER_WINDOWS_API +# elif defined(__MACH__)// && !defined(CLOCK_MONOTONIC) +# // we compile for all macs the same, CLOCK_MONOTONIC introduced in 10.12 +# define BOOST_TEST_TIMER_MACH_API +# else +# define BOOST_TEST_TIMER_POSIX_API +# if !defined(CLOCK_MONOTONIC) +# error "CLOCK_MONOTONIC not defined" +# endif +# endif + +# if defined(BOOST_TEST_TIMER_WINDOWS_API) +# include +# elif defined(BOOST_TEST_TIMER_MACH_API) +# include +//# include /* host_get_clock_service, mach_... */ +# else +# include +# endif + +# ifdef BOOST_NO_STDC_NAMESPACE + namespace std { using ::clock_t; using ::clock; } +# endif + +namespace boost { +namespace unit_test { +namespace timer { + + struct elapsed_time + { + typedef boost::int_least64_t nanosecond_type; + + nanosecond_type wall; + nanosecond_type system; + void clear() { + wall = 0; + system = 0; + } + }; + + inline double + microsecond_wall_time( elapsed_time const& elapsed ) + { + return elapsed.wall / 1E3; + } + + inline double + second_wall_time( elapsed_time const& elapsed ) + { + return elapsed.wall / 1E9; + } + + namespace details { + #if defined(BOOST_TEST_TIMER_WINDOWS_API) + elapsed_time::nanosecond_type get_tick_freq() { + LARGE_INTEGER freq; + ::QueryPerformanceFrequency( &freq ); + return static_cast(freq.QuadPart); + } + #elif defined(BOOST_TEST_TIMER_MACH_API) + std::pair get_time_base() { + mach_timebase_info_data_t timebase; + if(mach_timebase_info(&timebase) == 0) + return std::pair(timebase.numer, timebase.denom); + return std::pair(0, 1); + } + #endif + } + + //! Simple timing class + //! + //! This class measures the wall clock time. + class timer + { + public: + timer() + { + restart(); + } + void restart() + { + _start_time_clock = std::clock(); + #if defined(BOOST_TEST_TIMER_WINDOWS_API) + ::QueryPerformanceCounter(&_start_time_wall); + #elif defined(BOOST_TEST_TIMER_MACH_API) + _start_time_wall = mach_absolute_time(); + #else + if( ::clock_gettime( CLOCK_MONOTONIC, &_start_time_wall ) != 0 ) + { + _start_time_wall.tv_nsec = -1; + _start_time_wall.tv_sec = -1; + } + #endif + } + + // return elapsed time in seconds + elapsed_time elapsed() const + { + typedef elapsed_time::nanosecond_type nanosecond_type; + static const double clock_to_nano_seconds = 1E9 / CLOCKS_PER_SEC; + elapsed_time return_value; + + // processor / system time + return_value.system = static_cast(double(std::clock() - _start_time_clock) * clock_to_nano_seconds); + +#if defined(BOOST_TEST_TIMER_WINDOWS_API) + static const nanosecond_type tick_per_sec = details::get_tick_freq(); + LARGE_INTEGER end_time; + ::QueryPerformanceCounter(&end_time); + return_value.wall = static_cast(((end_time.QuadPart - _start_time_wall.QuadPart) * 1E9) / tick_per_sec); +#elif defined(BOOST_TEST_TIMER_MACH_API) + static std::pair timebase = details::get_time_base(); + nanosecond_type clock = mach_absolute_time() - _start_time_wall; + return_value.wall = static_cast((clock * timebase.first) / timebase.second); +#else + struct timespec end_time; + if( ::clock_gettime( CLOCK_MONOTONIC, &end_time ) == 0 ) + { + return_value.wall = static_cast((end_time.tv_sec - _start_time_wall.tv_sec) * 1E9 + (end_time.tv_nsec - _start_time_wall.tv_nsec)); + } +#endif + + return return_value; + } + + private: + std::clock_t _start_time_clock; + #if defined(BOOST_TEST_TIMER_WINDOWS_API) + LARGE_INTEGER _start_time_wall; + #elif defined(BOOST_TEST_TIMER_MACH_API) + elapsed_time::nanosecond_type _start_time_wall; + #else + struct timespec _start_time_wall; + #endif + }; + + +//____________________________________________________________________________// + +} // namespace timer +} // namespace unit_test +} // namespace boost + +#endif // BOOST_TEST_UTILS_TIMER_HPP + diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 054c1fccfb..62e32ba3b7 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -176,6 +176,10 @@ test-suite "writing-test-ts" [ boost.test-self-test run : writing-test-ts : test-fixture-detect-setup-teardown ] [ boost.test-self-test run : writing-test-ts : test-fixture-detect-setup-teardown-cpp11 : : : : : : [ requires cxx11_decltype cxx11_trailing_result_types ] ] [ boost.test-self-test run : writing-test-ts : test-with-precondition : : : : : : [ requires cxx11_auto_declarations cxx11_lambdas ] ] + [ boost.test-self-test run : writing-test-ts : test-timeout : : : : : : [ requires cxx11_hdr_thread cxx11_hdr_chrono ] ] + [ boost.test-self-test run-fail : writing-test-ts : test-timeout-fail : : : : : : [ requires cxx11_hdr_thread cxx11_hdr_chrono ] ] + [ boost.test-self-test run : writing-test-ts : test-timeout-suite : : : : : : $(requirements_datasets) [ requires cxx11_hdr_thread cxx11_hdr_chrono ] ] + [ boost.test-self-test run-fail : writing-test-ts : test-timeout-suite-fail : : : : : : $(requirements_datasets) [ requires cxx11_hdr_thread cxx11_hdr_chrono ] ] ; #_________________________________________________________________________________________________# diff --git a/test/baseline-outputs/log-formatter-test.pattern b/test/baseline-outputs/log-formatter-test.pattern index 6fec541311..bc2b3d70d4 100644 --- a/test/baseline-outputs/log-formatter-test.pattern +++ b/test/baseline-outputs/log-formatter-test.pattern @@ -7,14 +7,14 @@ xxx/log-formatter-test.cpp:210: Leaving test case "good_foo" xxx/log-formatter-test.cpp:209: Leaving test suite "1 test cases inside" * 2-format ******************************************************************* -ZZZ +ZZZ * 3-format ******************************************************************* @@ -30,19 +30,19 @@ xxx/log-formatter-test.cpp:209: Leaving test suite "1 test cases inside" Running 1 test case... xxx/log-formatter-test.cpp:212: Entering test suite "1 bad test case inside" xxx/log-formatter-test.cpp:213: Entering test case "bad_foo" -xxx/log-formatter-test.cpp:48: error: in "1 bad test case inside/bad_foo": +xxx/log-formatter-test.cpp:50: error: in "1 bad test case inside/bad_foo": this is a message -xxx/log-formatter-test.cpp:51: info: check true has passed -xxx/log-formatter-test.cpp:55: error: in "1 bad test case inside/bad_foo": with some message +xxx/log-formatter-test.cpp:53: info: check true has passed +xxx/log-formatter-test.cpp:57: error: in "1 bad test case inside/bad_foo": with some message Failure occurred in a following context: Context value=something Context value2=something different -xxx/log-formatter-test.cpp:57: error: in "1 bad test case inside/bad_foo": non sense +xxx/log-formatter-test.cpp:59: error: in "1 bad test case inside/bad_foo": non sense xxx/log-formatter-test.cpp:213: Leaving test case "bad_foo" xxx/log-formatter-test.cpp:212: Leaving test suite "1 bad test case inside" * 2-format ******************************************************************* -ZZZ +ZZZ * 3-format ******************************************************************* @@ -50,13 +50,13 @@ xxx/log-formatter-test.cpp:212: Leaving test suite "1 bad test case inside" @@ -94,13 +94,13 @@ INFO: 3 is not satisfied [2 <= 3] +xxx/log-formatter-test.cpp:46: warning: in "1 almost good test case inside/almost_good_foo": condition 2>3 is not satisfied [2 <= 3] Test case 1 almost good test case inside/almost_good_foo did not check any assertions xxx/log-formatter-test.cpp:216: Leaving test case "almost_good_foo" xxx/log-formatter-test.cpp:215: Leaving test suite "1 almost good test case inside" * 2-format ******************************************************************* -3 is not satisfied [2 <= 3]]]>ZZZ +3 is not satisfied [2 <= 3]]]>ZZZ * 3-format ******************************************************************* 3 is not satisfied [2 <= 3] MESSAGE: - file : boost.test framework -- line : 226 +- line : 244 - message: Test case 1 almost good test case inside/almost_good_foo did not check any assertions ]]> @@ -161,26 +161,26 @@ xxx/log-formatter-test.cpp:219: Entering test case "good_foo" Test case Fake Test Suite Hierarchy/2 test cases inside/good_foo did not check any assertions xxx/log-formatter-test.cpp:219: Leaving test case "good_foo" xxx/log-formatter-test.cpp:220: Entering test case "bad_foo" -xxx/log-formatter-test.cpp:48: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": +xxx/log-formatter-test.cpp:50: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": this is a message -xxx/log-formatter-test.cpp:51: info: check true has passed -xxx/log-formatter-test.cpp:55: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": with some message +xxx/log-formatter-test.cpp:53: info: check true has passed +xxx/log-formatter-test.cpp:57: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": with some message Failure occurred in a following context: Context value=something Context value2=something different -xxx/log-formatter-test.cpp:57: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": non sense +xxx/log-formatter-test.cpp:59: error: in "Fake Test Suite Hierarchy/2 test cases inside/bad_foo": non sense xxx/log-formatter-test.cpp:220: Leaving test case "bad_foo" xxx/log-formatter-test.cpp:218: Leaving test suite "2 test cases inside" * 2-format ******************************************************************* -ZZZZZZ +ZZZZZZ * 3-format ******************************************************************* @@ -189,13 +189,13 @@ xxx/log-formatter-test.cpp:218: Leaving test suite "2 test cases inside" @@ -235,13 +235,13 @@ INFO: * 1-format ******************************************************************* Running 3 test cases... -xxx/log-formatter-test.cpp:226: Entering test suite "3 test cases inside" +xxx/log-formatter-test.cpp:222: Entering test suite "3 test cases inside" xxx/log-formatter-test.cpp:223: Entering test case "bad_foo" -xxx/log-formatter-test.cpp:48: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": +xxx/log-formatter-test.cpp:50: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": this is a message -xxx/log-formatter-test.cpp:51: info: check true has passed -xxx/log-formatter-test.cpp:55: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": with some message +xxx/log-formatter-test.cpp:53: info: check true has passed +xxx/log-formatter-test.cpp:57: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": with some message Failure occurred in a following context: Context value=something Context value2=something different -xxx/log-formatter-test.cpp:57: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": non sense +xxx/log-formatter-test.cpp:59: error: in "Fake Test Suite Hierarchy/3 test cases inside/bad_foo": non sense xxx/log-formatter-test.cpp:223: Leaving test case "bad_foo" xxx/log-formatter-test.cpp:224: Entering test case "very_bad_foo" -xxx/log-formatter-test.cpp:62: fatal error: in "Fake Test Suite Hierarchy/3 test cases inside/very_bad_foo": very_bad_foo is fatal +xxx/log-formatter-test.cpp:64: fatal error: in "Fake Test Suite Hierarchy/3 test cases inside/very_bad_foo": very_bad_foo is fatal Failure occurred in a following context: some context xxx/log-formatter-test.cpp:224: Leaving test case "very_bad_foo" xxx/log-formatter-test.cpp:226: Test case "Fake Test Suite Hierarchy/3 test cases inside/bad_foo2" is skipped because dependency test case "Fake Test Suite Hierarchy/3 test cases inside/very_bad_foo" has failed -xxx/log-formatter-test.cpp:226: Leaving test suite "3 test cases inside" +xxx/log-formatter-test.cpp:222: Leaving test suite "3 test cases inside" * 2-format ******************************************************************* -ZZZZZZ +ZZZZZZ * 3-format ******************************************************************* @@ -291,13 +291,13 @@ xxx/log-formatter-test.cpp:226: Leaving test suite "3 test cases inside" @@ -331,7 +331,7 @@ INFO: ZZZZZZZZZZZZ +ZZZZZZZZZZZZ * 3-format ******************************************************************* @@ -454,13 +454,13 @@ xxx/log-formatter-test.cpp:230: Leaving test suite "4 test cases inside" @@ -494,7 +494,7 @@ INFO: @@ -589,13 +589,13 @@ INFO: ZZZZZZZZZZZZZZZZZZZZZZZZ +ZZZZZZZZZZZZZZZZZZZZZZZZ * 3-format ******************************************************************* @@ -785,13 +785,13 @@ xxx/log-formatter-test.cpp:236: Leaving test suite "Fake Test Suite Hierarchy" @@ -833,13 +833,13 @@ INFO: @@ -894,13 +894,13 @@ INFO: @@ -934,7 +934,7 @@ INFO: @@ -1031,13 +1031,13 @@ INFO: @@ -1062,13 +1062,13 @@ ASSERTION FAILURE: +* 1-format ******************************************************************* +Running 2 test cases... +xxx/log-formatter-test.cpp:243: Entering test suite "Timeout" +xxx/log-formatter-test.cpp:244: Entering test case "good_foo" +Test case Timeout/good_foo did not check any assertions +xxx/log-formatter-test.cpp:244: Leaving test case "good_foo" +xxx/log-formatter-test.cpp:245: Entering test case "timeout_foo" +unknown location:0: fatal error: in "Timeout/timeout_foo": fake timeout +xxx/log-formatter-test.cpp:245: Leaving test case "timeout_foo" +xxx/log-formatter-test.cpp:243: Leaving test suite "Timeout" + +* 2-format ******************************************************************* +ZZZZZZ +* 3-format ******************************************************************* + + + + + + + + + +* 3-format ******************************************************************* + + + + + + + + +* 1-format ******************************************************************* +Running 4 test cases... +xxx/log-formatter-test.cpp:248: Entering test suite "Timeout-nested" +xxx/log-formatter-test.cpp:249: Entering test case "good_foo" +Test case Timeout-nested/good_foo did not check any assertions +xxx/log-formatter-test.cpp:249: Leaving test case "good_foo" +xxx/log-formatter-test.cpp:250: Entering test suite "Timeout" +xxx/log-formatter-test.cpp:251: Entering test case "good_foo" +Test case Timeout-nested/Timeout/good_foo did not check any assertions +xxx/log-formatter-test.cpp:251: Leaving test case "good_foo" +xxx/log-formatter-test.cpp:252: Entering test case "timeout_foo" +unknown location:0: fatal error: in "Timeout-nested/Timeout/timeout_foo": fake timeout +xxx/log-formatter-test.cpp:252: Leaving test case "timeout_foo" +xxx/log-formatter-test.cpp:250: Leaving test suite "Timeout" +xxx/log-formatter-test.cpp:255: Entering test case "good_foo2" +Test case Timeout-nested/good_foo2 did not check any assertions +xxx/log-formatter-test.cpp:255: Leaving test case "good_foo2" +xxx/log-formatter-test.cpp:248: Leaving test suite "Timeout-nested" + +* 2-format ******************************************************************* +ZZZZZZZZZZZZ +* 3-format ******************************************************************* + + + + + + + + + + + + + + + +* 3-format ******************************************************************* + + + + + + + + + + + + diff --git a/test/baseline-outputs/log-formatter-test.pattern.junit b/test/baseline-outputs/log-formatter-test.pattern.junit index 11f2a2dd73..ea48c2df4d 100644 --- a/test/baseline-outputs/log-formatter-test.pattern.junit +++ b/test/baseline-outputs/log-formatter-test.pattern.junit @@ -10,14 +10,47 @@ 3 is not satisfied [2 <= 3] MESSAGE: - file : boost.test framework -- line : 226 +- line : 244 - message: Test case 1 test cases inside/almost_good_foo did not check any assertions ]]> +* 3-format ******************************************************************* + + + + + + + + + + + + + + + diff --git a/test/baseline-outputs/result-report-test.pattern b/test/baseline-outputs/result-report-test.pattern index 6cb8fb59fe..bb6087bd53 100644 --- a/test/baseline-outputs/result-report-test.pattern +++ b/test/baseline-outputs/result-report-test.pattern @@ -13,9 +13,9 @@ Test suite "Fake Test Suite Hierarchy/1 test cases inside" has passed with: Test case "Fake Test Suite Hierarchy/1 test cases inside/good_foo" has passed ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** No errors detected ************************************************************************* @@ -37,9 +37,9 @@ Test suite "1 bad test case inside" has passed with: 1 expected failure ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** No errors detected ************************************************************************* @@ -58,9 +58,9 @@ Test suite "1 almost good test case inside" has passed with: 1 failed warning ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** No errors detected ************************************************************************* @@ -84,9 +84,9 @@ Test suite "Fake Test Suite Hierarchy/2 test cases inside" has passed with: 1 expected failure ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** 2 failures are detected in the test suite "Fake Test Suite Hierarchy/3 test cases inside" ************************************************************************* @@ -113,9 +113,9 @@ Test suite "Fake Test Suite Hierarchy/3 test cases inside" has failed with: Test case "Fake Test Suite Hierarchy/3 test cases inside/bad_foo2" was skipped ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** 2 failures are detected (1 failure is expected) in the test suite "Fake Test Suite Hierarchy" ************************************************************************* @@ -159,9 +159,9 @@ Test suite "Fake Test Suite Hierarchy" has failed with: Test suite "Fake Test Suite Hierarchy/3 test cases inside" was skipped ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* *** 1 failure is detected in the test suite "Char escaping" ************************************************************************* @@ -184,6 +184,65 @@ Test suite "Char escaping" has failed with: 1 assertion out of 1 failed ************************************************************************* -************************************************************************* -************************************************************************* -************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* + +*** 1 failure is detected in the test suite "Timeout" +************************************************************************* + +Test suite "Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + +************************************************************************* + +Test suite "Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + + Test case "Timeout/good_foo" has passed + + Test case "Timeout/timeout_foo" has timed out with: + 1 assertion out of 1 failed + +************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* + +*** 1 failure is detected in the test suite "Timeout-nested" +************************************************************************* + +Test suite "Timeout-nested" has failed with: + 3 test cases out of 4 passed + 1 test case out of 4 timed-out + 1 assertion out of 1 failed + +************************************************************************* + +Test suite "Timeout-nested" has failed with: + 3 test cases out of 4 passed + 1 test case out of 4 timed-out + 1 assertion out of 1 failed + + Test case "Timeout-nested/good_foo" has passed + + Test suite "Timeout-nested/Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + + Test case "Timeout-nested/Timeout/good_foo" has passed + + Test case "Timeout-nested/Timeout/timeout_foo" has timed out with: + 1 assertion out of 1 failed + + Test case "Timeout-nested/good_foo2" has passed + +************************************************************************* +************************************************************************* +************************************************************************* +************************************************************************* diff --git a/test/baseline-outputs/result_report_test.pattern.default_behaviour b/test/baseline-outputs/result_report_test.pattern.default_behaviour index b59ba30cc9..733143b266 100644 --- a/test/baseline-outputs/result_report_test.pattern.default_behaviour +++ b/test/baseline-outputs/result_report_test.pattern.default_behaviour @@ -22,11 +22,11 @@ Test suite "Fake Test Suite Hierarchy/1 test cases inside" has passed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -56,11 +56,11 @@ Test suite "1 bad test case inside" has passed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -87,11 +87,11 @@ Test suite "1 almost good test case inside" has passed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -123,11 +123,11 @@ Test suite "Fake Test Suite Hierarchy/2 test cases inside" has passed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -162,11 +162,11 @@ Test suite "Fake Test Suite Hierarchy/3 test cases inside" has failed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -218,11 +218,11 @@ Test suite "Fake Test Suite Hierarchy" has failed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - + * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** @@ -253,8 +253,87 @@ Test suite "Char escaping" has failed with: * NO_REPORT ********************************************************************* * CONFIRMATION_REPORT *********************************************************** - + * SHORT_REPORT ****************************************************************** - + * DETAILED_REPORT *************************************************************** - \ No newline at end of file + +* NO_REPORT ********************************************************************* + +* CONFIRMATION_REPORT *********************************************************** + +*** 1 failure is detected in the test suite "Timeout" + +* SHORT_REPORT ****************************************************************** + +Test suite "Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + + +* DETAILED_REPORT *************************************************************** + +Test suite "Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + + Test case "Timeout/good_foo" has passed + + Test case "Timeout/timeout_foo" has timed out with: + 1 assertion out of 1 failed + + +* NO_REPORT ********************************************************************* + +* CONFIRMATION_REPORT *********************************************************** + +* SHORT_REPORT ****************************************************************** + +* DETAILED_REPORT *************************************************************** + +* NO_REPORT ********************************************************************* + +* CONFIRMATION_REPORT *********************************************************** + +*** 1 failure is detected in the test suite "Timeout-nested" + +* SHORT_REPORT ****************************************************************** + +Test suite "Timeout-nested" has failed with: + 3 test cases out of 4 passed + 1 test case out of 4 timed-out + 1 assertion out of 1 failed + + +* DETAILED_REPORT *************************************************************** + +Test suite "Timeout-nested" has failed with: + 3 test cases out of 4 passed + 1 test case out of 4 timed-out + 1 assertion out of 1 failed + + Test case "Timeout-nested/good_foo" has passed + + Test suite "Timeout-nested/Timeout" has failed with: + 1 test case out of 2 passed + 1 test case out of 2 timed-out + 1 assertion out of 1 failed + + Test case "Timeout-nested/Timeout/good_foo" has passed + + Test case "Timeout-nested/Timeout/timeout_foo" has timed out with: + 1 assertion out of 1 failed + + Test case "Timeout-nested/good_foo2" has passed + + +* NO_REPORT ********************************************************************* + +* CONFIRMATION_REPORT *********************************************************** + +* SHORT_REPORT ****************************************************************** + +* DETAILED_REPORT *************************************************************** + \ No newline at end of file diff --git a/test/framework-ts/log-formatter-test.cpp b/test/framework-ts/log-formatter-test.cpp index 72e886b9be..46e952e746 100644 --- a/test/framework-ts/log-formatter-test.cpp +++ b/test/framework-ts/log-formatter-test.cpp @@ -17,6 +17,8 @@ #include #include #include +#include + typedef boost::onullstream onullstream_type; // BOOST @@ -76,6 +78,16 @@ void very_bad_exception() { void bad_foo2() { bad_foo(); } // tests with clashing names +void timeout_foo() +{ + using boost::execution_exception; + execution_exception::location dummy; + throw execution_exception( + execution_exception::timeout_error, + "fake timeout", + dummy); +} + //____________________________________________________________________________// void check( output_test_stream& output, @@ -178,6 +190,20 @@ BOOST_AUTO_TEST_CASE( test_logs ) ts_main->add( ts_3 ); ts_main->add( ts_4 ); + test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); + ts_timeout->add( tc_timeout ); + + test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" ); + ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) ); + test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo ); + ts_timeout_internal->add( tc_timeout_internal ); + ts_timeout_nested->add( ts_timeout_internal ); + ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) ); + check( test_output, ts_1 ); check( test_output, ts_1b ); @@ -194,6 +220,10 @@ BOOST_AUTO_TEST_CASE( test_logs ) ts_3->depends_on( ts_1 ); check( test_output, ts_main ); + + check( test_output, ts_timeout ); + + check( test_output, ts_timeout_nested ); } //____________________________________________________________________________// @@ -226,6 +256,13 @@ BOOST_AUTO_TEST_CASE( test_logs_junit_info_closing_tags ) boost::unit_test::framework::impl::setup_for_execution( *ts_main ); check( test_output, OF_JUNIT, ts_main->p_id, log_successful_tests ); + + test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); + ts_timeout->add( tc_timeout ); + + check( test_output, OF_JUNIT, ts_timeout->p_id, log_successful_tests ); } // EOF diff --git a/test/framework-ts/result-report-test.cpp b/test/framework-ts/result-report-test.cpp index dd2057bece..c43bc65cdb 100644 --- a/test/framework-ts/result-report-test.cpp +++ b/test/framework-ts/result-report-test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include typedef boost::onullstream onullstream_type; // BOOST @@ -62,6 +63,20 @@ void very_bad_foo() { BOOST_FAIL( "" ); } +void timeout_foo() +{ + log_guard lg; + boost::ignore_unused( lg ); + onullstream_type null_out; + unit_test_log.set_stream( null_out ); + using boost::execution_exception; + execution_exception::location dummy; + throw execution_exception( + execution_exception::timeout_error, + "fake timeout", + dummy); +} + //____________________________________________________________________________// void check( output_test_stream& output, output_format report_format, test_unit_id id ) @@ -159,6 +174,20 @@ BOOST_AUTO_TEST_CASE( test_result_reports ) i_have_problems->p_name.set("bad_foo"); ts_char_escaping->add( i_have_problems ); + test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); + ts_timeout->add( tc_timeout ); + + test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" ); + ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) ); + test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo ); + ts_timeout_internal->add( tc_timeout_internal ); + ts_timeout_nested->add( ts_timeout_internal ); + ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) ); + check( test_output, ts_1 ); check( test_output, ts_1b ); @@ -175,6 +204,10 @@ BOOST_AUTO_TEST_CASE( test_result_reports ) check( test_output, ts_char_escaping ); + check( test_output, ts_timeout ); + + check( test_output, ts_timeout_nested ); + results_reporter::set_stream( std::cout ); } @@ -313,6 +346,20 @@ BOOST_AUTO_TEST_CASE( test_result_reports_default_behaviour ) i_have_problems->p_name.set("bad_foo"); ts_char_escaping->add( i_have_problems ); + test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); + ts_timeout->add( tc_timeout ); + + test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" ); + ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) ); + test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" ); + ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) ); + test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo ); + ts_timeout_internal->add( tc_timeout_internal ); + ts_timeout_nested->add( ts_timeout_internal ); + ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) ); + check2( test_output, ts_1 ); check2( test_output, ts_1b ); @@ -329,6 +376,10 @@ BOOST_AUTO_TEST_CASE( test_result_reports_default_behaviour ) check2( test_output, ts_char_escaping ); + check2( test_output, ts_timeout ); + + check2( test_output, ts_timeout_nested ); + results_reporter::set_stream( std::cout ); } diff --git a/test/writing-test-ts/test-timeout-fail.cpp b/test/writing-test-ts/test-timeout-fail.cpp new file mode 100644 index 0000000000..e1dea8d2f1 --- /dev/null +++ b/test/writing-test-ts/test-timeout-fail.cpp @@ -0,0 +1,19 @@ +// (C) Copyright Raffi Enficiaud, 2019 +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/test for the library home page. + +#define BOOST_TEST_MODULE timeout-error +#include +#include +#include + +namespace utf = boost::unit_test; + +BOOST_AUTO_TEST_CASE(test_fail, * utf::timeout(1)) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + BOOST_TEST(true); +} diff --git a/test/writing-test-ts/test-timeout-suite-fail.cpp b/test/writing-test-ts/test-timeout-suite-fail.cpp new file mode 100644 index 0000000000..5339026c80 --- /dev/null +++ b/test/writing-test-ts/test-timeout-suite-fail.cpp @@ -0,0 +1,42 @@ +// (C) Copyright Raffi Enficiaud, 2019 +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/test for the library home page. + +#define BOOST_TEST_MODULE timeout-error-suite +#include +#include +#include +#include + +namespace utf = boost::unit_test; + +// each of the test case of the data set has a timeout of 1 sec +// some of them will fail +BOOST_TEST_DECORATOR(* utf::timeout(1)) +BOOST_DATA_TEST_CASE(test_success, utf::data::make({0,1,2,3})) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(100*5*sample)); + BOOST_TEST(sample >= 0); +} + +// the full test suite has a timeout of 1s, some of the data test +// cases will not be executed +BOOST_TEST_DECORATOR(* utf::timeout(1)) + +BOOST_AUTO_TEST_SUITE(test_suite_timeout) +BOOST_DATA_TEST_CASE(test_success, utf::data::make({0,1,2,3,4,5,6,7,8})) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + BOOST_TEST((sample >= 0 && sample < 5)); +} + + +BOOST_AUTO_TEST_CASE(failure_should_not_be_executed) +{ + BOOST_TEST(false); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/writing-test-ts/test-timeout-suite.cpp b/test/writing-test-ts/test-timeout-suite.cpp new file mode 100644 index 0000000000..af327513de --- /dev/null +++ b/test/writing-test-ts/test-timeout-suite.cpp @@ -0,0 +1,33 @@ +// (C) Copyright Raffi Enficiaud, 2019 +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/test for the library home page. + +#define BOOST_TEST_MODULE timeout-error-suite +#include +#include +#include +#include + +namespace utf = boost::unit_test; + +// each of the test case of the data set has a timeout of 1 sec +BOOST_TEST_DECORATOR(* utf::timeout(1)) +BOOST_DATA_TEST_CASE(test_success, utf::data::make({0,1,2,3})) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + BOOST_TEST(sample >= 0); +} + +// the full test suite has a timeout of 1s +BOOST_TEST_DECORATOR(* utf::timeout(1)) + +BOOST_AUTO_TEST_SUITE(test_suite_success) +BOOST_DATA_TEST_CASE(test_success, utf::data::make({0,1,2,3,4,5,6,7,8})) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + BOOST_TEST(sample >= 0); +} +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/test/writing-test-ts/test-timeout.cpp b/test/writing-test-ts/test-timeout.cpp new file mode 100644 index 0000000000..7beb54a4b5 --- /dev/null +++ b/test/writing-test-ts/test-timeout.cpp @@ -0,0 +1,25 @@ +// (C) Copyright Raffi Enficiaud, 2019 +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/test for the library home page. + +#define BOOST_TEST_MODULE timeout-error +#include +#include +#include + +namespace utf = boost::unit_test; + +BOOST_AUTO_TEST_CASE(test1, * utf::timeout(1)) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test2, * utf::timeout(3)) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + BOOST_TEST(true); +}