Skip to content

testrunner: actually bail out on failed assert #7436

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 1 commit into from
Apr 7, 2025
Merged

testrunner: actually bail out on failed assert #7436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions test/fixture.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ void TestFixture::assert_(const char * const filename, const unsigned int linenr
errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
if (!msg.empty())
errmsg << "Hint:" << std::endl << msg << std::endl;
throw AssertFailedError();
}
}

Expand All @@ -183,6 +184,7 @@ void TestFixture::assertFailure(const char* const filename, const unsigned int l
if (!msg.empty())
errmsg << "Hint:" << std::endl << msg << std::endl;
errmsg << "_____" << std::endl;
throw AssertFailedError();
}

void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
Expand Down Expand Up @@ -289,25 +291,28 @@ void TestFixture::assertThrow(const char * const filename, const unsigned int li
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
<< "The expected exception was thrown" << std::endl << "_____" << std::endl;
throw AssertFailedError();
}

void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
{
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
<< "The expected exception was not thrown" << std::endl << "_____" << std::endl;
throw AssertFailedError();
}

void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr, bool bailout) const
{
++fails_counter;

std::string ex_msg;

try {
// cppcheck-suppress rethrowNoCurrentException
throw;
}
catch (const AssertFailedError&) {
return;
}
catch (const InternalError& e) {
ex_msg = e.errorMessage;
}
Expand All @@ -318,8 +323,11 @@ void TestFixture::assertNoThrowFail(const char * const filename, const unsigned
ex_msg = "unknown exception";
}

++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
<< "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;
if (bailout)
throw AssertFailedError();
}

void TestFixture::printHelp()
Expand Down
29 changes: 17 additions & 12 deletions test/fixture.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@

#include "check.h"
#include "color.h"
#include "config.h"
#include "errorlogger.h"
#include "platform.h"
#include "settings.h"
#include "standards.h"

#include <cstddef>
#include <cstdint>
#include <exception>
#include <list>
#include <memory>
#include <sstream>
Expand Down Expand Up @@ -68,6 +70,8 @@ class TestFixture : public ErrorLogger {
virtual void teardownTestInternal() {}
std::string getLocationStr(const char * filename, unsigned int linenr) const;

class AssertFailedError : public std::exception {};

void assert_(const char * filename, unsigned int linenr, bool condition, const std::string& msg = "") const;

template<typename T>
Expand Down Expand Up @@ -112,9 +116,9 @@ class TestFixture : public ErrorLogger {
const char current[], const std::string &actual) const;
void todoAssertEquals(const char * filename, unsigned int linenr, long long wanted,
long long current, long long actual) const;
void assertThrow(const char * filename, unsigned int linenr) const;
void assertThrowFail(const char * filename, unsigned int linenr) const;
void assertNoThrowFail(const char * filename, unsigned int linenr) const;
NORETURN void assertThrow(const char * filename, unsigned int linenr) const;
NORETURN void assertThrowFail(const char * filename, unsigned int linenr) const;
void assertNoThrowFail(const char * filename, unsigned int linenr, bool bailout) const;
static std::string deleteLineNumber(const std::string &message);

void setVerbose(bool v) {
Expand Down Expand Up @@ -278,7 +282,7 @@ class TestFixture : public ErrorLogger {
private:
//Helper function to be called when an assertEquals assertion fails.
//Writes the appropriate failure message to errmsg and increments fails_counter
void assertFailure(const char* filename, unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const;
NORETURN void assertFailure(const char* filename, unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const;

std::ostringstream mOutput;
std::ostringstream mErrout;
Expand Down Expand Up @@ -308,14 +312,13 @@ class TestInstance {
std::unique_ptr<TestFixture> impl;
};

#define TEST_CASE( NAME ) do { if (prepareTest(#NAME)) { setVerbose(false); try { NAME(); teardownTest(); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } } while (false)
#define TEST_CASE( NAME ) do { if (prepareTest(#NAME)) { setVerbose(false); try { NAME(); teardownTest(); } catch (const AssertFailedError&) {} catch (...) { assertNoThrowFail(__FILE__, __LINE__, false); } } } while (false)

// TODO: the asserts do not actually assert i.e. do stop executing the test
#define ASSERT( CONDITION ) assert_(__FILE__, __LINE__, (CONDITION))
#define ASSERT_LOC( CONDITION, FILE_, LINE_ ) assert_(FILE_, LINE_, (CONDITION))
#define ASSERT_LOC_MSG( CONDITION, MSG, FILE_, LINE_ ) assert_(FILE_, LINE_, (CONDITION), MSG)
// *INDENT-OFF*
#define ASSERT_EQUALS( EXPECTED, ACTUAL ) do { try { assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL)); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_EQUALS( EXPECTED, ACTUAL ) do { try { assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL)); } catch(const AssertFailedError&) { throw; } catch (...) { assertNoThrowFail(__FILE__, __LINE__, true); } } while (false)
Copy link
Copy Markdown
Collaborator Author

@firewave firewave Apr 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uncrustify is inconsistent with the whitespaces compared to other macros.

// *INDENT-ON*
#define ASSERT_EQUALS_WITHOUT_LINENUMBERS( EXPECTED, ACTUAL ) assertEqualsWithoutLineNumbers(__FILE__, __LINE__, EXPECTED, ACTUAL)
#define ASSERT_EQUALS_DOUBLE( EXPECTED, ACTUAL, TOLERANCE ) assertEqualsDouble(__FILE__, __LINE__, EXPECTED, ACTUAL, TOLERANCE)
Expand All @@ -327,15 +330,17 @@ class TestInstance {
#define ASSERT_EQUALS_ENUM_LOC_MSG( EXPECTED, ACTUAL, MSG, FILE_, LINE_ ) assertEqualsEnum(FILE_, LINE_, (EXPECTED), (ACTUAL), MSG)
#define TODO_ASSERT_EQUALS_ENUM( WANTED, CURRENT, ACTUAL ) todoAssertEqualsEnum(__FILE__, __LINE__, WANTED, CURRENT, ACTUAL)
#define ASSERT_THROW_EQUALS( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_EQUALS_2( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.what()); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_INTERNAL( CMD, TYPE ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_INTERNAL_EQUALS( CMD, TYPE, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_NO_THROW( CMD ) do { try { (void)(CMD); } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } while (false)
#define TODO_ASSERT_THROW( CMD, EXCEPTION ) do { try { (void)(CMD); } catch (const EXCEPTION&) {} catch (...) { assertThrow(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_EQUALS_2( CMD, EXCEPTION, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const EXCEPTION&e) { assertEquals(__FILE__, __LINE__, EXPECTED, e.what()); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_INTERNAL( CMD, TYPE ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_THROW_INTERNAL_EQUALS( CMD, TYPE, EXPECTED ) do { try { (void)(CMD); assertThrowFail(__FILE__, __LINE__); } catch (const AssertFailedError&) { throw; } catch (const InternalError& e) { assertEqualsEnum(__FILE__, __LINE__, InternalError::TYPE, e.type); assertEquals(__FILE__, __LINE__, EXPECTED, e.errorMessage); } catch (...) { assertThrowFail(__FILE__, __LINE__); } } while (false)
#define ASSERT_NO_THROW( CMD ) do { try { (void)(CMD); } catch (const AssertFailedError&) { throw; } catch (...) { assertNoThrowFail(__FILE__, __LINE__, true); } } while (false)
#define TODO_ASSERT_THROW( CMD, EXCEPTION ) do { try { (void)(CMD); } catch (const AssertFailedError&) { throw; } catch (const EXCEPTION&) {} catch (...) { assertThrow(__FILE__, __LINE__); } } while (false)
#define TODO_ASSERT( CONDITION ) do { const bool condition=(CONDITION); todoAssertEquals(__FILE__, __LINE__, true, false, condition); } while (false)
#define TODO_ASSERT_EQUALS( WANTED, CURRENT, ACTUAL ) todoAssertEquals(__FILE__, __LINE__, WANTED, CURRENT, ACTUAL)

// *INDENT-OFF*
#define REGISTER_TEST( CLASSNAME ) namespace { class CLASSNAME ## Instance : public TestInstance { public: CLASSNAME ## Instance() : TestInstance(#CLASSNAME) {} TestFixture* create() override { impl.reset(new CLASSNAME); return impl.get(); } }; CLASSNAME ## Instance instance_ ## CLASSNAME; }
// *INDENT-ON*

#define PLATFORM( P, T ) do { std::string errstr; assertEquals(__FILE__, __LINE__, true, P.set(Platform::toString(T), errstr, {exename}), errstr); } while (false)

Expand Down
Loading