QUnit.qm

# -*- mode: qore; indent-tabs-mode: nil -*-
#! @file QUnit.qm Qore user module for automatic testing

%requires qore >= 0.9
%new-style

%try-module xml >= 1.3
%define NO_XML
%endtry

%no-child-restrictions
%allow-injection

%requires Util

module QUnit {
    version = "0.4.1";
    desc = "User module for unit testing with dependency injection support";
    author = "Zdenek Behan <zdenek.behan@qoretechnologies.com>";
    url = "http://qore.org";
    license = "MIT";
}

/** @mainpage QUnit Module

    @tableofcontents

    @section qunitintro Introduction to the QUnit Module

    The %QUnit module provides a framework for automated testing.

    It contains base classes for creating test cases and test suites. It also provides a
    dependency injection helper for mocking pre-existing classes without modifying their code.

    It also provides a number of pre-defined testing functions for use in assertions.

    <b>Examples:</b>
    @code{.py}
#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires ../../qlib/QUnit.qm

#%include ./_some_module_to_test

%exec-class QUnitTest

public class QUnitTest inherits QUnit::Test {
    constructor() : Test("QUnitTest", "1.0") {
        addTestCase("What this method is testing", \testMethod(), NOTHING);
        addTestCase("Skipped test", \testSkipped(), NOTHING);

        # Return for compatibility with test harness that checks return value.
        set_return_value(main());
    }

    testMethod() {
        # Test against success
        testAssertion("success", \equals(), (True, True));
        # Test against something else
        testAssertion("failure", \equals(), (True, False), RESULT_FAILURE);
    }

    testSkipped() {
        # Skip this test
        testSkip("Because of the reason it skipped");
    }
}
    @endcode

    @code{.py}
#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires ../../qlib/QUnit.qm

#%include ./_MODULE_TO_TEST_

%exec-class MyTestClass

public class MyTestClass inherits QUnit::DependencyInjectedTest {
    constructor() : DependencyInjectedTest("MyTestClass", "1.0") {
        addTestCase("Test something", \testMethod(), NOTHING);

        # Mandatory for injected tests
        setScriptPath(get_script_path());
        # Return for compatibility with test harness that checks return value.
        set_return_value(main());
    }

    nothing performModuleInjections() {
        #injectIntoModule("_MODULE_TO_INJECT_INTO_(same as _MODULE_TO_TEST_?)");
    }

    nothing performInjections(Program p) {
        #p.importClass("_class_to_inject_", "_class_to_inject_into_", True);
    }

    testMethod() {
        # Same test style as in TestExample.qtest
        testAssertion("success", \equals(), (True, True));
    }
}
    @endcode

    @subsection qunit_run Running tests

    Tests are ran by simply executing the test script:
    @verbatim
    qore test.qtest [OPTIONS]
    @endverbatim

    A number of options is available, controlling the behaviour/output of the test
    @subsection unittestformats Supported output formats of test reports

    Currently the module provides the following output formats:
    - plainquiet - human readable quiet format, prints only failures and a short summary at the end, which is also the default
    - plaintext - human readable format, prints one statement per test
    - junit - machine readable format for further processing

    @section unittest_relnotes Release Notes

    @subsection qunit_v0_4_1 Version 0.4.1
    - allow tests to be nested
      (<a href="https://github.com/qorelanguage/qore/issues/3306">issue 3306</a>)

    @subsection qunit_v0_4 Version 0.4
    - updated @ref QUnit::Test::testSkip() "Test::testSkip()" to use the reason argument a format string with
      @ref Qore::vsprintf() "vsprintf()"
      (<a href="https://github.com/qorelanguage/qore/issues/3172">issue 3172</a>)
    - fixed error reporting with type errors with number values
      (<a href="https://github.com/qorelanguage/qore/issues/2984">issue 2984</a>)

    @subsection qunit_v0_3_3 Version 0.3.3
    - improved output in assertion failures for strings with special whitespace and for multi-line data structures (<a href="https://github.com/qorelanguage/qore/issues/2680">issue 2680</a>)

    @subsection qunit_v0_3_2 Version 0.3.2
    - improved error location reporting by providing all stack location information up until the QUnit call to cover the case when multiple code layers are used such as one or more test modules (<a href="https://github.com/qorelanguage/qore/issues/1720">issue 1720</a>)
    - overloaded testAssertionValue supports auto/number/float
    - more verbose output when number/float difference is found

    @subsection qunit_v0_3_1 Version 0.3.1
    - added the following methods:
      - @ref QUnit::Test::assertRegex()
      - @ref QUnit::Test::assertNRegex()
      - @ref QUnit::Test::assertNeq()
      - @ref QUnit::Test::assertNeqSoft()
      - @ref QUnit::Test::assertGt()
      - @ref QUnit::Test::assertGtSoft()
      - @ref QUnit::Test::assertGe()
      - @ref QUnit::Test::assertGeSoft()
      - @ref QUnit::Test::assertLt()
      - @ref QUnit::Test::assertLtSoft()
      - @ref QUnit::Test::assertLe()
      - @ref QUnit::Test::assertLeSoft()
      - @ref QUnit::Test::assertNothing()

    @subsection qunit_v0_3 Version 0.3
    - updated for complex types

    @subsection qunit_v0_2 Version 0.2
    - fixed showing the assertion location when there are test modules on top of QUnit.qm (<a href="https://github.com/qorelanguage/qore/issues/1046">issue 1046</a>)

    @subsection qunit_v0_1 Version 0.1
    - initial version of module
*/

#! the main namespace for all public definitions in the %QUnit module
public namespace QUnit;

#! An abstract class representing test result interface
public class QUnit::AbstractTestResult {
    public {
        string m_type;
        *string m_subType;
        auto m_value;
    }
    constructor(string type) {
        m_type = type;
    }
    constructor(string type, string subType) {
        m_type = type;
        m_subType = subType;
    }
    # test
    abstract public bool equals(AbstractTestResult r);
    abstract public string toString();
}

#! Class representing boolean True
public class QUnit::TestResultSuccess inherits QUnit::AbstractTestResult {
    constructor() : AbstractTestResult("True") {
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        return r.m_type == "True";
    }

    public string toString() {
        return "Success";
    }
}

#! Class representing test function failure, both unspecific and with detail
public class QUnit::TestResultFailure inherits QUnit::AbstractTestResult {
    private {
        *string m_detail;
    }

    #! Instantiate an unspecific failure, no detail.
    constructor() : AbstractTestResult("False") {
    }

    #! Instantiate an annotated failure, string detail.
    constructor(string s) : AbstractTestResult("False") {
        m_detail = s;
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        # All failures are equal. Annotation is just for description. For structured errors, Exceptions should be used.
        return r.m_type == "False";
    }

    public string toString() {
        return "Failure" + (m_detail ? ": " + m_detail : "");
    }
}

#! Class representing any non-boolean value
public class QUnit::TestResultValue inherits QUnit::AbstractTestResult {
    constructor(auto value) : AbstractTestResult("Value") {
        # treat NULL as NOTHING
        if (value === NULL)
            return;
        m_value = value;
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        return ((r.m_type == "Value") && (r.m_value === m_value));
    }

    public string toString() {
        switch (m_value.typeCode()) {
            case NT_LIST:
            case NT_HASH:
            case NT_BINARY:
                return sprintf("%N", m_value);
        }
        return sprintf("%s: %N", m_value.type(), m_value);
    }
}

#! Class representing a partial match of a hash value. Common keys have to be identical.
public class QUnit::TestResultPartialHashMatch inherits QUnit::AbstractTestResult {
    constructor(hash value) : AbstractTestResult("PartialHashMatch") {
        if (value == NULL)
            return;
        m_value = value;
    }

    public bool equals(QUnit::AbstractTestResult r) {
        # Check type
        if (r.m_type == "Value") {
            # For Value results, only match against hashes. The others make no sense.
            if (r.m_value.typeCode() != NT_HASH) {
                return False;
            }
        } else if (!r.m_type == "PartialHashMatch") {
            return False;
        }
        # All keys from this hash must be present in r and match.
        foreach string key in (m_value.keyIterator()) {
            if (!r.m_value.hasKey(key))
                return False;

            if (r.m_value{key} != m_value{key})
                return False;
        }
        return True;
    }

    public string toString() {
        return sprintf("Partial Hash match: %N", m_value);
    }
}

#! Class representing Exception of a particular type
public class QUnit::TestResultExceptionType inherits QUnit::AbstractTestResult {
    public {
        #! corresponds to the \c "err" key of @ref Qore::ExceptionInfo "ExceptionInfo" (the first value of a @ref throw "throw statement")
        string m_exceptionType;
    }

    #! creates the object from the exception arguments
    /** @param exceptionType corresponds to the \c "err" key of @ref Qore::ExceptionInfo "ExceptionInfo" (the first value of a @ref throw "throw statement")
    */
    constructor(string exceptionType) : AbstractTestResult("Exception") {
        m_exceptionType = exceptionType;
    }

    #! private constructor for subclasses
    private constructor(string exceptionType, string subType) : AbstractTestResult("Exception", subType) {
        m_exceptionType = exceptionType;
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        if (r.m_type != "Exception") {
            return False;
        }
        QUnit::TestResultExceptionType e = r;
        if (e.m_exceptionType != m_exceptionType) {
            return False;
        }
        return True;
    }

    #! returns a string describing the exception type
    public string toString() {
        return sprintf("Exception: %s", m_exceptionType);
    }
}

#! Class representing Exception of a particular type with a particular detail message
public class QUnit::TestResultExceptionDetail inherits QUnit::TestResultExceptionType {
    private {
        string m_exceptionDetail;
    }

    #! creates the object from the exception arguments
    /** @param exceptionType corresponds to the \c "err" key of @ref Qore::ExceptionInfo "ExceptionInfo"
        @param exceptionDetail corresponds to the \c "desc" key of @ref Qore::ExceptionInfo "ExceptionInfo"
    */
    constructor(string exceptionType, string exceptionDetail) : TestResultExceptionType(exceptionType, "Detail")  {
        m_exceptionDetail = exceptionDetail;
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        if (!TestResultExceptionType::equals(r)) {
            return False;
        }
        if (r.m_subType == "Regexp") {
            /* Ensure reflectiveness of regexp == operation.
               Note that this will only work if this == Detail and r == Regexp, otherwise we could get into infinite recursion.
             */
            return r.equals(self);
        }
        if (r.m_subType != "Detail") {
            return False;
        }
        QUnit::TestResultExceptionDetail e = cast<QUnit::TestResultExceptionDetail>(r);
        if (e.m_exceptionDetail != m_exceptionDetail) {
            return False;
        }
        return True;
    }

    public string toString() {
        return sprintf("Exception: %s: Detail '%s'", m_exceptionType, m_exceptionDetail);
    }

    string getDetail() {
        return m_exceptionDetail;
    }
}

#! Class representing Exception of a particular type and matching regexp for detail
public class QUnit::TestResultExceptionRegexp inherits QUnit::TestResultExceptionType {
    private {
        string m_exceptionRegexp;
    }

    constructor(string exceptionType, string exceptionRegexp) : TestResultExceptionType(exceptionType, "Regexp")  {
        m_exceptionRegexp = exceptionRegexp;
    }

    #! Equality operator
    public bool equals(QUnit::AbstractTestResult r) {
        if (!TestResultExceptionType::equals(r)) {
            return False;
        }
        if (r.m_subType != "Detail") {
            return False;
        }
        QUnit::TestResultExceptionDetail e = cast<QUnit::TestResultExceptionDetail>(r);
        if (!e.getDetail().regex(m_exceptionRegexp)) {
            return False;
        }
        return True;
    }

    public string toString() {
        return sprintf("Exception: %s: Regexp '%s'", m_exceptionType, m_exceptionRegexp);
    }
}

#! Class representing Exception of a particular type and substring for detail
public class QUnit::TestResultExceptionSubstring inherits QUnit::TestResultExceptionRegexp {
    constructor(string exceptionType, string exceptionSubstring) : TestResultExceptionRegexp(exceptionType, exceptionSubstring) {
    }

    public string toString() {
        return sprintf("Exception: %s: Substring '%s'", m_exceptionType, m_exceptionRegexp);
    }
}

#! Class containing the configuration for a test case
public class QUnit::TestCase {
    private {
        #! the name of the test case
        string m_name;

        #! the body of the test case
        code m_code;

        #! arguments to the above callable object
        *list<auto> m_args;

        #! number of assertions in current test case
        int num_asserts = 0;

        #! number of successful assertions in current test case
        int num_asserts_ok = 0;

        #! number of skipped assertions in current test case
        int num_asserts_skip = 0;

        #! any saved test case
        auto saved_tc;
    }

    #! creates the TestCase object from the given arguments
    constructor(string name, code code, *softlist<auto> args) {
        m_name = name;
        m_code = code;
        m_args = args;
    }

    #! runs the TestCase
    run(QUnit::Test test) {
        # Attempt to run the test function
        try {
            setupThread();
            on_exit restoreThread();
            # User code setup
            test.setUp();
            # User code cleanup
            on_exit test.tearDown();
            # Main test routine
            call_function_args(m_code, m_args);
            # Test success
            test.addTestResult(self, TestReporter::TEST_SUCCESS);
        } catch (hash<ExceptionInfo> e) {
            checkException(test, e);
        }
    }

    static list<string> getStackList(list stack, bool ok = False) {
        list<string> l();
        foreach hash callSite in (stack) {
            if (callSite.file.val() && callSite.line != -1) {
                if (callSite.file =~ /QUnit.qm$/) {
                    ok = True;
                    continue;
                }
                if (!ok)
                    continue;
                string loc = callSite.file;
                int ln = callSite.line + callSite.offset;
                if (ln > 0)
                    loc += sprintf(":%d", ln);
                # try to add function/method call name
                if (*string f = stack[$# + 1].function)
                    loc = sprintf("%s [%s()]", loc, f);
                l += loc;
            }
        }
        return l;
    }

    static string getPos(hash<auto> ex) {
        string pos = get_ex_pos(ex);
        bool qunit = (pos =~ /QUnit\.qm/);
        list<string> l = TestCase::getStackList(ex.callstack, qunit);
        if (!qunit)
            unshift l, pos;
        return (foldl $1 + " <- " + $2, l) ?? "<unknown>";
    }

    #! handles exceptions raised while running the TestCase
    checkException(QUnit::Test test, hash<auto> e) {
        if (e.err =~ /TEST-.*EXCEPTION/) {
            *string assertion_name = e.arg.name;
            *string pos = e.arg.pos;
            if (!pos)
                pos = TestCase::getPos(e);
            if (e.err == "TEST-SKIPPED-EXCEPTION") {
                string txt = sprintf("Skip %srequested", assertion_name ? sprintf("of %y ", assertion_name) : NOTHING);
                test.addTestResult(self, TestReporter::TEST_SKIPPED, txt, pos, e.desc);
            } else if (e.err == "TEST-EXCEPTION") {
                string txt = sprintf("Assertion %sfailure", assertion_name ? sprintf("%y ", assertion_name) : NOTHING);
                test.addTestResult(self, TestReporter::TEST_FAILURE, txt, pos, e.desc);
            }
        } else {
            # show little higher traceback record in this case
            string pos = get_ex_pos(e);
            string errtxt = sprintf("Unexpected exception: %s", e.err);
            test.addTestResult(self, TestReporter::TEST_ERROR, errtxt, pos, get_exception_string(e));
        }
    }

    #! returns the test case name
    string getName() {
        return m_name;
    }

    incAssertions() {
        ++num_asserts;
    }

    incAssertionsOk() {
        ++num_asserts_ok;
    }

    incAssertionsSkip() {
        ++num_asserts_skip;
    }

    int getAssertionCount() {
        return num_asserts;
    }

    int getAssertionOkCount() {
        return num_asserts_ok;
    }

    int getAssertionSkipCount() {
        return num_asserts_skip;
    }

    setupThread() {
        saved_tc = remove_thread_data("tc").tc;
        save_thread_data("tc", self);
    }

    restoreThread() {
        save_thread_data("tc", saved_tc);
    }

    #! renames the test case
    rename(string n_name) {
        m_name = n_name;
    }
}

#! Base class for collecting test results and reporting
public class QUnit::TestReporter {
    private {
        const PLAINQUIET = 0;
        const PLAIN = 1;
        const JUNIT = 2;

        const OUT_TYPES = (
            "plainquiet" : PLAINQUIET,
            "plain" : PLAIN,
            "junit" : JUNIT,
        );

        const RESULT_TYPE_DESCRIPTION = (
            TEST_SUCCESS : ("desc": "Success", "junittag" : NOTHING), # Success does not have a junit child tag
            TEST_FAILURE : ("desc": "FAILURE", "junittag" : "failure"),
            TEST_ERROR   : ("desc": "ERROR", "junittag" : "error"),
            TEST_SKIPPED : ("desc": "Skipped", "junittag" : "skipped"),
        );

        #! the default column offset for printing options used in printOption() and usageIntern()
        const OffsetColumn = 20;

        #! A map of print methods, categorised into three types: header, summary, testreport
        hash m_printMethods;

        list m_results = ();
        int m_output;
        *string m_comment;

        #! the result of parsing command-line options with @ref Qore::GetOpt::parse2()
        /** @see
            - Opts
            - constructor()
        */
        hash m_options;

        #! test case name
        string m_name;
        #! test case version
        string m_version;

        #! total number of assertions in script
        int num_asserts = 0;

        #! total number of successful assertions in script
        int num_asserts_ok = 0;

        #! total number of skipped assertions in script
        int num_asserts_skip = 0;
    }

    public {
        #! default options for @ref Qore::GetOpt::constructor()
        const Opts = (
            "help"    : "h,help",
            "verbose" : "v,verbose:i+",
            "quiet"   : "q,quiet",
            "format"  : "format=s",
            );

        const TEST_SUCCESS = 0;
        const TEST_FAILURE = 1;
        const TEST_ERROR = 2;
        const TEST_SKIPPED = 3;
    }

    private printOption(string left, string right, int offset = OffsetColumn) {
        printf(" %s", left);
        int blanks = offset - left.size();
        if (blanks > 0)
            print(strmul(" ", blanks));
        printf("%s\n", right);
    }

    private usageIntern(int offset = OffsetColumn) {
        printf("usage: %s [options]\n", get_script_name());
        printOption("-h,--help", "this help text", offset);
        printOption("   --format=type", "output format [default: plainquiet]", offset);
        print("        Format descriptions:
           plainquiet  only print failed tests and a summary at the end
           plain       print a status for each test performed
           junit       print a junit xml output\n");
        printOption("-v,--verbose", "shorthand for --format=plain", offset);
        printOption("-q,--quiet", "shorthand for --format=plainquiet", offset);
    }

    private usage() {
        usageIntern();
        exit(1);
    }

    private processOptions(reference<list<string>> p_argv) {
        if (m_options.help)
            usage();

        if (m_options.verbose && m_options.quiet && m_options.format) {
            printf("Please only select one format type\n");
            exit(1);
        }

        if (p_argv) {
            printf("Warning: excess arguments on command-line\n");
        }

        if (m_options.format) {
            if (!exists OUT_TYPES{m_options.format})
                throw "UNIT-TEST-ERROR", sprintf("Unknown output format: %s", m_options.format);
            m_output = OUT_TYPES{m_options.format};
        } else if (m_options.quiet) {
            m_output = PLAINQUIET;
        } else if (m_options.verbose) {
            m_output = PLAIN;
        } else {
            # Default
            m_output = PLAINQUIET;
        }
    }

    #! creates the object from the arguments
    /** @param name the name of the test
        @param version the version of the test
        @param p_argv an optional reference to a list of command-line arguments
        @param opts the option hash to be passed to @ref Qore::GetOpt::constructor()
    */
    constructor(string name, string version, *reference<list<string>> p_argv, hash opts = Opts) {
        m_name = name;
        m_version = version;
        if (!p_argv)
            p_argv = cast<list<string>>(ARGV);
        m_options = new GetOpt(opts).parse2(\p_argv);
        processOptions(\p_argv);

        # A map of print methods, categorised into three types: header, summary, testreport
        m_printMethods = (
            PLAINQUIET: (
                "header" : \self.printPlaintextHeader(),
                "testreport" : \self.printPlaintextOneTest(),
                "summary" : \self.printPlaintextSummary(),
            ),
            PLAIN: (
                "header" : \self.printPlaintextHeader(),
                "testreport" : \self.printPlaintextOneTest(),
                "summary" : \self.printPlaintextSummary(),
            ),
            JUNIT: (
                "summary" : \self.printJunitSummary(),
            )
        );
    }

    private callPrinterFunction(string type, *softlist<auto> args) {
        *code c = m_printMethods{m_output}{type};
        if (c) {
            call_function_args(c, args);
        }
    }

    private printHeader() { callPrinterFunction("header"); }
    private printSummary() { callPrinterFunction("summary"); }
    private printTestReport(hash testcase) { callPrinterFunction("testreport", testcase); }

    private printPlaintextHeader() {
        printf("QUnit Test %y v%s\n", m_name, m_version);
    }

    private printPlaintextSummary() {
        int err = errors();
        string serrors = sprintf(", %d error", err, err == 1 ? "" : "s");
        string sskipped = sprintf(", %d skipped", skipped());
        int succeeded = testCount() - errors() - skipped();
        string ssucceeded = sprintf(", %d succeeded", succeeded);
        string assert_str = sprintf(" (%d assertion%s", num_asserts, num_asserts == 1 ? "" : "s");
        if (num_asserts != num_asserts_ok)
            assert_str += sprintf(", %d succeeded", num_asserts_ok);
        if (num_asserts_skip > 0)
            assert_str += sprintf(", %d skipped", num_asserts_skip);
        assert_str += ")";
        int cnt = testCount();
        printf("Ran %d test case%s%s%s%s%s\n", cnt, cnt == 1 ? "" : "s", succeeded > 0 ? ssucceeded : "", errors() > 0 ? serrors : "", skipped() > 0 ? sskipped : "", assert_str);
    }

    private printPlaintextOneTest(hash testcase) {
        if (m_output == PLAINQUIET && (testcase.result == TEST_SUCCESS)) {
            return;
        }
        string resultType = RESULT_TYPE_DESCRIPTION{testcase.result}{"desc"};

        printf("%s: %s: %d assertion%s", resultType, testcase.testcase.getName(), testcase.testcase.getAssertionCount(), testcase.testcase.getAssertionCount() == 1 ? "" : "s");
        if (testcase.testcase.getAssertionCount() != testcase.testcase.getAssertionOkCount())
            printf(", %d succeeded", testcase.testcase.getAssertionOkCount());
        if (testcase.testcase.getAssertionSkipCount() > 0)
            printf(", %d skipped", testcase.testcase.getAssertionSkipCount());
        print("\n");
        if (testcase.result != TEST_SUCCESS) {
            string out = "";
            printf("%s at %s\n", testcase.error, testcase.pos);
            ListIterator errorDescription(testcase.detail.split("\n"));
            while (errorDescription.next()) {
                out += sprintf("\t>> %s\n", errorDescription.getValue());
            }
            printf("-----\n%s-----\n", out);
        }
    }

    private printJunitSummary() {
%ifdef NO_XML
        stderr.printf("No XML module found. Use --format=plain instead.\n");
        exit(1);
%else
        hash junit;
        junit.testsuites."^attributes^" = (
            "name" : m_name,
            "tests"  : testCount(),
            "errors" : errors(),
            "skipped" : skipped(),
        );
        junit.testsuites.testcase = ();

        ListIterator it(m_results);
        while (it.next()) {
            hash testcase;

            testcase."^attributes^".name = it.getValue().testcase.getName();
            string errmsg = sprintf("%s at %s:\n%s", it.getValue().error, it.getValue().pos, it.getValue().detail);

            int status = it.getValue().result;
            testcase."^attributes^"."status" = RESULT_TYPE_DESCRIPTION{status}.desc;

            if (status != TEST_SUCCESS) {
                string tag = RESULT_TYPE_DESCRIPTION{status}.junittag;
                hash result = ("message" : errmsg, "type" : RESULT_TYPE_DESCRIPTION{status}.desc );
                testcase{tag}."^attributes^" = result;
            }
            push junit."testsuites"."testcase", testcase;
        }

        printf("%s\n", make_xml(junit, XGF_ADD_FORMATTING));
%endif
    }

    #! returns the number of errors encountered during test execution
    int errors() {
        int errs = 0;
        ListIterator it(m_results);
        while (it.next()) {
            if (!(TEST_SUCCESS: "", TEST_SKIPPED: "").hasKey(it.getValue(){"result"})) {
                errs++;
            }
        }
        return errs;
    }

    #! returns the number of tests skipped
    int skipped() {
        int errs = 0;
        ListIterator it(m_results);
        while (it.next()) {
            if (!it.getValue(){"result"} == TEST_SKIPPED) {
                errs++;
            }
        }
        return errs;
    }

    #! returns the total number of test results
    int testCount() {
        return m_results.lsize();
    }

    #! adds a test result
    addTestResult(TestCase tc, int success, *string error, *string pos, *string detail) {
        if (!(RESULT_TYPE_DESCRIPTION.hasKey(success))) {
            throw "TESTING-EXCEPTION", "Invalid test result type!";
        }
        hash testcase = ("testcase": tc, "result": success, "error": error, "pos": pos, "detail": detail);

        printTestReport(testcase);

        push m_results, testcase;
    }
}

#! Base class representing a simple test, implements an implicit main() function and all utility functions for testing
public class QUnit::Test inherits QUnit::TestReporter {
    private {
        #! list of test cases
        list<TestCase> testCases();

        static QUnit::TestResultSuccess RESULT_SUCCESS = new QUnit::TestResultSuccess();
        static QUnit::TestResultFailure RESULT_FAILURE = new QUnit::TestResultFailure();
    }

    private:internal {
        #! equality comparitor
        const QUC_EQ = "eq";
        #! inequality comparitor
        const QUC_NEQ = "neq";
        #! greater than comparitor
        const QUC_GT = "gt";
        #! greater than or equals comparitor
        const QUC_GE = "ge";
        #! greater than comparitor
        const QUC_LT = "lt";
        #! greater than or equals comparitor
        const QUC_LE = "le";
        #! regex comparitor
        const QUC_RE = "re";
        #! negative regex comparitor
        const QUC_NRE = "nre";

        #! comparitor info
        const QUC_Map = (
            QUC_EQ: NOTHING,
            QUC_NEQ: "Not Equal",
            QUC_GT: "Greater Than",
            QUC_GE: "Greater Than Or Equal To",
            QUC_LT: "Less Than",
            QUC_LE: "Less Than Or Equal To",
            QUC_RE: "Regular Expression Match",
            QUC_NRE: "Negative Regular Expression Match",
        );

        #! default epsilon for number/float comparison
        const DEFAULT_EPSILON = 0.0000000001;
    }

    #! creates the object and sets the name of the test
    constructor(string name, string version, *reference<list<string>> p_argv, *hash opts) : QUnit::TestReporter(name, version, \p_argv, opts) {
    }

    #! global setup; will be called once before tests are run
    globalSetUp() {}

    #! global tear down; will be called once after all tests are run
    globalTearDown() {}

    #! Prototype function for setting up test environment. It will be called for each test individually.
    setUp() {}

    #! Prototype function for cleaning up test environemnt. It will be called after each test has executed.
    tearDown() {}

    #! facade for the system environment variables
    *string getEnv(string key, *string def) {
        string ekey = sprintf("QUNIT_%s", key);
        return ENV{ekey} ?? def;
    }

    #! adds a test case to run
    /** @par Example:
        @code{.py}
addTestCase("MyTest", \myTest());
        @endcode

        @param name the name of the test case
        @param call the code to call that executes the test case
        @param args any optional arguments to the test case call
    */
    addTestCase(string name, code call, *softlist<auto> args) {
        testCases += new TestCase(name, call, args);
    }

    #! adds a test case to run
    /** @par Example:
        @code{.py}
addTestCase(obj);
        @endcode

        @param tc the test case object
    */
    addTestCase(QUnit::TestCase tc) {
        testCases += tc;
    }

    private string escapeSpecialChars(string str) {
        str = replace(str, "\b", "<\\b>");
        str = replace(str, "\f", "<\\f>");
        str = replace(str, "\n", "<\\n>");
        str = replace(str, "\r", "<\\r>");
        str = replace(str, "\t", "<\\t>");
        str = replace(str, "\v", "<\\v>");
        return str;
    }

    private bool diffInSpecialCharsOnly(string a, string b) {
        a =~ s/[[:space:]]//;
        a =~ s/[[:cntrl:]]//;
        b =~ s/[[:space:]]//;
        b =~ s/[[:cntrl:]]//;
        return (a == b);
    }

    #! Helper function for printing out human-readable comparison of two values.
    private string printUnexpectedData(auto exp, auto act, *bool neg, *bool soft_comparisons, *string comparitor) {
        string comp = "Expected";
        if (comparitor)
            comp += " " + QUC_Map{comparitor};
        else
            comparitor = neg ? QUC_NEQ : QUC_EQ;

        string expected;
        string actual;
        # if values are "soft equal", then they must have different types, so we make the type differences explicit here
        if (!soft_comparisons && exp.typeCode() != act.typeCode()) {
            bool done = False;
            # if the values are strings that differ only in encodings, then show the encodings
            if (exp.typeCode() == NT_STRING && act.typeCode() == NT_STRING && exp.encoding() != act.encoding() && exp.length() == act.length()) {
                # in case of encoding errors, use exception handling
                try {
                    string nact = convert_encoding(exp.encoding(), act);
                    if (exp === nact) {
                        done = True;
                        expected = sprintf("(string %y) %N", exp.encoding(), exp);
                        actual = neg ? "<identical>" : sprintf("(string %y) %N", act.encoding(), act);
                    }
                } catch (hash<ExceptionInfo> ex) {
                }
            }
            if (!done) {
                switch (exp.typeCode()) {
                    case NT_NUMBER:
                        expected = sprintf("(%s) %N (%s)", exp.type(), exp, exp.toString(NF_Raw));
                        break;
                    case NT_FLOAT:
                        expected = sprintf("(%s) %N (%s)", exp.type(), exp, exp.toString());
                        break;
                    default:
                        expected = sprintf("(%s) %N", exp.type(), exp);
                }
                if (neg) {
                    actual = "<identical>";
                } else {
                    switch (act.typeCode()) {
                        case NT_NUMBER:
                            actual = sprintf("(%s) %N (%s)", act.type(), act, act.toString(NF_Raw));
                            break;
                        case NT_FLOAT:
                            actual = sprintf("(%s) %N (%s)", act.type(), act, act.toString());
                            break;
                        default:
                            actual = sprintf("(%s) %N", act.type(), act);
                    }
                }
            }
            comp += " (TYPE ERROR)";
        }
        else {
            switch (exp.typeCode()) {
                case NT_BINARY:
                    expected = sprintf("(binary: size %d) <%s>", exp.size(), make_hex_string(exp));
                    break;
                case NT_NUMBER:
                    expected = sprintf("(%s) %N (%s)", exp.type(), exp, exp.toString(NF_Raw));
                    break;
                case NT_FLOAT:
                    expected = sprintf("(%s) %N (%s)", exp.type(), exp, exp.toString());
                    break;
                default:
                    expected = sprintf("%N", exp);
            }
            if (neg) {
                actual = "<identical>";
            } else {
                switch (act.typeCode()) {
                    case NT_BINARY:
                        actual = sprintf("(binary: size %d) <%s>", act.size(), make_hex_string(act));
                        break;
                    case NT_NUMBER:
                        actual = sprintf("(%s) %N (%s)", act.type(), act, act.toString(NF_Raw));
                        break;
                    case NT_FLOAT:
                        actual = sprintf("(%s) %N (%s)", act.type(), act, act.toString());
                        break;
                    default:
                        actual = sprintf("%N", act);
                }
            }
        }
        if (diffInSpecialCharsOnly(expected, actual)) {
            expected = escapeSpecialChars(expected);
            actual = escapeSpecialChars(actual);
        }
        list<string> expectedLines = expected.split("\n");
        list<string> actualLines = actual.split("\n");

        # Distinguish between multiline
        bool multiline = ((expectedLines.size() + actualLines.size()) > 2);

        string result = "";
        if (!multiline) {
            # For some very simple values that fit on one line, just put it on one line. For values that would overflow the standard line size, do two lines.
            bool twoline = length(expected) + length(actual) > 40;
            result += sprintf("%s: %s%sActual: %s", comp, expected, twoline ? "\n" : ", ", actual);
            # For numeric values, when really verbose or difference not visible, compute and show the difference.
            if ((m_options.verbose > 1 || sprintf("%y", exp) == sprintf("%y", act)) &&
                    inlist (exp.typeCode(), (NT_INT, NT_FLOAT, NT_NUMBER)) &&
                    inlist (act.typeCode(), (NT_INT, NT_FLOAT, NT_NUMBER))) {
                result += sprintf("%sDifference: %y", twoline ? "\n" : ", ", abs(exp-act));
            }
        } else {
            string hr = "\t--------------------\n";
            result += sprintf("%s:\n", comp) + hr;
            map result += sprintf("\t%s\n", $1), expectedLines;
            result += hr + "Actual:\n" + hr;
            if (neg)
                result += "\t<same as above>\n";
            else
                map result += sprintf("\t%s\n", $1), actualLines;
            result += hr;

            list details = ();
            if (exp.typeCode() == NT_HASH && act.typeCode() == NT_HASH) {
                compareHashes(exp, act, \details, NOTHING, soft_comparisons);
            }
            if (exp.typeCode() == NT_LIST && act.typeCode() == NT_LIST) {
                compareLists(exp, act, \details, NOTHING, soft_comparisons);
            }
            if (details) {
                result += "Details:\n" + hr;
                map result += sprintf("\t%s\n", $1), details;
                result += hr;
            }
        }
        return result;
    }

    private string shorten(auto value) {
        # issue #2680: convert special whitespace in strings to escaped chars
        if (value.typeCode() == NT_STRING) {
            value =~ s/\n/\n/g;
            value =~ s/\r/\r/g;
            value =~ s/\t/\t/g;
        }
        string v = sprintf("%N", value);
        # issue #2680: when truncating complex data structures, indicate the truncation
        list l = v.split("\n");
        v = l[0];
        if (l.size() > 1) {
            return v + sprintf("...\" (%s truncated from %d lines)", value.type(), l.size());
        }
        else {
            return sprintf("%s (%s)", v, value.type());
        }
    }

    private compare(auto v1, auto v2, reference<list<string>> out, string path, *bool soft_comparisons) {
        if ((!soft_comparisons && v1 !== v2) || (soft_comparisons && v1 != v2)) {
            if (v1.typeCode() == NT_HASH && v2.typeCode() == NT_HASH) {
                compareHashes(v1, v2, \out, path);
            } else if (v1.typeCode() == NT_LIST && v2.typeCode() == NT_LIST) {
                compareLists(v1, v2, \out, path);
            } else {
                out += sprintf("The values of %s differ:", path);
                out += sprintf("    expected: %s", shorten(v1));
                out += sprintf("      actual: %s", shorten(v2));
            }
        }
    }

    private compareHashes(hash h1, hash h2, reference<list<string>> out, string path = "", *bool soft_comparisons) {
        foreach string key in (h1.keyIterator()) {
            if (h2.hasKey(key)) {
                compare(h1{key}, h2{key}, \out, path + "." + key, soft_comparisons);
            } else {
                out += sprintf("Missing expected key %s.%s (with value %N)", path, key, h1{key});
            }
        }
        foreach string key in (h2.keyIterator()) {
            if (!h1.hasKey(key)) {
                out += sprintf("Unexpected key %s.%s (with value %N)", path, key, h2{key});
            }
        }
    }

    private compareLists(list<auto> l1, list<auto> l2, reference<list<string>> out, string path = "", *bool soft_comparisons) {
        int len1 = elements l1;
        int len2 = elements l2;

        if (len1 != len2) {
            out += sprintf("Different lengths of lists %s%s- expected: %d, actual: %s", path, path ? " " : "", len1, len2);
        }

        for (int i = 0; i < min(len1, len2); ++i) {
            compare(l1[i], l2[i], \out, path + "[" + i + "]", soft_comparisons);
        }
    }

######### Assertions & test control functions

    #! Tests a value for equality to an expected value
    /** @par Example:
        @code{.py}
testAssertionValue("date > operator", now() > (now() - 1D), True);
        @endcode

        @param name the name or description of the assertion
        @param actual the value generated by the test
        @param expected the expected value

        @return the \a value argument

        @note make sure and use testAssertion() for any calls that could throw an exception
     */
    public auto testAssertionValue(*string name, auto actual, auto expected) {
        assertEq(expected, actual, name);
        return actual;
    }

    #! Tests a value for equality to an expected value of number type
    /**

        @param name the name or description of the assertion
        @param actual the value generated by the test
        @param expected the expected value
        @param epsilon accepted difference

        @return the \a value argument

        @note make sure and use testAssertion() for any calls that could throw an exception
     */
    public auto testAssertionValue(*string name, number actual, number expected, number epsilon = DEFAULT_EPSILON) {
        assertNumberEq(expected, actual, epsilon, name);
        return actual;
    }

    #! Tests a value for equality to an expected value of float type
    /**

        @param name the name or description of the assertion
        @param actual the value generated by the test
        @param expected the expected value
        @param epsilon accepted difference

        @return the \a value argument

        @note make sure and use testAssertion() for any calls that could throw an exception
     */
    public auto testAssertionValue(*string name, float actual, float expected, float epsilon = DEFAULT_EPSILON) {
        assertFloatEq(expected, actual, epsilon, name);
        return actual;
    }
    #! Tests that a test value passes a regular expression match
    /** @par Example:
        @code{.py}
assertRegex("regex string", string_function());
        @endcode

        @param regex_pattern the regular expression pattern string that the test value should match
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertRegex(string regex_pattern, string actual, *string name) {
        TestCase tc = getTestCase("assertRegex");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (!regex(actual, regex_pattern)) {
            throw "TEST-EXCEPTION", printUnexpectedData(regex_pattern, actual, NOTHING, NOTHING, QUC_RE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a test value does not pass a regular expression match
    /** @par Example:
        @code{.py}
assertNRegex("regex string", string_function());
        @endcode

        @param regex_pattern the regular expression pattern string that the test value should not match
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertNRegex(string regex_pattern, string actual, *string name) {
        TestCase tc = getTestCase("assertNRegex");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (regex(actual, regex_pattern)) {
            throw "TEST-EXCEPTION", printUnexpectedData(regex_pattern, actual, NOTHING, NOTHING, QUC_NRE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests for no value
    /** @par Example:
        @code{.py}
assertNothing(function_that_should_not_return_a value());
        @endcode

        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertNothing(auto actual, *string name) {
        TestCase tc = getTestCase("assertNothing");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (exists actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(NOTHING, actual), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a value for equality to an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertEqSoft("5", function_that_should_return_five());
        @endcode

        @param expected the expected value
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertEqSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertEqSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected != actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, True), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a value for inequality to an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertNeqSoft("5", function_that_should_not_return_five());
        @endcode

        @param expected the value that should not be equal to the test value with a soft comparison
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertNeqSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertNeqSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected == actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, True, True), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a value for equality to an expected value with hard comparisons (types and values must be identical)
    /** @par Example:
        @code{.py}
assertEq(5, function_that_should_return_five());
        @endcode

        @param expected the expected value
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertEq(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertEq");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected !== actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a value for inequality to an expected value with hard comparisons (types and values must be identical)
    /** @par Example:
        @code{.py}
assertNeq(5, function_that_should_return_five());
        @endcode

        @param expected the value that should not be equal to the test value with a hard comparison
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertNeq(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertNeq");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected === actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, True), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a float value for equality to an expected value with an allowed error
    /** @par Example:
        @code{.py}
assertFloatEq(2.5, 2.50001, 0.001);
        @endcode

        @param expected the expected value
        @param actual the value generated by the test
        @param epsilon the allowed error
        @param name the name or description of the assertion
     */
    public assertFloatEq(float expected, float actual, float epsilon = DEFAULT_EPSILON, *string name) {
        TestCase tc = getTestCase("assertFloatEq");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (abs(expected-actual) >= epsilon) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests a number value for equality to an expected value with an allowed error
    /** @par Example:
        @code{.py}
assertNumberEq(2.5n, 2.50001n, 0.001n);
        @endcode

        @param expected the expected value
        @param actual the value generated by the test
        @param epsilon the allowed error
        @param name the name or description of the assertion
     */
    public assertNumberEq(number expected, number actual, number epsilon = DEFAULT_EPSILON, *string name) {
        TestCase tc = getTestCase("assertNumberEq");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (abs(expected-actual) >= epsilon) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is greater than an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertGtSoft("5", function_that_should_rturn_a_value_greater_than_five());
        @endcode

        @param expected a value that should be less than \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertGtSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertGtSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected >= actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, True, QUC_GT), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is greater than an expected value with hard comparisons (types may not differ)
    /** @par Example:
        @code{.py}
assertGt(5, function_that_should_rturn_a_value_greater_than_five());
        @endcode

        @param expected a value that should be less than \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertGt(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertGt");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected >= actual || expected.typeCode() != actual.typeCode()) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, NOTHING, QUC_GT), ("name": name, "pos": NOTHING);
        }
        else {
            assertionOk(name);
        }
    }

    #! Tests that a value is greater than or equal to than an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertGeSoft("5", function_that_should_rturn_a_value_greater_than_or_equal_to_five());
        @endcode

        @param expected a value that should be less than or equal to \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertGeSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertGeSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected > actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, True, QUC_GE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is greater than or equal to an expected value with hard comparisons (types may not differ)
    /** @par Example:
        @code{.py}
assertGe(5, function_that_should_rturn_a_value_greater_than_or_equal_to_five());
        @endcode

        @param expected a value that should be less than or equal to \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertGe(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertGe");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected > actual || expected.typeCode() != actual.typeCode()) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, NOTHING, QUC_GE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is less than an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertLtSoft("5", function_that_should_rturn_a_value_less_than_five());
        @endcode

        @param expected a value that should be greater than \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertLtSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertLtSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected <= actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, True, QUC_LT), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is less than an expected value with hard comparisons (types may not differ)
    /** @par Example:
        @code{.py}
assertLt(5, function_that_should_rturn_a_value_less_than_five());
        @endcode

        @param expected a value that should be greater than \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertLt(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertLt");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected <= actual || expected.typeCode() != actual.typeCode()) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, NOTHING, QUC_LT), ("name": name, "pos": NOTHING);
        }
        else {
            assertionOk(name);
        }
    }

    #! Tests that a value is less than or equal to than an expected value with soft comparisons (types may differ)
    /** @par Example:
        @code{.py}
assertLeSoft("5", function_that_should_rturn_a_value_less_than_or_equal_to_five());
        @endcode

        @param expected a value that should be greater than or equal to \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertLeSoft(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertLeSoft");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected < actual) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, True, QUC_LE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! Tests that a value is less than or equal to an expected value with hard comparisons (types may not differ)
    /** @par Example:
        @code{.py}
assertLe(5, function_that_should_rturn_a_value_less_than_or_equal_to_five());
        @endcode

        @param expected a value that should be greater than or equal to \a actual
        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertLe(auto expected, auto actual, *string name) {
        TestCase tc = getTestCase("assertLe");
        # increment assertion count
        tc.incAssertions();
        ++num_asserts;

        if (expected < actual || expected.typeCode() != actual.typeCode()) {
            throw "TEST-EXCEPTION", printUnexpectedData(expected, actual, NOTHING, NOTHING, QUC_LE), ("name": name, "pos": NOTHING);
        } else {
            assertionOk(name);
        }
    }

    #! process an ok assertion
    private assertionOk(*string name) {
        get_thread_data("tc").incAssertionsOk();
        ++num_asserts_ok;
        if (m_options.verbose > 1)
            printf("+ OK: %y assertion %s\n", m_name, Test::getAssertionName(name));
    }

    #! Tests a boolean value
    /** @par Example:
        @code{.py}
assertTrue(functionThatShouldReturnTrue());
        @endcode

        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertTrue(auto actual, *string name) {
        testAssertionValue(name, actual, True);
    }

    #! Tests a boolean value
    /** @par Example:
        @code{.py}
assertFalse(functionThatShouldReturnFalse());
        @endcode

        @param actual the value generated by the test
        @param name the name or description of the assertion
     */
    public assertFalse(auto actual, *string name) {
        testAssertionValue(name, actual, False);
    }

    #! Tests that a piece of code throws an exception with given description
    /** @par Example:
        @code{.py}
assertThrows("DIVISION-BY-ZERO", "division by zero found in integer expression", sub(int a) {print(5/a);}, 0);
        @endcode

        @param expectedErr the expected exception type
        @param expectedDesc the expected exception detail (desc field), ignored if NOTHING
        @param theCode the code to execute
        @param args optional arguments to the code
        @param name an optional label for the assertion
     */
    public assertThrows(string expectedErr, *string expectedDesc, code theCode, *softlist<auto> args, *string name) {
        TestCase tc = getTestCase("assertEqThrows");
        tc.incAssertions();
        ++num_asserts;

        string exp = sprintf("Exception: %s", expectedErr) + (expectedDesc ? sprintf(": Detail '%s'", expectedDesc) : "");
        try {
            call_function_args(theCode, args);
        } catch (hash<ExceptionInfo> e) {
            if (e.err == expectedErr && (expectedDesc == NOTHING || e.desc.regex(expectedDesc))) {
                assertionOk(name);
                return;
            }
            throw "TEST-EXCEPTION", printUnexpectedData(exp, get_exception_string(e));
        }
        throw "TEST-EXCEPTION", sprintf("Expected %s was not thrown", exp);
    }

    #! Tests that a piece of code throws an exception
    /** @par Example:
        @code{.py}
assertThrows("DIVISION-BY-ZERO", sub(int a) {print(5/a);}, 0);
        @endcode

        @param expectedErr the expected exception type
        @param theCode the code to execute
        @param args optional arguments to the code
        @param name an optional label for the assertion
     */
    public assertThrows(string expectedErr, code theCode, *softlist<auto> args, *string name) {
        assertThrows(expectedErr, NOTHING, theCode, args, name);
    }

    #! Skips assertion on purpose
    /**
        @param name the name or description of the assertion

        @par Example:
        @code{.py}
assertSkip("assertion name or reason to skip");
        @endcode
     */
    public assertSkip (*string name) {
        TestCase tc = getTestCase("assertSkip");
        # increment assertion counters
        tc.incAssertions();
        ++num_asserts;
        tc.incAssertionsSkip();
        ++num_asserts_skip;
        # display optional message
        if (m_options.verbose > 1)
            printf("+ SKIP: %y assertion %s\n", m_name, Test::getAssertionName(name));
    }

    #! returns the current test case
    TestCase getTestCase(string meth) {
        *TestCase tc = get_thread_data("tc");
        if (!tc)
            throw "TEST-ERROR", sprintf("cannot test assertion while not executing a test case; %s() can only be called in code executed in a test case (added with %s::addTestCase())", meth, self.className());
        return tc;
    }

    #! Fails the test unconditionally
    /** @par Example:
        @code{.py}
fail("Unexpected code executed");
        @endcode

        @param msg the failure message
     */
    public fail(*string msg) {
        throw "TEST-EXCEPTION", msg;
    }

    #! Tests for a single assertion for a call returning no value (for example, to ensure that the call does not throw an exception)
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public testNullAssertion(string name, code condition, *softlist<auto> args) {
        testAssertion(name, condition, args, new TestResultValue());
    }

    #! Tests for a single assertion for a call returning an integer value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue an integer value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, int expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a floating-point value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a floating-point value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, float expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning an arbitrary-precision numeric value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue an arbitrary-precision numeric value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, number expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a boolean value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a boolean value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, bool expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a string value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a string value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, string expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a date value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a date value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, date expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a binary value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a binary value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, binary expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a hash value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a hash value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, hash<auto> expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion for a call returning a list value and returns the value generated
    /**
        @param name the name or description of the assertion
        @param condition A test function whose result we are asserting
        @param args Arguments passed to condition
        @param expectedResultValue a list value

        @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, list<auto> expectedResultValue) {
        return testAssertion(name, condition, args, new TestResultValue(expectedResultValue));
    }

    #! Tests for a single assertion and returns the value generated
    /**
     * @param name the name or description of the assertion
     * @param condition A test function whose result we are asserting
     * @param args Arguments passed to condition
     * @param expectedResult A class describing the expected result of condition; the default is QUnit::TestResultSuccess
     *
     * @return the result of the \a condition call, if the immediate value has any further use
     */
    public auto testAssertion(string name, code condition, *softlist<auto> args, QUnit::AbstractTestResult expectedResult = new QUnit::TestResultSuccess()) {
        TestCase tc = getTestCase("testAssertion");
        tc.incAssertions();
        ++num_asserts;

        *string pos;
        AbstractTestResult result;
        auto ret;
        try {
            ret = call_function_args(condition, args);
            if (ret.typeCode() == NT_BOOLEAN) {
                if (ret) {
                    result = new QUnit::TestResultSuccess();
                } else {
                    result = new QUnit::TestResultFailure();
                }
            } else {
                result = new QUnit::TestResultValue(ret);
            }
        } catch (hash<ExceptionInfo> e) {
            if (e.err == "TEST-FAILED-EXCEPTION") {
                # Since boolean can contain no detail, we abuse Exceptions this way to annotate a simple failure.
                result = new QUnit::TestResultFailure(e.desc);
            } else {
                pos = get_ex_pos(e);
                result = exists e.desc ? new QUnit::TestResultExceptionDetail(e.err, e.desc) : new QUnit::TestResultExceptionType(e.err);
                ret = e;
            }
        }

        return testAssertionIntern(tc, name, result, expectedResult, ret, pos);
    }

    private auto testAssertionIntern(TestCase tc, string name, QUnit::AbstractTestResult actualResult, QUnit::AbstractTestResult expectedResult, auto ret, *string pos) {
        if (!expectedResult.equals(actualResult)) {
            auto exp = expectedResult instanceof TestResultValue ? expectedResult.m_value : expectedResult.toString();
            auto act = actualResult instanceof TestResultValue ? actualResult.m_value : actualResult.toString();
            throw "TEST-EXCEPTION", printUnexpectedData (exp, act), ("name": name, "pos": pos);
        }
        else {
            assertionOk(name);
        }
        return ret;
    }

    #! Skips a given test, eg. because it may be missing some dependencies.
    /**
     * @param reason The reason for the test skip; used as the format argument with @ref Qore::vsprintf() "vsprintf()"
     * with any remaining arguments
     */
    public testSkip(string reason) {
        throw "TEST-SKIPPED-EXCEPTION", vsprintf(reason, argv);
    }

######### Test conditions passable to test assertions

    #! Compare two values for equality
    /**
     * @param a Argument 1
     * @param b Argument 2
     *
     * @returns a == b
     */
    bool equals(auto a, auto b) {
        if (a != b) {
            throw "TEST-FAILED-EXCEPTION", "Value mismatch, " + printUnexpectedData(a, b);
        }
        return True;
    }

    #! Compare two values for inequality
    /**
     * @param a Argument 1
     * @param b Argument 2
     *
     * @returns a != b
     */
    bool notEquals(auto a, auto b) {
        if (a == b) {
            throw "TEST-FAILED-EXCEPTION", "Value mismatch, " + printUnexpectedData(a, b, True);
        }
        return True;
    }

    #! Compare a string for match against a regexp
    /**
     * @param s String to match
     * @param regexp Regular expression to match against
     *
     * @returns a == b
     */
    bool regexpMatches(string s, string regexp) {
        if (!s.regex(regexp)) {
            throw "TEST-FAILED-EXCEPTION", sprintf("String does not match:\nString: %s\nRegexp: %s\n", s, regexp);
        }
        return True;
    }

    #! Compare two iterables, item by item, for equality of each index
    /**
     * @param a Iterable 1
     * @param b Iterable 2
     *
     * @returns a == b
     */
    bool equalsIterated(AbstractIterator a, AbstractIterator b) {
        int index = 0;
        while (a.next()) {
            if (!b.next()) {
                throw "TEST-FAILED-EXCEPTION", sprintf("Iterated mismatch, index %d:\n", index) + printUnexpectedData(a.getValue(), "Right hand side ran out of data!");
            }
            if (a.getValue() != b.getValue()) {
                throw "TEST-FAILED-EXCEPTION", sprintf("Iterated mismatch, index %d\n", index) + printUnexpectedData(a.getValue(), b.getValue());
            }
            index++;
        }
        if (b.next()) {
            throw "TEST-FAILED-EXCEPTION", sprintf("Iterated mismatch, index %d\n", index) + printUnexpectedData(b.getValue(), "Left hand side ran out of data!");
        }
        return True;
    }

######### Main function and test running helpers

    #! Run the whole suite, report results
    int main() {
        if (!testCases) {
            throw "TESTING-EXCEPTION", sprintf("Please define some tests first by calling %s::addTestCase()", self.className());
        }

        printHeader();

        # create initialization / shutdown test case
        TestCase gtc("globalSetUp", sub () {});
        gtc.setupThread();
        on_exit gtc.restoreThread();

        try {
            globalSetUp();
        } catch (hash<ExceptionInfo> ex) {
            gtc.checkException(self, ex);
        }

        foreach TestCase tc in (testCases) {
            code tinit = sub () {tc.setupThread();};
            set_thread_init(tinit);

            # run test case
            tc.run(self);
        }

        try {
            gtc.rename("globalTearDown");
            globalTearDown();
        } catch (hash<ExceptionInfo> ex) {
            gtc.checkException(self, ex);
        }

        printSummary();
        return errors();
    }

    #! returns the assertion name for display purposes
    static string getAssertionName(*string name) {
        if (exists name)
            return sprintf("%y", name);
        *string np;
        {
            list<string> l = TestCase::getStackList(get_thread_call_stack());
            if (l)
                np = foldl $1 + " <- " + $2, l;
        }
        return np ? sprintf("at %s", np) : "<no name>";
    }
}

#! A class representing a test with injected dependencies.
public class QUnit::DependencyInjectedTest inherits QUnit::Test {
    private {
        *string m_testedFile;
        Program m_child;
        *hash m_modules;

        static bool instantiated = False;
    }

    constructor(string name, string version, *reference<list<string>> p_argv, *hash opts) : QUnit::Test(name, version, \p_argv, opts) {
        if (instantiated) {
            throw "TESTING-EXCEPTION", "Only one instance of DependencyInjectedTest per program allowed!";
        }
        instantiated = True;
    }

    #! A helper function. It must be called once for a new instance with get_script_path() as argument.
    setScriptPath(*string testedFile) {
        if (testedFile) {
            m_testedFile = testedFile;
        }
    }

    #! A prototype function
    setUp() { }
    tearDown() { }

    private Program createInjectedProgram() {
        Program p(PO_NO_API|PO_NO_CHILD_PO_RESTRICTIONS|PO_ALLOW_INJECTION);
        p.setScriptPath(m_testedFile);
        # Replace oneself with the test runner
        p.importClass("QUnit::DependencyInjectedTestRunner", "QUnit::DependencyInjectedTest", True);
        performInjections(p);

        # Default classes
        p.importSystemApi();
        return p;
    }

    abstract performInjections(Program p);
    abstract performModuleInjections();

    private injectIntoModule(string module) {
        m_modules{module} = createInjectedProgram();
        m_modules{module}.loadApplyToUserModule(module, True);
    }

    destructor() {
        # Un-inject all injected modules.
        HashKeyIterator it(m_modules);
        while (it.next()) {
            string mn = basename(it.getValue());
            mn =~ s/\..+$//; # Strip suffix, if any
            reload_module(mn);
        }
    }

    int main() {
        if (!m_testedFile) {
            throw "TESTING-EXCEPTION", "You need to set the script path to run DependencyInjectedTest! Please run setScriptPath(get_script_path()) in the program.";
        }

        Program p = createInjectedProgram();

        # Inject the same into all modules
        performModuleInjections();

        # Load self, with replacements
        string file_data = ReadOnlyFile::readTextFile(m_testedFile);
        int eol = file_data.find("\n");
        if (eol) {
            string fl = file_data.substr(0, eol);
            if (fl =~ /(\\|\s)qr\s*$/)
                p.setParseOptions(PO_NEW_STYLE);
        }
        p.parse(file_data, m_testedFile);

        return p.run() ?? 0;
    }
}

public class QUnit::DependencyInjectedTestRunner inherits QUnit::Test {
    public setScriptPath(*string testedFile) {}
    private injectIntoModule(string module) {}

    constructor(string name, string version, *reference<list<string>> p_argv, *hash opts) : QUnit::Test(name, version, \p_argv, opts) {
    }
}