diff --git a/scripts/ci/check_compliance.py b/scripts/ci/check_compliance.py index 562852a57c515..9b466a8562814 100755 --- a/scripts/ci/check_compliance.py +++ b/scripts/ci/check_compliance.py @@ -607,6 +607,7 @@ def get_defined_syms(kconf): "MODVERSIONS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "SECURITY_LOADPIN", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "ZEPHYR_TRY_MASS_ERASE", # MCUBoot setting described in sysbuild documentation + "ZTEST_FAIL_TEST_", # regex in tests/ztest/fail/CMakeLists.txt } class KconfigBasicCheck(KconfigCheck, ComplianceTest): diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index f36328cff400d..7354a8f275e7b 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -46,15 +46,33 @@ enum ztest_phase { TEST_PHASE_FRAMEWORK }; +/** + * @brief The current status of the test binary + */ +enum ztest_status { + ZTEST_STATUS_OK, + ZTEST_STATUS_HAS_FAILURE, + ZTEST_STATUS_CRITICAL_ERROR +}; + /** * @brief Tracks the current phase that ztest is operating in. */ ZTEST_DMEM enum ztest_phase phase = TEST_PHASE_FRAMEWORK; -static ZTEST_BMEM int test_status; +static ZTEST_BMEM enum ztest_status test_status = ZTEST_STATUS_OK; extern ZTEST_DMEM const struct ztest_arch_api ztest_api; +void end_report(void) +{ + if (test_status) { + TC_END_REPORT(TC_FAIL); + } else { + TC_END_REPORT(TC_PASS); + } +} + static int cleanup_test(struct ztest_unit_test *test) { int ret = TC_PASS; @@ -247,27 +265,6 @@ static int get_final_test_result(const struct ztest_unit_test *test, int ret) return ret; } -#ifndef KERNEL - -/* Static code analysis tool can raise a violation that the standard header - * shall not be used. - * - * setjmp is using in a test code, not in a runtime code, it is acceptable. - * It is a deliberate deviation. - */ -#include /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/ -#include -#include -#include - -#define FAIL_FAST 0 - -static jmp_buf test_fail; -static jmp_buf test_pass; -static jmp_buf test_skip; -static jmp_buf stack_fail; -static jmp_buf test_suite_fail; - /** * @brief Get a friendly name string for a given test phrase. * @@ -294,6 +291,27 @@ static inline const char *get_friendly_phase_name(enum ztest_phase phase) } } +#ifndef KERNEL + +/* Static code analysis tool can raise a violation that the standard header + * shall not be used. + * + * setjmp is using in a test code, not in a runtime code, it is acceptable. + * It is a deliberate deviation. + */ +#include /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/ +#include +#include +#include + +#define FAIL_FAST 0 + +static jmp_buf test_fail; +static jmp_buf test_pass; +static jmp_buf test_skip; +static jmp_buf stack_fail; +static jmp_buf test_suite_fail; + void ztest_test_fail(void) { switch (phase) { @@ -302,25 +320,37 @@ void ztest_test_fail(void) longjmp(test_suite_fail, 1); case TEST_PHASE_BEFORE: case TEST_PHASE_TEST: - case TEST_PHASE_AFTER: - case TEST_PHASE_TEARDOWN: PRINT(" at %s function\n", get_friendly_phase_name(phase)); longjmp(test_fail, 1); + case TEST_PHASE_AFTER: + case TEST_PHASE_TEARDOWN: case TEST_PHASE_FRAMEWORK: - PRINT("\n"); + PRINT(" ERROR: cannot fail in test '%s()', bailing\n", + get_friendly_phase_name(phase)); longjmp(stack_fail, 1); } } -void ztest_test_pass(void) { longjmp(test_pass, 1); } - -void ztest_test_skip(void) { longjmp(test_skip, 1); } +void ztest_test_pass(void) +{ + if (phase == TEST_PHASE_TEST) { + longjmp(test_pass, 1); + } + PRINT(" ERROR: cannot pass in test '%s()', bailing\n", get_friendly_phase_name(phase)); + longjmp(stack_fail, 1); +} -static void init_testing(void) +void ztest_test_skip(void) { - if (setjmp(stack_fail)) { - PRINT("TESTSUITE crashed."); - exit(1); + switch (phase) { + case TEST_PHASE_SETUP: + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + longjmp(test_skip, 1); + default: + PRINT(" ERROR: cannot skip in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + longjmp(stack_fail, 1); } } @@ -329,6 +359,7 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test int ret = TC_PASS; TC_START(test->name); + phase = TEST_PHASE_BEFORE; if (test_result == ZTEST_RESULT_SUITE_FAIL) { ret = TC_FAIL; @@ -356,13 +387,15 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test } run_test_functions(suite, test, data); out: - ret |= cleanup_test(test); + phase = TEST_PHASE_AFTER; if (test_result != ZTEST_RESULT_SUITE_FAIL) { if (suite->after != NULL) { suite->after(data); } run_test_rules(/*is_before=*/false, test, data); } + phase = TEST_PHASE_FRAMEWORK; + ret |= cleanup_test(test); ret = get_final_test_result(test, ret); Z_TC_END_RESULT(ret, test->name); @@ -393,23 +426,56 @@ static void test_finalize(void) void ztest_test_fail(void) { - test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL; - if (phase != TEST_PHASE_SETUP) { + switch (phase) { + case TEST_PHASE_SETUP: + test_result = ZTEST_RESULT_SUITE_FAIL; + break; + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_FAIL; test_finalize(); + break; + default: + PRINT(" ERROR: cannot fail in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + break; } } void ztest_test_pass(void) { - test_result = ZTEST_RESULT_PASS; - test_finalize(); + switch (phase) { + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_PASS; + test_finalize(); + break; + default: + PRINT(" ERROR: cannot pass in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + if (phase == TEST_PHASE_BEFORE) { + test_finalize(); + } + } } void ztest_test_skip(void) { - test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP; - if (phase != TEST_PHASE_SETUP) { + switch (phase) { + case TEST_PHASE_SETUP: + test_result = ZTEST_RESULT_SUITE_SKIP; + break; + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_SKIP; test_finalize(); + break; + default: + PRINT(" ERROR: cannot skip in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + break; } } @@ -425,11 +491,6 @@ void ztest_simple_1cpu_after(void *data) z_test_1cpu_stop(); } -static void init_testing(void) -{ - k_object_access_all_grant(&ztest_thread); -} - static void test_cb(void *a, void *b, void *c) { struct ztest_suite_node *suite = a; @@ -576,11 +637,20 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } if (suite == NULL) { - test_status = 1; + test_status = ZTEST_STATUS_CRITICAL_ERROR; return -1; } - init_testing(); +#ifndef KERNEL + if (setjmp(stack_fail)) { + PRINT("TESTSUITE crashed.\n"); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + end_report(); + exit(1); + } +#else + k_object_access_all_grant(&ztest_thread); +#endif TC_SUITE_START(suite->name); test_result = ZTEST_RESULT_PENDING; @@ -624,7 +694,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } } - if (fail && FAIL_FAST) { + if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { break; } } @@ -646,13 +716,15 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } } - if (fail && FAIL_FAST) { + if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { break; } } #endif - test_status = (test_status || fail) ? 1 : 0; + if (test_status == ZTEST_STATUS_OK && fail != 0) { + test_status = ZTEST_STATUS_HAS_FAILURE; + } } TC_SUITE_END(suite->name, (fail > 0 ? TC_FAIL : TC_PASS)); @@ -669,15 +741,6 @@ int z_ztest_run_test_suite(const char *name) return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name)); } -void end_report(void) -{ - if (test_status) { - TC_END_REPORT(TC_FAIL); - } else { - TC_END_REPORT(TC_PASS); - } -} - #ifdef CONFIG_USERSPACE K_APPMEM_PARTITION_DEFINE(ztest_mem_partition); #endif @@ -842,6 +905,10 @@ int z_impl_ztest_run_test_suites(const void *state) { int count = 0; + if (test_status == ZTEST_STATUS_CRITICAL_ERROR) { + return count; + } + #ifdef CONFIG_ZTEST_SHUFFLE struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT]; @@ -850,11 +917,25 @@ int z_impl_ztest_run_test_suites(const void *state) ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { count += __ztest_run_test_suite(suites_to_run[i], state); + /* Stop running tests if we have a critical error or if we have a failure and + * FAIL_FAST was set + */ + if (test_status == ZTEST_STATUS_CRITICAL_ERROR || + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + break; + } } #else for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) { count += __ztest_run_test_suite(ptr, state); + /* Stop running tests if we have a critical error or if we have a failure and + * FAIL_FAST was set + */ + if (test_status == ZTEST_STATUS_CRITICAL_ERROR || + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + break; + } } #endif @@ -889,7 +970,7 @@ void ztest_verify_all_test_suites_ran(void) } if (!all_tests_run) { - test_status = 1; + test_status = ZTEST_STATUS_HAS_FAILURE; } } diff --git a/tests/ztest/fail/CMakeLists.txt b/tests/ztest/fail/CMakeLists.txt new file mode 100644 index 0000000000000..f7a070377740f --- /dev/null +++ b/tests/ztest/fail/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +include(ExternalProject) + +# Add the sources and set up the build for either unit testing or native_posix +list(APPEND SOURCES src/main.cpp) +if(BOARD STREQUAL unit_testing) + find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE}) + set(target testbinary) + # Set the target binary for the 'core' external project. The path to this must match the one set + # below in ExternalProject_Add's CMAKE_INSTALL_PREFIX + add_compile_definitions(FAIL_TARGET_BINARY="${CMAKE_BINARY_DIR}/core/bin/testbinary") +else() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + set(target app) + # Set the target binary for the 'core' external project. The path to this must match the one set + # below in ExternalProject_Add's CMAKE_INSTALL_PREFIX + add_compile_definitions(FAIL_TARGET_BINARY="${CMAKE_BINARY_DIR}/core/bin/zephyr.elf") +endif() + +# Create the project and set the sources for the target +project(fail) +target_sources(${target} PRIVATE ${SOURCES}) + +# Find which CONFIG_ZTEST_FAIL_TEST_* choice was set so we can pass it to the external project +# Once we find the config, we'll need to prepend a '-D' and append '=y' so we can pass it to the +# 'core' project as a cmake argument. +get_cmake_property(_vars VARIABLES) +string(REGEX MATCHALL "(^|;)CONFIG_ZTEST_FAIL_TEST_[A-Za-z0-9_]+" fail_test_config "${_vars}") +list(FILTER fail_test_config EXCLUDE REGEX "^$") +list(TRANSFORM fail_test_config PREPEND "-D") +list(TRANSFORM fail_test_config APPEND "=y") +string(REPLACE ";" " " fail_test_config "${fail_test_config}") + +# Add the 'core' external project which will mirror the configs of this project. +ExternalProject_Add(core + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/core + CMAKE_ARGS + -DBOARD:STRING=${BOARD} + -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/core + ${fail_test_config} +) +add_dependencies(${target} core) diff --git a/tests/ztest/fail/Kconfig b/tests/ztest/fail/Kconfig new file mode 100644 index 0000000000000..8f5527a23daea --- /dev/null +++ b/tests/ztest/fail/Kconfig @@ -0,0 +1,36 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +choice ZTEST_FAIL_TEST + prompt "Select the type of failure to test" + +config ZTEST_FAIL_TEST_ASSERT_AFTER + bool "Add a failed assert in the after phase" + +config ZTEST_FAIL_TEST_ASSERT_TEARDOWN + bool "Add a failed assert in the teardown phase" + +config ZTEST_FAIL_TEST_ASSUME_AFTER + bool "Add a failed assume in the after phase" + +config ZTEST_FAIL_TEST_ASSUME_TEARDOWN + bool "Add a failed assume in the teardown phase" + +config ZTEST_FAIL_TEST_PASS_AFTER + bool "Add a call to ztest_test_pass() in the after phase" + +config ZTEST_FAIL_TEST_PASS_TEARDOWN + bool "Add a call to ztest_test_pass() in the teardown phase" + +endchoice + +config TEST_ERROR_STRING + string + default "ERROR: cannot fail in test 'after()', bailing" if ZTEST_FAIL_TEST_ASSERT_AFTER + default "ERROR: cannot fail in test 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSERT_TEARDOWN + default "ERROR: cannot skip in test 'after()', bailing" if ZTEST_FAIL_TEST_ASSUME_AFTER + default "ERROR: cannot skip in test 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSUME_TEARDOWN + default "ERROR: cannot pass in test 'after()', bailing" if ZTEST_FAIL_TEST_PASS_AFTER + default "ERROR: cannot pass in test 'teardown()', bailing" if ZTEST_FAIL_TEST_PASS_TEARDOWN + +source "Kconfig.zephyr" diff --git a/tests/ztest/fail/README.rst b/tests/ztest/fail/README.rst new file mode 100644 index 0000000000000..3ed70d5229a5b --- /dev/null +++ b/tests/ztest/fail/README.rst @@ -0,0 +1,21 @@ +.. _ztest_framework_failure_tests: + +Ztest framework failure tests +############################# + +Overview +******** + +In order to test the actual framework's failure cases, this test suite has to do something unique. +There's a subdirectory to this test called 'core'. This project builds a sample ``native_posix`` or +``unit_testing`` binary which is expected to fail by calling one of the following: +- ``ztest_test_fail()`` during either the ``after`` or ``teardown`` phase of the test suite +- ``ztest_test_skip()`` during either the ``after`` or ``teardown`` phase of the test suite +- ``ztest_test_pass()`` during either the ``after`` or ``teardown`` phase of the test suite + +Note that these can be called indirectly through failed asserts or assumptions. + +The binary by itself, when executed, will fail to run and return a code of ``1``. The main test +binary will use ``popen()`` to run the failing test binary and will assert both the return code and +the output. The output itself cannot be printed to the log as it will confuse ``twister`` by +reporting a failure. diff --git a/tests/ztest/fail/core/CMakeLists.txt b/tests/ztest/fail/core/CMakeLists.txt new file mode 100644 index 0000000000000..70f884aa3d670 --- /dev/null +++ b/tests/ztest/fail/core/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) + +set(KCONFIG_ROOT ${CMAKE_CURRENT_LIST_DIR}/../Kconfig) + +# Add the sources +list(APPEND SOURCES src/main.cpp) +if(CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER) + list(APPEND SOURCES src/assert_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN) + list(APPEND SOURCES src/assert_teardown.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER) + list(APPEND SOURCES src/assume_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN) + list(APPEND SOURCES src/assume_teardown.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_PASS_AFTER) + list(APPEND SOURCES src/pass_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN) + list(APPEND SOURCES src/pass_teardown.cpp) +endif() + +if(BOARD STREQUAL unit_testing) + find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + target_include_directories(testbinary PRIVATE include) + install(TARGETS testbinary) +else() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + target_sources(app PRIVATE ${SOURCES}) + target_include_directories(app PRIVATE include) + install(TARGETS ${logical_target_for_zephyr_elf}) +endif() diff --git a/tests/ztest/fail/core/include/fail_test.hpp b/tests/ztest/fail/core/include/fail_test.hpp new file mode 100644 index 0000000000000..b4aa9788cde8a --- /dev/null +++ b/tests/ztest/fail/core/include/fail_test.hpp @@ -0,0 +1,9 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +void fail_test_after_impl(void); + +void fail_test_teardown_impl(void); diff --git a/tests/ztest/fail/core/prj.conf b/tests/ztest/fail/core/prj.conf new file mode 100644 index 0000000000000..6fa939ecf5081 --- /dev/null +++ b/tests/ztest/fail/core/prj.conf @@ -0,0 +1,8 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +CONFIG_ZTEST=y +CONFIG_ZTEST_NEW_API=y + +CONFIG_CPLUSPLUS=y +CONFIG_LIB_CPLUSPLUS=y diff --git a/tests/ztest/fail/core/src/assert_after.cpp b/tests/ztest/fail/core/src/assert_after.cpp new file mode 100644 index 0000000000000..79d7e35143cd6 --- /dev/null +++ b/tests/ztest/fail/core/src/assert_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + zassert_true(false, nullptr); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/assert_teardown.cpp b/tests/ztest/fail/core/src/assert_teardown.cpp new file mode 100644 index 0000000000000..e0134f1e31b3c --- /dev/null +++ b/tests/ztest/fail/core/src/assert_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + zassert_true(false, nullptr); +} diff --git a/tests/ztest/fail/core/src/assume_after.cpp b/tests/ztest/fail/core/src/assume_after.cpp new file mode 100644 index 0000000000000..2d7c9295a3f24 --- /dev/null +++ b/tests/ztest/fail/core/src/assume_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + zassume_true(false, nullptr); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/assume_teardown.cpp b/tests/ztest/fail/core/src/assume_teardown.cpp new file mode 100644 index 0000000000000..ffb0adf3ce9fd --- /dev/null +++ b/tests/ztest/fail/core/src/assume_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + zassume_true(false, nullptr); +} diff --git a/tests/ztest/fail/core/src/main.cpp b/tests/ztest/fail/core/src/main.cpp new file mode 100644 index 0000000000000..9fece93104c9c --- /dev/null +++ b/tests/ztest/fail/core/src/main.cpp @@ -0,0 +1,19 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +static void fail_after(void *) { + fail_test_after_impl(); +} + +static void fail_teardown(void *) { + fail_test_teardown_impl(); +} + +ZTEST_SUITE(fail, nullptr, nullptr, nullptr, fail_after, fail_teardown); + +ZTEST(fail, test_framework) {} diff --git a/tests/ztest/fail/core/src/pass_after.cpp b/tests/ztest/fail/core/src/pass_after.cpp new file mode 100644 index 0000000000000..447f082cf3cf4 --- /dev/null +++ b/tests/ztest/fail/core/src/pass_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + ztest_test_pass(); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/pass_teardown.cpp b/tests/ztest/fail/core/src/pass_teardown.cpp new file mode 100644 index 0000000000000..935f3020fb59f --- /dev/null +++ b/tests/ztest/fail/core/src/pass_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + ztest_test_pass(); +} diff --git a/tests/ztest/fail/prj.conf b/tests/ztest/fail/prj.conf new file mode 100644 index 0000000000000..6fa939ecf5081 --- /dev/null +++ b/tests/ztest/fail/prj.conf @@ -0,0 +1,8 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +CONFIG_ZTEST=y +CONFIG_ZTEST_NEW_API=y + +CONFIG_CPLUSPLUS=y +CONFIG_LIB_CPLUSPLUS=y diff --git a/tests/ztest/fail/src/main.cpp b/tests/ztest/fail/src/main.cpp new file mode 100644 index 0000000000000..7b57356b9cb41 --- /dev/null +++ b/tests/ztest/fail/src/main.cpp @@ -0,0 +1,56 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +ZTEST_SUITE(fail, nullptr, nullptr, nullptr, nullptr, nullptr); + +ZTEST(fail, test_framework) +{ + auto found_error_string = false; + char buffer[sizeof(CONFIG_TEST_ERROR_STRING)] = {0}; + std::string result; + + /* Start running the target binary. This binary is expected to fail. */ + auto pipe = popen(FAIL_TARGET_BINARY, "r"); + + zassert_not_null(pipe, "Failed to execute '" FAIL_TARGET_BINARY "'"); + + /* Wait for the binary to finish running and grab the output */ + while (!feof(pipe)) { + if (fgets(buffer, ARRAY_SIZE(buffer), pipe) != nullptr) { + if (found_error_string) { + /* Already found the error string, no need to do any more string + * manipulation. + */ + continue; + } + + /* Append the buffer to the result string */ + result += buffer; + + /* Check if result contains the right error string */ + found_error_string |= + (result.find(CONFIG_TEST_ERROR_STRING) != std::string::npos); + + /* If the result string is longer than the expected string, + * we can prune it + */ + auto prune_length = static_cast(result.length()) - + static_cast(sizeof(CONFIG_TEST_ERROR_STRING)); + if (prune_length > 0) { + result.erase(0, prune_length); + } + } + } + auto rc = WEXITSTATUS(pclose(pipe)); + + zassert_equal(1, rc, "Test binary expected to fail with return code 1, but got %d", rc); + zassert_true(found_error_string, "Test binary did not produce the expected error string \"" + CONFIG_TEST_ERROR_STRING "\""); +} diff --git a/tests/ztest/fail/testcase.yaml b/tests/ztest/fail/testcase.yaml new file mode 100644 index 0000000000000..67bb0986be6d3 --- /dev/null +++ b/tests/ztest/fail/testcase.yaml @@ -0,0 +1,52 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +tests: + testing.fail.unit.assert_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER=y + testing.fail.unit.assert_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN=y + testing.fail.unit.assume_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y + testing.fail.unit.assume_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y + testing.fail.unit.pass_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_AFTER=y + testing.fail.unit.pass_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN=y + testing.fail.zephyr.assert_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER=y + testing.fail.zephyr.assert_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN=y + testing.fail.zephyr.assume_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y + testing.fail.zephyr.assume_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y + testing.fail.zephyr.pass_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_AFTER=y + testing.fail.zephyr.pass_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN=y