From 74da8da4647de3dcb0ae791569fd9b27c21cfc9e Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 21:59:53 -0600 Subject: [PATCH 1/8] ztest: remove init_testing When KERNEL was not defined (unittest), the call to init_testing was used to set a longjump target using 'stack_fail'. When triggered, this was actually causing a segfault, because longjmp is only valid if going directly up the stack. Since init_testing returned, it was no longer on the stack. Instead, that logic MUST be inlined. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index f36328cff400d..2a59e5fa2e8e8 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -316,14 +316,6 @@ void ztest_test_pass(void) { longjmp(test_pass, 1); } void ztest_test_skip(void) { longjmp(test_skip, 1); } -static void init_testing(void) -{ - if (setjmp(stack_fail)) { - PRINT("TESTSUITE crashed."); - exit(1); - } -} - static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data) { int ret = TC_PASS; @@ -425,11 +417,6 @@ void ztest_simple_1cpu_after(void *data) z_test_1cpu_stop(); } -static void init_testing(void) -{ - k_object_access_all_grant(&ztest_thread); -} - static void test_cb(void *a, void *b, void *c) { struct ztest_suite_node *suite = a; @@ -580,7 +567,16 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) return -1; } - init_testing(); +#ifndef KERNEL + if (setjmp(stack_fail)) { + PRINT("TESTSUITE crashed.\n"); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + end_report(); + exit(1); + } +#else + k_object_access_all_grant(&ztest_thread); +#endif TC_SUITE_START(suite->name); test_result = ZTEST_RESULT_PENDING; From 1d7474471eeb98c7196144723a5ef533daaf9a64 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 22:06:26 -0600 Subject: [PATCH 2/8] ztest: move end_report implementation Move the implementation of end_report so it can be used by both code paths (with and without KERNEL). Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 2a59e5fa2e8e8..6b7aca84cf4bb 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -55,6 +55,15 @@ static ZTEST_BMEM int test_status; extern ZTEST_DMEM const struct ztest_arch_api ztest_api; +void end_report(void) +{ + if (test_status) { + TC_END_REPORT(TC_FAIL); + } else { + TC_END_REPORT(TC_PASS); + } +} + static int cleanup_test(struct ztest_unit_test *test) { int ret = TC_PASS; @@ -665,15 +674,6 @@ int z_ztest_run_test_suite(const char *name) return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name)); } -void end_report(void) -{ - if (test_status) { - TC_END_REPORT(TC_FAIL); - } else { - TC_END_REPORT(TC_PASS); - } -} - #ifdef CONFIG_USERSPACE K_APPMEM_PARTITION_DEFINE(ztest_mem_partition); #endif From f989fc87975d4d43604e461d6949f4bf6a17ed6f Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 22:08:26 -0600 Subject: [PATCH 3/8] ztest: unittest: Add before/after phase Add missing setters for the before/after phase when running without KERNEL. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 6b7aca84cf4bb..d896899303c8a 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -330,6 +330,7 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test int ret = TC_PASS; TC_START(test->name); + phase = TEST_PHASE_BEFORE; if (test_result == ZTEST_RESULT_SUITE_FAIL) { ret = TC_FAIL; @@ -358,6 +359,7 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test run_test_functions(suite, test, data); out: ret |= cleanup_test(test); + phase = TEST_PHASE_AFTER; if (test_result != ZTEST_RESULT_SUITE_FAIL) { if (suite->after != NULL) { suite->after(data); From a99daccd3a2904e3a64c50df9b5e0af4555c6c91 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 22:11:06 -0600 Subject: [PATCH 4/8] ztest: unittest: Fix fail/skip/pass outside of tests When not running in setup, before, or in the test. Calling fail, skip, or pass is invalid and should be considered an error. Properly handle these cases by printing a more detailed error message and bailing the process. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 31 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index d896899303c8a..3ca659a9ef8ad 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -311,19 +311,39 @@ void ztest_test_fail(void) longjmp(test_suite_fail, 1); case TEST_PHASE_BEFORE: case TEST_PHASE_TEST: - case TEST_PHASE_AFTER: - case TEST_PHASE_TEARDOWN: PRINT(" at %s function\n", get_friendly_phase_name(phase)); longjmp(test_fail, 1); + case TEST_PHASE_AFTER: + case TEST_PHASE_TEARDOWN: case TEST_PHASE_FRAMEWORK: - PRINT("\n"); + PRINT(" ERROR: cannot fail in test '%s()', bailing\n", + get_friendly_phase_name(phase)); longjmp(stack_fail, 1); } } -void ztest_test_pass(void) { longjmp(test_pass, 1); } +void ztest_test_pass(void) +{ + if (phase == TEST_PHASE_TEST) { + longjmp(test_pass, 1); + } + PRINT(" ERROR: cannot pass in test '%s()', bailing\n", get_friendly_phase_name(phase)); + longjmp(stack_fail, 1); +} -void ztest_test_skip(void) { longjmp(test_skip, 1); } +void ztest_test_skip(void) +{ + switch (phase) { + case TEST_PHASE_SETUP: + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + longjmp(test_skip, 1); + default: + PRINT(" ERROR: cannot skip in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + longjmp(stack_fail, 1); + } +} static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data) { @@ -358,6 +378,7 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test } run_test_functions(suite, test, data); out: + phase = TEST_PHASE_FRAMEWORK; ret |= cleanup_test(test); phase = TEST_PHASE_AFTER; if (test_result != ZTEST_RESULT_SUITE_FAIL) { From e771a6f43146a4f785276118af0435c691f68e64 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 22:12:37 -0600 Subject: [PATCH 5/8] ztest: move get_friendly_phase_name to common code Move the function used for printing the phase name up so its available for both unittest and KERNEL mode of tests. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 3ca659a9ef8ad..67596f2720cb9 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -256,27 +256,6 @@ static int get_final_test_result(const struct ztest_unit_test *test, int ret) return ret; } -#ifndef KERNEL - -/* Static code analysis tool can raise a violation that the standard header - * shall not be used. - * - * setjmp is using in a test code, not in a runtime code, it is acceptable. - * It is a deliberate deviation. - */ -#include /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/ -#include -#include -#include - -#define FAIL_FAST 0 - -static jmp_buf test_fail; -static jmp_buf test_pass; -static jmp_buf test_skip; -static jmp_buf stack_fail; -static jmp_buf test_suite_fail; - /** * @brief Get a friendly name string for a given test phrase. * @@ -303,6 +282,27 @@ static inline const char *get_friendly_phase_name(enum ztest_phase phase) } } +#ifndef KERNEL + +/* Static code analysis tool can raise a violation that the standard header + * shall not be used. + * + * setjmp is using in a test code, not in a runtime code, it is acceptable. + * It is a deliberate deviation. + */ +#include /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/ +#include +#include +#include + +#define FAIL_FAST 0 + +static jmp_buf test_fail; +static jmp_buf test_pass; +static jmp_buf test_skip; +static jmp_buf stack_fail; +static jmp_buf test_suite_fail; + void ztest_test_fail(void) { switch (phase) { From ab3c083f9db90d10629ed946297063977efc7a7e Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 9 Aug 2022 22:14:48 -0600 Subject: [PATCH 6/8] ztest: Safely handle fail/skip/pass outside tests Updates the implementation when KERNEL is available to safely bail on the test when the test calls fail, skip, or pass during invalid test phases. Print a detailed message, and skip all other tests. The test run will be marked as failed. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 86 ++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 67596f2720cb9..1d0aa6c5dcb14 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -46,12 +46,21 @@ enum ztest_phase { TEST_PHASE_FRAMEWORK }; +/** + * @brief The current status of the test binary + */ +enum ztest_status { + ZTEST_STATUS_OK, + ZTEST_STATUS_HAS_FAILURE, + ZTEST_STATUS_CRITICAL_ERROR +}; + /** * @brief Tracks the current phase that ztest is operating in. */ ZTEST_DMEM enum ztest_phase phase = TEST_PHASE_FRAMEWORK; -static ZTEST_BMEM int test_status; +static ZTEST_BMEM enum ztest_status test_status = ZTEST_STATUS_OK; extern ZTEST_DMEM const struct ztest_arch_api ztest_api; @@ -417,23 +426,56 @@ static void test_finalize(void) void ztest_test_fail(void) { - test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL; - if (phase != TEST_PHASE_SETUP) { + switch (phase) { + case TEST_PHASE_SETUP: + test_result = ZTEST_RESULT_SUITE_FAIL; + break; + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_FAIL; test_finalize(); + break; + default: + PRINT(" ERROR: cannot fail in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + break; } } void ztest_test_pass(void) { - test_result = ZTEST_RESULT_PASS; - test_finalize(); + switch (phase) { + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_PASS; + test_finalize(); + break; + default: + PRINT(" ERROR: cannot pass in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + if (phase == TEST_PHASE_BEFORE) { + test_finalize(); + } + } } void ztest_test_skip(void) { - test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP; - if (phase != TEST_PHASE_SETUP) { + switch (phase) { + case TEST_PHASE_SETUP: + test_result = ZTEST_RESULT_SUITE_SKIP; + break; + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + test_result = ZTEST_RESULT_SKIP; test_finalize(); + break; + default: + PRINT(" ERROR: cannot skip in test '%s()', bailing\n", + get_friendly_phase_name(phase)); + test_status = ZTEST_STATUS_CRITICAL_ERROR; + break; } } @@ -595,7 +637,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } if (suite == NULL) { - test_status = 1; + test_status = ZTEST_STATUS_CRITICAL_ERROR; return -1; } @@ -652,7 +694,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } } - if (fail && FAIL_FAST) { + if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { break; } } @@ -674,13 +716,15 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } } - if (fail && FAIL_FAST) { + if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { break; } } #endif - test_status = (test_status || fail) ? 1 : 0; + if (test_status == ZTEST_STATUS_OK && fail != 0) { + test_status = ZTEST_STATUS_HAS_FAILURE; + } } TC_SUITE_END(suite->name, (fail > 0 ? TC_FAIL : TC_PASS)); @@ -861,6 +905,10 @@ int z_impl_ztest_run_test_suites(const void *state) { int count = 0; + if (test_status == ZTEST_STATUS_CRITICAL_ERROR) { + return count; + } + #ifdef CONFIG_ZTEST_SHUFFLE struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT]; @@ -869,11 +917,25 @@ int z_impl_ztest_run_test_suites(const void *state) ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { count += __ztest_run_test_suite(suites_to_run[i], state); + /* Stop running tests if we have a critical error or if we have a failure and + * FAIL_FAST was set + */ + if (test_status == ZTEST_STATUS_CRITICAL_ERROR || + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + break; + } } #else for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) { count += __ztest_run_test_suite(ptr, state); + /* Stop running tests if we have a critical error or if we have a failure and + * FAIL_FAST was set + */ + if (test_status == ZTEST_STATUS_CRITICAL_ERROR || + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + break; + } } #endif @@ -908,7 +970,7 @@ void ztest_verify_all_test_suites_ran(void) } if (!all_tests_run) { - test_status = 1; + test_status = ZTEST_STATUS_HAS_FAILURE; } } From 71c9945ae51ca808bf27050dda1b252d14ec3cb4 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Wed, 10 Aug 2022 22:50:01 -0600 Subject: [PATCH 7/8] ztest: Match cleanup ordering between unittest and normal tests Tests with KERNEL enabled perform their cleanup logic after the suite's after and test rules are executed. Unittests should do the same. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/src/ztest_new.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 1d0aa6c5dcb14..7354a8f275e7b 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -387,8 +387,6 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test } run_test_functions(suite, test, data); out: - phase = TEST_PHASE_FRAMEWORK; - ret |= cleanup_test(test); phase = TEST_PHASE_AFTER; if (test_result != ZTEST_RESULT_SUITE_FAIL) { if (suite->after != NULL) { @@ -396,6 +394,8 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test } run_test_rules(/*is_before=*/false, test, data); } + phase = TEST_PHASE_FRAMEWORK; + ret |= cleanup_test(test); ret = get_final_test_result(test, ret); Z_TC_END_RESULT(ret, test->name); From 7b08bdd1f4ce3ab3912b5a3fb71dfabe19fc64d3 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Thu, 18 Aug 2022 23:31:36 -0600 Subject: [PATCH 8/8] ztest: add framework tests for failed states Add specialized tests that execute another test indirectly and ensure that the results report errors correctly. See the README.rst file provided with this commit for details. Signed-off-by: Yuval Peress --- scripts/ci/check_compliance.py | 1 + tests/ztest/fail/CMakeLists.txt | 45 +++++++++++++++ tests/ztest/fail/Kconfig | 36 ++++++++++++ tests/ztest/fail/README.rst | 21 +++++++ tests/ztest/fail/core/CMakeLists.txt | 37 ++++++++++++ tests/ztest/fail/core/include/fail_test.hpp | 9 +++ tests/ztest/fail/core/prj.conf | 8 +++ tests/ztest/fail/core/src/assert_after.cpp | 14 +++++ tests/ztest/fail/core/src/assert_teardown.cpp | 14 +++++ tests/ztest/fail/core/src/assume_after.cpp | 14 +++++ tests/ztest/fail/core/src/assume_teardown.cpp | 14 +++++ tests/ztest/fail/core/src/main.cpp | 19 +++++++ tests/ztest/fail/core/src/pass_after.cpp | 14 +++++ tests/ztest/fail/core/src/pass_teardown.cpp | 14 +++++ tests/ztest/fail/prj.conf | 8 +++ tests/ztest/fail/src/main.cpp | 56 +++++++++++++++++++ tests/ztest/fail/testcase.yaml | 52 +++++++++++++++++ 17 files changed, 376 insertions(+) create mode 100644 tests/ztest/fail/CMakeLists.txt create mode 100644 tests/ztest/fail/Kconfig create mode 100644 tests/ztest/fail/README.rst create mode 100644 tests/ztest/fail/core/CMakeLists.txt create mode 100644 tests/ztest/fail/core/include/fail_test.hpp create mode 100644 tests/ztest/fail/core/prj.conf create mode 100644 tests/ztest/fail/core/src/assert_after.cpp create mode 100644 tests/ztest/fail/core/src/assert_teardown.cpp create mode 100644 tests/ztest/fail/core/src/assume_after.cpp create mode 100644 tests/ztest/fail/core/src/assume_teardown.cpp create mode 100644 tests/ztest/fail/core/src/main.cpp create mode 100644 tests/ztest/fail/core/src/pass_after.cpp create mode 100644 tests/ztest/fail/core/src/pass_teardown.cpp create mode 100644 tests/ztest/fail/prj.conf create mode 100644 tests/ztest/fail/src/main.cpp create mode 100644 tests/ztest/fail/testcase.yaml diff --git a/scripts/ci/check_compliance.py b/scripts/ci/check_compliance.py index 562852a57c515..9b466a8562814 100755 --- a/scripts/ci/check_compliance.py +++ b/scripts/ci/check_compliance.py @@ -607,6 +607,7 @@ def get_defined_syms(kconf): "MODVERSIONS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "SECURITY_LOADPIN", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "ZEPHYR_TRY_MASS_ERASE", # MCUBoot setting described in sysbuild documentation + "ZTEST_FAIL_TEST_", # regex in tests/ztest/fail/CMakeLists.txt } class KconfigBasicCheck(KconfigCheck, ComplianceTest): diff --git a/tests/ztest/fail/CMakeLists.txt b/tests/ztest/fail/CMakeLists.txt new file mode 100644 index 0000000000000..f7a070377740f --- /dev/null +++ b/tests/ztest/fail/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +include(ExternalProject) + +# Add the sources and set up the build for either unit testing or native_posix +list(APPEND SOURCES src/main.cpp) +if(BOARD STREQUAL unit_testing) + find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE}) + set(target testbinary) + # Set the target binary for the 'core' external project. The path to this must match the one set + # below in ExternalProject_Add's CMAKE_INSTALL_PREFIX + add_compile_definitions(FAIL_TARGET_BINARY="${CMAKE_BINARY_DIR}/core/bin/testbinary") +else() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + set(target app) + # Set the target binary for the 'core' external project. The path to this must match the one set + # below in ExternalProject_Add's CMAKE_INSTALL_PREFIX + add_compile_definitions(FAIL_TARGET_BINARY="${CMAKE_BINARY_DIR}/core/bin/zephyr.elf") +endif() + +# Create the project and set the sources for the target +project(fail) +target_sources(${target} PRIVATE ${SOURCES}) + +# Find which CONFIG_ZTEST_FAIL_TEST_* choice was set so we can pass it to the external project +# Once we find the config, we'll need to prepend a '-D' and append '=y' so we can pass it to the +# 'core' project as a cmake argument. +get_cmake_property(_vars VARIABLES) +string(REGEX MATCHALL "(^|;)CONFIG_ZTEST_FAIL_TEST_[A-Za-z0-9_]+" fail_test_config "${_vars}") +list(FILTER fail_test_config EXCLUDE REGEX "^$") +list(TRANSFORM fail_test_config PREPEND "-D") +list(TRANSFORM fail_test_config APPEND "=y") +string(REPLACE ";" " " fail_test_config "${fail_test_config}") + +# Add the 'core' external project which will mirror the configs of this project. +ExternalProject_Add(core + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/core + CMAKE_ARGS + -DBOARD:STRING=${BOARD} + -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/core + ${fail_test_config} +) +add_dependencies(${target} core) diff --git a/tests/ztest/fail/Kconfig b/tests/ztest/fail/Kconfig new file mode 100644 index 0000000000000..8f5527a23daea --- /dev/null +++ b/tests/ztest/fail/Kconfig @@ -0,0 +1,36 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +choice ZTEST_FAIL_TEST + prompt "Select the type of failure to test" + +config ZTEST_FAIL_TEST_ASSERT_AFTER + bool "Add a failed assert in the after phase" + +config ZTEST_FAIL_TEST_ASSERT_TEARDOWN + bool "Add a failed assert in the teardown phase" + +config ZTEST_FAIL_TEST_ASSUME_AFTER + bool "Add a failed assume in the after phase" + +config ZTEST_FAIL_TEST_ASSUME_TEARDOWN + bool "Add a failed assume in the teardown phase" + +config ZTEST_FAIL_TEST_PASS_AFTER + bool "Add a call to ztest_test_pass() in the after phase" + +config ZTEST_FAIL_TEST_PASS_TEARDOWN + bool "Add a call to ztest_test_pass() in the teardown phase" + +endchoice + +config TEST_ERROR_STRING + string + default "ERROR: cannot fail in test 'after()', bailing" if ZTEST_FAIL_TEST_ASSERT_AFTER + default "ERROR: cannot fail in test 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSERT_TEARDOWN + default "ERROR: cannot skip in test 'after()', bailing" if ZTEST_FAIL_TEST_ASSUME_AFTER + default "ERROR: cannot skip in test 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSUME_TEARDOWN + default "ERROR: cannot pass in test 'after()', bailing" if ZTEST_FAIL_TEST_PASS_AFTER + default "ERROR: cannot pass in test 'teardown()', bailing" if ZTEST_FAIL_TEST_PASS_TEARDOWN + +source "Kconfig.zephyr" diff --git a/tests/ztest/fail/README.rst b/tests/ztest/fail/README.rst new file mode 100644 index 0000000000000..3ed70d5229a5b --- /dev/null +++ b/tests/ztest/fail/README.rst @@ -0,0 +1,21 @@ +.. _ztest_framework_failure_tests: + +Ztest framework failure tests +############################# + +Overview +******** + +In order to test the actual framework's failure cases, this test suite has to do something unique. +There's a subdirectory to this test called 'core'. This project builds a sample ``native_posix`` or +``unit_testing`` binary which is expected to fail by calling one of the following: +- ``ztest_test_fail()`` during either the ``after`` or ``teardown`` phase of the test suite +- ``ztest_test_skip()`` during either the ``after`` or ``teardown`` phase of the test suite +- ``ztest_test_pass()`` during either the ``after`` or ``teardown`` phase of the test suite + +Note that these can be called indirectly through failed asserts or assumptions. + +The binary by itself, when executed, will fail to run and return a code of ``1``. The main test +binary will use ``popen()`` to run the failing test binary and will assert both the return code and +the output. The output itself cannot be printed to the log as it will confuse ``twister`` by +reporting a failure. diff --git a/tests/ztest/fail/core/CMakeLists.txt b/tests/ztest/fail/core/CMakeLists.txt new file mode 100644 index 0000000000000..70f884aa3d670 --- /dev/null +++ b/tests/ztest/fail/core/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) + +set(KCONFIG_ROOT ${CMAKE_CURRENT_LIST_DIR}/../Kconfig) + +# Add the sources +list(APPEND SOURCES src/main.cpp) +if(CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER) + list(APPEND SOURCES src/assert_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN) + list(APPEND SOURCES src/assert_teardown.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER) + list(APPEND SOURCES src/assume_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN) + list(APPEND SOURCES src/assume_teardown.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_PASS_AFTER) + list(APPEND SOURCES src/pass_after.cpp) +elseif(CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN) + list(APPEND SOURCES src/pass_teardown.cpp) +endif() + +if(BOARD STREQUAL unit_testing) + find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + target_include_directories(testbinary PRIVATE include) + install(TARGETS testbinary) +else() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + target_sources(app PRIVATE ${SOURCES}) + target_include_directories(app PRIVATE include) + install(TARGETS ${logical_target_for_zephyr_elf}) +endif() diff --git a/tests/ztest/fail/core/include/fail_test.hpp b/tests/ztest/fail/core/include/fail_test.hpp new file mode 100644 index 0000000000000..b4aa9788cde8a --- /dev/null +++ b/tests/ztest/fail/core/include/fail_test.hpp @@ -0,0 +1,9 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +void fail_test_after_impl(void); + +void fail_test_teardown_impl(void); diff --git a/tests/ztest/fail/core/prj.conf b/tests/ztest/fail/core/prj.conf new file mode 100644 index 0000000000000..6fa939ecf5081 --- /dev/null +++ b/tests/ztest/fail/core/prj.conf @@ -0,0 +1,8 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +CONFIG_ZTEST=y +CONFIG_ZTEST_NEW_API=y + +CONFIG_CPLUSPLUS=y +CONFIG_LIB_CPLUSPLUS=y diff --git a/tests/ztest/fail/core/src/assert_after.cpp b/tests/ztest/fail/core/src/assert_after.cpp new file mode 100644 index 0000000000000..79d7e35143cd6 --- /dev/null +++ b/tests/ztest/fail/core/src/assert_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + zassert_true(false, nullptr); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/assert_teardown.cpp b/tests/ztest/fail/core/src/assert_teardown.cpp new file mode 100644 index 0000000000000..e0134f1e31b3c --- /dev/null +++ b/tests/ztest/fail/core/src/assert_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + zassert_true(false, nullptr); +} diff --git a/tests/ztest/fail/core/src/assume_after.cpp b/tests/ztest/fail/core/src/assume_after.cpp new file mode 100644 index 0000000000000..2d7c9295a3f24 --- /dev/null +++ b/tests/ztest/fail/core/src/assume_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + zassume_true(false, nullptr); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/assume_teardown.cpp b/tests/ztest/fail/core/src/assume_teardown.cpp new file mode 100644 index 0000000000000..ffb0adf3ce9fd --- /dev/null +++ b/tests/ztest/fail/core/src/assume_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + zassume_true(false, nullptr); +} diff --git a/tests/ztest/fail/core/src/main.cpp b/tests/ztest/fail/core/src/main.cpp new file mode 100644 index 0000000000000..9fece93104c9c --- /dev/null +++ b/tests/ztest/fail/core/src/main.cpp @@ -0,0 +1,19 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +static void fail_after(void *) { + fail_test_after_impl(); +} + +static void fail_teardown(void *) { + fail_test_teardown_impl(); +} + +ZTEST_SUITE(fail, nullptr, nullptr, nullptr, fail_after, fail_teardown); + +ZTEST(fail, test_framework) {} diff --git a/tests/ztest/fail/core/src/pass_after.cpp b/tests/ztest/fail/core/src/pass_after.cpp new file mode 100644 index 0000000000000..447f082cf3cf4 --- /dev/null +++ b/tests/ztest/fail/core/src/pass_after.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) +{ + ztest_test_pass(); +} + +void fail_test_teardown_impl(void) {} diff --git a/tests/ztest/fail/core/src/pass_teardown.cpp b/tests/ztest/fail/core/src/pass_teardown.cpp new file mode 100644 index 0000000000000..935f3020fb59f --- /dev/null +++ b/tests/ztest/fail/core/src/pass_teardown.cpp @@ -0,0 +1,14 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "fail_test.hpp" + +void fail_test_after_impl(void) {} + +void fail_test_teardown_impl(void) +{ + ztest_test_pass(); +} diff --git a/tests/ztest/fail/prj.conf b/tests/ztest/fail/prj.conf new file mode 100644 index 0000000000000..6fa939ecf5081 --- /dev/null +++ b/tests/ztest/fail/prj.conf @@ -0,0 +1,8 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +CONFIG_ZTEST=y +CONFIG_ZTEST_NEW_API=y + +CONFIG_CPLUSPLUS=y +CONFIG_LIB_CPLUSPLUS=y diff --git a/tests/ztest/fail/src/main.cpp b/tests/ztest/fail/src/main.cpp new file mode 100644 index 0000000000000..7b57356b9cb41 --- /dev/null +++ b/tests/ztest/fail/src/main.cpp @@ -0,0 +1,56 @@ +/* Copyright (c) 2022 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +ZTEST_SUITE(fail, nullptr, nullptr, nullptr, nullptr, nullptr); + +ZTEST(fail, test_framework) +{ + auto found_error_string = false; + char buffer[sizeof(CONFIG_TEST_ERROR_STRING)] = {0}; + std::string result; + + /* Start running the target binary. This binary is expected to fail. */ + auto pipe = popen(FAIL_TARGET_BINARY, "r"); + + zassert_not_null(pipe, "Failed to execute '" FAIL_TARGET_BINARY "'"); + + /* Wait for the binary to finish running and grab the output */ + while (!feof(pipe)) { + if (fgets(buffer, ARRAY_SIZE(buffer), pipe) != nullptr) { + if (found_error_string) { + /* Already found the error string, no need to do any more string + * manipulation. + */ + continue; + } + + /* Append the buffer to the result string */ + result += buffer; + + /* Check if result contains the right error string */ + found_error_string |= + (result.find(CONFIG_TEST_ERROR_STRING) != std::string::npos); + + /* If the result string is longer than the expected string, + * we can prune it + */ + auto prune_length = static_cast(result.length()) - + static_cast(sizeof(CONFIG_TEST_ERROR_STRING)); + if (prune_length > 0) { + result.erase(0, prune_length); + } + } + } + auto rc = WEXITSTATUS(pclose(pipe)); + + zassert_equal(1, rc, "Test binary expected to fail with return code 1, but got %d", rc); + zassert_true(found_error_string, "Test binary did not produce the expected error string \"" + CONFIG_TEST_ERROR_STRING "\""); +} diff --git a/tests/ztest/fail/testcase.yaml b/tests/ztest/fail/testcase.yaml new file mode 100644 index 0000000000000..67bb0986be6d3 --- /dev/null +++ b/tests/ztest/fail/testcase.yaml @@ -0,0 +1,52 @@ +# Copyright (c) 2022 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +tests: + testing.fail.unit.assert_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER=y + testing.fail.unit.assert_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN=y + testing.fail.unit.assume_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y + testing.fail.unit.assume_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y + testing.fail.unit.pass_after: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_AFTER=y + testing.fail.unit.pass_teardown: + type: unit + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN=y + testing.fail.zephyr.assert_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_AFTER=y + testing.fail.zephyr.assert_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN=y + testing.fail.zephyr.assume_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y + testing.fail.zephyr.assume_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y + testing.fail.zephyr.pass_after: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_AFTER=y + testing.fail.zephyr.pass_teardown: + platform_allow: native_posix + extra_configs: + - CONFIG_ZTEST_FAIL_TEST_PASS_TEARDOWN=y