Skip to content

Safely handle asserts/assumptions in invalid test states #48846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 29, 2022
Merged

Safely handle asserts/assumptions in invalid test states #48846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
74da8da
ztest: remove init_testing
Aug 10, 2022
1d74744
ztest: move end_report implementation
Aug 10, 2022
f989fc8
ztest: unittest: Add before/after phase
Aug 10, 2022
a99dacc
ztest: unittest: Fix fail/skip/pass outside of tests
Aug 10, 2022
e771a6f
ztest: move get_friendly_phase_name to common code
Aug 10, 2022
ab3c083
ztest: Safely handle fail/skip/pass outside tests
Aug 10, 2022
71c9945
ztest: Match cleanup ordering between unittest and normal tests
Aug 11, 2022
7b08bdd
ztest: add framework tests for failed states
Aug 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/ci/check_compliance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,7 @@ def get_defined_syms(kconf):
"MODVERSIONS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc
"SECURITY_LOADPIN", # Linux, in boards/xtensa/intel_adsp_cavs25/doc
"ZEPHYR_TRY_MASS_ERASE", # MCUBoot setting described in sysbuild documentation
"ZTEST_FAIL_TEST_", # regex in tests/ztest/fail/CMakeLists.txt
}

class KconfigBasicCheck(KconfigCheck, ComplianceTest):
Expand Down
199 changes: 140 additions & 59 deletions subsys/testsuite/ztest/src/ztest_new.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,33 @@ enum ztest_phase {
TEST_PHASE_FRAMEWORK
};

/**
* @brief The current status of the test binary
*/
enum ztest_status {
ZTEST_STATUS_OK,
ZTEST_STATUS_HAS_FAILURE,
ZTEST_STATUS_CRITICAL_ERROR
};

/**
* @brief Tracks the current phase that ztest is operating in.
*/
ZTEST_DMEM enum ztest_phase phase = TEST_PHASE_FRAMEWORK;

static ZTEST_BMEM int test_status;
static ZTEST_BMEM enum ztest_status test_status = ZTEST_STATUS_OK;

extern ZTEST_DMEM const struct ztest_arch_api ztest_api;

void end_report(void)
{
if (test_status) {
TC_END_REPORT(TC_FAIL);
} else {
TC_END_REPORT(TC_PASS);
}
}

static int cleanup_test(struct ztest_unit_test *test)
{
int ret = TC_PASS;
Expand Down Expand Up @@ -247,27 +265,6 @@ static int get_final_test_result(const struct ztest_unit_test *test, int ret)
return ret;
}

#ifndef KERNEL

/* Static code analysis tool can raise a violation that the standard header
* <setjmp.h> shall not be used.
*
* setjmp is using in a test code, not in a runtime code, it is acceptable.
* It is a deliberate deviation.
*/
#include <setjmp.h> /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/
#include <signal.h>
#include <stdlib.h>
#include <string.h>

#define FAIL_FAST 0

static jmp_buf test_fail;
static jmp_buf test_pass;
static jmp_buf test_skip;
static jmp_buf stack_fail;
static jmp_buf test_suite_fail;

/**
* @brief Get a friendly name string for a given test phrase.
*
Expand All @@ -294,6 +291,27 @@ static inline const char *get_friendly_phase_name(enum ztest_phase phase)
}
}

#ifndef KERNEL

/* Static code analysis tool can raise a violation that the standard header
* <setjmp.h> shall not be used.
*
* setjmp is using in a test code, not in a runtime code, it is acceptable.
* It is a deliberate deviation.
*/
#include <setjmp.h> /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/
#include <signal.h>
#include <stdlib.h>
#include <string.h>

#define FAIL_FAST 0

static jmp_buf test_fail;
static jmp_buf test_pass;
static jmp_buf test_skip;
static jmp_buf stack_fail;
static jmp_buf test_suite_fail;

void ztest_test_fail(void)
{
switch (phase) {
Expand All @@ -302,25 +320,37 @@ void ztest_test_fail(void)
longjmp(test_suite_fail, 1);
case TEST_PHASE_BEFORE:
case TEST_PHASE_TEST:
case TEST_PHASE_AFTER:
case TEST_PHASE_TEARDOWN:
PRINT(" at %s function\n", get_friendly_phase_name(phase));
longjmp(test_fail, 1);
case TEST_PHASE_AFTER:
case TEST_PHASE_TEARDOWN:
case TEST_PHASE_FRAMEWORK:
PRINT("\n");
PRINT(" ERROR: cannot fail in test '%s()', bailing\n",
get_friendly_phase_name(phase));
longjmp(stack_fail, 1);
}
}

void ztest_test_pass(void) { longjmp(test_pass, 1); }

void ztest_test_skip(void) { longjmp(test_skip, 1); }
void ztest_test_pass(void)
{
if (phase == TEST_PHASE_TEST) {
longjmp(test_pass, 1);
}
PRINT(" ERROR: cannot pass in test '%s()', bailing\n", get_friendly_phase_name(phase));
longjmp(stack_fail, 1);
}

static void init_testing(void)
void ztest_test_skip(void)
{
if (setjmp(stack_fail)) {
PRINT("TESTSUITE crashed.");
exit(1);
switch (phase) {
case TEST_PHASE_SETUP:
case TEST_PHASE_BEFORE:
case TEST_PHASE_TEST:
longjmp(test_skip, 1);
default:
PRINT(" ERROR: cannot skip in test '%s()', bailing\n",
get_friendly_phase_name(phase));
longjmp(stack_fail, 1);
}
}

Expand All @@ -329,6 +359,7 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
int ret = TC_PASS;

TC_START(test->name);
phase = TEST_PHASE_BEFORE;

if (test_result == ZTEST_RESULT_SUITE_FAIL) {
ret = TC_FAIL;
Expand Down Expand Up @@ -356,13 +387,15 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
}
run_test_functions(suite, test, data);
out:
ret |= cleanup_test(test);
phase = TEST_PHASE_AFTER;
if (test_result != ZTEST_RESULT_SUITE_FAIL) {
if (suite->after != NULL) {
suite->after(data);
}
run_test_rules(/*is_before=*/false, test, data);
}
phase = TEST_PHASE_FRAMEWORK;
ret |= cleanup_test(test);

ret = get_final_test_result(test, ret);
Z_TC_END_RESULT(ret, test->name);
Expand Down Expand Up @@ -393,23 +426,56 @@ static void test_finalize(void)

void ztest_test_fail(void)
{
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL;
if (phase != TEST_PHASE_SETUP) {
switch (phase) {
case TEST_PHASE_SETUP:
test_result = ZTEST_RESULT_SUITE_FAIL;
break;
case TEST_PHASE_BEFORE:
case TEST_PHASE_TEST:
test_result = ZTEST_RESULT_FAIL;
test_finalize();
break;
default:
PRINT(" ERROR: cannot fail in test '%s()', bailing\n",
get_friendly_phase_name(phase));
test_status = ZTEST_STATUS_CRITICAL_ERROR;
break;
}
}

void ztest_test_pass(void)
{
test_result = ZTEST_RESULT_PASS;
test_finalize();
switch (phase) {
case TEST_PHASE_TEST:
test_result = ZTEST_RESULT_PASS;
test_finalize();
break;
default:
PRINT(" ERROR: cannot pass in test '%s()', bailing\n",
get_friendly_phase_name(phase));
test_status = ZTEST_STATUS_CRITICAL_ERROR;
if (phase == TEST_PHASE_BEFORE) {
test_finalize();
}
}
}

void ztest_test_skip(void)
{
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP;
if (phase != TEST_PHASE_SETUP) {
switch (phase) {
case TEST_PHASE_SETUP:
test_result = ZTEST_RESULT_SUITE_SKIP;
break;
case TEST_PHASE_BEFORE:
case TEST_PHASE_TEST:
test_result = ZTEST_RESULT_SKIP;
test_finalize();
break;
default:
PRINT(" ERROR: cannot skip in test '%s()', bailing\n",
get_friendly_phase_name(phase));
test_status = ZTEST_STATUS_CRITICAL_ERROR;
break;
}
}

Expand All @@ -425,11 +491,6 @@ void ztest_simple_1cpu_after(void *data)
z_test_1cpu_stop();
}

static void init_testing(void)
{
k_object_access_all_grant(&ztest_thread);
}

static void test_cb(void *a, void *b, void *c)
{
struct ztest_suite_node *suite = a;
Expand Down Expand Up @@ -576,11 +637,20 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
}

if (suite == NULL) {
test_status = 1;
test_status = ZTEST_STATUS_CRITICAL_ERROR;
return -1;
}

init_testing();
#ifndef KERNEL
if (setjmp(stack_fail)) {
PRINT("TESTSUITE crashed.\n");
test_status = ZTEST_STATUS_CRITICAL_ERROR;
end_report();
exit(1);
}
#else
k_object_access_all_grant(&ztest_thread);
#endif

TC_SUITE_START(suite->name);
test_result = ZTEST_RESULT_PENDING;
Expand Down Expand Up @@ -624,7 +694,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
}
}

if (fail && FAIL_FAST) {
if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) {
break;
}
}
Expand All @@ -646,13 +716,15 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
}
}

if (fail && FAIL_FAST) {
if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) {
break;
}
}
#endif

test_status = (test_status || fail) ? 1 : 0;
if (test_status == ZTEST_STATUS_OK && fail != 0) {
test_status = ZTEST_STATUS_HAS_FAILURE;
}
}

TC_SUITE_END(suite->name, (fail > 0 ? TC_FAIL : TC_PASS));
Expand All @@ -669,15 +741,6 @@ int z_ztest_run_test_suite(const char *name)
return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name));
}

void end_report(void)
{
if (test_status) {
TC_END_REPORT(TC_FAIL);
} else {
TC_END_REPORT(TC_PASS);
}
}

#ifdef CONFIG_USERSPACE
K_APPMEM_PARTITION_DEFINE(ztest_mem_partition);
#endif
Expand Down Expand Up @@ -842,6 +905,10 @@ int z_impl_ztest_run_test_suites(const void *state)
{
int count = 0;

if (test_status == ZTEST_STATUS_CRITICAL_ERROR) {
return count;
}

#ifdef CONFIG_ZTEST_SHUFFLE
struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT];

Expand All @@ -850,11 +917,25 @@ int z_impl_ztest_run_test_suites(const void *state)
ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node));
for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
count += __ztest_run_test_suite(suites_to_run[i], state);
/* Stop running tests if we have a critical error or if we have a failure and
* FAIL_FAST was set
*/
if (test_status == ZTEST_STATUS_CRITICAL_ERROR ||
(test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
break;
}
}
#else
for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
ptr < _ztest_suite_node_list_end; ++ptr) {
count += __ztest_run_test_suite(ptr, state);
/* Stop running tests if we have a critical error or if we have a failure and
* FAIL_FAST was set
*/
if (test_status == ZTEST_STATUS_CRITICAL_ERROR ||
(test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
break;
}
}
#endif

Expand Down Expand Up @@ -889,7 +970,7 @@ void ztest_verify_all_test_suites_ran(void)
}

if (!all_tests_run) {
test_status = 1;
test_status = ZTEST_STATUS_HAS_FAILURE;
}
}

Expand Down
Loading