ztest: allow asserts anywhere
Updates the ztest_test_fail() function to allow failures in setup. When executed, a failed assert will fail every test in the suite owning the setup function. This was verified by adding a suite which asserts in the setup function and has a test that should pass. During exeuction, ztest marks the test as failing. In order to verify exection I also added 2 new APIs: - ZTEST_EXPECT_FAIL(suite_name, test_name) - ZTEST_EXPECT_SKIP(suite_name, test_name) Signed-off-by: Yuval Peress <peress@google.com>
This commit is contained in:
parent
e526b407d2
commit
84dfb8edf8
7 changed files with 194 additions and 17 deletions
|
@ -111,4 +111,5 @@ endif()
|
|||
if(CONFIG_ZTEST_NEW_API)
|
||||
zephyr_iterable_section(NAME ztest_unit_test GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
|
||||
zephyr_iterable_section(NAME ztest_test_rule GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
|
||||
zephyr_iterable_section(NAME ztest_expected_result_entry GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
|
||||
endif()
|
||||
|
|
|
@ -110,6 +110,35 @@ This is achieved via fixtures in the following way::
|
|||
Advanced features
|
||||
*****************
|
||||
|
||||
Test result expectations
|
||||
========================
|
||||
|
||||
Some tests were made to be broken. In cases where the test is expected to fail or skip due to the
|
||||
nature of the code, it's possible to annotate the test as such. For example::
|
||||
|
||||
.. code-block:: C
|
||||
|
||||
#include <zephyr/ztest.h>
|
||||
|
||||
ZTEST_SUITE(my_suite, NULL, NULL, NULL, NULL, NULL);
|
||||
|
||||
ZTEST_EXPECT_FAIL(my_suite, test_fail)
|
||||
ZTEST(my_suite, test_fail)
|
||||
{
|
||||
/** This will fail the test */
|
||||
zassert_true(false, NULL);
|
||||
}
|
||||
|
||||
ZTEST_EXPECT_SKIP(my_suite, test_fail)
|
||||
ZTEST(my_suite, test_skip)
|
||||
{
|
||||
/** This will skip the test */
|
||||
zassume_true(false, NULL);
|
||||
}
|
||||
|
||||
In this example, the above tests should be marked as failed and skipped respectively. Instead,
|
||||
Ztest will mark both as passed due to the expectation.
|
||||
|
||||
Test rules
|
||||
==========
|
||||
|
||||
|
|
|
@ -6,6 +6,8 @@
|
|||
|
||||
SECTION_PROLOGUE(ztest,,)
|
||||
{
|
||||
Z_LINK_ITERABLE(ztest_expected_result_entry);
|
||||
|
||||
Z_LINK_ITERABLE(ztest_suite_node);
|
||||
|
||||
Z_LINK_ITERABLE(ztest_unit_test);
|
||||
|
|
|
@ -6,6 +6,12 @@
|
|||
|
||||
SECTIONS
|
||||
{
|
||||
.data.ztest_expected_result_area : ALIGN(4)
|
||||
{
|
||||
_ztest_expected_result_entry_list_start = .;
|
||||
KEEP(*(SORT_BY_NAME(._ztest_expected_result_entry.static.*)))
|
||||
_ztest_expected_result_entry_list_end = .;
|
||||
}
|
||||
.data.ztest_suite_node_area : ALIGN(4)
|
||||
{
|
||||
_ztest_suite_node_list_start = .;
|
||||
|
|
|
@ -27,6 +27,73 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief The expected result of a test.
|
||||
*
|
||||
* @see ZTEST_EXPECT_FAIL
|
||||
* @see ZTEST_EXPECT_SKIP
|
||||
*/
|
||||
enum ztest_expected_result {
|
||||
ZTEST_EXPECTED_RESULT_FAIL = 0, /**< Expect a test to fail */
|
||||
ZTEST_EXPECTED_RESULT_SKIP, /**< Expect a test to pass */
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A single expectation entry allowing tests to fail/skip and be considered passing.
|
||||
*
|
||||
* @see ZTEST_EXPECT_FAIL
|
||||
* @see ZTEST_EXPECT_SKIP
|
||||
*/
|
||||
struct ztest_expected_result_entry {
|
||||
const char *test_suite_name; /**< The test suite's name for the expectation */
|
||||
const char *test_name; /**< The test's name for the expectation */
|
||||
enum ztest_expected_result expected_result; /**< The expectation */
|
||||
};
|
||||
|
||||
extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_start[];
|
||||
extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_end[];
|
||||
|
||||
#define __ZTEST_EXPECT(_suite_name, _test_name, expectation) \
|
||||
static const STRUCT_SECTION_ITERABLE( \
|
||||
ztest_expected_result_entry, \
|
||||
UTIL_CAT(UTIL_CAT(z_ztest_expected_result_, _suite_name), _test_name)) = { \
|
||||
.test_suite_name = STRINGIFY(_suite_name), \
|
||||
.test_name = STRINGIFY(_test_name), \
|
||||
.expected_result = expectation, \
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Expect a test to fail (mark it passing if it failed)
|
||||
*
|
||||
* Adding this macro to your logic will allow the failing test to be considered passing, example:
|
||||
*
|
||||
* ZTEST_EXPECT_FAIL(my_suite, test_x);
|
||||
* ZTEST(my_suite, text_x) {
|
||||
* zassert_true(false, NULL);
|
||||
* }
|
||||
*
|
||||
* @param _suite_name The name of the suite
|
||||
* @param _test_name The name of the test
|
||||
*/
|
||||
#define ZTEST_EXPECT_FAIL(_suite_name, _test_name) \
|
||||
__ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_FAIL)
|
||||
|
||||
/**
|
||||
* @brief Expect a test to skip (mark it passing if it failed)
|
||||
*
|
||||
* Adding this macro to your logic will allow the failing test to be considered passing, example:
|
||||
*
|
||||
* ZTEST_EXPECT_SKIP(my_suite, test_x);
|
||||
* ZTEST(my_suite, text_x) {
|
||||
* zassume_true(false, NULL);
|
||||
* }
|
||||
*
|
||||
* @param _suite_name The name of the suite
|
||||
* @param _test_name The name of the test
|
||||
*/
|
||||
#define ZTEST_EXPECT_SKIP(_suite_name, _test_name) \
|
||||
__ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_SKIP)
|
||||
|
||||
struct ztest_unit_test {
|
||||
const char *test_suite_name;
|
||||
const char *name;
|
||||
|
|
|
@ -211,9 +211,42 @@ enum ztest_result {
|
|||
ZTEST_RESULT_FAIL,
|
||||
ZTEST_RESULT_SKIP,
|
||||
ZTEST_RESULT_SUITE_SKIP,
|
||||
ZTEST_RESULT_SUITE_FAIL,
|
||||
};
|
||||
COND_CODE_1(KERNEL, (ZTEST_BMEM), ()) static enum ztest_result test_result;
|
||||
|
||||
static int get_final_test_result(const struct ztest_unit_test *test, int ret)
|
||||
{
|
||||
enum ztest_expected_result expected_result = -1;
|
||||
|
||||
for (struct ztest_expected_result_entry *expectation =
|
||||
_ztest_expected_result_entry_list_start;
|
||||
expectation < _ztest_expected_result_entry_list_end; ++expectation) {
|
||||
if (strcmp(expectation->test_name, test->name) == 0 &&
|
||||
strcmp(expectation->test_suite_name, test->test_suite_name) == 0) {
|
||||
expected_result = expectation->expected_result;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (expected_result == ZTEST_EXPECTED_RESULT_FAIL) {
|
||||
/* Expected a failure:
|
||||
* - If we got a failure, return TC_PASS
|
||||
* - Otherwise force a failure
|
||||
*/
|
||||
return (ret == TC_FAIL) ? TC_PASS : TC_FAIL;
|
||||
}
|
||||
if (expected_result == ZTEST_EXPECTED_RESULT_SKIP) {
|
||||
/* Expected a skip:
|
||||
* - If we got a skip, return TC_PASS
|
||||
* - Otherwise force a failure
|
||||
*/
|
||||
return (ret == TC_SKIP) ? TC_PASS : TC_FAIL;
|
||||
}
|
||||
/* No expectation was made, no change is needed. */
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifndef KERNEL
|
||||
|
||||
/* Static code analysis tool can raise a violation that the standard header
|
||||
|
@ -233,6 +266,7 @@ static jmp_buf test_fail;
|
|||
static jmp_buf test_pass;
|
||||
static jmp_buf test_skip;
|
||||
static jmp_buf stack_fail;
|
||||
static jmp_buf test_suite_fail;
|
||||
|
||||
/**
|
||||
* @brief Get a friendly name string for a given test phrase.
|
||||
|
@ -264,6 +298,8 @@ void ztest_test_fail(void)
|
|||
{
|
||||
switch (phase) {
|
||||
case TEST_PHASE_SETUP:
|
||||
PRINT(" at %s function\n", get_friendly_phase_name(phase));
|
||||
longjmp(test_suite_fail, 1);
|
||||
case TEST_PHASE_BEFORE:
|
||||
case TEST_PHASE_TEST:
|
||||
case TEST_PHASE_AFTER:
|
||||
|
@ -294,6 +330,11 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
|
|||
|
||||
TC_START(test->name);
|
||||
|
||||
if (test_result == ZTEST_RESULT_SUITE_FAIL) {
|
||||
ret = TC_FAIL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (setjmp(test_fail)) {
|
||||
ret = TC_FAIL;
|
||||
goto out;
|
||||
|
@ -316,10 +357,14 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
|
|||
run_test_functions(suite, test, data);
|
||||
out:
|
||||
ret |= cleanup_test(test);
|
||||
if (suite->after != NULL) {
|
||||
suite->after(data);
|
||||
if (test_result != ZTEST_RESULT_SUITE_FAIL) {
|
||||
if (suite->after != NULL) {
|
||||
suite->after(data);
|
||||
}
|
||||
run_test_rules(/*is_before=*/false, test, data);
|
||||
}
|
||||
run_test_rules(/*is_before=*/false, test, data);
|
||||
|
||||
ret = get_final_test_result(test, ret);
|
||||
Z_TC_END_RESULT(ret, test->name);
|
||||
|
||||
return ret;
|
||||
|
@ -348,8 +393,10 @@ static void test_finalize(void)
|
|||
|
||||
void ztest_test_fail(void)
|
||||
{
|
||||
test_result = ZTEST_RESULT_FAIL;
|
||||
test_finalize();
|
||||
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL;
|
||||
if (phase != TEST_PHASE_SETUP) {
|
||||
test_finalize();
|
||||
}
|
||||
}
|
||||
|
||||
void ztest_test_pass(void)
|
||||
|
@ -360,9 +407,8 @@ void ztest_test_pass(void)
|
|||
|
||||
void ztest_test_skip(void)
|
||||
{
|
||||
test_result = ZTEST_RESULT_SUITE_SKIP;
|
||||
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP;
|
||||
if (phase != TEST_PHASE_SETUP) {
|
||||
test_result = ZTEST_RESULT_SKIP;
|
||||
test_finalize();
|
||||
}
|
||||
}
|
||||
|
@ -379,7 +425,10 @@ void ztest_simple_1cpu_after(void *data)
|
|||
z_test_1cpu_stop();
|
||||
}
|
||||
|
||||
static void init_testing(void) { k_object_access_all_grant(&ztest_thread); }
|
||||
static void init_testing(void)
|
||||
{
|
||||
k_object_access_all_grant(&ztest_thread);
|
||||
}
|
||||
|
||||
static void test_cb(void *a, void *b, void *c)
|
||||
{
|
||||
|
@ -418,11 +467,13 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
|
|||
k_thread_name_set(&ztest_thread, test->name);
|
||||
}
|
||||
/* Only start the thread if we're not skipping the suite */
|
||||
if (test_result != ZTEST_RESULT_SUITE_SKIP) {
|
||||
if (test_result != ZTEST_RESULT_SUITE_SKIP &&
|
||||
test_result != ZTEST_RESULT_SUITE_FAIL) {
|
||||
k_thread_start(&ztest_thread);
|
||||
k_thread_join(&ztest_thread, K_FOREVER);
|
||||
}
|
||||
} else if (test_result != ZTEST_RESULT_SUITE_SKIP) {
|
||||
} else if (test_result != ZTEST_RESULT_SUITE_SKIP &&
|
||||
test_result != ZTEST_RESULT_SUITE_FAIL) {
|
||||
test_result = ZTEST_RESULT_PENDING;
|
||||
run_test_rules(/*is_before=*/true, test, data);
|
||||
if (suite->before) {
|
||||
|
@ -444,19 +495,18 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
|
|||
k_msleep(100);
|
||||
}
|
||||
|
||||
if (test_result == ZTEST_RESULT_FAIL) {
|
||||
if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL) {
|
||||
ret = TC_FAIL;
|
||||
} else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) {
|
||||
ret = TC_SKIP;
|
||||
}
|
||||
|
||||
if (test_result == ZTEST_RESULT_PASS || !FAIL_FAST) {
|
||||
ret |= cleanup_test(test);
|
||||
}
|
||||
|
||||
if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) {
|
||||
Z_TC_END_RESULT(TC_SKIP, test->name);
|
||||
} else {
|
||||
Z_TC_END_RESULT(ret, test->name);
|
||||
}
|
||||
ret = get_final_test_result(test, ret);
|
||||
Z_TC_END_RESULT(ret, test->name);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
@ -526,7 +576,12 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
|
|||
TC_SUITE_START(suite->name);
|
||||
test_result = ZTEST_RESULT_PENDING;
|
||||
phase = TEST_PHASE_SETUP;
|
||||
if (suite->setup != NULL) {
|
||||
#ifndef KERNEL
|
||||
if (setjmp(test_suite_fail)) {
|
||||
test_result = ZTEST_RESULT_SUITE_FAIL;
|
||||
}
|
||||
#endif
|
||||
if (test_result != ZTEST_RESULT_SUITE_FAIL && suite->setup != NULL) {
|
||||
data = suite->setup();
|
||||
}
|
||||
|
||||
|
|
|
@ -35,12 +35,14 @@ ZTEST(framework_tests, test_assert_mem_equal)
|
|||
zassert_mem_equal(actual, expected, sizeof(expected), NULL);
|
||||
}
|
||||
|
||||
ZTEST_EXPECT_SKIP(framework_tests, test_skip_config);
|
||||
ZTEST(framework_tests, test_skip_config)
|
||||
{
|
||||
Z_TEST_SKIP_IFDEF(CONFIG_BUGxxxxx);
|
||||
ztest_test_fail();
|
||||
}
|
||||
|
||||
ZTEST_EXPECT_SKIP(framework_tests, test_skip_no_config);
|
||||
ZTEST(framework_tests, test_skip_no_config)
|
||||
{
|
||||
Z_TEST_SKIP_IFNDEF(CONFIG_BUGyyyyy);
|
||||
|
@ -149,3 +151,18 @@ ZTEST_F(rules_tests, test_rules_before_after)
|
|||
fixture->state = RULE_STATE_TEST;
|
||||
fixture->run_count++;
|
||||
}
|
||||
|
||||
static void *fail_in_setup_setup(void)
|
||||
{
|
||||
zassert_true(false, NULL);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ZTEST_EXPECT_FAIL(fail_in_setup, test_should_never_run);
|
||||
ZTEST(fail_in_setup, test_should_never_run)
|
||||
{
|
||||
/* The following should pass, but the setup function will cause it to fail */
|
||||
zassert_true(true, NULL);
|
||||
}
|
||||
|
||||
ZTEST_SUITE(fail_in_setup, NULL, fail_in_setup_setup, NULL, NULL, NULL);
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue