michaelh/zephyr
1
0
Fork 0
Code Releases Activity

ztest: allow asserts anywhere

Browse source 
Updates the ztest_test_fail() function to allow failures in setup.
When executed, a failed assert will fail every test in the suite owning
the setup function. This was verified by adding a suite which asserts
in the setup function and has a test that should pass. During
exeuction, ztest marks the test as failing.

In order to verify exection I also added 2 new APIs:
- ZTEST_EXPECT_FAIL(suite_name, test_name)
- ZTEST_EXPECT_SKIP(suite_name, test_name)

Signed-off-by: Yuval Peress <peress@google.com>
This commit is contained in:
Yuval Peress 2022-08-07 21:16:25 -06:00 • committed by Anas Nashif
commit 84dfb8edf8
7 changed files with 194 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

1
cmake/linker_script/common/common-ram.cmake
View file

@ -111,4 +111,5 @@ endif()
if(CONFIG_ZTEST_NEW_API)
zephyr_iterable_section(NAME ztest_unit_test GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
zephyr_iterable_section(NAME ztest_test_rule GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
zephyr_iterable_section(NAME ztest_expected_result_entry GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4)
endif()

29
doc/develop/test/ztest.rst
View file

@ -110,6 +110,35 @@ This is achieved via fixtures in the following way::
Advanced features
*****************
Test result expectations
========================
Some tests were made to be broken. In cases where the test is expected to fail or skip due to the
nature of the code, it's possible to annotate the test as such. For example::
.. code-block:: C
#include <zephyr/ztest.h>
ZTEST_SUITE(my_suite, NULL, NULL, NULL, NULL, NULL);
ZTEST_EXPECT_FAIL(my_suite, test_fail)
ZTEST(my_suite, test_fail)
{
/** This will fail the test */
zassert_true(false, NULL);
}
ZTEST_EXPECT_SKIP(my_suite, test_fail)
ZTEST(my_suite, test_skip)
{
/** This will skip the test */
zassume_true(false, NULL);
}
In this example, the above tests should be marked as failed and skipped respectively. Instead,
Ztest will mark both as passed due to the expectation.
Test rules
==========

2
include/zephyr/linker/common-rom/common-rom-ztest.ld
View file

@ -6,6 +6,8 @@
SECTION_PROLOGUE(ztest,,)
{
Z_LINK_ITERABLE(ztest_expected_result_entry);
Z_LINK_ITERABLE(ztest_suite_node);
Z_LINK_ITERABLE(ztest_unit_test);

6
subsys/testsuite/include/zephyr/ztest_unittest.ld
View file

@ -6,6 +6,12 @@
SECTIONS
{
.data.ztest_expected_result_area : ALIGN(4)
{
_ztest_expected_result_entry_list_start = .;
KEEP(*(SORT_BY_NAME(._ztest_expected_result_entry.static.*)))
_ztest_expected_result_entry_list_end = .;
}
.data.ztest_suite_node_area : ALIGN(4)
{
_ztest_suite_node_list_start = .;

67
subsys/testsuite/ztest/include/zephyr/ztest_test_new.h
View file

@ -27,6 +27,73 @@
extern "C" {
#endif
/**
* @brief The expected result of a test.
*
* @see ZTEST_EXPECT_FAIL
* @see ZTEST_EXPECT_SKIP
*/
enum ztest_expected_result {
ZTEST_EXPECTED_RESULT_FAIL = 0, /**< Expect a test to fail */
ZTEST_EXPECTED_RESULT_SKIP, /**< Expect a test to pass */
};
/**
* @brief A single expectation entry allowing tests to fail/skip and be considered passing.
*
* @see ZTEST_EXPECT_FAIL
* @see ZTEST_EXPECT_SKIP
*/
struct ztest_expected_result_entry {
const char *test_suite_name; /**< The test suite's name for the expectation */
const char *test_name; /**< The test's name for the expectation */
enum ztest_expected_result expected_result; /**< The expectation */
};
extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_start[];
extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_end[];
#define __ZTEST_EXPECT(_suite_name, _test_name, expectation) \
static const STRUCT_SECTION_ITERABLE( \
ztest_expected_result_entry, \
UTIL_CAT(UTIL_CAT(z_ztest_expected_result_, _suite_name), _test_name)) = { \
.test_suite_name = STRINGIFY(_suite_name), \
.test_name = STRINGIFY(_test_name), \
.expected_result = expectation, \
}
/**
* @brief Expect a test to fail (mark it passing if it failed)
*
* Adding this macro to your logic will allow the failing test to be considered passing, example:
*
* ZTEST_EXPECT_FAIL(my_suite, test_x);
* ZTEST(my_suite, text_x) {
* zassert_true(false, NULL);
* }
*
* @param _suite_name The name of the suite
* @param _test_name The name of the test
*/
#define ZTEST_EXPECT_FAIL(_suite_name, _test_name) \
__ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_FAIL)
/**
* @brief Expect a test to skip (mark it passing if it failed)
*
* Adding this macro to your logic will allow the failing test to be considered passing, example:
*
* ZTEST_EXPECT_SKIP(my_suite, test_x);
* ZTEST(my_suite, text_x) {
* zassume_true(false, NULL);
* }
*
* @param _suite_name The name of the suite
* @param _test_name The name of the test
*/
#define ZTEST_EXPECT_SKIP(_suite_name, _test_name) \
__ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_SKIP)
struct ztest_unit_test {
const char *test_suite_name;
const char *name;

89
subsys/testsuite/ztest/src/ztest_new.c
View file

@ -211,9 +211,42 @@ enum ztest_result {
ZTEST_RESULT_FAIL,
ZTEST_RESULT_SKIP,
ZTEST_RESULT_SUITE_SKIP,
ZTEST_RESULT_SUITE_FAIL,
};
COND_CODE_1(KERNEL, (ZTEST_BMEM), ()) static enum ztest_result test_result;
static int get_final_test_result(const struct ztest_unit_test *test, int ret)
{
enum ztest_expected_result expected_result = -1;
for (struct ztest_expected_result_entry *expectation =
_ztest_expected_result_entry_list_start;
expectation < _ztest_expected_result_entry_list_end; ++expectation) {
if (strcmp(expectation->test_name, test->name) == 0 &&
strcmp(expectation->test_suite_name, test->test_suite_name) == 0) {
expected_result = expectation->expected_result;
break;
}
}
if (expected_result == ZTEST_EXPECTED_RESULT_FAIL) {
/* Expected a failure:
* - If we got a failure, return TC_PASS
* - Otherwise force a failure
*/
return (ret == TC_FAIL) ? TC_PASS : TC_FAIL;
}
if (expected_result == ZTEST_EXPECTED_RESULT_SKIP) {
/* Expected a skip:
* - If we got a skip, return TC_PASS
* - Otherwise force a failure
*/
return (ret == TC_SKIP) ? TC_PASS : TC_FAIL;
}
/* No expectation was made, no change is needed. */
return ret;
}
#ifndef KERNEL
/* Static code analysis tool can raise a violation that the standard header
@ -233,6 +266,7 @@ static jmp_buf test_fail;
static jmp_buf test_pass;
static jmp_buf test_skip;
static jmp_buf stack_fail;
static jmp_buf test_suite_fail;
/**
* @brief Get a friendly name string for a given test phrase.
@ -264,6 +298,8 @@ void ztest_test_fail(void)
{
switch (phase) {
case TEST_PHASE_SETUP:
PRINT(" at %s function\n", get_friendly_phase_name(phase));
longjmp(test_suite_fail, 1);
case TEST_PHASE_BEFORE:
case TEST_PHASE_TEST:
case TEST_PHASE_AFTER:
@ -294,6 +330,11 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
TC_START(test->name);
if (test_result == ZTEST_RESULT_SUITE_FAIL) {
ret = TC_FAIL;
goto out;
}
if (setjmp(test_fail)) {
ret = TC_FAIL;
goto out;
@ -316,10 +357,14 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
run_test_functions(suite, test, data);
out:
ret |= cleanup_test(test);
if (suite->after != NULL) {
suite->after(data);
if (test_result != ZTEST_RESULT_SUITE_FAIL) {
if (suite->after != NULL) {
suite->after(data);
}
run_test_rules(/*is_before=*/false, test, data);
}
run_test_rules(/*is_before=*/false, test, data);
ret = get_final_test_result(test, ret);
Z_TC_END_RESULT(ret, test->name);
return ret;
@ -348,8 +393,10 @@ static void test_finalize(void)
void ztest_test_fail(void)
{
test_result = ZTEST_RESULT_FAIL;
test_finalize();
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL;
if (phase != TEST_PHASE_SETUP) {
test_finalize();
}
}
void ztest_test_pass(void)
@ -360,9 +407,8 @@ void ztest_test_pass(void)
void ztest_test_skip(void)
{
test_result = ZTEST_RESULT_SUITE_SKIP;
test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP;
if (phase != TEST_PHASE_SETUP) {
test_result = ZTEST_RESULT_SKIP;
test_finalize();
}
}
@ -379,7 +425,10 @@ void ztest_simple_1cpu_after(void *data)
z_test_1cpu_stop();
}
static void init_testing(void) { k_object_access_all_grant(&ztest_thread); }
static void init_testing(void)
{
k_object_access_all_grant(&ztest_thread);
}
static void test_cb(void *a, void *b, void *c)
{
@ -418,11 +467,13 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
k_thread_name_set(&ztest_thread, test->name);
}
/* Only start the thread if we're not skipping the suite */
if (test_result != ZTEST_RESULT_SUITE_SKIP) {
if (test_result != ZTEST_RESULT_SUITE_SKIP &&
test_result != ZTEST_RESULT_SUITE_FAIL) {
k_thread_start(&ztest_thread);
k_thread_join(&ztest_thread, K_FOREVER);
}
} else if (test_result != ZTEST_RESULT_SUITE_SKIP) {
} else if (test_result != ZTEST_RESULT_SUITE_SKIP &&
test_result != ZTEST_RESULT_SUITE_FAIL) {
test_result = ZTEST_RESULT_PENDING;
run_test_rules(/*is_before=*/true, test, data);
if (suite->before) {
@ -444,19 +495,18 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
k_msleep(100);
}
if (test_result == ZTEST_RESULT_FAIL) {
if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL) {
ret = TC_FAIL;
} else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) {
ret = TC_SKIP;
}
if (test_result == ZTEST_RESULT_PASS || !FAIL_FAST) {
ret |= cleanup_test(test);
}
if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) {
Z_TC_END_RESULT(TC_SKIP, test->name);
} else {
Z_TC_END_RESULT(ret, test->name);
}
ret = get_final_test_result(test, ret);
Z_TC_END_RESULT(ret, test->name);
return ret;
}
@ -526,7 +576,12 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
TC_SUITE_START(suite->name);
test_result = ZTEST_RESULT_PENDING;
phase = TEST_PHASE_SETUP;
if (suite->setup != NULL) {
#ifndef KERNEL
if (setjmp(test_suite_fail)) {
test_result = ZTEST_RESULT_SUITE_FAIL;
}
#endif
if (test_result != ZTEST_RESULT_SUITE_FAIL && suite->setup != NULL) {
data = suite->setup();
}

17
tests/ztest/base/src/main.c
View file

@ -35,12 +35,14 @@ ZTEST(framework_tests, test_assert_mem_equal)
zassert_mem_equal(actual, expected, sizeof(expected), NULL);
}
ZTEST_EXPECT_SKIP(framework_tests, test_skip_config);
ZTEST(framework_tests, test_skip_config)
{
Z_TEST_SKIP_IFDEF(CONFIG_BUGxxxxx);
ztest_test_fail();
}
ZTEST_EXPECT_SKIP(framework_tests, test_skip_no_config);
ZTEST(framework_tests, test_skip_no_config)
{
Z_TEST_SKIP_IFNDEF(CONFIG_BUGyyyyy);
@ -149,3 +151,18 @@ ZTEST_F(rules_tests, test_rules_before_after)
fixture->state = RULE_STATE_TEST;
fixture->run_count++;
}
static void *fail_in_setup_setup(void)
{
zassert_true(false, NULL);
return NULL;
}
ZTEST_EXPECT_FAIL(fail_in_setup, test_should_never_run);
ZTEST(fail_in_setup, test_should_never_run)
{
/* The following should pass, but the setup function will cause it to fail */
zassert_true(true, NULL);
}
ZTEST_SUITE(fail_in_setup, NULL, fail_in_setup_setup, NULL, NULL, NULL);