From 84dfb8edf86bc26c80cb2b8987c8a3dc164c803b Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Sun, 7 Aug 2022 21:16:25 -0600 Subject: [PATCH] ztest: allow asserts anywhere Updates the ztest_test_fail() function to allow failures in setup. When executed, a failed assert will fail every test in the suite owning the setup function. This was verified by adding a suite which asserts in the setup function and has a test that should pass. During exeuction, ztest marks the test as failing. In order to verify exection I also added 2 new APIs: - ZTEST_EXPECT_FAIL(suite_name, test_name) - ZTEST_EXPECT_SKIP(suite_name, test_name) Signed-off-by: Yuval Peress --- cmake/linker_script/common/common-ram.cmake | 1 + doc/develop/test/ztest.rst | 29 ++++++ .../linker/common-rom/common-rom-ztest.ld | 2 + .../include/zephyr/ztest_unittest.ld | 6 ++ .../ztest/include/zephyr/ztest_test_new.h | 67 ++++++++++++++ subsys/testsuite/ztest/src/ztest_new.c | 89 +++++++++++++++---- tests/ztest/base/src/main.c | 17 ++++ 7 files changed, 194 insertions(+), 17 deletions(-) diff --git a/cmake/linker_script/common/common-ram.cmake b/cmake/linker_script/common/common-ram.cmake index baedefe2797..32ffb357406 100644 --- a/cmake/linker_script/common/common-ram.cmake +++ b/cmake/linker_script/common/common-ram.cmake @@ -111,4 +111,5 @@ endif() if(CONFIG_ZTEST_NEW_API) zephyr_iterable_section(NAME ztest_unit_test GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4) zephyr_iterable_section(NAME ztest_test_rule GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4) + zephyr_iterable_section(NAME ztest_expected_result_entry GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT} SUBALIGN 4) endif() diff --git a/doc/develop/test/ztest.rst b/doc/develop/test/ztest.rst index 48d708f700c..5b07799793b 100644 --- a/doc/develop/test/ztest.rst +++ b/doc/develop/test/ztest.rst @@ -110,6 +110,35 @@ This is achieved via fixtures in the following way:: Advanced features ***************** +Test result expectations +======================== + +Some tests were made to be broken. In cases where the test is expected to fail or skip due to the +nature of the code, it's possible to annotate the test as such. For example:: + + .. code-block:: C + + #include + + ZTEST_SUITE(my_suite, NULL, NULL, NULL, NULL, NULL); + + ZTEST_EXPECT_FAIL(my_suite, test_fail) + ZTEST(my_suite, test_fail) + { + /** This will fail the test */ + zassert_true(false, NULL); + } + + ZTEST_EXPECT_SKIP(my_suite, test_fail) + ZTEST(my_suite, test_skip) + { + /** This will skip the test */ + zassume_true(false, NULL); + } + +In this example, the above tests should be marked as failed and skipped respectively. Instead, +Ztest will mark both as passed due to the expectation. + Test rules ========== diff --git a/include/zephyr/linker/common-rom/common-rom-ztest.ld b/include/zephyr/linker/common-rom/common-rom-ztest.ld index 3fba470be0b..62d952b89dd 100644 --- a/include/zephyr/linker/common-rom/common-rom-ztest.ld +++ b/include/zephyr/linker/common-rom/common-rom-ztest.ld @@ -6,6 +6,8 @@ SECTION_PROLOGUE(ztest,,) { + Z_LINK_ITERABLE(ztest_expected_result_entry); + Z_LINK_ITERABLE(ztest_suite_node); Z_LINK_ITERABLE(ztest_unit_test); diff --git a/subsys/testsuite/include/zephyr/ztest_unittest.ld b/subsys/testsuite/include/zephyr/ztest_unittest.ld index af73dfe7e7a..4a3e7d783cc 100644 --- a/subsys/testsuite/include/zephyr/ztest_unittest.ld +++ b/subsys/testsuite/include/zephyr/ztest_unittest.ld @@ -6,6 +6,12 @@ SECTIONS { + .data.ztest_expected_result_area : ALIGN(4) + { + _ztest_expected_result_entry_list_start = .; + KEEP(*(SORT_BY_NAME(._ztest_expected_result_entry.static.*))) + _ztest_expected_result_entry_list_end = .; + } .data.ztest_suite_node_area : ALIGN(4) { _ztest_suite_node_list_start = .; diff --git a/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h b/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h index 09095befc61..24e567ad40b 100644 --- a/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h +++ b/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h @@ -27,6 +27,73 @@ extern "C" { #endif +/** + * @brief The expected result of a test. + * + * @see ZTEST_EXPECT_FAIL + * @see ZTEST_EXPECT_SKIP + */ +enum ztest_expected_result { + ZTEST_EXPECTED_RESULT_FAIL = 0, /**< Expect a test to fail */ + ZTEST_EXPECTED_RESULT_SKIP, /**< Expect a test to pass */ +}; + +/** + * @brief A single expectation entry allowing tests to fail/skip and be considered passing. + * + * @see ZTEST_EXPECT_FAIL + * @see ZTEST_EXPECT_SKIP + */ +struct ztest_expected_result_entry { + const char *test_suite_name; /**< The test suite's name for the expectation */ + const char *test_name; /**< The test's name for the expectation */ + enum ztest_expected_result expected_result; /**< The expectation */ +}; + +extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_start[]; +extern struct ztest_expected_result_entry _ztest_expected_result_entry_list_end[]; + +#define __ZTEST_EXPECT(_suite_name, _test_name, expectation) \ + static const STRUCT_SECTION_ITERABLE( \ + ztest_expected_result_entry, \ + UTIL_CAT(UTIL_CAT(z_ztest_expected_result_, _suite_name), _test_name)) = { \ + .test_suite_name = STRINGIFY(_suite_name), \ + .test_name = STRINGIFY(_test_name), \ + .expected_result = expectation, \ + } + +/** + * @brief Expect a test to fail (mark it passing if it failed) + * + * Adding this macro to your logic will allow the failing test to be considered passing, example: + * + * ZTEST_EXPECT_FAIL(my_suite, test_x); + * ZTEST(my_suite, text_x) { + * zassert_true(false, NULL); + * } + * + * @param _suite_name The name of the suite + * @param _test_name The name of the test + */ +#define ZTEST_EXPECT_FAIL(_suite_name, _test_name) \ + __ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_FAIL) + +/** + * @brief Expect a test to skip (mark it passing if it failed) + * + * Adding this macro to your logic will allow the failing test to be considered passing, example: + * + * ZTEST_EXPECT_SKIP(my_suite, test_x); + * ZTEST(my_suite, text_x) { + * zassume_true(false, NULL); + * } + * + * @param _suite_name The name of the suite + * @param _test_name The name of the test + */ +#define ZTEST_EXPECT_SKIP(_suite_name, _test_name) \ + __ZTEST_EXPECT(_suite_name, _test_name, ZTEST_EXPECTED_RESULT_SKIP) + struct ztest_unit_test { const char *test_suite_name; const char *name; diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index f7825c73a15..fb8ec4b7b1b 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -211,9 +211,42 @@ enum ztest_result { ZTEST_RESULT_FAIL, ZTEST_RESULT_SKIP, ZTEST_RESULT_SUITE_SKIP, + ZTEST_RESULT_SUITE_FAIL, }; COND_CODE_1(KERNEL, (ZTEST_BMEM), ()) static enum ztest_result test_result; +static int get_final_test_result(const struct ztest_unit_test *test, int ret) +{ + enum ztest_expected_result expected_result = -1; + + for (struct ztest_expected_result_entry *expectation = + _ztest_expected_result_entry_list_start; + expectation < _ztest_expected_result_entry_list_end; ++expectation) { + if (strcmp(expectation->test_name, test->name) == 0 && + strcmp(expectation->test_suite_name, test->test_suite_name) == 0) { + expected_result = expectation->expected_result; + break; + } + } + + if (expected_result == ZTEST_EXPECTED_RESULT_FAIL) { + /* Expected a failure: + * - If we got a failure, return TC_PASS + * - Otherwise force a failure + */ + return (ret == TC_FAIL) ? TC_PASS : TC_FAIL; + } + if (expected_result == ZTEST_EXPECTED_RESULT_SKIP) { + /* Expected a skip: + * - If we got a skip, return TC_PASS + * - Otherwise force a failure + */ + return (ret == TC_SKIP) ? TC_PASS : TC_FAIL; + } + /* No expectation was made, no change is needed. */ + return ret; +} + #ifndef KERNEL /* Static code analysis tool can raise a violation that the standard header @@ -233,6 +266,7 @@ static jmp_buf test_fail; static jmp_buf test_pass; static jmp_buf test_skip; static jmp_buf stack_fail; +static jmp_buf test_suite_fail; /** * @brief Get a friendly name string for a given test phrase. @@ -264,6 +298,8 @@ void ztest_test_fail(void) { switch (phase) { case TEST_PHASE_SETUP: + PRINT(" at %s function\n", get_friendly_phase_name(phase)); + longjmp(test_suite_fail, 1); case TEST_PHASE_BEFORE: case TEST_PHASE_TEST: case TEST_PHASE_AFTER: @@ -294,6 +330,11 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test TC_START(test->name); + if (test_result == ZTEST_RESULT_SUITE_FAIL) { + ret = TC_FAIL; + goto out; + } + if (setjmp(test_fail)) { ret = TC_FAIL; goto out; @@ -316,10 +357,14 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test run_test_functions(suite, test, data); out: ret |= cleanup_test(test); - if (suite->after != NULL) { - suite->after(data); + if (test_result != ZTEST_RESULT_SUITE_FAIL) { + if (suite->after != NULL) { + suite->after(data); + } + run_test_rules(/*is_before=*/false, test, data); } - run_test_rules(/*is_before=*/false, test, data); + + ret = get_final_test_result(test, ret); Z_TC_END_RESULT(ret, test->name); return ret; @@ -348,8 +393,10 @@ static void test_finalize(void) void ztest_test_fail(void) { - test_result = ZTEST_RESULT_FAIL; - test_finalize(); + test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_FAIL : ZTEST_RESULT_FAIL; + if (phase != TEST_PHASE_SETUP) { + test_finalize(); + } } void ztest_test_pass(void) @@ -360,9 +407,8 @@ void ztest_test_pass(void) void ztest_test_skip(void) { - test_result = ZTEST_RESULT_SUITE_SKIP; + test_result = (phase == TEST_PHASE_SETUP) ? ZTEST_RESULT_SUITE_SKIP : ZTEST_RESULT_SKIP; if (phase != TEST_PHASE_SETUP) { - test_result = ZTEST_RESULT_SKIP; test_finalize(); } } @@ -379,7 +425,10 @@ void ztest_simple_1cpu_after(void *data) z_test_1cpu_stop(); } -static void init_testing(void) { k_object_access_all_grant(&ztest_thread); } +static void init_testing(void) +{ + k_object_access_all_grant(&ztest_thread); +} static void test_cb(void *a, void *b, void *c) { @@ -418,11 +467,13 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test k_thread_name_set(&ztest_thread, test->name); } /* Only start the thread if we're not skipping the suite */ - if (test_result != ZTEST_RESULT_SUITE_SKIP) { + if (test_result != ZTEST_RESULT_SUITE_SKIP && + test_result != ZTEST_RESULT_SUITE_FAIL) { k_thread_start(&ztest_thread); k_thread_join(&ztest_thread, K_FOREVER); } - } else if (test_result != ZTEST_RESULT_SUITE_SKIP) { + } else if (test_result != ZTEST_RESULT_SUITE_SKIP && + test_result != ZTEST_RESULT_SUITE_FAIL) { test_result = ZTEST_RESULT_PENDING; run_test_rules(/*is_before=*/true, test, data); if (suite->before) { @@ -444,19 +495,18 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test k_msleep(100); } - if (test_result == ZTEST_RESULT_FAIL) { + if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL) { ret = TC_FAIL; + } else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) { + ret = TC_SKIP; } if (test_result == ZTEST_RESULT_PASS || !FAIL_FAST) { ret |= cleanup_test(test); } - if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) { - Z_TC_END_RESULT(TC_SKIP, test->name); - } else { - Z_TC_END_RESULT(ret, test->name); - } + ret = get_final_test_result(test, ret); + Z_TC_END_RESULT(ret, test->name); return ret; } @@ -526,7 +576,12 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) TC_SUITE_START(suite->name); test_result = ZTEST_RESULT_PENDING; phase = TEST_PHASE_SETUP; - if (suite->setup != NULL) { +#ifndef KERNEL + if (setjmp(test_suite_fail)) { + test_result = ZTEST_RESULT_SUITE_FAIL; + } +#endif + if (test_result != ZTEST_RESULT_SUITE_FAIL && suite->setup != NULL) { data = suite->setup(); } diff --git a/tests/ztest/base/src/main.c b/tests/ztest/base/src/main.c index becf1fc3f62..71a46541fc3 100644 --- a/tests/ztest/base/src/main.c +++ b/tests/ztest/base/src/main.c @@ -35,12 +35,14 @@ ZTEST(framework_tests, test_assert_mem_equal) zassert_mem_equal(actual, expected, sizeof(expected), NULL); } +ZTEST_EXPECT_SKIP(framework_tests, test_skip_config); ZTEST(framework_tests, test_skip_config) { Z_TEST_SKIP_IFDEF(CONFIG_BUGxxxxx); ztest_test_fail(); } +ZTEST_EXPECT_SKIP(framework_tests, test_skip_no_config); ZTEST(framework_tests, test_skip_no_config) { Z_TEST_SKIP_IFNDEF(CONFIG_BUGyyyyy); @@ -149,3 +151,18 @@ ZTEST_F(rules_tests, test_rules_before_after) fixture->state = RULE_STATE_TEST; fixture->run_count++; } + +static void *fail_in_setup_setup(void) +{ + zassert_true(false, NULL); + return NULL; +} + +ZTEST_EXPECT_FAIL(fail_in_setup, test_should_never_run); +ZTEST(fail_in_setup, test_should_never_run) +{ + /* The following should pass, but the setup function will cause it to fail */ + zassert_true(true, NULL); +} + +ZTEST_SUITE(fail_in_setup, NULL, fail_in_setup_setup, NULL, NULL, NULL);