diff --git a/subsys/testsuite/include/zephyr/tc_util.h b/subsys/testsuite/include/zephyr/tc_util.h index 232bc7d6374..69c9359b50f 100644 --- a/subsys/testsuite/include/zephyr/tc_util.h +++ b/subsys/testsuite/include/zephyr/tc_util.h @@ -67,6 +67,7 @@ #define TC_PASS 0 #define TC_FAIL 1 #define TC_SKIP 2 +#define TC_FLAKY 3 #ifndef TC_PASS_STR #define TC_PASS_STR "PASS" @@ -77,6 +78,9 @@ #ifndef TC_SKIP_STR #define TC_SKIP_STR "SKIP" #endif +#ifndef TC_FLAKY_STR +#define TC_FLAKY_STR "FLAKY" +#endif static inline const char *TC_RESULT_TO_STR(int result) { @@ -87,6 +91,8 @@ static inline const char *TC_RESULT_TO_STR(int result) return TC_FAIL_STR; case TC_SKIP: return TC_SKIP_STR; + case TC_FLAKY: + return TC_FLAKY_STR; default: return "?"; } diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 6119a6905e0..d0e0752519f 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -675,7 +675,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) int fail = 0; int tc_result = TC_PASS; - if (test_status < 0) { + if (test_status != ZTEST_STATUS_OK) { return test_status; } @@ -766,7 +766,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } #endif - if (test_status == ZTEST_STATUS_OK && fail != 0) { + if (test_status == ZTEST_STATUS_OK && fail != 0 && FAIL_FAST) { test_status = ZTEST_STATUS_HAS_FAILURE; } } @@ -882,15 +882,28 @@ static void __ztest_show_suite_summary_verbose(struct ztest_suite_node *suite) tc_result = TC_SKIP; } else if (test->stats->pass_count == test->stats->run_count) { tc_result = TC_PASS; - } else { + } else if (test->stats->pass_count == 0) { tc_result = TC_FAIL; + } else { + tc_result = TC_FLAKY; } - TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n", - TC_RESULT_TO_STR(tc_result), - test->test_suite_name, test->name, - test->stats->duration_worst_ms / 1000, - test->stats->duration_worst_ms % 1000); + if (tc_result == TC_FLAKY) { + TC_SUMMARY_PRINT(" - %s - [%s.%s] - (Failed %d of %d attempts)" + " - duration = %u.%03u seconds\n", + TC_RESULT_TO_STR(tc_result), + test->test_suite_name, test->name, + test->stats->run_count - test->stats->pass_count, + test->stats->run_count, + test->stats->duration_worst_ms / 1000, + test->stats->duration_worst_ms % 1000); + } else { + TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n", + TC_RESULT_TO_STR(tc_result), + test->test_suite_name, test->name, + test->stats->duration_worst_ms / 1000, + test->stats->duration_worst_ms % 1000); + } if (flush_frequency % 3 == 0) { /** Reduce the flush frequencey a bit to speed up the output */ @@ -930,7 +943,6 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat for (int i = 0; i < NUM_ITER_PER_SUITE; i++) { if (ztest_api.should_suite_run(state, ptr)) { - __ztest_init_unit_test_result_for_suite(ptr); int fail = z_ztest_run_test_suite_ptr(ptr); count++; @@ -958,6 +970,9 @@ int z_impl_ztest_run_test_suites(const void *state) memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *)); z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start, ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); + for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { + __ztest_init_unit_test_result_for_suite(suites_to_run[i]); + } for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { count += __ztest_run_test_suite(suites_to_run[i], state); /* Stop running tests if we have a critical error or if we have a failure and @@ -971,6 +986,7 @@ int z_impl_ztest_run_test_suites(const void *state) #else for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) { + __ztest_init_unit_test_result_for_suite(ptr); count += __ztest_run_test_suite(ptr, state); /* Stop running tests if we have a critical error or if we have a failure and * FAIL_FAST was set