michaelh/zephyr
1
0
Fork 0
Code Releases Activity

ztest: Fix test statistics reporting

Browse source 
Flaky tests give innacurate test summary indicating a misleading
passing test suite.  Add new status for flaky tests.

Signed-off-by: Al Semjonovs <asemjonovs@google.com>
This commit is contained in:
Al Semjonovs 2023-08-02 10:38:31 -06:00 • committed by Anas Nashif
commit 2af5ac8fbb
2 changed files with 31 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

6
subsys/testsuite/include/zephyr/tc_util.h
View file

@ -67,6 +67,7 @@
#define TC_PASS 0
#define TC_FAIL 1
#define TC_SKIP 2
#define TC_FLAKY 3
#ifndef TC_PASS_STR
#define TC_PASS_STR "PASS"
@ -77,6 +78,9 @@
#ifndef TC_SKIP_STR
#define TC_SKIP_STR "SKIP"
#endif
#ifndef TC_FLAKY_STR
#define TC_FLAKY_STR "FLAKY"
#endif
static inline const char *TC_RESULT_TO_STR(int result)
{
@ -87,6 +91,8 @@ static inline const char *TC_RESULT_TO_STR(int result)
return TC_FAIL_STR;
case TC_SKIP:
return TC_SKIP_STR;
case TC_FLAKY:
return TC_FLAKY_STR;
default:
return "?";
}

24
subsys/testsuite/ztest/src/ztest_new.c
View file

@ -675,7 +675,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
int fail = 0;
int tc_result = TC_PASS;
if (test_status < 0) {
if (test_status != ZTEST_STATUS_OK) {
return test_status;
}
@ -766,7 +766,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
}
#endif
if (test_status == ZTEST_STATUS_OK && fail != 0) {
if (test_status == ZTEST_STATUS_OK && fail != 0 && FAIL_FAST) {
test_status = ZTEST_STATUS_HAS_FAILURE;
}
}
@ -882,15 +882,28 @@ static void __ztest_show_suite_summary_verbose(struct ztest_suite_node *suite)
tc_result = TC_SKIP;
} else if (test->stats->pass_count == test->stats->run_count) {
tc_result = TC_PASS;
} else {
} else if (test->stats->pass_count == 0) {
tc_result = TC_FAIL;
} else {
tc_result = TC_FLAKY;
}
if (tc_result == TC_FLAKY) {
TC_SUMMARY_PRINT(" - %s - [%s.%s] - (Failed %d of %d attempts)"
" - duration = %u.%03u seconds\n",
TC_RESULT_TO_STR(tc_result),
test->test_suite_name, test->name,
test->stats->run_count - test->stats->pass_count,
test->stats->run_count,
test->stats->duration_worst_ms / 1000,
test->stats->duration_worst_ms % 1000);
} else {
TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n",
TC_RESULT_TO_STR(tc_result),
test->test_suite_name, test->name,
test->stats->duration_worst_ms / 1000,
test->stats->duration_worst_ms % 1000);
}
if (flush_frequency % 3 == 0) {
/** Reduce the flush frequencey a bit to speed up the output */
@ -930,7 +943,6 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat
for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
if (ztest_api.should_suite_run(state, ptr)) {
__ztest_init_unit_test_result_for_suite(ptr);
int fail = z_ztest_run_test_suite_ptr(ptr);
count++;
@ -958,6 +970,9 @@ int z_impl_ztest_run_test_suites(const void *state)
memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *));
z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start,
ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node));
for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
__ztest_init_unit_test_result_for_suite(suites_to_run[i]);
}
for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
count += __ztest_run_test_suite(suites_to_run[i], state);
/* Stop running tests if we have a critical error or if we have a failure and
@ -971,6 +986,7 @@ int z_impl_ztest_run_test_suites(const void *state)
#else
for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
ptr < _ztest_suite_node_list_end; ++ptr) {
__ztest_init_unit_test_result_for_suite(ptr);
count += __ztest_run_test_suite(ptr, state);
/* Stop running tests if we have a critical error or if we have a failure and
* FAIL_FAST was set