michaelh/zephyr
1
0
Fork 0
Code Releases Activity

ztest: Fix test statistics reporting

Browse source 
Flaky tests give innacurate test summary indicating a misleading
passing test suite.  Add new status for flaky tests.

Signed-off-by: Al Semjonovs <asemjonovs@google.com>
This commit is contained in:
Al Semjonovs 2023-08-02 10:38:31 -06:00 • committed by Anas Nashif
commit 2af5ac8fbb
2 changed files with 31 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

6
subsys/testsuite/include/zephyr/tc_util.h
View file

@ -67,6 +67,7 @@
#define TC_PASS 0 #define TC_PASS 0
#define TC_FAIL 1 #define TC_FAIL 1
#define TC_SKIP 2 #define TC_SKIP 2
#define TC_FLAKY 3
#ifndef TC_PASS_STR #ifndef TC_PASS_STR
#define TC_PASS_STR "PASS" #define TC_PASS_STR "PASS"
@ -77,6 +78,9 @@
#ifndef TC_SKIP_STR #ifndef TC_SKIP_STR
#define TC_SKIP_STR "SKIP" #define TC_SKIP_STR "SKIP"
#endif #endif
#ifndef TC_FLAKY_STR
#define TC_FLAKY_STR "FLAKY"
#endif
static inline const char *TC_RESULT_TO_STR(int result) static inline const char *TC_RESULT_TO_STR(int result)
{ {
@ -87,6 +91,8 @@ static inline const char *TC_RESULT_TO_STR(int result)
return TC_FAIL_STR; return TC_FAIL_STR;
case TC_SKIP: case TC_SKIP:
return TC_SKIP_STR; return TC_SKIP_STR;
case TC_FLAKY:
return TC_FLAKY_STR;
default: default:
return "?"; return "?";
} }

34
subsys/testsuite/ztest/src/ztest_new.c
View file

@ -675,7 +675,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
int fail = 0; int fail = 0;
int tc_result = TC_PASS; int tc_result = TC_PASS;
if (test_status < 0) { if (test_status != ZTEST_STATUS_OK) {
return test_status; return test_status;
} }
@ -766,7 +766,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
} }
#endif #endif
if (test_status == ZTEST_STATUS_OK && fail != 0) { if (test_status == ZTEST_STATUS_OK && fail != 0 && FAIL_FAST) {
test_status = ZTEST_STATUS_HAS_FAILURE; test_status = ZTEST_STATUS_HAS_FAILURE;
} }
} }
@ -882,15 +882,28 @@ static void __ztest_show_suite_summary_verbose(struct ztest_suite_node *suite)
tc_result = TC_SKIP; tc_result = TC_SKIP;
} else if (test->stats->pass_count == test->stats->run_count) { } else if (test->stats->pass_count == test->stats->run_count) {
tc_result = TC_PASS; tc_result = TC_PASS;
} else { } else if (test->stats->pass_count == 0) {
tc_result = TC_FAIL; tc_result = TC_FAIL;
} else {
tc_result = TC_FLAKY;
} }
TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n", if (tc_result == TC_FLAKY) {
TC_RESULT_TO_STR(tc_result), TC_SUMMARY_PRINT(" - %s - [%s.%s] - (Failed %d of %d attempts)"
test->test_suite_name, test->name, " - duration = %u.%03u seconds\n",
test->stats->duration_worst_ms / 1000, TC_RESULT_TO_STR(tc_result),
test->stats->duration_worst_ms % 1000); test->test_suite_name, test->name,
test->stats->run_count - test->stats->pass_count,
test->stats->run_count,
test->stats->duration_worst_ms / 1000,
test->stats->duration_worst_ms % 1000);
} else {
TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n",
TC_RESULT_TO_STR(tc_result),
test->test_suite_name, test->name,
test->stats->duration_worst_ms / 1000,
test->stats->duration_worst_ms % 1000);
}
if (flush_frequency % 3 == 0) { if (flush_frequency % 3 == 0) {
/** Reduce the flush frequencey a bit to speed up the output */ /** Reduce the flush frequencey a bit to speed up the output */
@ -930,7 +943,6 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat
for (int i = 0; i < NUM_ITER_PER_SUITE; i++) { for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
if (ztest_api.should_suite_run(state, ptr)) { if (ztest_api.should_suite_run(state, ptr)) {
__ztest_init_unit_test_result_for_suite(ptr);
int fail = z_ztest_run_test_suite_ptr(ptr); int fail = z_ztest_run_test_suite_ptr(ptr);
count++; count++;
@ -958,6 +970,9 @@ int z_impl_ztest_run_test_suites(const void *state)
memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *)); memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *));
z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start, z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start,
ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node));
for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
__ztest_init_unit_test_result_for_suite(suites_to_run[i]);
}
for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
count += __ztest_run_test_suite(suites_to_run[i], state); count += __ztest_run_test_suite(suites_to_run[i], state);
/* Stop running tests if we have a critical error or if we have a failure and /* Stop running tests if we have a critical error or if we have a failure and
@ -971,6 +986,7 @@ int z_impl_ztest_run_test_suites(const void *state)
#else #else
for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
ptr < _ztest_suite_node_list_end; ++ptr) { ptr < _ztest_suite_node_list_end; ++ptr) {
__ztest_init_unit_test_result_for_suite(ptr);
count += __ztest_run_test_suite(ptr, state); count += __ztest_run_test_suite(ptr, state);
/* Stop running tests if we have a critical error or if we have a failure and /* Stop running tests if we have a critical error or if we have a failure and
* FAIL_FAST was set * FAIL_FAST was set