ztest: Add Support for Multiple Test Runs.

Introduces new kconfig options to enable multiple test runs in the ztest.

Signed-off-by: Arkadiusz Cholewinski <arkadiuszx.cholewinski@intel.com>
This commit is contained in:
Arkadiusz Cholewinski 2024-06-26 15:24:10 +02:00 committed by Henrik Brix Andersen
commit 9d9089edd0
9 changed files with 53 additions and 19 deletions

View file

@ -602,11 +602,15 @@ randomize the order. The output from the test will display the seed for failed
tests. For native simulator builds you can provide the seed as an argument to
twister with ``--seed``.
Static configuration of ZTEST_SHUFFLE contains:
- :kconfig:option:`CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT` - Number of iterations the test suite will run.
- :kconfig:option:`CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT` - Number of iterations the test will run.
Repeating Tests
***********************
By default the tests are executed once. The test cases and test suites
may be executed multiple times. Enable :kconfig:option:`CONFIG_ZTEST_REPEAT` to
execute the tests multiple times. By default the multiplication factors are 3, which
means every test suite is executed 3 times and every test case is executed 3 times. This can
be changed by the :kconfig:option:`CONFIG_ZTEST_SUITE_REPEAT_COUNT` and
:kconfig:option:`CONFIG_ZTEST_TEST_REPEAT_COUNT` Kconfig options.
Test Selection
**************

View file

@ -150,21 +150,40 @@ config ZTEST_SHUFFLE
if ZTEST_SHUFFLE
config ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
int "[DEPRECATED] Number of iterations the test suite will run"
default 3
help
This is used to execute a test suite N number of times.
[DEPRECATED] use ZTEST_SUITE_REPEAT_COUNT instead.
config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
int "[DEPRECATED] Number of iterations the test will run"
default 3
help
This is used to execute a test case N number of times.
[DEPRECATED] use ZTEST_TEST_REPEAT_COUNT instead.
endif #ZTEST_SHUFFLE
config ZTEST_REPEAT
bool "Repeat the tests and suites"
help
This rule will repeat the tests and the test suites.
if ZTEST_REPEAT
config ZTEST_SUITE_REPEAT_COUNT
int "Number of iterations the test suite will run"
default 3
help
This rule will execute a test suite N number of times. The tests
per suite will be shuffled on each iteration. The test order will likely
be different per iteration.
This rule will execute a test suite N number of times.
config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
config ZTEST_TEST_REPEAT_COUNT
int "Number of iterations the test will run"
default 3
help
This rule will execute a test N number of times. The test order will
likely be different per iteration.
This rule will execute a test N number of times.
endif #ZTEST_SHUFFLE
endif #ZTEST_REPEAT
config ZTEST_SUMMARY
bool "Display test summary"

View file

@ -30,9 +30,15 @@ static bool failed_expectation;
#include <stdlib.h>
#include <time.h>
#include <zephyr/random/random.h>
#ifndef CONFIG_ZTEST_REPEAT
#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
#define NUM_ITER_PER_TEST CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT
#endif
#endif /* CONFIG_ZTEST_SHUFFLE */
#ifdef CONFIG_ZTEST_REPEAT
#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SUITE_REPEAT_COUNT
#define NUM_ITER_PER_TEST CONFIG_ZTEST_TEST_REPEAT_COUNT
#else
#define NUM_ITER_PER_SUITE 1
#define NUM_ITER_PER_TEST 1

View file

@ -8,6 +8,7 @@ CONFIG_TEST_RANDOM_GENERATOR=y
CONFIG_LWM2M_LOG_ENCODE_BUFFER_ALLOCATIONS=y
CONFIG_ZTEST_SHUFFLE=y
CONFIG_ZTEST_REPEAT=y
CONFIG_LWM2M=y
CONFIG_LWM2M_COAP_BLOCK_TRANSFER=y

View file

@ -1,6 +1,7 @@
CONFIG_ZTEST=y
CONFIG_ZTRESS=y
CONFIG_ZTEST_SHUFFLE=y
CONFIG_ZTEST_REPEAT=y
CONFIG_IPC_SERVICE=y
CONFIG_IPC_SERVICE_ICMSG=y

View file

@ -7,4 +7,5 @@ CONFIG_SERIAL=y
CONFIG_ZTEST=y
CONFIG_LOG=y
CONFIG_ZTEST_SHUFFLE=y
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=3
CONFIG_ZTEST_REPEAT=y
CONFIG_ZTEST_TEST_REPEAT_COUNT=3

View file

@ -6,5 +6,6 @@ CONFIG_TEST_RANDOM_GENERATOR=y
CONFIG_TIMER_RANDOM_GENERATOR=y
CONFIG_ZTEST_SHUFFLE=y
CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
CONFIG_ZTEST_REPEAT=y
CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
CONFIG_ZTEST_TEST_REPEAT_COUNT=2

View file

@ -2,6 +2,7 @@ CONFIG_ZTEST=y
CONFIG_ZTEST_ASSERT_VERBOSE=1
CONFIG_ZTEST_SHUFFLE=y
CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
CONFIG_ZTEST_REPEAT=y
CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
CONFIG_ZTEST_TEST_REPEAT_COUNT=2
CONFIG_ENTROPY_GENERATOR=y

View file

@ -153,8 +153,8 @@ static void rule_test_teardown(void *data)
* after_each function was called.
*/
zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
#ifdef CONFIG_ZTEST_SHUFFLE
zassert_equal(fixture->run_count, CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT);
#ifdef CONFIG_ZTEST_REPEAT
zassert_equal(fixture->run_count, CONFIG_ZTEST_TEST_REPEAT_COUNT);
#endif
}