ztest: Add Support for Multiple Test Runs.
Introduces new kconfig options to enable multiple test runs in the ztest. Signed-off-by: Arkadiusz Cholewinski <arkadiuszx.cholewinski@intel.com>
This commit is contained in:
Arkadiusz Cholewinski
Henrik Brix Andersen
parent
80bf9bf5c8
commit
9d9089edd0
9 changed files with 53 additions and 19 deletions
|
|
@ -602,11 +602,15 @@ randomize the order. The output from the test will display the seed for failed
|
|||
tests. For native simulator builds you can provide the seed as an argument to
|
||||
twister with ``--seed``.
|
||||
|
||||
Static configuration of ZTEST_SHUFFLE contains:
|
||||
|
||||
- :kconfig:option:`CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT` - Number of iterations the test suite will run.
|
||||
- :kconfig:option:`CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT` - Number of iterations the test will run.
|
||||
|
||||
Repeating Tests
|
||||
***********************
|
||||
By default the tests are executed once. The test cases and test suites
|
||||
may be executed multiple times. Enable :kconfig:option:`CONFIG_ZTEST_REPEAT` to
|
||||
execute the tests multiple times. By default the multiplication factors are 3, which
|
||||
means every test suite is executed 3 times and every test case is executed 3 times. This can
|
||||
be changed by the :kconfig:option:`CONFIG_ZTEST_SUITE_REPEAT_COUNT` and
|
||||
:kconfig:option:`CONFIG_ZTEST_TEST_REPEAT_COUNT` Kconfig options.
|
||||
|
||||
Test Selection
|
||||
**************
|
||||
|
|
|
|||
|
|
@ -150,21 +150,40 @@ config ZTEST_SHUFFLE
|
|||
|
||||
if ZTEST_SHUFFLE
|
||||
config ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
|
||||
int "[DEPRECATED] Number of iterations the test suite will run"
|
||||
default 3
|
||||
help
|
||||
This is used to execute a test suite N number of times.
|
||||
[DEPRECATED] use ZTEST_SUITE_REPEAT_COUNT instead.
|
||||
|
||||
config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
|
||||
int "[DEPRECATED] Number of iterations the test will run"
|
||||
default 3
|
||||
help
|
||||
This is used to execute a test case N number of times.
|
||||
[DEPRECATED] use ZTEST_TEST_REPEAT_COUNT instead.
|
||||
|
||||
endif #ZTEST_SHUFFLE
|
||||
|
||||
config ZTEST_REPEAT
|
||||
bool "Repeat the tests and suites"
|
||||
help
|
||||
This rule will repeat the tests and the test suites.
|
||||
|
||||
if ZTEST_REPEAT
|
||||
config ZTEST_SUITE_REPEAT_COUNT
|
||||
int "Number of iterations the test suite will run"
|
||||
default 3
|
||||
help
|
||||
This rule will execute a test suite N number of times. The tests
|
||||
per suite will be shuffled on each iteration. The test order will likely
|
||||
be different per iteration.
|
||||
This rule will execute a test suite N number of times.
|
||||
|
||||
config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
|
||||
config ZTEST_TEST_REPEAT_COUNT
|
||||
int "Number of iterations the test will run"
|
||||
default 3
|
||||
help
|
||||
This rule will execute a test N number of times. The test order will
|
||||
likely be different per iteration.
|
||||
This rule will execute a test N number of times.
|
||||
|
||||
endif #ZTEST_SHUFFLE
|
||||
endif #ZTEST_REPEAT
|
||||
|
||||
config ZTEST_SUMMARY
|
||||
bool "Display test summary"
|
||||
|
|
|
|||
|
|
@ -30,9 +30,15 @@ static bool failed_expectation;
|
|||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <zephyr/random/random.h>
|
||||
|
||||
#ifndef CONFIG_ZTEST_REPEAT
|
||||
#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
|
||||
#define NUM_ITER_PER_TEST CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT
|
||||
#endif
|
||||
#endif /* CONFIG_ZTEST_SHUFFLE */
|
||||
|
||||
#ifdef CONFIG_ZTEST_REPEAT
|
||||
#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SUITE_REPEAT_COUNT
|
||||
#define NUM_ITER_PER_TEST CONFIG_ZTEST_TEST_REPEAT_COUNT
|
||||
#else
|
||||
#define NUM_ITER_PER_SUITE 1
|
||||
#define NUM_ITER_PER_TEST 1
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ CONFIG_TEST_RANDOM_GENERATOR=y
|
|||
CONFIG_LWM2M_LOG_ENCODE_BUFFER_ALLOCATIONS=y
|
||||
|
||||
CONFIG_ZTEST_SHUFFLE=y
|
||||
CONFIG_ZTEST_REPEAT=y
|
||||
|
||||
CONFIG_LWM2M=y
|
||||
CONFIG_LWM2M_COAP_BLOCK_TRANSFER=y
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
CONFIG_ZTEST=y
|
||||
CONFIG_ZTRESS=y
|
||||
CONFIG_ZTEST_SHUFFLE=y
|
||||
CONFIG_ZTEST_REPEAT=y
|
||||
|
||||
CONFIG_IPC_SERVICE=y
|
||||
CONFIG_IPC_SERVICE_ICMSG=y
|
||||
|
|
|
|||
|
|
@ -7,4 +7,5 @@ CONFIG_SERIAL=y
|
|||
CONFIG_ZTEST=y
|
||||
CONFIG_LOG=y
|
||||
CONFIG_ZTEST_SHUFFLE=y
|
||||
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=3
|
||||
CONFIG_ZTEST_REPEAT=y
|
||||
CONFIG_ZTEST_TEST_REPEAT_COUNT=3
|
||||
|
|
|
|||
|
|
@ -6,5 +6,6 @@ CONFIG_TEST_RANDOM_GENERATOR=y
|
|||
CONFIG_TIMER_RANDOM_GENERATOR=y
|
||||
|
||||
CONFIG_ZTEST_SHUFFLE=y
|
||||
CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_REPEAT=y
|
||||
CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_TEST_REPEAT_COUNT=2
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ CONFIG_ZTEST=y
|
|||
CONFIG_ZTEST_ASSERT_VERBOSE=1
|
||||
|
||||
CONFIG_ZTEST_SHUFFLE=y
|
||||
CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_REPEAT=y
|
||||
CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
|
||||
CONFIG_ZTEST_TEST_REPEAT_COUNT=2
|
||||
CONFIG_ENTROPY_GENERATOR=y
|
||||
|
|
|
|||
|
|
@ -153,8 +153,8 @@ static void rule_test_teardown(void *data)
|
|||
* after_each function was called.
|
||||
*/
|
||||
zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
|
||||
#ifdef CONFIG_ZTEST_SHUFFLE
|
||||
zassert_equal(fixture->run_count, CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT);
|
||||
#ifdef CONFIG_ZTEST_REPEAT
|
||||
zassert_equal(fixture->run_count, CONFIG_ZTEST_TEST_REPEAT_COUNT);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue