diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index 6290d4a18ad..36894ada8cf 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -902,14 +902,15 @@ using an external J-Link probe. The "probe_id" keyword overrides the Quarantine ++++++++++ -Twister allows using user-defined yaml files defining the list of tests to be put -under quarantine. Such tests will be skipped and marked accordingly in the output -reports. This feature is especially useful when running larger test suits, where -a failure of one test can affect the execution of other tests (e.g. putting the -physical board in a corrupted state). +Twister allows user to provide onfiguration files defining a list of tests or +platforms to be put under quarantine. Such tests will be skipped and marked +accordingly in the output reports. This feature is especially useful when +running larger test suits, where a failure of one test can affect the execution +of other tests (e.g. putting the physical board in a corrupted state). To use the quarantine feature one has to add the argument ``--quarantine-list `` to a twister call. +Multiple quarantine files can be used. The current status of tests on the quarantine list can also be verified by adding ``--quarantine-verify`` to the above argument. This will make twister skip all tests which are not on the given list. @@ -920,22 +921,33 @@ to put under quarantine. In addition, an optional entry "comment" can be used, w some more details can be given (e.g. link to a reported issue). These comments will also be added to the output reports. +When quarantining a class of tests or many scenarios in a single testsuite or +when dealing with multiple issues within a subsystem, it is possible to use +regular expressions, for example, **kernel.*** would quarantine +all kernel tests. + An example of entries in a quarantine yaml:: - scenarios: - sample.basic.helloworld - platforms: - - all comment: "Link to the issue: https://github.com/zephyrproject-rtos/zephyr/pull/33287" - scenarios: - kernel.common - - kernel.common.misra + - kernel.common.(misra|tls) - kernel.common.nano64 platforms: - - qemu_cortex_m3 + - .*_cortex_.* - native_posix +To exclude a platform, use the following syntax:: + + - platforms: + - qemu_x86 + comment: "broken qemu" + +Additionally you can quarantine entire architectures or a specific simulator for executing tests. + Running in Tests in Random Order ******************************** Enable ZTEST framework's :kconfig:option:`CONFIG_ZTEST_SHUFFLE` config option to diff --git a/scripts/pylib/twister/twisterlib/quarantine.py b/scripts/pylib/twister/twisterlib/quarantine.py index 0e645aab35e..cb05a920ca6 100644 --- a/scripts/pylib/twister/twisterlib/quarantine.py +++ b/scripts/pylib/twister/twisterlib/quarantine.py @@ -40,7 +40,7 @@ class QuarantineElement: platforms: list[str] = field(default_factory=list) architectures: list[str] = field(default_factory=list) simulations: list[str] = field(default_factory=list) - comment: str = 'under quarantine' + comment: str = 'NA' def __post_init__(self): # If there is no entry in filters then take all possible values. diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index e49dd1d35f8..3abfcf2af65 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -753,7 +753,7 @@ class TestPlan: instance.testsuite.id, plat.name, plat.arch, plat.simulation ) if matched_quarantine and not self.options.quarantine_verify: - instance.add_filter(matched_quarantine, Filters.QUARENTINE) + instance.add_filter("Quarantine: " + matched_quarantine, Filters.QUARENTINE) if not matched_quarantine and self.options.quarantine_verify: instance.add_filter("Not under quarantine", Filters.QUARENTINE) diff --git a/scripts/schemas/twister/quarantine-schema.yaml b/scripts/schemas/twister/quarantine-schema.yaml index 2b7edab4535..f7a145a3f0f 100644 --- a/scripts/schemas/twister/quarantine-schema.yaml +++ b/scripts/schemas/twister/quarantine-schema.yaml @@ -25,19 +25,19 @@ sequence: type: seq sequence: - type: str - - unique: True + - unique: true "architectures": required: false type: seq sequence: - type: str - - unique: True + - unique: true "simulations": required: false type: seq sequence: - type: str - - unique: True + - unique: true "comment": type: str required: false diff --git a/scripts/tests/twister/test_data/quarantines/platform.yaml b/scripts/tests/twister/test_data/quarantines/platform.yaml index ef5faf77652..4d47a3fe262 100644 --- a/scripts/tests/twister/test_data/quarantines/platform.yaml +++ b/scripts/tests/twister/test_data/quarantines/platform.yaml @@ -1,4 +1,3 @@ - - platforms: - demo_board_3 comment: "all on board_3" diff --git a/scripts/tests/twister/test_data/quarantines/with_regexp.yaml b/scripts/tests/twister/test_data/quarantines/with_regexp.yaml index 626463443ad..95cf767a690 100644 --- a/scripts/tests/twister/test_data/quarantines/with_regexp.yaml +++ b/scripts/tests/twister/test_data/quarantines/with_regexp.yaml @@ -1,4 +1,3 @@ - - scenarios: - test_(a|c).check_2 architectures: diff --git a/scripts/tests/twister/test_testplan_class.py b/scripts/tests/twister/test_testplan_class.py index 3b276d5d624..3a5e6e649c5 100644 --- a/scripts/tests/twister/test_testplan_class.py +++ b/scripts/tests/twister/test_testplan_class.py @@ -332,6 +332,6 @@ def test_quarantine(class_testplan, platforms_list, test_data, else: if testname in expected_val: assert instance.status == 'filtered' - assert instance.reason == expected_val[testname] + assert instance.reason == "Quarantine: " + expected_val[testname] else: assert not instance.status