twister: quarantine: Added pytests

Implemented scenarios to test the quarantine in twister
Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>
This commit is contained in:
Grzegorz Chwierut 2023-01-13 12:32:19 +01:00 committed by Anas Nashif
commit 6809c05d09
7 changed files with 106 additions and 0 deletions

View file

@ -80,6 +80,9 @@ class QuarantineData:
with open(filename, 'r', encoding='UTF-8') as yaml_fd:
qlist_raw_data: list[dict] = safe_load(yaml_fd)
try:
if not qlist_raw_data:
# in case of loading empty quarantine file
return cls()
return cls(qlist_raw_data)
except Exception as e:

View file

@ -134,6 +134,9 @@ class TestPlan:
logger.error("No quarantine list given to be verified")
raise TwisterRuntimeError("No quarantine list given to be verified")
if ql:
for quarantine_file in ql:
# validate quarantine yaml file against the provided schema
scl.yaml_load_verify(quarantine_file, self.quarantine_schema)
self.quarantine = Quarantine(ql)
def load(self):

View file

@ -0,0 +1,6 @@
- scenarios:
- test_a.check_1
platforms:
- demo_board_1
- demo_board_3
comment: "a1 on board_1 and board_3"

View file

@ -0,0 +1 @@
# empty quarantine file

View file

@ -0,0 +1,4 @@
- platforms:
- demo_board_3
comment: "all on board_3"

View file

@ -0,0 +1,10 @@
- scenarios:
- test_(a|c).check_2
architectures:
- x86.*
comment: "a2 and c2 on x86"
- scenarios:
- test_d.*
comment: "all test_d"

View file

@ -17,6 +17,7 @@ from twisterlib.testplan import TestPlan
from twisterlib.testinstance import TestInstance
from twisterlib.testsuite import TestSuite
from twisterlib.platform import Platform
from twisterlib.quarantine import Quarantine
def test_testplan_add_testsuites(class_testplan):
@ -256,3 +257,81 @@ def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list
[platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
assert list(plan.instances.values()) == instance_list
QUARANTINE_BASIC = {
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
}
QUARANTINE_WITH_REGEXP = {
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
}
QUARANTINE_PLATFORM = {
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3'
}
QUARANTINE_MULTIFILES = {
**QUARANTINE_BASIC,
**QUARANTINE_WITH_REGEXP
}
@pytest.mark.parametrize(
("quarantine_files, quarantine_verify, expected_val"),
[
(['basic.yaml'], False, QUARANTINE_BASIC),
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
(['platform.yaml'], False, QUARANTINE_PLATFORM),
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
(['empty.yaml'], False, {})
],
ids=[
'basic',
'with_regexp',
'quarantine_verify',
'platform',
'multifiles',
'empty'
])
def test_quarantine(class_testplan, platforms_list, test_data,
quarantine_files, quarantine_verify, expected_val):
""" Testing quarantine feature in Twister
"""
class_testplan.options.all = True
class_testplan.platforms = platforms_list
class_testplan.platform_names = [p.name for p in platforms_list]
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
class_testplan.add_testsuites()
quarantine_list = [
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
]
class_testplan.quarantine = Quarantine(quarantine_list)
class_testplan.options.quarantine_verify = quarantine_verify
class_testplan.apply_filters()
for testname, instance in class_testplan.instances.items():
if quarantine_verify:
if testname in expected_val:
assert not instance.status
else:
assert instance.status == 'filtered'
assert instance.reason == "Not under quarantine"
else:
if testname in expected_val:
assert instance.status == 'filtered'
assert instance.reason == expected_val[testname]
else:
assert not instance.status