twister: quarantine: Added pytests
Implemented scenarios to test the quarantine in twister Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>
This commit is contained in:
parent
bac1a955d0
commit
6809c05d09
7 changed files with 106 additions and 0 deletions
|
@ -80,6 +80,9 @@ class QuarantineData:
|
|||
with open(filename, 'r', encoding='UTF-8') as yaml_fd:
|
||||
qlist_raw_data: list[dict] = safe_load(yaml_fd)
|
||||
try:
|
||||
if not qlist_raw_data:
|
||||
# in case of loading empty quarantine file
|
||||
return cls()
|
||||
return cls(qlist_raw_data)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
@ -134,6 +134,9 @@ class TestPlan:
|
|||
logger.error("No quarantine list given to be verified")
|
||||
raise TwisterRuntimeError("No quarantine list given to be verified")
|
||||
if ql:
|
||||
for quarantine_file in ql:
|
||||
# validate quarantine yaml file against the provided schema
|
||||
scl.yaml_load_verify(quarantine_file, self.quarantine_schema)
|
||||
self.quarantine = Quarantine(ql)
|
||||
|
||||
def load(self):
|
||||
|
|
6
scripts/tests/twister/test_data/quarantines/basic.yaml
Normal file
6
scripts/tests/twister/test_data/quarantines/basic.yaml
Normal file
|
@ -0,0 +1,6 @@
|
|||
- scenarios:
|
||||
- test_a.check_1
|
||||
platforms:
|
||||
- demo_board_1
|
||||
- demo_board_3
|
||||
comment: "a1 on board_1 and board_3"
|
1
scripts/tests/twister/test_data/quarantines/empty.yaml
Normal file
1
scripts/tests/twister/test_data/quarantines/empty.yaml
Normal file
|
@ -0,0 +1 @@
|
|||
# empty quarantine file
|
|
@ -0,0 +1,4 @@
|
|||
|
||||
- platforms:
|
||||
- demo_board_3
|
||||
comment: "all on board_3"
|
10
scripts/tests/twister/test_data/quarantines/with_regexp.yaml
Normal file
10
scripts/tests/twister/test_data/quarantines/with_regexp.yaml
Normal file
|
@ -0,0 +1,10 @@
|
|||
|
||||
- scenarios:
|
||||
- test_(a|c).check_2
|
||||
architectures:
|
||||
- x86.*
|
||||
comment: "a2 and c2 on x86"
|
||||
|
||||
- scenarios:
|
||||
- test_d.*
|
||||
comment: "all test_d"
|
|
@ -17,6 +17,7 @@ from twisterlib.testplan import TestPlan
|
|||
from twisterlib.testinstance import TestInstance
|
||||
from twisterlib.testsuite import TestSuite
|
||||
from twisterlib.platform import Platform
|
||||
from twisterlib.quarantine import Quarantine
|
||||
|
||||
|
||||
def test_testplan_add_testsuites(class_testplan):
|
||||
|
@ -256,3 +257,81 @@ def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list
|
|||
[platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
|
||||
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
|
||||
assert list(plan.instances.values()) == instance_list
|
||||
|
||||
|
||||
QUARANTINE_BASIC = {
|
||||
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
|
||||
}
|
||||
|
||||
QUARANTINE_WITH_REGEXP = {
|
||||
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
|
||||
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
||||
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
||||
'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
|
||||
}
|
||||
|
||||
QUARANTINE_PLATFORM = {
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
||||
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3'
|
||||
}
|
||||
|
||||
QUARANTINE_MULTIFILES = {
|
||||
**QUARANTINE_BASIC,
|
||||
**QUARANTINE_WITH_REGEXP
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("quarantine_files, quarantine_verify, expected_val"),
|
||||
[
|
||||
(['basic.yaml'], False, QUARANTINE_BASIC),
|
||||
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
|
||||
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
|
||||
(['platform.yaml'], False, QUARANTINE_PLATFORM),
|
||||
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
|
||||
(['empty.yaml'], False, {})
|
||||
],
|
||||
ids=[
|
||||
'basic',
|
||||
'with_regexp',
|
||||
'quarantine_verify',
|
||||
'platform',
|
||||
'multifiles',
|
||||
'empty'
|
||||
])
|
||||
def test_quarantine(class_testplan, platforms_list, test_data,
|
||||
quarantine_files, quarantine_verify, expected_val):
|
||||
""" Testing quarantine feature in Twister
|
||||
"""
|
||||
class_testplan.options.all = True
|
||||
class_testplan.platforms = platforms_list
|
||||
class_testplan.platform_names = [p.name for p in platforms_list]
|
||||
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
||||
class_testplan.add_testsuites()
|
||||
|
||||
quarantine_list = [
|
||||
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
|
||||
]
|
||||
class_testplan.quarantine = Quarantine(quarantine_list)
|
||||
class_testplan.options.quarantine_verify = quarantine_verify
|
||||
class_testplan.apply_filters()
|
||||
|
||||
for testname, instance in class_testplan.instances.items():
|
||||
if quarantine_verify:
|
||||
if testname in expected_val:
|
||||
assert not instance.status
|
||||
else:
|
||||
assert instance.status == 'filtered'
|
||||
assert instance.reason == "Not under quarantine"
|
||||
else:
|
||||
if testname in expected_val:
|
||||
assert instance.status == 'filtered'
|
||||
assert instance.reason == expected_val[testname]
|
||||
else:
|
||||
assert not instance.status
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue