From 6809c05d09c3cc85716a354cecbd2229a4c17db7 Mon Sep 17 00:00:00 2001 From: Grzegorz Chwierut Date: Fri, 13 Jan 2023 12:32:19 +0100 Subject: [PATCH] twister: quarantine: Added pytests Implemented scenarios to test the quarantine in twister Signed-off-by: Grzegorz Chwierut --- .../pylib/twister/twisterlib/quarantine.py | 3 + scripts/pylib/twister/twisterlib/testplan.py | 3 + .../twister/test_data/quarantines/basic.yaml | 6 ++ .../twister/test_data/quarantines/empty.yaml | 1 + .../test_data/quarantines/platform.yaml | 4 + .../test_data/quarantines/with_regexp.yaml | 10 +++ scripts/tests/twister/test_testplan_class.py | 79 +++++++++++++++++++ 7 files changed, 106 insertions(+) create mode 100644 scripts/tests/twister/test_data/quarantines/basic.yaml create mode 100644 scripts/tests/twister/test_data/quarantines/empty.yaml create mode 100644 scripts/tests/twister/test_data/quarantines/platform.yaml create mode 100644 scripts/tests/twister/test_data/quarantines/with_regexp.yaml diff --git a/scripts/pylib/twister/twisterlib/quarantine.py b/scripts/pylib/twister/twisterlib/quarantine.py index 92f86911b6b..0e645aab35e 100644 --- a/scripts/pylib/twister/twisterlib/quarantine.py +++ b/scripts/pylib/twister/twisterlib/quarantine.py @@ -80,6 +80,9 @@ class QuarantineData: with open(filename, 'r', encoding='UTF-8') as yaml_fd: qlist_raw_data: list[dict] = safe_load(yaml_fd) try: + if not qlist_raw_data: + # in case of loading empty quarantine file + return cls() return cls(qlist_raw_data) except Exception as e: diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 3f8403d8997..e49dd1d35f8 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -134,6 +134,9 @@ class TestPlan: logger.error("No quarantine list given to be verified") raise TwisterRuntimeError("No quarantine list given to be verified") if ql: + for quarantine_file in ql: + # validate quarantine yaml file against the provided schema + scl.yaml_load_verify(quarantine_file, self.quarantine_schema) self.quarantine = Quarantine(ql) def load(self): diff --git a/scripts/tests/twister/test_data/quarantines/basic.yaml b/scripts/tests/twister/test_data/quarantines/basic.yaml new file mode 100644 index 00000000000..a03e06c0b1c --- /dev/null +++ b/scripts/tests/twister/test_data/quarantines/basic.yaml @@ -0,0 +1,6 @@ +- scenarios: + - test_a.check_1 + platforms: + - demo_board_1 + - demo_board_3 + comment: "a1 on board_1 and board_3" diff --git a/scripts/tests/twister/test_data/quarantines/empty.yaml b/scripts/tests/twister/test_data/quarantines/empty.yaml new file mode 100644 index 00000000000..6d797101945 --- /dev/null +++ b/scripts/tests/twister/test_data/quarantines/empty.yaml @@ -0,0 +1 @@ +# empty quarantine file diff --git a/scripts/tests/twister/test_data/quarantines/platform.yaml b/scripts/tests/twister/test_data/quarantines/platform.yaml new file mode 100644 index 00000000000..ef5faf77652 --- /dev/null +++ b/scripts/tests/twister/test_data/quarantines/platform.yaml @@ -0,0 +1,4 @@ + +- platforms: + - demo_board_3 + comment: "all on board_3" diff --git a/scripts/tests/twister/test_data/quarantines/with_regexp.yaml b/scripts/tests/twister/test_data/quarantines/with_regexp.yaml new file mode 100644 index 00000000000..626463443ad --- /dev/null +++ b/scripts/tests/twister/test_data/quarantines/with_regexp.yaml @@ -0,0 +1,10 @@ + +- scenarios: + - test_(a|c).check_2 + architectures: + - x86.* + comment: "a2 and c2 on x86" + +- scenarios: + - test_d.* + comment: "all test_d" diff --git a/scripts/tests/twister/test_testplan_class.py b/scripts/tests/twister/test_testplan_class.py index 8b5d8e2979d..3b276d5d624 100644 --- a/scripts/tests/twister/test_testplan_class.py +++ b/scripts/tests/twister/test_testplan_class.py @@ -17,6 +17,7 @@ from twisterlib.testplan import TestPlan from twisterlib.testinstance import TestInstance from twisterlib.testsuite import TestSuite from twisterlib.platform import Platform +from twisterlib.quarantine import Quarantine def test_testplan_add_testsuites(class_testplan): @@ -256,3 +257,81 @@ def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list [platform.name + '/' + s for s in list(all_testsuites_dict.keys())] assert all(isinstance(n, TestInstance) for n in list(plan.instances.values())) assert list(plan.instances.values()) == instance_list + + +QUARANTINE_BASIC = { + 'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3' +} + +QUARANTINE_WITH_REGEXP = { + 'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86', + 'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', + 'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d', + 'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86' +} + +QUARANTINE_PLATFORM = { + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3', + 'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3' +} + +QUARANTINE_MULTIFILES = { + **QUARANTINE_BASIC, + **QUARANTINE_WITH_REGEXP +} + +@pytest.mark.parametrize( + ("quarantine_files, quarantine_verify, expected_val"), + [ + (['basic.yaml'], False, QUARANTINE_BASIC), + (['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP), + (['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP), + (['platform.yaml'], False, QUARANTINE_PLATFORM), + (['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES), + (['empty.yaml'], False, {}) + ], + ids=[ + 'basic', + 'with_regexp', + 'quarantine_verify', + 'platform', + 'multifiles', + 'empty' + ]) +def test_quarantine(class_testplan, platforms_list, test_data, + quarantine_files, quarantine_verify, expected_val): + """ Testing quarantine feature in Twister + """ + class_testplan.options.all = True + class_testplan.platforms = platforms_list + class_testplan.platform_names = [p.name for p in platforms_list] + class_testplan.TESTSUITE_FILENAME = 'test_data.yaml' + class_testplan.add_testsuites() + + quarantine_list = [ + os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files + ] + class_testplan.quarantine = Quarantine(quarantine_list) + class_testplan.options.quarantine_verify = quarantine_verify + class_testplan.apply_filters() + + for testname, instance in class_testplan.instances.items(): + if quarantine_verify: + if testname in expected_val: + assert not instance.status + else: + assert instance.status == 'filtered' + assert instance.reason == "Not under quarantine" + else: + if testname in expected_val: + assert instance.status == 'filtered' + assert instance.reason == expected_val[testname] + else: + assert not instance.status