diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index c9eccbf9cae..50f422180d2 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -734,7 +734,7 @@ class Reporting: f'.' ) - built_only = results.total - run - results.filtered_configs + built_only = results.total - run - results.filtered_configs - results.skipped logger.info( f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms," f" {TwisterStatus.get_color(TwisterStatus.NOTRUN)}{built_only}{Fore.RESET}" diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 5b1a782eca9..059679d5bcd 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1885,7 +1885,7 @@ class TwisterRunner: self.results.done -= self.results.error self.results.error = 0 else: - self.results.done = self.results.filtered_static + self.results.done = self.results.filtered_static + self.results.skipped self.execute(pipeline, done_queue) @@ -1923,6 +1923,10 @@ class TwisterRunner: self.results.filtered_configs_increment() self.results.filtered_cases_increment(len(instance.testsuite.testcases)) self.results.cases_increment(len(instance.testsuite.testcases)) + elif instance.status == TwisterStatus.SKIP and "overflow" not in instance.reason: + self.results.skipped_increment() + self.results.skipped_cases_increment(len(instance.testsuite.testcases)) + self.results.cases_increment(len(instance.testsuite.testcases)) elif instance.status == TwisterStatus.ERROR: self.results.error_increment() diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 12c10ee1b14..042c0019c67 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -605,10 +605,12 @@ class TestPlan: sim_name ) if matched_quarantine and not self.options.quarantine_verify: - instance.add_filter("Quarantine: " + matched_quarantine, Filters.QUARANTINE) + instance.status = TwisterStatus.SKIP + instance.reason = "Quarantine: " + matched_quarantine return if not matched_quarantine and self.options.quarantine_verify: - instance.add_filter("Not under quarantine", Filters.QUARANTINE) + instance.status = TwisterStatus.SKIP + instance.reason = "Not under quarantine" def load_from_file(self, file, filter_platform=None): if filter_platform is None: diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index 5184bd62d33..5d2fecc735a 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -104,20 +104,26 @@ def twister(options: argparse.Namespace, default_options: argparse.Namespace): logger.error(f"{e}") return 1 - if options.verbose > 1: - # if we are using command line platform filter, no need to list every - # other platform as excluded, we know that already. - # Show only the discards that apply to the selected platforms on the - # command line + # if we are using command line platform filter, no need to list every + # other platform as excluded, we know that already. + # Show only the discards that apply to the selected platforms on the + # command line + if options.verbose > 0: for i in tplan.instances.values(): - if i.status == TwisterStatus.FILTER: + if i.status in [TwisterStatus.SKIP,TwisterStatus.FILTER]: if options.platform and not tplan.check_platform(i.platform, options.platform): continue + # Filtered tests should be visable only when verbosity > 1 + if options.verbose < 2 and i.status == TwisterStatus.FILTER: + continue + res = i.reason + if "Quarantine" in i.reason: + res = "Quarantined" logger.info( f"{i.platform.name:<25} {i.testsuite.name:<50}" - f" {Fore.YELLOW}FILTERED{Fore.RESET}: {i.reason}" - ) + f" {Fore.YELLOW}{i.status.upper()}{Fore.RESET}: {res}" + ) report = Reporting(tplan, env) plan_file = os.path.join(options.outdir, "testplan.json") diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 0e6227acefe..27c4b3cb3bf 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -2545,6 +2545,8 @@ def test_twisterrunner_run( results_mock().iteration = 0 results_mock().failed = 2 results_mock().total = 9 + results_mock().filtered_static = 0 + results_mock().skipped = 0 def iteration_increment(value=1, decrement=False): results_mock().iteration += value * (-1 if decrement else 1) @@ -2608,7 +2610,7 @@ def test_twisterrunner_update_counting_before_pipeline(): ), 'dummy5': mock.Mock( status=TwisterStatus.SKIP, - reason=None, + reason="Quarantine", testsuite=mock.Mock( testcases=[mock.Mock()] ) @@ -2629,6 +2631,7 @@ def test_twisterrunner_update_counting_before_pipeline(): error = 0, cases = 0, filtered_cases = 0, + skipped = 0, skipped_cases = 0, failed_cases = 0, error_cases = 0, @@ -2652,14 +2655,22 @@ def test_twisterrunner_update_counting_before_pipeline(): def filtered_cases_increment(value=1, decrement=False): tr.results.filtered_cases += value * (-1 if decrement else 1) tr.results.filtered_cases_increment = filtered_cases_increment + def skipped_increment(value=1, decrement=False): + tr.results.skipped += value * (-1 if decrement else 1) + tr.results.skipped_increment = skipped_increment + def skipped_cases_increment(value=1, decrement=False): + tr.results.skipped_cases += value * (-1 if decrement else 1) + tr.results.skipped_cases_increment = skipped_cases_increment tr.update_counting_before_pipeline() assert tr.results.filtered_static == 1 assert tr.results.filtered_configs == 1 assert tr.results.filtered_cases == 4 - assert tr.results.cases == 4 + assert tr.results.cases == 5 assert tr.results.error == 1 + assert tr.results.skipped == 1 + assert tr.results.skipped_cases == 1 def test_twisterrunner_show_brief(caplog): diff --git a/scripts/tests/twister/test_testplan.py b/scripts/tests/twister/test_testplan.py index 73b2673b08e..00218750390 100644 --- a/scripts/tests/twister/test_testplan.py +++ b/scripts/tests/twister/test_testplan.py @@ -342,11 +342,11 @@ def test_quarantine_short(class_testplan, platforms_list, test_data, if testname in expected_val: assert instance.status == TwisterStatus.NONE else: - assert instance.status == TwisterStatus.FILTER + assert instance.status == TwisterStatus.SKIP assert instance.reason == "Not under quarantine" else: if testname in expected_val: - assert instance.status == TwisterStatus.FILTER + assert instance.status == TwisterStatus.SKIP assert instance.reason == "Quarantine: " + expected_val[testname] else: assert instance.status == TwisterStatus.NONE diff --git a/scripts/tests/twister_blackbox/test_quarantine.py b/scripts/tests/twister_blackbox/test_quarantine.py index 61cfa9634e3..0b4c6cdafec 100644 --- a/scripts/tests/twister_blackbox/test_quarantine.py +++ b/scripts/tests/twister_blackbox/test_quarantine.py @@ -15,6 +15,7 @@ import sys import json # pylint: disable=duplicate-code +# pylint: disable=no-name-in-module from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan @@ -49,10 +50,15 @@ class TestQuarantine: with open(os.path.join(out_path, 'testplan.json')) as f: j = json.load(f) + + # Quarantine-verify "swaps" statuses. The ones that are in quarantine list + # should no longer be quarantined, and the ones that are not in the list + # should be quarantined. Remove "quarantined" tests from "verify" testplan + # to count what should be verified. filtered_j = [ - (ts['platform'], ts['name'], tc['identifier']) \ + (ts['platform'], ts['name']) \ for ts in j['testsuites'] \ - for tc in ts['testcases'] if 'reason' not in tc + if ts['status'] != "skipped" ] assert str(sys_exit.value) == '0' @@ -89,26 +95,26 @@ class TestQuarantine: sys.stdout.write(out) sys.stderr.write(err) - board1_match1 = re.search('agnostic/group2/dummy.agnostic.group2 FILTERED: Quarantine: test ' + board1_match1 = re.search('agnostic/group2/dummy.agnostic.group2 SKIPPED: Quarantine: test ' 'intel_adl_crb', err) board1_match2 = re.search( - 'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 FILTERED: Quarantine: test ' + 'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test ' 'intel_adl_crb', err) qemu_64_match = re.search( - 'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 FILTERED: Quarantine: test ' + 'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test ' 'qemu_x86_64', err) all_platforms_match = re.search( - 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test ' + 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test ' 'all platforms', err) all_platforms_match2 = re.search( - 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test ' + 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test ' 'all platforms', err) all_platforms_match3 = re.search( - 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 FILTERED: Quarantine: test ' + 'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test ' 'all platforms', err)