From 5f4d330db66e6e6daba7db8f7223f3bef02ef830 Mon Sep 17 00:00:00 2001 From: Lukasz Mrugala Date: Tue, 30 Apr 2024 14:17:47 +0000 Subject: [PATCH] scripts: twister: Do not report filtered test instances by default Filtered testcases are removed by default from Twister tests. Older functionality is preserved via a new Twister flag: --report-filtered. Old tests were adjusted and a new test for that flag added. Signed-off-by: Lukasz Mrugala --- .../pylib/twister/twisterlib/environment.py | 2 + scripts/pylib/twister/twisterlib/reports.py | 2 + scripts/tests/twister_blackbox/test_report.py | 60 ++++++++++++++++--- scripts/tests/twister_blackbox/test_runner.py | 8 +-- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 85b7cdb371e..3d818e4e964 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -547,6 +547,8 @@ structure in the main Zephyr tree: boards///""") parser.add_argument("--overflow-as-errors", action="store_true", help="Treat RAM/SRAM overflows as errors.") + parser.add_argument("--report-filtered", action="store_true", + help="Include filtered tests in the reports.") parser.add_argument("-P", "--exclude-platform", action="append", default=[], help="""Exclude platforms and do not build or run any tests diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 48c9df082c4..4bc352e4f1d 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -259,6 +259,8 @@ class Reporting: for instance in self.instances.values(): if platform and platform != instance.platform.name: continue + if instance.status == "filtered" and not self.env.options.report_filtered: + continue suite = {} handler_log = os.path.join(instance.build_dir, "handler.log") pytest_log = os.path.join(instance.build_dir, "twister_harness.log") diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index e116bea3da1..5b30394ef60 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -344,14 +344,25 @@ class TestReport: assert str(sys_exit.value) == '0' @pytest.mark.parametrize( - 'test_path, expected_testcase_count', - [(os.path.join(TEST_DATA, 'tests', 'dummy'), 6),], - ids=['dummy tests'] + 'test_path, flags, expected_testcase_counts', + [ + ( + os.path.join(TEST_DATA, 'tests', 'dummy'), + ['--detailed-skipped-report'], + {'qemu_x86': 5, 'frdm_k64f': 1} + ), + ( + os.path.join(TEST_DATA, 'tests', 'dummy'), + ['--detailed-skipped-report', '--report-filtered'], + {'qemu_x86': 6, 'frdm_k64f': 6} + ), + ], + ids=['dummy tests', 'dummy tests with filtered'] ) - def test_detailed_skipped_report(self, out_path, test_path, expected_testcase_count): + def test_detailed_skipped_report(self, out_path, test_path, flags, expected_testcase_counts): test_platforms = ['qemu_x86', 'frdm_k64f'] args = ['-i', '--outdir', out_path, '-T', test_path] + \ - ['--detailed-skipped-report'] + \ + flags + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] @@ -367,12 +378,47 @@ class TestReport: for ts in xml_data.iter('testsuite'): testsuite_counter += 1 # Without the tested flag, filtered testcases would be missing from the report - assert len(list(ts.iter('testcase'))) == expected_testcase_count, \ - 'Not all expected testcases appear in the report.' + testcase_count = len(list(ts.iter('testcase'))) + expected_tc_count = expected_testcase_counts[ts.get('name')] + assert testcase_count == expected_tc_count, \ + f'Not all expected testcases appear in the report.' \ + f' (In {ts.get("name")}, expected {expected_tc_count}, got {testcase_count}.)' assert testsuite_counter == len(test_platforms), \ 'Some platforms are missing from the XML report.' + @pytest.mark.parametrize( + 'test_path, report_filtered, expected_filtered_count', + [ + (os.path.join(TEST_DATA, 'tests', 'dummy'), False, 0), + (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 4), + ], + ids=['no filtered', 'with filtered'] + ) + def test_report_filtered(self, out_path, test_path, report_filtered, expected_filtered_count): + test_platforms = ['qemu_x86', 'frdm_k64f'] + args = ['-i', '--outdir', out_path, '-T', test_path] + \ + (['--report-filtered'] if report_filtered else []) + \ + [val for pair in zip( + ['-p'] * len(test_platforms), test_platforms + ) for val in pair] + + with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ + pytest.raises(SystemExit) as sys_exit: + self.loader.exec_module(self.twister_module) + + assert str(sys_exit.value) == '0' + + with open(os.path.join(out_path, 'twister.json')) as f: + j = json.load(f) + + testsuites = j.get('testsuites') + assert testsuites, 'No testsuites found.' + statuses = [testsuite.get('status') for testsuite in testsuites] + filtered_status_count = statuses.count("filtered") + assert filtered_status_count == expected_filtered_count, \ + f'Expected {expected_filtered_count} filtered statuses, got {filtered_status_count}.' + def test_enable_size_report(self, out_path): test_platforms = ['qemu_x86', 'frdm_k64f'] path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 97e36d8b8a6..c93b93eb74e 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -45,15 +45,15 @@ class TestRunner: ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], { 'selected_test_scenarios': 3, - 'selected_test_instances': 9, - 'skipped_configurations': 3, - 'skipped_by_static_filter': 3, + 'selected_test_instances': 6, + 'skipped_configurations': 0, + 'skipped_by_static_filter': 0, 'skipped_at_runtime': 0, 'passed_configurations': 4, 'failed_configurations': 0, 'errored_configurations': 0, 'executed_test_cases': 8, - 'skipped_test_cases': 5, + 'skipped_test_cases': 0, 'platform_count': 0, 'executed_on_platform': 4, 'only_built': 2