From dd651c0323d0c17c13f2cdb50fa3517c8a464ab4 Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Tue, 30 Nov 2021 10:59:13 -0500 Subject: [PATCH] twister: add option to ignore skipped tests in reports With --no-skipped-report, twister will skip over all tests marked as skipped (filtered) in the junit report. This is useful for CI where have 1000s of filtered tests that appear in the report and in some cases cause tools parsing the output to fail or provide incomplete results. Fixes #38179 Signed-off-by: Anas Nashif --- scripts/pylib/twister/twisterlib.py | 38 ++++++++++++++--------------- scripts/twister | 8 +++++- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/scripts/pylib/twister/twisterlib.py b/scripts/pylib/twister/twisterlib.py index 032f9bed273..3d24e26863e 100755 --- a/scripts/pylib/twister/twisterlib.py +++ b/scripts/pylib/twister/twisterlib.py @@ -2980,7 +2980,7 @@ class TestSuite(DisablePyTestCollectionMixin): logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \ {Fore.RED}{results.total - run - results.skipped_configs}{Fore.RESET} test configurations were only built.") - def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report): + def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report, report_skipped): if not self.instances: return @@ -3003,9 +3003,9 @@ class TestSuite(DisablePyTestCollectionMixin): if not no_update: self.xunit_report(filename + ".xml", full_report=False, - append=only_failed, version=self.version) + append=only_failed, version=self.version, report_skipped=report_skipped) self.xunit_report(filename + "_report.xml", full_report=True, - append=only_failed, version=self.version) + append=only_failed, version=self.version, report_skipped=report_skipped) self.csv_report(filename + ".csv") if json_report: @@ -3588,7 +3588,7 @@ class TestSuite(DisablePyTestCollectionMixin): "reason": reason} cw.writerow(rowdict) - def target_report(self, outdir, suffix, append=False): + def target_report(self, outdir, suffix, append=False, report_skipped=True): platforms = {inst.platform.name for _, inst in self.instances.items()} for platform in platforms: if suffix: @@ -3596,7 +3596,7 @@ class TestSuite(DisablePyTestCollectionMixin): else: filename = os.path.join(outdir,"{}.xml".format(platform)) self.xunit_report(filename, platform, full_report=True, - append=append, version=self.version) + append=append, version=self.version, report_skipped=report_skipped) @staticmethod @@ -3610,7 +3610,7 @@ class TestSuite(DisablePyTestCollectionMixin): return filtered_string - def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA"): + def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA", report_skipped=True): total = 0 fails = passes = errors = skips = 0 if platform: @@ -3670,6 +3670,8 @@ class TestSuite(DisablePyTestCollectionMixin): run = p eleTestsuite = None + if not report_skipped and total == skips: + continue # When we re-run the tests, we re-use the results and update only with # the newly run tests. @@ -3687,23 +3689,21 @@ class TestSuite(DisablePyTestCollectionMixin): tests="%d" % (total), failures="%d" % fails, errors="%d" % (errors), skipped="%s" % (skips)) - eleTSPropetries = ET.SubElement(eleTestsuite, 'properties') - # Multiple 'property' can be added to 'properties' - # differing by name and value - ET.SubElement(eleTSPropetries, 'property', name="version", value=version) - else: eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite', - name=run, time="%f" % duration, - tests="%d" % (total), - failures="%d" % fails, - errors="%d" % (errors), skipped="%s" % (skips)) - eleTSPropetries = ET.SubElement(eleTestsuite, 'properties') - # Multiple 'property' can be added to 'properties' - # differing by name and value - ET.SubElement(eleTSPropetries, 'property', name="version", value=version) + name=run, time="%f" % duration, + tests="%d" % (total), + failures="%d" % fails, + errors="%d" % (errors), skipped="%s" % (skips)) + + eleTSPropetries = ET.SubElement(eleTestsuite, 'properties') + # Multiple 'property' can be added to 'properties' + # differing by name and value + ET.SubElement(eleTSPropetries, 'property', name="version", value=version) for _, instance in inst.items(): + if instance.status == 'skipped' and not report_skipped: + continue if full_report: tname = os.path.basename(instance.testcase.name) else: diff --git a/scripts/twister b/scripts/twister index 47304b3a843..937d82668e1 100755 --- a/scripts/twister +++ b/scripts/twister @@ -577,6 +577,11 @@ structure in the main Zephyr tree: boards///""") help="Re-use the outdir before building. Will result in " "faster compilation since builds will be incremental.") + parser.add_argument( + "--no-skipped-report", action="store_true", + help="""Do not report skipped test cases in junit output. [Experimental] + """) + parser.add_argument( "-O", "--outdir", default=os.path.join(os.getcwd(), "twister-out"), @@ -1359,7 +1364,8 @@ def main(): options.release, options.only_failed, options.platform_reports, - options.json_report + options.json_report, + not options.no_skipped_report ) # FIXME: remove later