michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: add option to ignore skipped tests in reports

Browse source 
With --no-skipped-report, twister will skip over all tests marked as
skipped (filtered) in the junit report.

This is useful for CI where have 1000s of filtered tests that appear in
the report and in some cases cause tools parsing the output to fail or
provide incomplete results.

Fixes #38179

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2021-11-30 10:59:13 -05:00
commit dd651c0323
2 changed files with 26 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

22
scripts/pylib/twister/twisterlib.py
View file

@ -2980,7 +2980,7 @@ class TestSuite(DisablePyTestCollectionMixin):
logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \
{Fore.RED}{results.total - run - results.skipped_configs}{Fore.RESET} test configurations were only built.")
def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report):
def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report, report_skipped):
if not self.instances:
return
@ -3003,9 +3003,9 @@ class TestSuite(DisablePyTestCollectionMixin):
if not no_update:
self.xunit_report(filename + ".xml", full_report=False,
append=only_failed, version=self.version)
append=only_failed, version=self.version, report_skipped=report_skipped)
self.xunit_report(filename + "_report.xml", full_report=True,
append=only_failed, version=self.version)
append=only_failed, version=self.version, report_skipped=report_skipped)
self.csv_report(filename + ".csv")
if json_report:
@ -3588,7 +3588,7 @@ class TestSuite(DisablePyTestCollectionMixin):
"reason": reason}
cw.writerow(rowdict)
def target_report(self, outdir, suffix, append=False):
def target_report(self, outdir, suffix, append=False, report_skipped=True):
platforms = {inst.platform.name for _, inst in self.instances.items()}
for platform in platforms:
if suffix:
@ -3596,7 +3596,7 @@ class TestSuite(DisablePyTestCollectionMixin):
else:
filename = os.path.join(outdir,"{}.xml".format(platform))
self.xunit_report(filename, platform, full_report=True,
append=append, version=self.version)
append=append, version=self.version, report_skipped=report_skipped)
@staticmethod
@ -3610,7 +3610,7 @@ class TestSuite(DisablePyTestCollectionMixin):
return filtered_string
def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA"):
def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA", report_skipped=True):
total = 0
fails = passes = errors = skips = 0
if platform:
@ -3670,6 +3670,8 @@ class TestSuite(DisablePyTestCollectionMixin):
run = p
eleTestsuite = None
if not report_skipped and total == skips:
continue
# When we re-run the tests, we re-use the results and update only with
# the newly run tests.
@ -3687,23 +3689,21 @@ class TestSuite(DisablePyTestCollectionMixin):
tests="%d" % (total),
failures="%d" % fails,
errors="%d" % (errors), skipped="%s" % (skips))
eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
# Multiple 'property' can be added to 'properties'
# differing by name and value
ET.SubElement(eleTSPropetries, 'property', name="version", value=version)
else:
eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
name=run, time="%f" % duration,
tests="%d" % (total),
failures="%d" % fails,
errors="%d" % (errors), skipped="%s" % (skips))
eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
# Multiple 'property' can be added to 'properties'
# differing by name and value
ET.SubElement(eleTSPropetries, 'property', name="version", value=version)
for _, instance in inst.items():
if instance.status == 'skipped' and not report_skipped:
continue
if full_report:
tname = os.path.basename(instance.testcase.name)
else:

8
scripts/twister
View file

@ -577,6 +577,11 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
help="Re-use the outdir before building. Will result in "
"faster compilation since builds will be incremental.")
parser.add_argument(
"--no-skipped-report", action="store_true",
help="""Do not report skipped test cases in junit output. [Experimental]
""")
parser.add_argument(
"-O", "--outdir",
default=os.path.join(os.getcwd(), "twister-out"),
@ -1359,7 +1364,8 @@ def main():
options.release,
options.only_failed,
options.platform_reports,
options.json_report
options.json_report,
not options.no_skipped_report
)
# FIXME: remove later