From b89a3d955ab377270aa2d900f0c21495b6e68cec Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Mon, 24 Oct 2022 13:34:48 -0400 Subject: [PATCH] twister: count retries after failures or errors When twister is set to retry any failures, count the number of retries and record the number in the json file. This will help us identify unstable tests or tests requiring attention. Signed-off-by: Anas Nashif --- scripts/pylib/twister/twisterlib/reports.py | 2 ++ scripts/pylib/twister/twisterlib/runner.py | 3 +++ scripts/pylib/twister/twisterlib/testinstance.py | 1 + scripts/pylib/twister/twisterlib/testplan.py | 1 + 4 files changed, 7 insertions(+) diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 6c805ddb6ec..70c64d5f0ed 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -267,6 +267,8 @@ class Reporting: if rom_size: suite["rom_size"] = rom_size + suite['retries'] = instance.retries + if instance.status in ["error", "failed"]: suite['status'] = instance.status suite["reason"] = instance.reason diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index f93e67472ee..12ee53c0899 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -972,6 +972,9 @@ class TwisterRunner: if instance.status not in no_retry_statuses: logger.debug(f"adding {instance.name}") + if instance.status: + instance.retries += 1 + instance.status = None if test_only and instance.run: pipeline.put({"op": "run", "test": instance}) diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index 03f59dffad1..cafea02a161 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -40,6 +40,7 @@ class TestInstance: self.handler = None self.outdir = outdir self.execution_time = 0 + self.retries = 0 self.name = os.path.join(platform.name, testsuite.name) self.run_id = self._get_run_id() diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 9fd035798e6..0910f980866 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -524,6 +524,7 @@ class TestPlan: if status in ["error", "failed"]: instance.status = None instance.reason = None + instance.retries += 1 # test marked as passed (built only) but can run when # --test-only is used. Reset status to capture new results. elif status == 'passed' and instance.run and self.options.test_only: