twister: add --report-summary switch

Added a switch that show failed/error report from the last run.
Default shows all items found. However, you can specify the number of items
(e.g. --report-summary 15).
It also works well with the --outdir switch

Signed-off-by: Kamil Paszkiet <kamilx.paszkiet@intel.com>

scripts/pylib/twister/twisterlib/environment.py

@@ -602,6 +602,12 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
         help="""Create a report with a custom name.
         """)
 
+    parser.add_argument(
+        "--report-summary", action="store", nargs='?', type=int, const=0,
+        help="Show failed/error report from latest run. Default shows all items found. "
+             "However, you can specify the number of items (e.g. --report-summary 15). "
+             "It also works well with the --outdir switch.")
+
     parser.add_argument(
         "--report-suffix",
         help="""Add a suffix to all generated file names, for example to add a

scripts/pylib/twister/twisterlib/reports.py

@@ -26,6 +26,7 @@ class Reporting:
         self.env = env
         self.timestamp = datetime.now().isoformat()
         self.outdir = os.path.abspath(env.options.outdir)
+        self.instance_fail_count = plan.instance_fail_count
 
     @staticmethod
     def process_log(log_file):
@@ -432,20 +433,36 @@ class Reporting:
                            (report if not last_metrics else "the last twister run.")))
 
     def synopsis(self):
+        if self.env.options.report_summary == 0:
+            count = self.instance_fail_count
+            log_txt = f"The following issues were found (showing the all {count} items):"
+        elif self.env.options.report_summary:
+            count = self.env.options.report_summary
+            log_txt = f"The following issues were found "
+            if count > self.instance_fail_count:
+                log_txt += f"(presenting {self.instance_fail_count} out of the {count} items requested):"
+            else:
+                log_txt += f"(showing the {count} of {self.instance_fail_count} items):"
+        else:
+            count = 10
+            log_txt = f"The following issues were found (showing the top {count} items):"
         cnt = 0
         example_instance = None
         detailed_test_id = self.env.options.detailed_test_id
         for instance in self.instances.values():
             if instance.status not in ["passed", "filtered", "skipped"]:
-                cnt = cnt + 1
+                cnt += 1
                 if cnt == 1:
                     logger.info("-+" * 40)
-                    logger.info("The following issues were found (showing the top 10 items):")
+                    logger.info(log_txt)
 
                 logger.info(f"{cnt}) {instance.testsuite.name} on {instance.platform.name} {instance.status} ({instance.reason})")
                 example_instance = instance
-            if cnt == 10:
+            if cnt == count:
                 break
+        if cnt == 0 and self.env.options.report_summary is not None:
+            logger.info("-+" * 40)
+            logger.info(f"No errors/fails found")
 
         if cnt and example_instance:
             logger.info("")

scripts/pylib/twister/twisterlib/testplan.py

@@ -17,6 +17,7 @@ import copy
 import shutil
 import random
 import snippets
+from colorama import Fore
 from pathlib import Path
 from argparse import Namespace
 
@@ -107,6 +108,7 @@ class TestPlan:
         self.default_platforms = []
         self.load_errors = 0
         self.instances = dict()
+        self.instance_fail_count = 0
         self.warnings = 0
 
         self.scenarios = []
@@ -217,7 +219,7 @@ class TestPlan:
         else:
             last_run = os.path.join(self.options.outdir, "twister.json")
 
-        if self.options.only_failed:
+        if self.options.only_failed or self.options.report_summary is not None:
             self.load_from_file(last_run)
             self.selected_platforms = set(p.platform.name for p in self.instances.values())
         elif self.options.load_tests:
@@ -581,6 +583,7 @@ class TestPlan:
                 instance.add_filter("Not under quarantine", Filters.QUARANTINE)
 
     def load_from_file(self, file, filter_platform=[]):
+        try:
             with open(file, "r") as json_test_plan:
                 jtp = json.load(json_test_plan)
                 instance_list = []
@@ -615,6 +618,13 @@ class TestPlan:
                     status = ts.get('status', None)
                     reason = ts.get("reason", "Unknown")
                     if status in ["error", "failed"]:
+                        if self.options.report_summary is not None:
+                            if status == "error": status = "ERROR"
+                            elif status == "failed": status = "FAILED"
+                            instance.status = Fore.RED + status + Fore.RESET
+                            instance.reason = reason
+                            self.instance_fail_count += 1
+                        else:
                             instance.status = None
                             instance.reason = None
                             instance.retries += 1
@@ -647,6 +657,9 @@ class TestPlan:
                     instance.create_overlay(platform, self.options.enable_asan, self.options.enable_ubsan, self.options.enable_coverage, self.options.coverage_platform)
                     instance_list.append(instance)
                 self.add_instances(instance_list)
+        except FileNotFoundError as e:
+            logger.error(f"{e}")
+            return 1
 
     def apply_filters(self, **kwargs):
 

scripts/pylib/twister/twisterlib/twister_main.py

@@ -73,7 +73,7 @@ def main(options):
 
     previous_results = None
     # Cleanup
-    if options.no_clean or options.only_failed or options.test_only:
+    if options.no_clean or options.only_failed or options.test_only or options.report_summary is not None:
         if os.path.exists(options.outdir):
             print("Keeping artifacts untouched")
     elif options.last_metrics:
@@ -160,6 +160,13 @@ def main(options):
         report.json_report(options.save_tests)
         return 0
 
+    if options.report_summary is not None:
+        if options.report_summary < 0:
+            logger.error("The report summary value cannot be less than 0")
+            return 1
+        report.synopsis()
+        return 0
+
     if options.device_testing and not options.build_only:
         print("\nDevice testing on:")
         hwm.dump(filtered=tplan.selected_platforms)