sanitycheck: fail on faults/panics/oopses

Fail in tests where we have an OOPS or a panic. Right now and in many
cases we continue and test case might be reported as PASS.

Cases that have the tag ignore_faults will ignore those faults (cases
that are testing faults for example).

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

scripts/sanity_chk/harness.py

@@ -11,10 +11,14 @@ class Harness:
         self.repeat = 1
         self.tests = {}
         self.id = None
+        self.fail_on_fault = True
 
     def configure(self, instance):
         config = instance.test.harness_config
         self.id = instance.test.id
+        if "ignore_faults" in instance.test.tags:
+            self.fail_on_fault = False
+
         if config:
             self.type = config.get('type', None)
             self.regex = config.get('regex', [] )
@@ -55,6 +59,14 @@ class Test(Harness):
     RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
     RUN_FAILED = "PROJECT EXECUTION FAILED"
 
+    faults = [
+            "Unknown Fatal Error",
+            "MPU FAULT",
+            "Kernel Panic",
+            "Kernel OOPS",
+            "BUS FAULT"
+            ]
+
     def handle(self, line):
         result = re.compile("(PASS|FAIL|SKIP) - (test_)?(.*)")
         match = result.match(line)
@@ -67,3 +79,9 @@ class Test(Harness):
 
         if self.RUN_FAILED in line:
             self.state = "failed"
+
+        if self.fail_on_fault:
+            for fault in self.faults:
+                if fault in line:
+                    self.state = "failed"
+

tests/kernel/pipe/pipe/testcase.yaml

@@ -1,3 +1,3 @@
 tests:
   kernel.pipe:
-      tags: kernel userspace
+      tags: kernel userspace ignore_faults