twister: generate and verify test run id

Set run id (md5 of instance and a random number) for each test and match
what was set during build with the output on the screen. This will
verify that we are evaluating the same test we have uploaded and not
some previous output on the screen.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

scripts/pylib/twister/harness.py

@@ -13,6 +13,8 @@ class Harness:
     FAULT = "ZEPHYR FATAL ERROR"
     RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
     RUN_FAILED = "PROJECT EXECUTION FAILED"
+    run_id_pattern = r"RunID: (?P<run_id>.*)"
+
 
     def __init__(self):
         self.state = None
@@ -33,10 +35,14 @@ class Harness:
         self.ztest = False
         self.is_pytest = False
         self.detected_suite_names = []
+        self.run_id = None
+        self.matched_run_id = False
+        self.run_id_exists = False
 
     def configure(self, instance):
         config = instance.testcase.harness_config
         self.id = instance.testcase.id
+        self.run_id = instance.run_id
         if "ignore_faults" in instance.testcase.tags:
             self.fail_on_fault = False
 
@@ -49,6 +55,13 @@ class Harness:
 
     def process_test(self, line):
 
+        runid_match = re.search(self.run_id_pattern, line)
+        if runid_match:
+            run_id = runid_match.group("run_id")
+            self.run_id_exists = True
+            if run_id == str(self.run_id):
+                self.matched_run_id = True
+
         if self.RUN_PASSED in line:
             if self.fault:
                 self.state = "failed"

scripts/pylib/twister/twisterlib.py

@@ -14,12 +14,14 @@ import select
 import shutil
 import shlex
 import signal
+import hashlib
 import threading
 from collections import OrderedDict
 import queue
 import time
 import csv
 import glob
+import random
 import xml.etree.ElementTree as ET
 import logging
 from pathlib import Path
@@ -529,6 +531,12 @@ class Handler:
         if harness_class_name == "Test":  # only for ZTest tests
             self._verify_ztest_suite_name(harness.state, harness.detected_suite_names, handler_time)
 
+            if not harness.matched_run_id and harness.run_id_exists:
+                self.set_state("failed", handler_time)
+                self.instance.reason = "RunID mismatch"
+                for k in self.instance.testcase.cases:
+                    self.instance.results[k] = "FAIL"
+
         self.record(harness)
 
 
@@ -2085,12 +2093,23 @@ class TestInstance(DisablePyTestCollectionMixin):
         self.outdir = outdir
 
         self.name = os.path.join(platform.name, testcase.name)
+        self.run_id = self._get_run_id()
         self.build_dir = os.path.join(outdir, platform.name, testcase.name)
 
         self.run = False
 
         self.results = {}
 
+    def _get_run_id(self):
+        """ generate run id from instance unique identifier and a random
+        number"""
+
+        hash_object = hashlib.md5(self.name.encode())
+        random_str = f"{random.getrandbits(64)}".encode()
+        hash_object.update(random_str)
+        return hash_object.hexdigest()
+
+
     def __getstate__(self):
         d = self.__dict__.copy()
         return d
@@ -2335,6 +2354,7 @@ class CMake():
         cmake_args = [
             f'-B{self.build_dir}',
             f'-S{self.source_dir}',
+            f'-DTC_RUNID={self.instance.run_id}',
             f'-DEXTRA_CFLAGS={cflags}',
             f'-DEXTRA_AFLAGS={aflags}',
             f'-DEXTRA_LDFLAGS={ldflags}',