twister: generate and verify test run id

Set run id (md5 of instance and a random number) for each test and match
what was set during build with the output on the screen. This will
verify that we are evaluating the same test we have uploaded and not
some previous output on the screen.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-04-07 16:11:01 -04:00 committed by Marti Bolivar
commit 540327ee4f
2 changed files with 33 additions and 0 deletions

View file

@ -13,6 +13,8 @@ class Harness:
FAULT = "ZEPHYR FATAL ERROR"
RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
RUN_FAILED = "PROJECT EXECUTION FAILED"
run_id_pattern = r"RunID: (?P<run_id>.*)"
def __init__(self):
self.state = None
@ -33,10 +35,14 @@ class Harness:
self.ztest = False
self.is_pytest = False
self.detected_suite_names = []
self.run_id = None
self.matched_run_id = False
self.run_id_exists = False
def configure(self, instance):
config = instance.testcase.harness_config
self.id = instance.testcase.id
self.run_id = instance.run_id
if "ignore_faults" in instance.testcase.tags:
self.fail_on_fault = False
@ -49,6 +55,13 @@ class Harness:
def process_test(self, line):
runid_match = re.search(self.run_id_pattern, line)
if runid_match:
run_id = runid_match.group("run_id")
self.run_id_exists = True
if run_id == str(self.run_id):
self.matched_run_id = True
if self.RUN_PASSED in line:
if self.fault:
self.state = "failed"

View file

@ -14,12 +14,14 @@ import select
import shutil
import shlex
import signal
import hashlib
import threading
from collections import OrderedDict
import queue
import time
import csv
import glob
import random
import xml.etree.ElementTree as ET
import logging
from pathlib import Path
@ -529,6 +531,12 @@ class Handler:
if harness_class_name == "Test": # only for ZTest tests
self._verify_ztest_suite_name(harness.state, harness.detected_suite_names, handler_time)
if not harness.matched_run_id and harness.run_id_exists:
self.set_state("failed", handler_time)
self.instance.reason = "RunID mismatch"
for k in self.instance.testcase.cases:
self.instance.results[k] = "FAIL"
self.record(harness)
@ -2085,12 +2093,23 @@ class TestInstance(DisablePyTestCollectionMixin):
self.outdir = outdir
self.name = os.path.join(platform.name, testcase.name)
self.run_id = self._get_run_id()
self.build_dir = os.path.join(outdir, platform.name, testcase.name)
self.run = False
self.results = {}
def _get_run_id(self):
""" generate run id from instance unique identifier and a random
number"""
hash_object = hashlib.md5(self.name.encode())
random_str = f"{random.getrandbits(64)}".encode()
hash_object.update(random_str)
return hash_object.hexdigest()
def __getstate__(self):
d = self.__dict__.copy()
return d
@ -2335,6 +2354,7 @@ class CMake():
cmake_args = [
f'-B{self.build_dir}',
f'-S{self.source_dir}',
f'-DTC_RUNID={self.instance.run_id}',
f'-DEXTRA_CFLAGS={cflags}',
f'-DEXTRA_AFLAGS={aflags}',
f'-DEXTRA_LDFLAGS={ldflags}',