michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: cleanup status and reason setting

Browse source 
Remove some legacy code and set status and failure/skip reason directly
without set_state and get_state.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-05-07 18:18:09 -04:00
commit d5a5cc131f
2 changed files with 43 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

108
scripts/pylib/twister/twisterlib.py
View file

@ -415,7 +415,6 @@ class Handler:
""" """
self.state = "waiting" self.state = "waiting"
self.run = False self.run = False
self.duration = 0
self.type_str = type_str self.type_str = type_str
self.binary = None self.binary = None
@ -429,7 +428,6 @@ class Handler:
self.build_dir = instance.build_dir self.build_dir = instance.build_dir
self.log = os.path.join(self.build_dir, "handler.log") self.log = os.path.join(self.build_dir, "handler.log")
self.returncode = 0 self.returncode = 0
self.set_state("running", self.duration)
self.generator = None self.generator = None
self.generator_cmd = None self.generator_cmd = None
self.suite_name_check = True self.suite_name_check = True
@ -437,14 +435,6 @@ class Handler:
self.args = [] self.args = []
self.terminated = False self.terminated = False
def set_state(self, state, duration):
self.state = state
self.duration = duration
def get_state(self):
ret = (self.state, self.duration)
return ret
def record(self, harness): def record(self, harness):
if harness.recording: if harness.recording:
filename = os.path.join(self.build_dir, "recording.csv") filename = os.path.join(self.build_dir, "recording.csv")
@ -493,7 +483,8 @@ class Handler:
Change result of performed test if problem with missing or unpropper Change result of performed test if problem with missing or unpropper
suite name was occurred. suite name was occurred.
""" """
self.set_state("failed", handler_time) self.instance.status = "failed"
self.instance.execution_time = handler_time
for tc in self.instance.testcases: for tc in self.instance.testcases:
tc.status = "failed" tc.status = "failed"
self.instance.reason = f"Testsuite mismatch" self.instance.reason = f"Testsuite mismatch"
@ -509,7 +500,8 @@ class Handler:
self._verify_ztest_suite_name(harness.state, harness.detected_suite_names, handler_time) self._verify_ztest_suite_name(harness.state, harness.detected_suite_names, handler_time)
if not harness.matched_run_id and harness.run_id_exists: if not harness.matched_run_id and harness.run_id_exists:
self.set_state("failed", handler_time) self.instance.status = "failed"
self.instance.execution_time = handler_time
self.instance.reason = "RunID mismatch" self.instance.reason = "RunID mismatch"
for tc in self.instance.testcases: for tc in self.instance.testcases:
tc.status = "failed" tc.status = "failed"
@ -660,21 +652,21 @@ class BinaryHandler(Handler):
if harness.is_pytest: if harness.is_pytest:
harness.pytest_run(self.log) harness.pytest_run(self.log)
self.instance.execution_time = handler_time
if not self.terminated and self.returncode != 0: if not self.terminated and self.returncode != 0:
self.instance.status = "failed"
if run_valgrind and self.returncode == 2: if run_valgrind and self.returncode == 2:
self.set_state("failed", handler_time)
self.instance.reason = "Valgrind error" self.instance.reason = "Valgrind error"
else: else:
# When a process is killed, the default handler returns 128 + SIGTERM # When a process is killed, the default handler returns 128 + SIGTERM
# so in that case the return code itself is not meaningful # so in that case the return code itself is not meaningful
self.set_state("failed", handler_time)
self.instance.reason = "Failed" self.instance.reason = "Failed"
elif harness.state: elif harness.state:
self.set_state(harness.state, handler_time) self.instance.status = harness.state
if harness.state == "failed": if harness.state == "failed":
self.instance.reason = "Failed" self.instance.reason = "Failed"
else: else:
self.set_state("timeout", handler_time) self.instance.status = "failed"
self.instance.reason = "Timeout" self.instance.reason = "Timeout"
self.instance.add_missing_testscases("blocked", "Timeout") self.instance.add_missing_testscases("blocked", "Timeout")
@ -786,7 +778,6 @@ class DeviceHandler(Handler):
logger.error("{} timed out".format(script)) logger.error("{} timed out".format(script))
def handle(self): def handle(self):
out_state = "failed"
runner = None runner = None
hardware = self.device_is_available(self.instance) hardware = self.device_is_available(self.instance)
@ -884,7 +875,7 @@ class DeviceHandler(Handler):
timeout=self.timeout timeout=self.timeout
) )
except serial.SerialException as e: except serial.SerialException as e:
self.set_state("failed", 0) self.instance.status = "failed"
self.instance.reason = "Serial Device Error" self.instance.reason = "Serial Device Error"
logger.error("Serial device error: %s" % (str(e))) logger.error("Serial device error: %s" % (str(e)))
@ -924,14 +915,15 @@ class DeviceHandler(Handler):
logger.debug(stdout.decode(errors = "ignore")) logger.debug(stdout.decode(errors = "ignore"))
if proc.returncode != 0: if proc.returncode != 0:
self.instance.reason = "Device issue (Flash?)" self.instance.status = "error"
self.instance.reason = "Device issue (Flash error?)"
with open(d_log, "w") as dlog_fp: with open(d_log, "w") as dlog_fp:
dlog_fp.write(stderr.decode()) dlog_fp.write(stderr.decode())
os.write(write_pipe, b'x') # halt the thread os.write(write_pipe, b'x') # halt the thread
out_state = "flash_error"
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
proc.kill() proc.kill()
(stdout, stderr) = proc.communicate() (stdout, stderr) = proc.communicate()
self.instance.status = "error"
self.instance.reason = "Device issue (Timeout)" self.instance.reason = "Device issue (Timeout)"
with open(d_log, "w") as dlog_fp: with open(d_log, "w") as dlog_fp:
@ -946,7 +938,6 @@ class DeviceHandler(Handler):
t.join(self.timeout) t.join(self.timeout)
if t.is_alive(): if t.is_alive():
logger.debug("Timed out while monitoring serial output on {}".format(self.instance.platform.name)) logger.debug("Timed out while monitoring serial output on {}".format(self.instance.platform.name))
out_state = "timeout"
if ser.isOpen(): if ser.isOpen():
ser.close() ser.close()
@ -961,13 +952,8 @@ class DeviceHandler(Handler):
handler_time = time.time() - start_time handler_time = time.time() - start_time
if out_state in ["timeout", "flash_error"]: if self.instance.status == "error":
self.instance.add_missing_testscases("blocked") self.instance.add_missing_testscases("blocked", self.instance.reason)
if out_state == "timeout":
self.instance.reason = "Timeout"
elif out_state == "flash_error":
self.instance.reason = "Flash error"
if harness.is_pytest: if harness.is_pytest:
harness.pytest_run(self.log) harness.pytest_run(self.log)
@ -978,11 +964,11 @@ class DeviceHandler(Handler):
self.instance.add_missing_testscases("blocked") self.instance.add_missing_testscases("blocked")
if harness.state: if harness.state:
self.set_state(harness.state, handler_time) self.instance.status = harness.state
if harness.state == "failed": if harness.state == "failed":
self.instance.reason = "Failed" self.instance.reason = "Failed"
else: else:
self.set_state(out_state, handler_time) self.instance.execution_time = handler_time
self._final_handle_actions(harness, handler_time) self._final_handle_actions(harness, handler_time)
@ -1146,17 +1132,19 @@ class QEMUHandler(Handler):
handler_time = time.time() - start_time handler_time = time.time() - start_time
logger.debug(f"QEMU ({pid}) complete ({out_state}) after {handler_time} seconds") logger.debug(f"QEMU ({pid}) complete ({out_state}) after {handler_time} seconds")
handler.instance.execution_time = handler_time
if out_state == "timeout": if out_state == "timeout":
handler.instance.status = "failed"
handler.instance.reason = "Timeout" handler.instance.reason = "Timeout"
handler.set_state("failed", handler_time)
elif out_state == "failed": elif out_state == "failed":
handler.instance.status = "failed"
handler.instance.reason = "Failed" handler.instance.reason = "Failed"
handler.set_state("failed", handler_time)
elif out_state in ['unexpected eof', 'unexpected byte']: elif out_state in ['unexpected eof', 'unexpected byte']:
handler.instance.status = "failed"
handler.instance.reason = out_state handler.instance.reason = out_state
handler.set_state("failed", handler_time)
else: else:
handler.set_state(out_state, handler_time) handler.instance.status = out_state
handler.instance.reason = "Unknown"
log_out_fp.close() log_out_fp.close()
out_fp.close() out_fp.close()
@ -1245,7 +1233,7 @@ class QEMUHandler(Handler):
logger.debug(f"return code from QEMU ({qemu_pid}): {self.returncode}") logger.debug(f"return code from QEMU ({qemu_pid}): {self.returncode}")
if (self.returncode != 0 and not self.ignore_qemu_crash) or not harness.state: if (self.returncode != 0 and not self.ignore_qemu_crash) or not harness.state:
self.set_state("failed", 0) self.instance.status = "failed"
if is_timeout: if is_timeout:
self.instance.reason = "Timeout" self.instance.reason = "Timeout"
else: else:
@ -2108,6 +2096,7 @@ class TestInstance(DisablePyTestCollectionMixin):
self.metrics = dict() self.metrics = dict()
self.handler = None self.handler = None
self.outdir = outdir self.outdir = outdir
self.execution_time = 0
self.name = os.path.join(platform.name, testsuite.name) self.name = os.path.join(platform.name, testsuite.name)
self.run_id = self._get_run_id() self.run_id = self._get_run_id()
@ -2137,6 +2126,8 @@ class TestInstance(DisablePyTestCollectionMixin):
case.status = status case.status = status
if reason: if reason:
case.reason = reason case.reason = reason
else:
case.reason = self.reason
def __getstate__(self): def __getstate__(self):
d = self.__dict__.copy() d = self.__dict__.copy()
@ -2363,14 +2354,14 @@ class CMake():
log.write(log_msg) log.write(log_msg)
if log_msg: if log_msg:
res = re.findall("region `(FLASH|ROM|RAM|ICCM|DCCM|SRAM)' overflowed by", log_msg) overflow_found = re.findall("region `(FLASH|ROM|RAM|ICCM|DCCM|SRAM)' overflowed by", log_msg)
if res and not self.overflow_as_errors: if overflow_found and not self.overflow_as_errors:
logger.debug("Test skipped due to {} Overflow".format(res[0])) logger.debug("Test skipped due to {} Overflow".format(overflow_found[0]))
self.instance.status = "skipped" self.instance.status = "skipped"
self.instance.reason = "{} overflow".format(res[0]) self.instance.reason = "{} overflow".format(overflow_found[0])
else: else:
self.instance.status = "error" self.instance.status = "error"
self.instance.reason = "Overflow failure" self.instance.reason = "Build failure"
results = { results = {
"returncode": p.returncode, "returncode": p.returncode,
@ -2739,7 +2730,6 @@ class ProjectBuilder(FilterBuilder):
elif op == "run": elif op == "run":
logger.debug("run test: %s" % self.instance.name) logger.debug("run test: %s" % self.instance.name)
self.run() self.run()
self.instance.status, _ = self.instance.handler.get_state()
logger.debug(f"run status: {self.instance.name} {self.instance.status}") logger.debug(f"run status: {self.instance.name} {self.instance.status}")
# to make it work with pickle # to make it work with pickle
@ -2832,7 +2822,7 @@ class ProjectBuilder(FilterBuilder):
results.done += 1 results.done += 1
instance = self.instance instance = self.instance
if instance.status in ["error", "failed", "timeout", "flash_error"]: if instance.status in ["error", "failed"]:
if instance.status == "error": if instance.status == "error":
results.error += 1 results.error += 1
else: else:
@ -2872,7 +2862,7 @@ class ProjectBuilder(FilterBuilder):
else: else:
if instance.handler and instance.run: if instance.handler and instance.run:
more_info = instance.handler.type_str more_info = instance.handler.type_str
htime = instance.handler.duration htime = instance.execution_time
if htime: if htime:
more_info += " {:.3f}s".format(htime) more_info += " {:.3f}s".format(htime)
else: else:
@ -2990,7 +2980,7 @@ class ProjectBuilder(FilterBuilder):
instance.metrics["rom_size"] = 0 instance.metrics["rom_size"] = 0
instance.metrics["unrecognized"] = [] instance.metrics["unrecognized"] = []
instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0 instance.metrics["handler_time"] = instance.execution_time
class TestPlan(DisablePyTestCollectionMixin): class TestPlan(DisablePyTestCollectionMixin):
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
@ -3881,12 +3871,12 @@ class TestPlan(DisablePyTestCollectionMixin):
if status in ['skipped', 'filtered']: if status in ['skipped', 'filtered']:
skips += 1 skips += 1
ET.SubElement(eleTestcase, 'skipped', type=f"{status}", message=f"{reason}") ET.SubElement(eleTestcase, 'skipped', type=f"{status}", message=f"{reason}")
elif status in ["failed", "timeout", "blocked"]: elif status in ["failed", "blocked"]:
fails += 1 fails += 1
el = ET.SubElement(eleTestcase, 'failure', type="failure", message=f"{reason}") el = ET.SubElement(eleTestcase, 'failure', type="failure", message=f"{reason}")
if log: if log:
el.text = log el.text = log
elif status in ["error"]: elif status == "error":
errors += 1 errors += 1
el = ET.SubElement(eleTestcase, 'error', type="failure", message=f"{reason}") el = ET.SubElement(eleTestcase, 'error', type="failure", message=f"{reason}")
if log: if log:
@ -4025,12 +4015,8 @@ class TestPlan(DisablePyTestCollectionMixin):
if rom_size: if rom_size:
suite["rom_size"] = rom_size suite["rom_size"] = rom_size
if instance.status in ["error", "failed", "timeout", "flash_error"]: if instance.status in ["error", "failed"]:
if instance.status == 'failed': suite['status'] = instance.status
suite['status'] = instance.status
else:
suite['status'] = "error"
suite["reason"] = instance.reason suite["reason"] = instance.reason
# FIXME # FIXME
if os.path.exists(handler_log): if os.path.exists(handler_log):
@ -4077,20 +4063,10 @@ class TestPlan(DisablePyTestCollectionMixin):
else: else:
testcase["status"] = "skipped" testcase["status"] = "skipped"
testcase["reason"] = case.reason or instance.reason testcase["reason"] = case.reason or instance.reason
else:
elif case.status == 'passed': testcase["status"] = case.status
testcase["status"] = "passed" if case.reason:
elif case.status == 'blocked': testcase["reason"] = case.reason
testcase["reason"] = case.reason
testcase["status"] = "blocked"
elif case.status in ['failed', 'timeout']:
testcase["status"] = "failed"
testcase["reason"] = instance.reason
elif case.status in ["error", "flash_error"]:
testcase["status"] = "error"
testcase["reason"] = instance.reason
elif instance.status == "filtered":
testcase["status"] = "filtered"
testcases.append(testcase) testcases.append(testcase)

2
scripts/twister
View file

@ -1280,7 +1280,7 @@ def main():
break break
else: else:
inst.metrics.update(tplan.instances[inst.name].metrics) inst.metrics.update(tplan.instances[inst.name].metrics)
inst.metrics["handler_time"] = inst.handler.duration if inst.handler else 0 inst.metrics["handler_time"] = inst.execution_time
inst.metrics["unrecognized"] = [] inst.metrics["unrecognized"] = []
tplan.instances[inst.name] = inst tplan.instances[inst.name] = inst