michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: twister: Unify Twister Statuses

Browse source 
Various different Statuses were joined into a single class,
TwisterStatus. This change anticipates further streamlining
of the Twister's approach to Status.

Code guarding Twister's Properties was shortened to a
value check only.

QEMUOutputStatus was left separate, as doubts were cast
whether it should remain a status. Leaving it separate makes
its removal easier.

Signed-off-by: Lukasz Mrugala <lukaszx.mrugala@intel.com>
This commit is contained in:
Lukasz Mrugala 2024-05-28 12:31:53 +00:00 • committed by Anas Nashif
commit 212f48c146
18 changed files with 570 additions and 554 deletions
Download patch file Download diff file Expand all files Collapse all files

76
scripts/pylib/twister/twisterlib/runner.py
View file

@ -26,7 +26,7 @@ from domains import Domains
from twisterlib.cmakecache import CMakeCache
from twisterlib.environment import canonical_zephyr_base
from twisterlib.error import BuildError, ConfigurationError
from twisterlib.statuses import TestCaseStatus, TestInstanceStatus
from twisterlib.statuses import TwisterStatus
import elftools
from elftools.elf.elffile import ELFFile
@ -285,9 +285,9 @@ class CMake:
msg = f"Finished building {self.source_dir} for {self.platform.name} in {duration:.2f} seconds"
logger.debug(msg)
self.instance.status = TestInstanceStatus.PASS
self.instance.status = TwisterStatus.PASS
if not self.instance.run:
self.instance.add_missing_case_status(TestCaseStatus.SKIP, "Test was built only")
self.instance.add_missing_case_status(TwisterStatus.SKIP, "Test was built only")
ret = {"returncode": p.returncode}
if out:
@ -309,15 +309,15 @@ class CMake:
imgtool_overflow_found = re.findall(r"Error: Image size \(.*\) \+ trailer \(.*\) exceeds requested size", log_msg)
if overflow_found and not self.options.overflow_as_errors:
logger.debug("Test skipped due to {} Overflow".format(overflow_found[0]))
self.instance.status = TestInstanceStatus.SKIP
self.instance.status = TwisterStatus.SKIP
self.instance.reason = "{} overflow".format(overflow_found[0])
change_skip_to_error_if_integration(self.options, self.instance)
elif imgtool_overflow_found and not self.options.overflow_as_errors:
self.instance.status = TestInstanceStatus.SKIP
self.instance.status = TwisterStatus.SKIP
self.instance.reason = "imgtool overflow"
change_skip_to_error_if_integration(self.options, self.instance)
else:
self.instance.status = TestInstanceStatus.ERROR
self.instance.status = TwisterStatus.ERROR
self.instance.reason = "Build failure"
ret = {
@ -416,7 +416,7 @@ class CMake:
'filter': filter_results
}
else:
self.instance.status = TestInstanceStatus.ERROR
self.instance.status = TwisterStatus.ERROR
self.instance.reason = "Cmake build failure"
for tc in self.instance.testcases:
@ -608,16 +608,16 @@ class ProjectBuilder(FilterBuilder):
if op == "filter":
ret = self.cmake(filter_stages=self.instance.filter_stages)
if self.instance.status in [TestInstanceStatus.FAIL, TestInstanceStatus.ERROR]:
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
pipeline.put({"op": "report", "test": self.instance})
else:
# Here we check the dt/kconfig filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TestInstanceStatus.FILTER
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TestCaseStatus.SKIP)
self.instance.add_missing_case_status(TwisterStatus.SKIP)
pipeline.put({"op": "report", "test": self.instance})
else:
pipeline.put({"op": "cmake", "test": self.instance})
@ -625,20 +625,20 @@ class ProjectBuilder(FilterBuilder):
# The build process, call cmake and build with configured generator
elif op == "cmake":
ret = self.cmake()
if self.instance.status in [TestInstanceStatus.FAIL, TestInstanceStatus.ERROR]:
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
pipeline.put({"op": "report", "test": self.instance})
elif self.options.cmake_only:
if self.instance.status == TestInstanceStatus.NONE:
self.instance.status = TestInstanceStatus.PASS
if self.instance.status == TwisterStatus.NONE:
self.instance.status = TwisterStatus.PASS
pipeline.put({"op": "report", "test": self.instance})
else:
# Here we check the runtime filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TestInstanceStatus.FILTER
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TestCaseStatus.SKIP)
self.instance.add_missing_case_status(TwisterStatus.SKIP)
pipeline.put({"op": "report", "test": self.instance})
else:
pipeline.put({"op": "build", "test": self.instance})
@ -647,18 +647,18 @@ class ProjectBuilder(FilterBuilder):
logger.debug("build test: %s" % self.instance.name)
ret = self.build()
if not ret:
self.instance.status = TestInstanceStatus.ERROR
self.instance.status = TwisterStatus.ERROR
self.instance.reason = "Build Failure"
pipeline.put({"op": "report", "test": self.instance})
else:
# Count skipped cases during build, for example
# due to ram/rom overflow.
if self.instance.status == TestInstanceStatus.SKIP:
if self.instance.status == TwisterStatus.SKIP:
results.skipped_runtime += 1
self.instance.add_missing_case_status(TestCaseStatus.SKIP, self.instance.reason)
self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
if ret.get('returncode', 1) > 0:
self.instance.add_missing_case_status(TestCaseStatus.BLOCK, self.instance.reason)
self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
pipeline.put({"op": "report", "test": self.instance})
else:
if self.instance.testsuite.harness in ['ztest', 'test']:
@ -668,7 +668,7 @@ class ProjectBuilder(FilterBuilder):
pipeline.put({"op": "gather_metrics", "test": self.instance})
except BuildError as e:
logger.error(str(e))
self.instance.status = TestInstanceStatus.ERROR
self.instance.status = TwisterStatus.ERROR
self.instance.reason = str(e)
pipeline.put({"op": "report", "test": self.instance})
else:
@ -714,7 +714,7 @@ class ProjectBuilder(FilterBuilder):
if not self.options.coverage:
if self.options.prep_artifacts_for_testing:
pipeline.put({"op": "cleanup", "mode": "device", "test": self.instance})
elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TestInstanceStatus.PASS:
elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TwisterStatus.PASS:
pipeline.put({"op": "cleanup", "mode": "passed", "test": self.instance})
elif self.options.runtime_artifact_cleanup == "all":
pipeline.put({"op": "cleanup", "mode": "all", "test": self.instance})
@ -967,8 +967,8 @@ class ProjectBuilder(FilterBuilder):
if results.iteration == 1:
results.cases += len(instance.testcases)
if instance.status in [TestInstanceStatus.ERROR, TestInstanceStatus.FAIL]:
if instance.status == TestInstanceStatus.ERROR:
if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
if instance.status == TwisterStatus.ERROR:
results.error += 1
txt = " ERROR "
else:
@ -987,17 +987,17 @@ class ProjectBuilder(FilterBuilder):
instance.reason))
if not self.options.verbose:
self.log_info_file(self.options.inline_logs)
elif instance.status in [TestInstanceStatus.SKIP, TestInstanceStatus.FILTER]:
elif instance.status in [TwisterStatus.SKIP, TwisterStatus.FILTER]:
status = Fore.YELLOW + "SKIPPED" + Fore.RESET
results.skipped_configs += 1
# test cases skipped at the test instance level
results.skipped_cases += len(instance.testsuite.testcases)
elif instance.status == TestInstanceStatus.PASS:
elif instance.status == TwisterStatus.PASS:
status = Fore.GREEN + "PASSED" + Fore.RESET
results.passed += 1
for case in instance.testcases:
# test cases skipped at the test case level
if case.status == TestCaseStatus.SKIP:
if case.status == TwisterStatus.SKIP:
results.skipped_cases += 1
else:
logger.debug(f"Unknown status = {instance.status}")
@ -1006,7 +1006,7 @@ class ProjectBuilder(FilterBuilder):
if self.options.verbose:
if self.options.cmake_only:
more_info = "cmake"
elif instance.status in [TestInstanceStatus.SKIP, TestInstanceStatus.FILTER]:
elif instance.status in [TwisterStatus.SKIP, TwisterStatus.FILTER]:
more_info = instance.reason
else:
if instance.handler.ready and instance.run:
@ -1019,7 +1019,7 @@ class ProjectBuilder(FilterBuilder):
else:
more_info = "build"
if ( instance.status in [TestInstanceStatus.ERROR, TestInstanceStatus.FAIL]
if ( instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]
and hasattr(self.instance.handler, 'seed')
and self.instance.handler.seed is not None ):
more_info += "/seed: " + str(self.options.seed)
@ -1027,7 +1027,7 @@ class ProjectBuilder(FilterBuilder):
results.done, total_tests_width, total_to_do , instance.platform.name,
instance.testsuite.name, status, more_info))
if instance.status in [TestInstanceStatus.ERROR, TestInstanceStatus.FAIL]:
if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
self.log_info_file(self.options.inline_logs)
else:
completed_perc = 0
@ -1110,7 +1110,7 @@ class ProjectBuilder(FilterBuilder):
harness.instance = self.instance
harness.build()
except ConfigurationError as error:
self.instance.status = TestInstanceStatus.ERROR
self.instance.status = TwisterStatus.ERROR
self.instance.reason = str(error)
logger.error(self.instance.reason)
return
@ -1122,7 +1122,7 @@ class ProjectBuilder(FilterBuilder):
if instance.handler.ready:
logger.debug(f"Reset instance status from '{instance.status}' to None before run.")
instance.status = TestInstanceStatus.NONE
instance.status = TwisterStatus.NONE
if instance.handler.type_str == "device":
instance.handler.duts = self.duts
@ -1140,7 +1140,7 @@ class ProjectBuilder(FilterBuilder):
try:
harness.configure(instance)
except ConfigurationError as error:
instance.status = TestInstanceStatus.ERROR
instance.status = TwisterStatus.ERROR
instance.reason = str(error)
logger.error(instance.reason)
return
@ -1168,7 +1168,7 @@ class ProjectBuilder(FilterBuilder):
@staticmethod
def calc_size(instance: TestInstance, from_buildlog: bool):
if instance.status not in [TestInstanceStatus.ERROR, TestInstanceStatus.FAIL, TestInstanceStatus.SKIP]:
if instance.status not in [TwisterStatus.ERROR, TwisterStatus.FAIL, TwisterStatus.SKIP]:
if not instance.platform.type in ["native", "qemu", "unit"]:
generate_warning = bool(instance.platform.type == "mcu")
size_calc = instance.calculate_sizes(from_buildlog=from_buildlog, generate_warning=generate_warning)
@ -1279,12 +1279,12 @@ class TwisterRunner:
the static filter stats. So need to prepare them before pipline starts.
'''
for instance in self.instances.values():
if instance.status == TestInstanceStatus.FILTER and not instance.reason == 'runtime filter':
if instance.status == TwisterStatus.FILTER and not instance.reason == 'runtime filter':
self.results.skipped_filter += 1
self.results.skipped_configs += 1
self.results.skipped_cases += len(instance.testsuite.testcases)
self.results.cases += len(instance.testsuite.testcases)
elif instance.status == TestInstanceStatus.ERROR:
elif instance.status == TwisterStatus.ERROR:
self.results.error += 1
def show_brief(self):
@ -1300,15 +1300,15 @@ class TwisterRunner:
if build_only:
instance.run = False
no_retry_statuses = [TestInstanceStatus.PASS, TestInstanceStatus.SKIP, TestInstanceStatus.FILTER]
no_retry_statuses = [TwisterStatus.PASS, TwisterStatus.SKIP, TwisterStatus.FILTER]
if not retry_build_errors:
no_retry_statuses.append("error")
if instance.status not in no_retry_statuses:
logger.debug(f"adding {instance.name}")
if instance.status != TestInstanceStatus.NONE:
if instance.status != TwisterStatus.NONE:
instance.retries += 1
instance.status = TestInstanceStatus.NONE
instance.status = TwisterStatus.NONE
# Check if cmake package_helper script can be run in advance.
instance.filter_stages = []