michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: twister: Elevate Status Error

Browse source 
Status errors previously logged an error, but didn't fail the running test.
This commit changes that
 and introduces a new StatusAttributeError to use there.

One test is modified so it follows proper status form.
One test for the new error has been added.

Status errors now will properly mark the Instance as ERROR
and not run TestCases as SKIP.
This necessitated some code layout changes in runner.py

Signed-off-by: Lukasz Mrugala <lukaszx.mrugala@intel.com>
This commit is contained in:
Lukasz Mrugala 2024-08-14 13:01:51 +00:00 • committed by Carles Cufí
commit 345eb8e94a
7 changed files with 201 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/pylib/twister/twisterlib/error.py
View file

@ -22,3 +22,9 @@ class BuildError(TwisterException):
class ExecutionError(TwisterException): class ExecutionError(TwisterException):
pass pass
class StatusAttributeError(TwisterException):
def __init__(self, cls : type, value):
msg = f'{cls.__name__} assigned status {value}, which could not be cast to a TwisterStatus.'
super().__init__(msg)

6
scripts/pylib/twister/twisterlib/harness.py
View file

@ -18,7 +18,7 @@ import json
from pytest import ExitCode from pytest import ExitCode
from twisterlib.reports import ReportStatus from twisterlib.reports import ReportStatus
from twisterlib.error import ConfigurationError from twisterlib.error import ConfigurationError, StatusAttributeError
from twisterlib.environment import ZEPHYR_BASE, PYTEST_PLUGIN_INSTALLED from twisterlib.environment import ZEPHYR_BASE, PYTEST_PLUGIN_INSTALLED
from twisterlib.handlers import Handler, terminate_process, SUPPORTED_SIMS_IN_PYTEST from twisterlib.handlers import Handler, terminate_process, SUPPORTED_SIMS_IN_PYTEST
from twisterlib.statuses import TwisterStatus from twisterlib.statuses import TwisterStatus
@ -76,9 +76,7 @@ class Harness:
key = value.name if isinstance(value, Enum) else value key = value.name if isinstance(value, Enum) else value
self._status = TwisterStatus[key] self._status = TwisterStatus[key]
except KeyError: except KeyError:
logger.error(f'Harness assigned status "{value}"' raise StatusAttributeError(self.__class__, value)
f' without an equivalent in TwisterStatus.'
f' Assignment was ignored.')
def configure(self, instance): def configure(self, instance):
self.instance = instance self.instance = instance

266
scripts/pylib/twister/twisterlib/runner.py
View file

@ -25,7 +25,7 @@ from colorama import Fore
from domains import Domains from domains import Domains
from twisterlib.cmakecache import CMakeCache from twisterlib.cmakecache import CMakeCache
from twisterlib.environment import canonical_zephyr_base from twisterlib.environment import canonical_zephyr_base
from twisterlib.error import BuildError, ConfigurationError from twisterlib.error import BuildError, ConfigurationError, StatusAttributeError
from twisterlib.statuses import TwisterStatus from twisterlib.statuses import TwisterStatus
import elftools import elftools
@ -601,130 +601,212 @@ class ProjectBuilder(FilterBuilder):
self.log_info("{}".format(b_log), inline_logs) self.log_info("{}".format(b_log), inline_logs)
def _add_to_pipeline(self, pipeline, op: str, additionals: dict={}):
try:
if op:
task = dict({'op': op, 'test': self.instance}, **additionals)
pipeline.put(task)
# Only possible RuntimeError source here is a mutation of the pipeline during iteration.
# If that happens, we ought to consider the whole pipeline corrupted.
except RuntimeError as e:
logger.error(f"RuntimeError: {e}")
traceback.print_exc()
def process(self, pipeline, done, message, lock, results): def process(self, pipeline, done, message, lock, results):
op = message.get('op') op = message.get('op')
self.instance.setup_handler(self.env) self.instance.setup_handler(self.env)
if op == "filter": if op == "filter":
ret = self.cmake(filter_stages=self.instance.filter_stages) try:
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]: ret = self.cmake(filter_stages=self.instance.filter_stages)
pipeline.put({"op": "report", "test": self.instance}) if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
else: next_op = 'report'
# Here we check the dt/kconfig filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TwisterStatus.SKIP)
pipeline.put({"op": "report", "test": self.instance})
else: else:
pipeline.put({"op": "cmake", "test": self.instance}) # Here we check the dt/kconfig filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TwisterStatus.SKIP)
next_op = 'report'
else:
next_op = 'cmake'
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR
reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
next_op = 'report'
finally:
self._add_to_pipeline(pipeline, next_op)
# The build process, call cmake and build with configured generator # The build process, call cmake and build with configured generator
elif op == "cmake": elif op == "cmake":
ret = self.cmake() try:
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]: ret = self.cmake()
pipeline.put({"op": "report", "test": self.instance}) if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
elif self.options.cmake_only: next_op = 'report'
if self.instance.status == TwisterStatus.NONE: elif self.options.cmake_only:
self.instance.status = TwisterStatus.PASS if self.instance.status == TwisterStatus.NONE:
pipeline.put({"op": "report", "test": self.instance}) self.instance.status = TwisterStatus.PASS
else: next_op = 'report'
# Here we check the runtime filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TwisterStatus.SKIP)
pipeline.put({"op": "report", "test": self.instance})
else: else:
pipeline.put({"op": "build", "test": self.instance}) # Here we check the runtime filter results coming from running cmake
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.skipped_runtime += 1
self.instance.add_missing_case_status(TwisterStatus.SKIP)
next_op = 'report'
else:
next_op = 'build'
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR
reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
next_op = 'report'
finally:
self._add_to_pipeline(pipeline, next_op)
elif op == "build": elif op == "build":
logger.debug("build test: %s" % self.instance.name) try:
ret = self.build() logger.debug("build test: %s" % self.instance.name)
if not ret: ret = self.build()
self.instance.status = TwisterStatus.ERROR if not ret:
self.instance.reason = "Build Failure" self.instance.status = TwisterStatus.ERROR
pipeline.put({"op": "report", "test": self.instance}) self.instance.reason = "Build Failure"
else: next_op = 'report'
# Count skipped cases during build, for example
# due to ram/rom overflow.
if self.instance.status == TwisterStatus.SKIP:
results.skipped_runtime += 1
self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
if ret.get('returncode', 1) > 0:
self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
pipeline.put({"op": "report", "test": self.instance})
else: else:
if self.instance.testsuite.harness in ['ztest', 'test']: # Count skipped cases during build, for example
logger.debug(f"Determine test cases for test instance: {self.instance.name}") # due to ram/rom overflow.
try: if self.instance.status == TwisterStatus.SKIP:
self.determine_testcases(results) results.skipped_runtime += 1
pipeline.put({"op": "gather_metrics", "test": self.instance}) self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
except BuildError as e:
logger.error(str(e)) if ret.get('returncode', 1) > 0:
self.instance.status = TwisterStatus.ERROR self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
self.instance.reason = str(e) next_op = 'report'
pipeline.put({"op": "report", "test": self.instance})
else: else:
pipeline.put({"op": "gather_metrics", "test": self.instance}) if self.instance.testsuite.harness in ['ztest', 'test']:
logger.debug(f"Determine test cases for test instance: {self.instance.name}")
try:
self.determine_testcases(results)
next_op = 'gather_metrics'
except BuildError as e:
logger.error(str(e))
self.instance.status = TwisterStatus.ERROR
self.instance.reason = str(e)
next_op = 'report'
else:
next_op = 'gather_metrics'
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR
reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
next_op = 'report'
finally:
self._add_to_pipeline(pipeline, next_op)
elif op == "gather_metrics": elif op == "gather_metrics":
ret = self.gather_metrics(self.instance) try:
if not ret or ret.get('returncode', 1) > 0: ret = self.gather_metrics(self.instance)
if not ret or ret.get('returncode', 1) > 0:
self.instance.status = TwisterStatus.ERROR
self.instance.reason = "Build Failure at gather_metrics."
next_op = 'report'
elif self.instance.run and self.instance.handler.ready:
next_op = 'run'
else:
next_op = 'report'
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR self.instance.status = TwisterStatus.ERROR
self.instance.reason = "Build Failure at gather_metrics." reason = 'Incorrect status assignment'
pipeline.put({"op": "report", "test": self.instance}) self.instance.reason = reason
elif self.instance.run and self.instance.handler.ready: self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
pipeline.put({"op": "run", "test": self.instance}) next_op = 'report'
else: finally:
pipeline.put({"op": "report", "test": self.instance}) self._add_to_pipeline(pipeline, next_op)
# Run the generated binary using one of the supported handlers # Run the generated binary using one of the supported handlers
elif op == "run": elif op == "run":
logger.debug("run test: %s" % self.instance.name)
self.run()
logger.debug(f"run status: {self.instance.name} {self.instance.status}")
try: try:
logger.debug("run test: %s" % self.instance.name)
self.run()
logger.debug(f"run status: {self.instance.name} {self.instance.status}")
# to make it work with pickle # to make it work with pickle
self.instance.handler.thread = None self.instance.handler.thread = None
self.instance.handler.duts = None self.instance.handler.duts = None
pipeline.put({
"op": "report", next_op = 'report'
"test": self.instance, additionals = {
"status": self.instance.status, "status": self.instance.status,
"reason": self.instance.reason "reason": self.instance.reason
} }
) except StatusAttributeError as sae:
except RuntimeError as e: logger.error(str(sae))
logger.error(f"RuntimeError: {e}") self.instance.status = TwisterStatus.ERROR
traceback.print_exc() reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
next_op = 'report'
additionals = {}
finally:
self._add_to_pipeline(pipeline, next_op, additionals)
# Report results and output progress to screen # Report results and output progress to screen
elif op == "report": elif op == "report":
with lock: try:
done.put(self.instance) with lock:
self.report_out(results) done.put(self.instance)
self.report_out(results)
if not self.options.coverage: next_op = None
if self.options.prep_artifacts_for_testing: additionals = {}
pipeline.put({"op": "cleanup", "mode": "device", "test": self.instance}) if not self.options.coverage:
elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TwisterStatus.PASS: if self.options.prep_artifacts_for_testing:
pipeline.put({"op": "cleanup", "mode": "passed", "test": self.instance}) next_op = 'cleanup'
elif self.options.runtime_artifact_cleanup == "all": additionals = {"mode": "device"}
pipeline.put({"op": "cleanup", "mode": "all", "test": self.instance}) elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TwisterStatus.PASS:
next_op = 'cleanup'
additionals = {"mode": "passed"}
elif self.options.runtime_artifact_cleanup == "all":
next_op = 'cleanup'
additionals = {"mode": "all"}
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR
reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
next_op = None
additionals = {}
finally:
self._add_to_pipeline(pipeline, next_op, additionals)
elif op == "cleanup": elif op == "cleanup":
mode = message.get("mode") try:
if mode == "device": mode = message.get("mode")
self.cleanup_device_testing_artifacts() if mode == "device":
elif mode == "passed" or (mode == "all" and self.instance.reason != "Cmake build failure"): self.cleanup_device_testing_artifacts()
self.cleanup_artifacts() elif mode == "passed" or (mode == "all" and self.instance.reason != "Cmake build failure"):
self.cleanup_artifacts()
except StatusAttributeError as sae:
logger.error(str(sae))
self.instance.status = TwisterStatus.ERROR
reason = 'Incorrect status assignment'
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
def determine_testcases(self, results): def determine_testcases(self, results):
yaml_testsuite_name = self.instance.testsuite.id yaml_testsuite_name = self.instance.testsuite.id

6
scripts/pylib/twister/twisterlib/testinstance.py
View file

@ -15,7 +15,7 @@ import csv
from twisterlib.testsuite import TestCase, TestSuite from twisterlib.testsuite import TestCase, TestSuite
from twisterlib.platform import Platform from twisterlib.platform import Platform
from twisterlib.error import BuildError from twisterlib.error import BuildError, StatusAttributeError
from twisterlib.size_calc import SizeCalculator from twisterlib.size_calc import SizeCalculator
from twisterlib.statuses import TwisterStatus from twisterlib.statuses import TwisterStatus
from twisterlib.handlers import ( from twisterlib.handlers import (
@ -105,9 +105,7 @@ class TestInstance:
key = value.name if isinstance(value, Enum) else value key = value.name if isinstance(value, Enum) else value
self._status = TwisterStatus[key] self._status = TwisterStatus[key]
except KeyError: except KeyError:
logger.error(f'TestInstance assigned status "{value}"' raise StatusAttributeError(self.__class__, value)
f' without an equivalent in TwisterStatus.'
f' Assignment was ignored.')
def add_filter(self, reason, filter_type): def add_filter(self, reason, filter_type):
self.filters.append({'type': filter_type, 'reason': reason }) self.filters.append({'type': filter_type, 'reason': reason })

10
scripts/pylib/twister/twisterlib/testsuite.py
View file

@ -15,7 +15,7 @@ from typing import List
from twisterlib.mixins import DisablePyTestCollectionMixin from twisterlib.mixins import DisablePyTestCollectionMixin
from twisterlib.environment import canonical_zephyr_base from twisterlib.environment import canonical_zephyr_base
from twisterlib.error import TwisterException, TwisterRuntimeError from twisterlib.error import StatusAttributeError, TwisterException, TwisterRuntimeError
from twisterlib.statuses import TwisterStatus from twisterlib.statuses import TwisterStatus
logger = logging.getLogger('twister') logger = logging.getLogger('twister')
@ -377,9 +377,7 @@ class TestCase(DisablePyTestCollectionMixin):
key = value.name if isinstance(value, Enum) else value key = value.name if isinstance(value, Enum) else value
self._status = TwisterStatus[key] self._status = TwisterStatus[key]
except KeyError: except KeyError:
logger.error(f'TestCase assigned status "{value}"' raise StatusAttributeError(self.__class__, value)
f' without an equivalent in TwisterStatus.'
f' Assignment was ignored.')
def __lt__(self, other): def __lt__(self, other):
return self.name < other.name return self.name < other.name
@ -445,9 +443,7 @@ class TestSuite(DisablePyTestCollectionMixin):
key = value.name if isinstance(value, Enum) else value key = value.name if isinstance(value, Enum) else value
self._status = TwisterStatus[key] self._status = TwisterStatus[key]
except KeyError: except KeyError:
logger.error(f'TestSuite assigned status "{value}"' raise StatusAttributeError(self.__class__, value)
f' without an equivalent in TwisterStatus.'
f' Assignment was ignored.')
def load(self, data): def load(self, data):
for k, v in data.items(): for k, v in data.items():

13
scripts/tests/twister/test_errors.py
View file

@ -10,7 +10,10 @@ import os
import pytest import pytest
from pathlib import Path from pathlib import Path
from twisterlib.error import StatusAttributeError
from twisterlib.error import ConfigurationError from twisterlib.error import ConfigurationError
from twisterlib.harness import Test
def test_configurationerror(): def test_configurationerror():
cfile = Path('some') / 'path' cfile = Path('some') / 'path'
@ -20,3 +23,13 @@ def test_configurationerror():
with pytest.raises(ConfigurationError, match=expected_err): with pytest.raises(ConfigurationError, match=expected_err):
raise ConfigurationError(cfile, message) raise ConfigurationError(cfile, message)
def test_status_value_error():
harness = Test()
expected_err = 'Test assigned status None,' \
' which could not be cast to a TwisterStatus.'
with pytest.raises(StatusAttributeError, match=expected_err):
harness.status = None

2
scripts/tests/twister/test_handlers.py
View file

@ -130,7 +130,7 @@ def test_handler_final_handle_actions(mocked_instance):
handler.suite_name_check = True handler.suite_name_check = True
harness = twisterlib.harness.Test() harness = twisterlib.harness.Test()
harness.status = mock.Mock() harness.status = 'NONE'
harness.detected_suite_names = mock.Mock() harness.detected_suite_names = mock.Mock()
harness.matched_run_id = False harness.matched_run_id = False
harness.run_id_exists = True harness.run_id_exists = True