scripts: twister: Add CTest harness
Introduce a twister harness for CMake's CTest. Signed-off-by: Pieter De Gendt <pieter.degendt@basalte.be>
This commit is contained in:
parent
33f257b12b
commit
0b67255b0f
6 changed files with 158 additions and 2 deletions
|
@ -275,6 +275,12 @@ Artificially long but functional example:
|
|||
will extend the pytest_args from the harness_config in YAML file.
|
||||
""")
|
||||
|
||||
parser.add_argument(
|
||||
"--ctest-args", action="append",
|
||||
help="""Pass additional arguments to the ctest subprocess. This parameter
|
||||
will extend the ctest_args from the harness_config in YAML file.
|
||||
""")
|
||||
|
||||
valgrind_asan_group.add_argument(
|
||||
"--enable-valgrind", action="store_true",
|
||||
help="""Run binary through valgrind and check for several memory access
|
||||
|
|
|
@ -16,6 +16,7 @@ import xml.etree.ElementTree as ET
|
|||
from collections import OrderedDict
|
||||
from enum import Enum
|
||||
|
||||
import junitparser.junitparser as junit
|
||||
from pytest import ExitCode
|
||||
from twisterlib.constants import SUPPORTED_SIMS_IN_PYTEST
|
||||
from twisterlib.environment import PYTEST_PLUGIN_INSTALLED, ZEPHYR_BASE
|
||||
|
@ -955,6 +956,142 @@ class Bsim(Harness):
|
|||
logger.debug(f'Copying executable from {original_exe_path} to {new_exe_path}')
|
||||
shutil.copy(original_exe_path, new_exe_path)
|
||||
|
||||
class Ctest(Harness):
|
||||
def configure(self, instance: TestInstance):
|
||||
super().configure(instance)
|
||||
self.running_dir = instance.build_dir
|
||||
self.report_file = os.path.join(self.running_dir, 'report.xml')
|
||||
self.ctest_log_file_path = os.path.join(self.running_dir, 'twister_harness.log')
|
||||
self._output = []
|
||||
|
||||
def ctest_run(self, timeout):
|
||||
assert self.instance is not None
|
||||
try:
|
||||
cmd = self.generate_command()
|
||||
self.run_command(cmd, timeout)
|
||||
except Exception as err:
|
||||
logger.error(str(err))
|
||||
self.status = TwisterStatus.FAIL
|
||||
self.instance.reason = str(err)
|
||||
finally:
|
||||
self.instance.record(self.recording)
|
||||
self._update_test_status()
|
||||
|
||||
def generate_command(self):
|
||||
config = self.instance.testsuite.harness_config
|
||||
handler: Handler = self.instance.handler
|
||||
ctest_args_yaml = config.get('ctest_args', []) if config else []
|
||||
command = [
|
||||
'ctest',
|
||||
'--build-nocmake',
|
||||
'--test-dir',
|
||||
self.running_dir,
|
||||
'--output-junit',
|
||||
self.report_file,
|
||||
'--output-log',
|
||||
self.ctest_log_file_path,
|
||||
'--output-on-failure',
|
||||
]
|
||||
base_timeout = handler.get_test_timeout()
|
||||
command.extend(['--timeout', str(base_timeout)])
|
||||
command.extend(ctest_args_yaml)
|
||||
|
||||
if handler.options.ctest_args:
|
||||
command.extend(handler.options.ctest_args)
|
||||
|
||||
return command
|
||||
|
||||
def run_command(self, cmd, timeout):
|
||||
with subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as proc:
|
||||
try:
|
||||
reader_t = threading.Thread(target=self._output_reader, args=(proc,), daemon=True)
|
||||
reader_t.start()
|
||||
reader_t.join(timeout)
|
||||
if reader_t.is_alive():
|
||||
terminate_process(proc)
|
||||
logger.warning('Timeout has occurred. Can be extended in testspec file. '
|
||||
f'Currently set to {timeout} seconds.')
|
||||
self.instance.reason = 'Ctest timeout'
|
||||
self.status = TwisterStatus.FAIL
|
||||
proc.wait(timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.status = TwisterStatus.FAIL
|
||||
proc.kill()
|
||||
|
||||
if proc.returncode in (ExitCode.INTERRUPTED, ExitCode.USAGE_ERROR, ExitCode.INTERNAL_ERROR):
|
||||
self.status = TwisterStatus.ERROR
|
||||
self.instance.reason = f'Ctest error - return code {proc.returncode}'
|
||||
with open(self.ctest_log_file_path, 'w') as log_file:
|
||||
log_file.write(shlex.join(cmd) + '\n\n')
|
||||
log_file.write('\n'.join(self._output))
|
||||
|
||||
def _output_reader(self, proc):
|
||||
self._output = []
|
||||
while proc.stdout.readable() and proc.poll() is None:
|
||||
line = proc.stdout.readline().decode().strip()
|
||||
if not line:
|
||||
continue
|
||||
self._output.append(line)
|
||||
logger.debug(f'CTEST: {line}')
|
||||
self.parse_record(line)
|
||||
proc.communicate()
|
||||
|
||||
def _update_test_status(self):
|
||||
if self.status == TwisterStatus.NONE:
|
||||
self.instance.testcases = []
|
||||
try:
|
||||
self._parse_report_file(self.report_file)
|
||||
except Exception as e:
|
||||
logger.error(f'Error when parsing file {self.report_file}: {e}')
|
||||
self.status = TwisterStatus.FAIL
|
||||
finally:
|
||||
if not self.instance.testcases:
|
||||
self.instance.init_cases()
|
||||
|
||||
self.instance.status = self.status if self.status != TwisterStatus.NONE else \
|
||||
TwisterStatus.FAIL
|
||||
if self.instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]:
|
||||
self.instance.reason = self.instance.reason or 'Ctest failed'
|
||||
self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
|
||||
|
||||
def _parse_report_file(self, report):
|
||||
suite = junit.JUnitXml.fromfile(report)
|
||||
if suite is None:
|
||||
self.status = TwisterStatus.SKIP
|
||||
self.instance.reason = 'No tests collected'
|
||||
return
|
||||
|
||||
assert isinstance(suite, junit.TestSuite)
|
||||
|
||||
if suite.failures and suite.failures > 0:
|
||||
self.status = TwisterStatus.FAIL
|
||||
self.instance.reason = f"{suite.failures}/{suite.tests} ctest scenario(s) failed"
|
||||
elif suite.errors and suite.errors > 0:
|
||||
self.status = TwisterStatus.ERROR
|
||||
self.instance.reason = 'Error during ctest execution'
|
||||
elif suite.skipped and suite.skipped > 0:
|
||||
self.status = TwisterStatus.SKIP
|
||||
else:
|
||||
self.status = TwisterStatus.PASS
|
||||
self.instance.execution_time = suite.time
|
||||
|
||||
for case in suite:
|
||||
tc = self.instance.add_testcase(f"{self.id}.{case.name}")
|
||||
tc.duration = case.time
|
||||
if any(isinstance(r, junit.Failure) for r in case.result):
|
||||
tc.status = TwisterStatus.FAIL
|
||||
tc.output = case.system_out
|
||||
elif any(isinstance(r, junit.Error) for r in case.result):
|
||||
tc.status = TwisterStatus.ERROR
|
||||
tc.output = case.system_out
|
||||
elif any(isinstance(r, junit.Skipped) for r in case.result):
|
||||
tc.status = TwisterStatus.SKIP
|
||||
else:
|
||||
tc.status = TwisterStatus.PASS
|
||||
|
||||
class HarnessImporter:
|
||||
|
||||
|
|
|
@ -43,7 +43,7 @@ from twisterlib.environment import ZEPHYR_BASE
|
|||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers"))
|
||||
from domains import Domains
|
||||
from twisterlib.environment import TwisterEnv
|
||||
from twisterlib.harness import HarnessImporter, Pytest
|
||||
from twisterlib.harness import Ctest, HarnessImporter, Pytest
|
||||
from twisterlib.log_helper import log_command
|
||||
from twisterlib.platform import Platform
|
||||
from twisterlib.testinstance import TestInstance
|
||||
|
@ -1745,6 +1745,8 @@ class ProjectBuilder(FilterBuilder):
|
|||
#
|
||||
if isinstance(harness, Pytest):
|
||||
harness.pytest_run(instance.handler.get_test_timeout())
|
||||
elif isinstance(harness, Ctest):
|
||||
harness.ctest_run(instance.handler.get_test_timeout())
|
||||
else:
|
||||
instance.handler.handle(harness)
|
||||
|
||||
|
|
|
@ -213,7 +213,7 @@ class TestInstance:
|
|||
def testsuite_runnable(testsuite, fixtures):
|
||||
can_run = False
|
||||
# console harness allows us to run the test and capture data.
|
||||
if testsuite.harness in [ 'console', 'ztest', 'pytest', 'test', 'gtest', 'robot']:
|
||||
if testsuite.harness in ['console', 'ztest', 'pytest', 'test', 'gtest', 'robot', 'ctest']:
|
||||
can_run = True
|
||||
# if we have a fixture that is also being supplied on the
|
||||
# command-line, then we need to run the test, not just build it.
|
||||
|
@ -256,6 +256,8 @@ class TestInstance:
|
|||
handler.ready = True
|
||||
else:
|
||||
handler = Handler(self, "", *common_args)
|
||||
if self.testsuite.harness == "ctest":
|
||||
handler.ready = True
|
||||
|
||||
self.handler = handler
|
||||
|
||||
|
@ -291,6 +293,7 @@ class TestInstance:
|
|||
|
||||
target_ready = bool(self.testsuite.type == "unit" or \
|
||||
self.platform.type == "native" or \
|
||||
self.testsuite.harness == "ctest" or \
|
||||
(simulator and simulator.name in SUPPORTED_SIMS and \
|
||||
simulator.name not in self.testsuite.simulation_exclude) or \
|
||||
device_testing)
|
||||
|
|
|
@ -19,3 +19,6 @@ mypy
|
|||
|
||||
# used for mocking functions in pytest
|
||||
mock>=4.0.1
|
||||
|
||||
# used for JUnit XML parsing in CTest harness
|
||||
junitparser
|
||||
|
|
|
@ -130,6 +130,11 @@ schema;scenario-schema:
|
|||
type: str
|
||||
enum: ["function", "class", "module", "package", "session"]
|
||||
required: false
|
||||
"ctest_args":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"regex":
|
||||
type: seq
|
||||
required: false
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue