Twister: Rework integration with renode-test

Initial integration with renode-test was introduced in commit bdf02ff,
which added support for calling the `renode-test` command from both west
and twister.

This commit removes the custom run_renode_test target used for running
Robot tests with the `west build` command and makes twister call
`renode-test` directly instead.

Signed-off-by: Michał Szprejda <mszprejda@antmicro.com>
This commit is contained in:
Michał Szprejda 2024-03-04 17:47:49 +01:00 committed by Alberto Escolar
commit 7bc9a98a70
13 changed files with 59 additions and 50 deletions

View file

@ -12,4 +12,7 @@ testing:
ignore_tags:
- net
- bluetooth
renode:
uart: sysbus.uart
resc: boards/microchip/m2gl025_miv/support/m2gl025_miv.resc
vendor: microchip

View file

@ -12,5 +12,8 @@ testing:
ignore_tags:
- net
- bluetooth
renode:
uart: sysbus.uart0
resc: boards/renode/riscv32_virtual/support/riscv32_virtual.resc
supported:
- uart

View file

@ -19,4 +19,7 @@ testing:
- bluetooth
- flash
- newlib
renode:
uart: sysbus.uart0
resc: boards/sifive/hifive1/support/hifive1.resc
vendor: sifive

View file

@ -14,6 +14,9 @@ testing:
- flash
- newlib
- crypto
renode:
uart: sysbus.uart0
resc: boards/sifive/hifive_unleashed/support/hifive_unleashed.resc
supported:
- gpio
- spi

View file

@ -11,6 +11,9 @@ testing:
ignore_tags:
- net
- bluetooth
renode:
uart: sysbus.uart0
resc: boards/sifive/hifive_unmatched/support/hifive_unmatched.resc
supported:
- spi
- memc

View file

@ -22,40 +22,7 @@ add_custom_target(run_renode
COMMAND
${RENODE}
${RENODE_FLAGS}
-e '$$bin=@${APPLICATION_BINARY_DIR}/zephyr/${KERNEL_ELF_NAME}\; include @${RENODE_SCRIPT}\; ${RENODE_OVERLAY} s'
WORKING_DIRECTORY ${APPLICATION_BINARY_DIR}
DEPENDS ${logical_target_for_zephyr_elf}
USES_TERMINAL
)
find_program(
RENODE_TEST
renode-test
)
set(RENODE_TEST_FLAGS
--variable ELF:@${APPLICATION_BINARY_DIR}/zephyr/${KERNEL_ELF_NAME}
--variable RESC:@${RENODE_SCRIPT}
--variable UART:${RENODE_UART}
--variable KEYWORDS:${ZEPHYR_BASE}/tests/robot/common.robot
--results-dir ${APPLICATION_BINARY_DIR}
)
add_custom_target(run_renode_test
COMMAND /bin/sh -c "\
if [ -z $$ROBOT_FILES ] \;\
then\
echo ''\;\
echo '--- Error: Robot file path is required to run Robot tests in Renode. To provide the path please set the ROBOT_FILES variable.'\;\
echo '--- To rerun the test with west execute:'\;\
echo '--- ROBOT_FILES=\\<FILES\\> west build -p -b \\<BOARD\\> -s \\<SOURCE_DIR\\> -t run_renode_test'\;\
echo ''\;\
exit 1\;\
fi\;"
COMMAND
${RENODE_TEST}
${RENODE_TEST_FLAGS}
${APPLICATION_SOURCE_DIR}/$$ROBOT_FILES
-e '$$bin=@${PROJECT_BINARY_DIR}/${KERNEL_ELF_NAME}\; include @${RENODE_SCRIPT}\; ${RENODE_OVERLAY} s'
WORKING_DIRECTORY ${APPLICATION_BINARY_DIR}
DEPENDS ${logical_target_for_zephyr_elf}
USES_TERMINAL

View file

@ -1347,12 +1347,6 @@ To execute a Robot test suite with twister, run the following command:
python .\scripts\twister --platform hifive1 --test samples/subsys/shell/shell_module/sample.shell.shell_module.robot
It's also possible to run it by `west` directly, with:
.. code-block:: bash
$ ROBOT_FILES=shell_module.robot west build -p -b hifive1 -s samples/subsys/shell/shell_module -t run_renode_test
Writing Robot tests
===================

View file

@ -21,6 +21,7 @@ import time
from queue import Queue, Empty
from twisterlib.environment import ZEPHYR_BASE, strip_ansi_sequences
from twisterlib.error import TwisterException
from twisterlib.platform import Platform
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers"))
from domains import Domains
@ -232,7 +233,21 @@ class BinaryHandler(Handler):
def _create_command(self, robot_test):
if robot_test:
command = [self.generator_cmd, "run_renode_test"]
keywords = os.path.join(self.options.coverage_basedir, 'tests/robot/common.robot')
elf = os.path.join(self.build_dir, "zephyr/zephyr.elf")
command = [self.generator_cmd]
resc = ""
uart = ""
# os.path.join cannot be used on a Mock object, so we are
# explicitly checking the type
if isinstance(self.instance.platform, Platform):
resc = os.path.join(self.options.coverage_basedir, self.instance.platform.resc)
uart = self.instance.platform.uart
command = ["renode-test",
"--variable", "KEYWORDS:" + keywords,
"--variable", "ELF:@" + elf,
"--variable", "RESC:@" + resc,
"--variable", "UART:" + uart]
elif self.call_make_run:
command = [self.generator_cmd, "run"]
elif self.instance.testsuite.type == "unit":

View file

@ -152,18 +152,17 @@ class Robot(Harness):
tc.status = "passed"
def run_robot_test(self, command, handler):
start_time = time.time()
env = os.environ.copy()
env["ROBOT_FILES"] = self.path
command.append(os.path.join(handler.sourcedir, self.path))
with subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=self.instance.build_dir, env=env) as cmake_proc:
out, _ = cmake_proc.communicate()
stderr=subprocess.STDOUT, cwd=self.instance.build_dir, env=env) as renode_test_proc:
out, _ = renode_test_proc.communicate()
self.instance.execution_time = time.time() - start_time
if cmake_proc.returncode == 0:
if renode_test_proc.returncode == 0:
self.instance.status = "passed"
# all tests in one Robot file are treated as a single test case,
# so its status should be set accordingly to the instance status

View file

@ -46,6 +46,8 @@ class Platform:
self.env = []
self.env_satisfied = True
self.filter_data = dict()
self.uart = ""
self.resc = ""
def load(self, platform_file):
scp = TwisterConfigParser(platform_file, self.platform_schema)
@ -63,6 +65,9 @@ class Platform:
self.only_tags = testing.get("only_tags", [])
self.default = testing.get("default", False)
self.binaries = testing.get("binaries", [])
renode = testing.get("renode", {})
self.uart = renode.get("uart", "")
self.resc = renode.get("resc", "")
# if no flash size is specified by the board, take a default of 512K
self.flash = data.get("flash", 512)
self.supported = set()

View file

@ -100,3 +100,10 @@ mapping:
type: seq
seq:
- type: str
"renode":
type: map
mapping:
"uart":
type: str
"resc":
type: str

View file

@ -18,10 +18,12 @@ from contextlib import nullcontext
from importlib import reload
from serial import SerialException
from subprocess import CalledProcessError, TimeoutExpired
from types import SimpleNamespace
import twisterlib.harness
from conftest import ZEPHYR_BASE
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
from twisterlib.error import TwisterException
from twisterlib.handlers import (
Handler,
@ -413,7 +415,7 @@ TESTDATA_4 = [
['valgrind', '--error-exitcode=2', '--leak-check=full',
f'--suppressions={ZEPHYR_BASE}/scripts/valgrind.supp',
'--log-file=build_dir/valgrind.log', '--track-origins=yes',
'generator', 'run_renode_test']),
'generator']),
(False, True, False, 123, None, ['generator', 'run', '--seed=123']),
(False, False, False, None, ['ex1', 'ex2'], ['build_dir/zephyr/zephyr.exe', 'ex1', 'ex2']),
]
@ -437,11 +439,16 @@ def test_binaryhandler_create_command(
handler.generator_cmd = 'generator'
handler.binary = 'bin'
handler.call_make_run = call_make_run
handler.options = mock.Mock(enable_valgrind=enable_valgrind)
handler.options = SimpleNamespace()
handler.options.enable_valgrind = enable_valgrind
handler.options.coverage_basedir = "coverage_basedir"
handler.seed = seed
handler.extra_test_args = extra_args
handler.build_dir = 'build_dir'
handler.instance.testsuite.sysbuild = False
handler.platform = SimpleNamespace()
handler.platform.resc = "file.resc"
handler.platform.uart = "uart"
command = handler._create_command(robot_test)

View file

@ -166,7 +166,7 @@ TEST_DATA_2 = [("", 0, "passed"), ("Robot test failure: sourcedir for mock_platf
)
def test_robot_run_robot_test(tmp_path, caplog, exp_out, returncode, expected_status):
# Arrange
command = "command"
command = ["command"]
handler = mock.Mock()
handler.sourcedir = "sourcedir"