Twister: Rework integration with renode-test

Initial integration with renode-test was introduced in commit bdf02ff, which added support for calling the `renode-test` command from both west and twister. This commit removes the custom run_renode_test target used for running Robot tests with the `west build` command and makes twister call `renode-test` directly instead. Signed-off-by: Michał Szprejda <mszprejda@antmicro.com>
2024-03-04 17:47:49 +01:00 · 2024-03-04 17:47:49 +01:00 · 7bc9a98a70
commit 7bc9a98a70
parent 0b654c1502
13 changed files with 59 additions and 50 deletions
--- a/boards/microchip/m2gl025_miv/m2gl025_miv.yaml
+++ b/boards/microchip/m2gl025_miv/m2gl025_miv.yaml
@ -12,4 +12,7 @@ testing:
  ignore_tags:
    - net
    - bluetooth
+  renode:
+    uart: sysbus.uart
+    resc: boards/microchip/m2gl025_miv/support/m2gl025_miv.resc
 vendor: microchip
--- a/boards/renode/riscv32_virtual/riscv32_virtual.yaml
+++ b/boards/renode/riscv32_virtual/riscv32_virtual.yaml
@ -12,5 +12,8 @@ testing:
  ignore_tags:
    - net
    - bluetooth
+  renode:
+    uart: sysbus.uart0
+    resc: boards/renode/riscv32_virtual/support/riscv32_virtual.resc
 supported:
  - uart
--- a/boards/sifive/hifive1/hifive1.yaml
+++ b/boards/sifive/hifive1/hifive1.yaml
@ -19,4 +19,7 @@ testing:
    - bluetooth
    - flash
    - newlib
+  renode:
+    uart: sysbus.uart0
+    resc: boards/sifive/hifive1/support/hifive1.resc
 vendor: sifive
--- a/boards/sifive/hifive_unleashed/hifive_unleashed.yaml
+++ b/boards/sifive/hifive_unleashed/hifive_unleashed.yaml
@ -14,6 +14,9 @@ testing:
    - flash
    - newlib
    - crypto
+  renode:
+    uart: sysbus.uart0
+    resc: boards/sifive/hifive_unleashed/support/hifive_unleashed.resc
 supported:
  - gpio
  - spi
--- a/boards/sifive/hifive_unmatched/hifive_unmatched.yaml
+++ b/boards/sifive/hifive_unmatched/hifive_unmatched.yaml
@ -11,6 +11,9 @@ testing:
  ignore_tags:
    - net
    - bluetooth
+  renode:
+    uart: sysbus.uart0
+    resc: boards/sifive/hifive_unmatched/support/hifive_unmatched.resc
 supported:
  - spi
  - memc
--- a/cmake/emu/renode.cmake
+++ b/cmake/emu/renode.cmake
@ -22,40 +22,7 @@ add_custom_target(run_renode
  COMMAND
  ${RENODE}
  ${RENODE_FLAGS}
-  -e '$$bin=@${APPLICATION_BINARY_DIR}/zephyr/${KERNEL_ELF_NAME}\; include @${RENODE_SCRIPT}\; ${RENODE_OVERLAY} s'
-  WORKING_DIRECTORY ${APPLICATION_BINARY_DIR}
-  DEPENDS ${logical_target_for_zephyr_elf}
-  USES_TERMINAL
-  )
-
-find_program(
-  RENODE_TEST
-  renode-test
-  )
-
-set(RENODE_TEST_FLAGS
-  --variable ELF:@${APPLICATION_BINARY_DIR}/zephyr/${KERNEL_ELF_NAME}
-  --variable RESC:@${RENODE_SCRIPT}
-  --variable UART:${RENODE_UART}
-  --variable KEYWORDS:${ZEPHYR_BASE}/tests/robot/common.robot
-  --results-dir ${APPLICATION_BINARY_DIR}
-  )
-
-add_custom_target(run_renode_test
-  COMMAND /bin/sh -c "\
-    if [ -z $$ROBOT_FILES ] \;\
-    then\
-        echo ''\;\
-        echo '--- Error: Robot file path is required to run Robot tests in Renode. To provide the path please set the ROBOT_FILES variable.'\;\
-        echo '--- To rerun the test with west execute:'\;\
-        echo '--- ROBOT_FILES=\\<FILES\\> west build -p -b \\<BOARD\\> -s \\<SOURCE_DIR\\> -t run_renode_test'\;\
-        echo ''\;\
-        exit 1\;\
-    fi\;"
-  COMMAND
-  ${RENODE_TEST}
-  ${RENODE_TEST_FLAGS}
-  ${APPLICATION_SOURCE_DIR}/$$ROBOT_FILES
+  -e '$$bin=@${PROJECT_BINARY_DIR}/${KERNEL_ELF_NAME}\; include @${RENODE_SCRIPT}\; ${RENODE_OVERLAY} s'
  WORKING_DIRECTORY ${APPLICATION_BINARY_DIR}
  DEPENDS ${logical_target_for_zephyr_elf}
  USES_TERMINAL
--- a/doc/develop/test/twister.rst
+++ b/doc/develop/test/twister.rst
@ -1347,12 +1347,6 @@ To execute a Robot test suite with twister, run the following command:

         python .\scripts\twister --platform hifive1 --test samples/subsys/shell/shell_module/sample.shell.shell_module.robot

-It's also possible to run it by `west` directly, with:
-
-.. code-block:: bash
-
-   $ ROBOT_FILES=shell_module.robot west build -p -b hifive1 -s samples/subsys/shell/shell_module -t run_renode_test
-
 Writing Robot tests
 ===================

--- a/scripts/pylib/twister/twisterlib/handlers.py
+++ b/scripts/pylib/twister/twisterlib/handlers.py
@ -21,6 +21,7 @@ import time
 from queue import Queue, Empty
 from twisterlib.environment import ZEPHYR_BASE, strip_ansi_sequences
 from twisterlib.error import TwisterException
+from twisterlib.platform import Platform
 sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers"))
 from domains import Domains

@ -232,7 +233,21 @@ class BinaryHandler(Handler):

    def _create_command(self, robot_test):
        if robot_test:
-            command = [self.generator_cmd, "run_renode_test"]
+            keywords = os.path.join(self.options.coverage_basedir, 'tests/robot/common.robot')
+            elf = os.path.join(self.build_dir, "zephyr/zephyr.elf")
+            command = [self.generator_cmd]
+            resc = ""
+            uart = ""
+            # os.path.join cannot be used on a Mock object, so we are
+            # explicitly checking the type
+            if isinstance(self.instance.platform, Platform):
+                resc = os.path.join(self.options.coverage_basedir, self.instance.platform.resc)
+                uart = self.instance.platform.uart
+                command = ["renode-test",
+                            "--variable", "KEYWORDS:" + keywords,
+                            "--variable", "ELF:@" + elf,
+                            "--variable", "RESC:@" + resc,
+                            "--variable", "UART:" + uart]
        elif self.call_make_run:
            command = [self.generator_cmd, "run"]
        elif self.instance.testsuite.type == "unit":
--- a/scripts/pylib/twister/twisterlib/harness.py
+++ b/scripts/pylib/twister/twisterlib/harness.py
@ -152,18 +152,17 @@ class Robot(Harness):
        tc.status = "passed"

    def run_robot_test(self, command, handler):
-
        start_time = time.time()
        env = os.environ.copy()
-        env["ROBOT_FILES"] = self.path

+        command.append(os.path.join(handler.sourcedir, self.path))
        with subprocess.Popen(command, stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT, cwd=self.instance.build_dir, env=env) as cmake_proc:
-            out, _ = cmake_proc.communicate()
+                stderr=subprocess.STDOUT, cwd=self.instance.build_dir, env=env) as renode_test_proc:
+            out, _ = renode_test_proc.communicate()

            self.instance.execution_time = time.time() - start_time

-            if cmake_proc.returncode == 0:
+            if renode_test_proc.returncode == 0:
                self.instance.status = "passed"
                # all tests in one Robot file are treated as a single test case,
                # so its status should be set accordingly to the instance status
--- a/scripts/pylib/twister/twisterlib/platform.py
+++ b/scripts/pylib/twister/twisterlib/platform.py
@ -46,6 +46,8 @@ class Platform:
        self.env = []
        self.env_satisfied = True
        self.filter_data = dict()
+        self.uart = ""
+        self.resc = ""

    def load(self, platform_file):
        scp = TwisterConfigParser(platform_file, self.platform_schema)
@ -63,6 +65,9 @@ class Platform:
        self.only_tags = testing.get("only_tags", [])
        self.default = testing.get("default", False)
        self.binaries = testing.get("binaries", [])
+        renode = testing.get("renode", {})
+        self.uart = renode.get("uart", "")
+        self.resc = renode.get("resc", "")
        # if no flash size is specified by the board, take a default of 512K
        self.flash = data.get("flash", 512)
        self.supported = set()
--- a/scripts/schemas/twister/platform-schema.yaml
+++ b/scripts/schemas/twister/platform-schema.yaml
@ -100,3 +100,10 @@ mapping:
        type: seq
        seq:
          - type: str
+      "renode":
+        type: map
+        mapping:
+          "uart":
+            type: str
+          "resc":
+            type: str
--- a/scripts/tests/twister/test_handlers.py
+++ b/scripts/tests/twister/test_handlers.py
@ -18,10 +18,12 @@ from contextlib import nullcontext
 from importlib import reload
 from serial import SerialException
 from subprocess import CalledProcessError, TimeoutExpired
+from types import SimpleNamespace

 import twisterlib.harness

-from conftest import ZEPHYR_BASE
+ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
+
 from twisterlib.error import TwisterException
 from twisterlib.handlers import (
    Handler,
@ -413,7 +415,7 @@ TESTDATA_4 = [
     ['valgrind', '--error-exitcode=2', '--leak-check=full',
      f'--suppressions={ZEPHYR_BASE}/scripts/valgrind.supp',
      '--log-file=build_dir/valgrind.log', '--track-origins=yes',
-      'generator', 'run_renode_test']),
+      'generator']),
    (False, True, False, 123, None, ['generator', 'run', '--seed=123']),
    (False, False, False, None, ['ex1', 'ex2'], ['build_dir/zephyr/zephyr.exe', 'ex1', 'ex2']),
 ]
@ -437,11 +439,16 @@ def test_binaryhandler_create_command(
    handler.generator_cmd = 'generator'
    handler.binary = 'bin'
    handler.call_make_run = call_make_run
-    handler.options = mock.Mock(enable_valgrind=enable_valgrind)
+    handler.options = SimpleNamespace()
+    handler.options.enable_valgrind = enable_valgrind
+    handler.options.coverage_basedir = "coverage_basedir"
    handler.seed = seed
    handler.extra_test_args = extra_args
    handler.build_dir = 'build_dir'
    handler.instance.testsuite.sysbuild = False
+    handler.platform = SimpleNamespace()
+    handler.platform.resc = "file.resc"
+    handler.platform.uart = "uart"

    command = handler._create_command(robot_test)

--- a/scripts/tests/twister/test_harness.py
+++ b/scripts/tests/twister/test_harness.py
@ -166,7 +166,7 @@ TEST_DATA_2 = [("", 0, "passed"), ("Robot test failure: sourcedir for mock_platf
 )
 def test_robot_run_robot_test(tmp_path, caplog, exp_out, returncode, expected_status):
    # Arrange
-    command = "command"
+    command = ["command"]

    handler = mock.Mock()
    handler.sourcedir = "sourcedir"