twister: Fields for Kconfig and DT overlay files in testcase.yaml
This is an implementation of issue #48334 and adds support for specifying additional config and device tree overlays through fields in the testcase.yaml file, which is more readable than having to cram these in to `extra_args`. Consider this example which shows the original and new ways to add config and DT overlays: ``` common: extra_args: "CONF_FILE=a.conf;b.conf DTC_OVERLAY_FILE=w.overlay;x.overlay OVERLAY_CONFIG=e.conf UNRELATED=abc" tests: utilities.base64: extra_conf_files: - "c.conf" - "d.conf" extra_overlay_confs: - "extra_overlay.conf" extra_dtc_overlay_files: - "y.overlay" - "z.overlay" extra_configs: - CONFIG_SAMPLE=y tags: base64 type: unit ``` The new fields are `extra_conf_files`, `extra_overlay_confs, `extra_dtc_overlay_files`. Files specified in these sections are appended to any `CONF_FILE`, `OVERLAY_CONFIG`, or `DTC_OVERLAY_FILE` fields in `extra_args`, causing the following args being passed in to `self.run_cmake` at `runner.py:850`: ``` ['-DUNRELATED=abc', '-DCONF_FILE=a.conf;b.conf;c.conf;d.conf', '-DDTC_OVERLAY_FILE=w.overlay;x.overlay;y.overlay;z.overlay', '-DOVERLAY_CONFIG=e.conf extra_overlay.conf ' '<build_dir>/twister/testsuite_extra.conf'] ``` These fields can be used in the common or scenario-specific YAML sections and will be merged in order of least to most specific: 1. config files extracted from common's extra_args 2. files listed in common's {extra_conf_files or extra_overlay_confs or extra_dtc_overlay_files} 3. config files extracted from test scenario's extra_args 4. files listed in test scenario's {extra_conf_files or extra_overlay_confs or extra_dtc_overlay_files} Specifying these files in extra_args now triggers a deprecation warning, as the direct YAML fields are preferred for readability. They will still function for now but support will be dropped in the future. One testcase.yaml (`zephyr/tests/cmake/overlays/var_expansions/testcase.yaml`) is converted to use the new fields. A follow-up PR will convert the remaining files to the new format. Signed-off-by: Tristan Honscheid <honscheid@google.com>
This commit is contained in:
parent
166efee65e
commit
1eb0452fcb
9 changed files with 268 additions and 43 deletions
|
@ -4,8 +4,33 @@
|
|||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import scl
|
||||
import warnings
|
||||
from twisterlib.error import ConfigurationError
|
||||
|
||||
def extract_fields_from_arg_list(target_fields: set, arg_list: str):
|
||||
"""
|
||||
Given a list of "FIELD=VALUE" args, extract values of args with a
|
||||
given field name and return the remaining args separately.
|
||||
"""
|
||||
extracted_fields = {f : list() for f in target_fields}
|
||||
other_fields = []
|
||||
|
||||
for field in arg_list.split(" "):
|
||||
try:
|
||||
name, val = field.split("=", 1)
|
||||
except ValueError:
|
||||
# Can't parse this. Just pass it through
|
||||
other_fields.append(field)
|
||||
continue
|
||||
|
||||
if name in target_fields:
|
||||
extracted_fields[name].append(val.strip('\'"'))
|
||||
else:
|
||||
# Move to other_fields
|
||||
other_fields.append(field)
|
||||
|
||||
return extracted_fields, " ".join(other_fields)
|
||||
|
||||
class TwisterConfigParser:
|
||||
"""Class to read testsuite yaml files with semantic checking
|
||||
"""
|
||||
|
@ -14,6 +39,9 @@ class TwisterConfigParser:
|
|||
"type": {"type": "str", "default": "integration"},
|
||||
"extra_args": {"type": "list"},
|
||||
"extra_configs": {"type": "list"},
|
||||
"extra_conf_files": {"type": "list", "default": []},
|
||||
"extra_overlay_confs" : {"type": "list", "default": []},
|
||||
"extra_dtc_overlay_files": {"type": "list", "default": []},
|
||||
"build_only": {"type": "bool", "default": False},
|
||||
"build_on_all": {"type": "bool", "default": False},
|
||||
"skip": {"type": "bool", "default": False},
|
||||
|
@ -107,11 +135,29 @@ class TwisterConfigParser:
|
|||
type conversion and default values filled in per valid_keys
|
||||
"""
|
||||
|
||||
# "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
|
||||
# of the extra_args lines
|
||||
extracted_common = {}
|
||||
extracted_testsuite = {}
|
||||
|
||||
d = {}
|
||||
for k, v in self.common.items():
|
||||
d[k] = v
|
||||
if k == "extra_args":
|
||||
# Pull out these fields and leave the rest
|
||||
extracted_common, d[k] = extract_fields_from_arg_list(
|
||||
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"},
|
||||
v.strip()
|
||||
)
|
||||
else:
|
||||
d[k] = v
|
||||
|
||||
for k, v in self.scenarios[name].items():
|
||||
if k == "extra_args":
|
||||
# Pull out these fields and leave the rest
|
||||
extracted_testsuite, v = extract_fields_from_arg_list(
|
||||
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"},
|
||||
v.strip()
|
||||
)
|
||||
if k in d:
|
||||
if isinstance(d[k], str):
|
||||
# By default, we just concatenate string values of keys
|
||||
|
@ -125,6 +171,40 @@ class TwisterConfigParser:
|
|||
else:
|
||||
d[k] = v
|
||||
|
||||
# Compile conf files in to a single list. The order to apply them is:
|
||||
# (1) CONF_FILEs extracted from common['extra_args']
|
||||
# (2) common['extra_conf_files']
|
||||
# (3) CONF_FILES extracted from scenarios[name]['extra_args']
|
||||
# (4) scenarios[name]['extra_conf_files']
|
||||
d["extra_conf_files"] = \
|
||||
extracted_common.get("CONF_FILE", []) + \
|
||||
self.common.get("extra_conf_files", []) + \
|
||||
extracted_testsuite.get("CONF_FILE", []) + \
|
||||
self.scenarios[name].get("extra_conf_files", [])
|
||||
|
||||
# Repeat the above for overlay confs and DTC overlay files
|
||||
d["extra_overlay_confs"] = \
|
||||
extracted_common.get("OVERLAY_CONFIG", []) + \
|
||||
self.common.get("extra_overlay_confs", []) + \
|
||||
extracted_testsuite.get("OVERLAY_CONFIG", []) + \
|
||||
self.scenarios[name].get("extra_overlay_confs", [])
|
||||
|
||||
d["extra_dtc_overlay_files"] = \
|
||||
extracted_common.get("DTC_OVERLAY_FILE", []) + \
|
||||
self.common.get("extra_dtc_overlay_files", []) + \
|
||||
extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
|
||||
self.scenarios[name].get("extra_dtc_overlay_files", [])
|
||||
|
||||
if any({len(x) > 0 for x in extracted_common.values()}) or \
|
||||
any({len(x) > 0 for x in extracted_testsuite.values()}):
|
||||
warnings.warn(
|
||||
"Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
|
||||
"in extra_args. This feature is deprecated and will soon "
|
||||
"result in an error. Use extra_conf_files, extra_overlay_confs "
|
||||
"or extra_dtc_overlay_files YAML fields instead",
|
||||
DeprecationWarning
|
||||
)
|
||||
|
||||
for k, kinfo in self.testsuite_valid_keys.items():
|
||||
if k not in d:
|
||||
if "required" in kinfo:
|
||||
|
|
|
@ -814,43 +814,49 @@ class ProjectBuilder(FilterBuilder):
|
|||
)
|
||||
sys.stdout.flush()
|
||||
|
||||
def cmake(self):
|
||||
@staticmethod
|
||||
def cmake_assemble_args(args, handler, extra_conf_files, extra_overlay_confs,
|
||||
extra_dtc_overlay_files, cmake_extra_args,
|
||||
build_dir):
|
||||
if handler.ready:
|
||||
args.extend(handler.args)
|
||||
|
||||
instance = self.instance
|
||||
args = self.testsuite.extra_args[:]
|
||||
if extra_conf_files:
|
||||
args.append(f"CONF_FILE=\"{';'.join(extra_conf_files)}\"")
|
||||
|
||||
if instance.handler.ready:
|
||||
args += instance.handler.args
|
||||
if extra_dtc_overlay_files:
|
||||
args.append(f"DTC_OVERLAY_FILE=\"{';'.join(extra_dtc_overlay_files)}\"")
|
||||
|
||||
# merge overlay files into one variable
|
||||
# overlays with prefixes won't be merged but pass to cmake as they are
|
||||
def extract_overlays(args):
|
||||
re_overlay = re.compile(r'^\s*OVERLAY_CONFIG=(.*)')
|
||||
other_args = []
|
||||
overlays = []
|
||||
for arg in args:
|
||||
match = re_overlay.search(arg)
|
||||
if match:
|
||||
overlays.append(match.group(1).strip('\'"'))
|
||||
else:
|
||||
other_args.append(arg)
|
||||
overlays = extra_overlay_confs.copy()
|
||||
|
||||
args[:] = other_args
|
||||
return overlays
|
||||
|
||||
overlays = extract_overlays(args)
|
||||
|
||||
if os.path.exists(os.path.join(instance.build_dir,
|
||||
"twister", "testsuite_extra.conf")):
|
||||
overlays.append(os.path.join(instance.build_dir,
|
||||
"twister", "testsuite_extra.conf"))
|
||||
additional_overlay_path = os.path.join(
|
||||
build_dir, "twister", "testsuite_extra.conf"
|
||||
)
|
||||
if os.path.exists(additional_overlay_path):
|
||||
overlays.append(additional_overlay_path)
|
||||
|
||||
if overlays:
|
||||
args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays)))
|
||||
|
||||
args_expanded = ["-D{}".format(a.replace('"', '\"')) for a in self.options.extra_args]
|
||||
args_expanded = args_expanded + ["-D{}".format(a.replace('"', '')) for a in args]
|
||||
res = self.run_cmake(args_expanded)
|
||||
# Build the final argument list
|
||||
args_expanded = ["-D{}".format(a.replace('"', '\"')) for a in cmake_extra_args]
|
||||
args_expanded.extend(["-D{}".format(a.replace('"', '')) for a in args])
|
||||
|
||||
return args_expanded
|
||||
|
||||
def cmake(self):
|
||||
args = self.cmake_assemble_args(
|
||||
self.testsuite.extra_args.copy(), # extra_args from YAML
|
||||
self.instance.handler,
|
||||
self.testsuite.extra_conf_files,
|
||||
self.testsuite.extra_overlay_confs,
|
||||
self.testsuite.extra_dtc_overlay_files,
|
||||
self.options.extra_args, # CMake extra args
|
||||
self.instance.build_dir,
|
||||
)
|
||||
|
||||
res = self.run_cmake(args)
|
||||
return res
|
||||
|
||||
def build(self):
|
||||
|
|
|
@ -448,7 +448,7 @@ class TestPlan:
|
|||
self.testsuites[suite.name] = suite
|
||||
|
||||
except Exception as e:
|
||||
logger.error("%s: can't load (skipping): %s" % (suite_path, e))
|
||||
logger.error(f"{suite_path}: can't load (skipping): {e!r}")
|
||||
self.load_errors += 1
|
||||
return len(self.testsuites)
|
||||
|
||||
|
|
|
@ -32,6 +32,21 @@ mapping:
|
|||
"extra_args":
|
||||
type: str
|
||||
required: false
|
||||
"extra_conf_files":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_overlay_confs":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_dtc_overlay_files":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_sections":
|
||||
type: str
|
||||
required: false
|
||||
|
@ -201,6 +216,21 @@ mapping:
|
|||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_conf_files":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_overlay_confs":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_dtc_overlay_files":
|
||||
type: seq
|
||||
required: false
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_sections":
|
||||
type: str
|
||||
required: false
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
common:
|
||||
extra_args: >
|
||||
CONF_FILE=conf1;conf2 DTC_OVERLAY_FILE=overlay1;overlay2
|
||||
OVERLAY_CONFIG=oc1.conf UNRELATED1=abc
|
||||
extra_conf_files:
|
||||
- "conf3"
|
||||
- "conf4"
|
||||
extra_overlay_confs:
|
||||
- "oc2.conf"
|
||||
extra_dtc_overlay_files:
|
||||
- "overlay3"
|
||||
- "overlay4"
|
||||
tests:
|
||||
test_config.main:
|
||||
extra_args: >
|
||||
CONF_FILE=conf5;conf6 DTC_OVERLAY_FILE=overlay5;overlay6
|
||||
OVERLAY_CONFIG=oc3.conf UNRELATED2=xyz
|
||||
extra_conf_files:
|
||||
- "conf7"
|
||||
- "conf8"
|
||||
extra_overlay_confs:
|
||||
- "oc4.conf"
|
||||
extra_dtc_overlay_files:
|
||||
- "overlay7"
|
||||
- "overlay8"
|
||||
extra_configs:
|
||||
- CONFIG_FOO=y
|
46
scripts/tests/twister/test_runner.py
Normal file
46
scripts/tests/twister/test_runner.py
Normal file
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2023 Google LLC
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Tests for runner.py classes
|
||||
"""
|
||||
|
||||
import mock
|
||||
import os
|
||||
import sys
|
||||
|
||||
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
||||
|
||||
from twisterlib.runner import ProjectBuilder
|
||||
|
||||
@mock.patch("os.path.exists")
|
||||
def test_projectbuilder_cmake_assemble_args(m):
|
||||
# Causes the additional_overlay_path to be appended
|
||||
m.return_value = True
|
||||
|
||||
class MockHandler:
|
||||
pass
|
||||
|
||||
handler = MockHandler()
|
||||
handler.args = ["handler_arg1", "handler_arg2"]
|
||||
handler.ready = True
|
||||
|
||||
assert(ProjectBuilder.cmake_assemble_args(
|
||||
["basearg1"],
|
||||
handler,
|
||||
["a.conf;b.conf", "c.conf"],
|
||||
["extra_overlay.conf"],
|
||||
["x.overlay;y.overlay", "z.overlay"],
|
||||
["cmake1=foo", "cmake2=bar"],
|
||||
"/builddir/",
|
||||
) == [
|
||||
"-Dcmake1=foo", "-Dcmake2=bar",
|
||||
"-Dbasearg1",
|
||||
"-Dhandler_arg1", "-Dhandler_arg2",
|
||||
"-DCONF_FILE=a.conf;b.conf;c.conf",
|
||||
"-DDTC_OVERLAY_FILE=x.overlay;y.overlay;z.overlay",
|
||||
"-DOVERLAY_CONFIG=extra_overlay.conf "
|
||||
"/builddir/twister/testsuite_extra.conf",
|
||||
])
|
|
@ -35,7 +35,8 @@ def test_testplan_add_testsuites(class_testplan):
|
|||
'test_a.check_1',
|
||||
'test_a.check_2',
|
||||
'test_d.check_1',
|
||||
'sample_test.app']
|
||||
'sample_test.app',
|
||||
'test_config.main']
|
||||
testsuite_list = []
|
||||
for key in sorted(class_testplan.testsuites.keys()):
|
||||
testsuite_list.append(os.path.basename(os.path.normpath(key)))
|
||||
|
@ -75,11 +76,8 @@ def test_get_all_testsuites(class_env, all_testsuites_dict):
|
|||
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
||||
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
||||
'test_c.check_2', 'test_d.check_1.unit_1a',
|
||||
'test_d.check_1.unit_1b']
|
||||
tests = plan.get_all_tests()
|
||||
result = [c for c in tests]
|
||||
assert len(plan.get_all_tests()) == len(expected_tests)
|
||||
assert sorted(result) == sorted(expected_tests)
|
||||
'test_d.check_1.unit_1b', 'test_config.main']
|
||||
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
|
||||
|
||||
def test_get_platforms(class_env, platforms_list):
|
||||
""" Testing get_platforms function of TestPlan class in Twister """
|
||||
|
@ -279,7 +277,8 @@ QUARANTINE_PLATFORM = {
|
|||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
||||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
||||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
||||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3'
|
||||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
|
||||
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
|
||||
}
|
||||
|
||||
QUARANTINE_MULTIFILES = {
|
||||
|
@ -330,6 +329,7 @@ def test_quarantine(class_testplan, platforms_list, test_data,
|
|||
assert instance.status == 'filtered'
|
||||
assert instance.reason == "Not under quarantine"
|
||||
else:
|
||||
print(testname)
|
||||
if testname in expected_val:
|
||||
assert instance.status == 'filtered'
|
||||
assert instance.reason == "Quarantine: " + expected_val[testname]
|
||||
|
|
|
@ -10,6 +10,8 @@ import os
|
|||
import sys
|
||||
import pytest
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
||||
|
||||
|
@ -45,3 +47,33 @@ def test_incorrect_schema(filename, schema, test_data):
|
|||
with pytest.raises(Exception) as exception:
|
||||
scl.yaml_load_verify(filename, schema)
|
||||
assert str(exception.value) == "Schema validation failed"
|
||||
|
||||
def test_testsuite_config_files():
|
||||
""" Test to validate conf and overlay files are extracted properly """
|
||||
filename = Path(ZEPHYR_BASE) / "scripts/tests/twister/test_data/testsuites/tests/test_config/test_data.yaml"
|
||||
schema = scl.yaml_load(Path(ZEPHYR_BASE) / "scripts/schemas/twister/testsuite-schema.yaml")
|
||||
data = TwisterConfigParser(filename, schema)
|
||||
data.load()
|
||||
|
||||
# Load and validate the specific scenario from testcases.yaml
|
||||
scenario = data.get_scenario("test_config.main")
|
||||
assert scenario
|
||||
|
||||
# CONF_FILE, DTC_OVERLAY_FILE, OVERLAY_CONFIG fields should be stripped out
|
||||
# of extra_args. Other fields should remain untouched.
|
||||
assert scenario["extra_args"] == ["UNRELATED1=abc", "UNRELATED2=xyz"]
|
||||
|
||||
# Check that all conf files have been assembled in the correct order
|
||||
assert ";".join(scenario["extra_conf_files"]) == \
|
||||
"conf1;conf2;conf3;conf4;conf5;conf6;conf7;conf8"
|
||||
|
||||
# Check that all DTC overlay files have been assembled in the correct order
|
||||
assert ";".join(scenario["extra_dtc_overlay_files"]) == \
|
||||
"overlay1;overlay2;overlay3;overlay4;overlay5;overlay6;overlay7;overlay8"
|
||||
|
||||
# Check that all overlay conf files have been assembled in the correct order
|
||||
assert scenario["extra_overlay_confs"] == \
|
||||
["oc1.conf", "oc2.conf", "oc3.conf", "oc4.conf"]
|
||||
|
||||
# Check extra kconfig statements, too
|
||||
assert scenario["extra_configs"] == ["CONFIG_FOO=y"]
|
||||
|
|
|
@ -4,11 +4,15 @@ common:
|
|||
platform_allow: native_posix
|
||||
tests:
|
||||
cmake.overlays.var_expansions.CONF_FILE:
|
||||
extra_args:
|
||||
CONF_FILE=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf;prj.conf
|
||||
extra_conf_files:
|
||||
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf
|
||||
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
|
||||
- prj.conf
|
||||
cmake.overlays.var_expansions.OVERLAY_CONFIG:
|
||||
extra_args:
|
||||
OVERLAY_CONFIG=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
|
||||
extra_overlay_confs:
|
||||
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf
|
||||
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
|
||||
cmake.overlays.var_expansions.DTC_OVERLAY_FILE:
|
||||
extra_args:
|
||||
DTC_OVERLAY_FILE=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-board.overlay;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-board.overlay
|
||||
extra_dtc_overlay_files:
|
||||
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-board.overlay
|
||||
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-board.overlay
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue