michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: Fields for Kconfig and DT overlay files in testcase.yaml

Browse source 
This is an implementation of issue #48334 and adds support for
specifying additional config and device tree overlays through fields in
the testcase.yaml file, which is more readable than having to cram these
in to `extra_args`.

Consider this example which shows the original and new ways to add
config and DT overlays:

```
common:
  extra_args: "CONF_FILE=a.conf;b.conf
DTC_OVERLAY_FILE=w.overlay;x.overlay OVERLAY_CONFIG=e.conf
UNRELATED=abc"
tests:
  utilities.base64:
    extra_conf_files:
      - "c.conf"
      - "d.conf"
    extra_overlay_confs:
      - "extra_overlay.conf"
    extra_dtc_overlay_files:
      - "y.overlay"
      - "z.overlay"
    extra_configs:
      - CONFIG_SAMPLE=y
    tags: base64
    type: unit
```

The new fields are `extra_conf_files`, `extra_overlay_confs,
`extra_dtc_overlay_files`. Files specified in these sections are appended
to any `CONF_FILE`, `OVERLAY_CONFIG`, or `DTC_OVERLAY_FILE` fields in
`extra_args`, causing the following args being passed in to
`self.run_cmake` at `runner.py:850`:

```
['-DUNRELATED=abc',
 '-DCONF_FILE=a.conf;b.conf;c.conf;d.conf',
 '-DDTC_OVERLAY_FILE=w.overlay;x.overlay;y.overlay;z.overlay',
 '-DOVERLAY_CONFIG=e.conf extra_overlay.conf '
 '<build_dir>/twister/testsuite_extra.conf']
```

These fields can be used in the common or scenario-specific YAML
sections and will be merged in order of least to most specific:

 1. config files extracted from common's extra_args
 2. files listed in common's {extra_conf_files or extra_overlay_confs
    or extra_dtc_overlay_files}
 3. config files extracted from test scenario's extra_args
 4. files listed in test scenario's {extra_conf_files or
    extra_overlay_confs or extra_dtc_overlay_files}

Specifying these files in extra_args now triggers a deprecation warning,
as the direct YAML fields are preferred for readability. They will still
function for now but support will be dropped in the future.  One
testcase.yaml
(`zephyr/tests/cmake/overlays/var_expansions/testcase.yaml`) is
converted to use the new fields. A follow-up PR will convert the
remaining files to the new format.

Signed-off-by: Tristan Honscheid <honscheid@google.com>
This commit is contained in:
Tristan Honscheid 2022-08-09 16:51:17 -06:00 • committed by Anas Nashif
commit 1eb0452fcb
9 changed files with 268 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

82
scripts/pylib/twister/twisterlib/config_parser.py
View file

@ -4,8 +4,33 @@
# SPDX-License-Identifier: Apache-2.0
import scl
import warnings
from twisterlib.error import ConfigurationError
def extract_fields_from_arg_list(target_fields: set, arg_list: str):
"""
Given a list of "FIELD=VALUE" args, extract values of args with a
given field name and return the remaining args separately.
"""
extracted_fields = {f : list() for f in target_fields}
other_fields = []
for field in arg_list.split(" "):
try:
name, val = field.split("=", 1)
except ValueError:
# Can't parse this. Just pass it through
other_fields.append(field)
continue
if name in target_fields:
extracted_fields[name].append(val.strip('\'"'))
else:
# Move to other_fields
other_fields.append(field)
return extracted_fields, " ".join(other_fields)
class TwisterConfigParser:
"""Class to read testsuite yaml files with semantic checking
"""
@ -14,6 +39,9 @@ class TwisterConfigParser:
"type": {"type": "str", "default": "integration"},
"extra_args": {"type": "list"},
"extra_configs": {"type": "list"},
"extra_conf_files": {"type": "list", "default": []},
"extra_overlay_confs" : {"type": "list", "default": []},
"extra_dtc_overlay_files": {"type": "list", "default": []},
"build_only": {"type": "bool", "default": False},
"build_on_all": {"type": "bool", "default": False},
"skip": {"type": "bool", "default": False},
@ -107,11 +135,29 @@ class TwisterConfigParser:
type conversion and default values filled in per valid_keys
"""
# "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
# of the extra_args lines
extracted_common = {}
extracted_testsuite = {}
d = {}
for k, v in self.common.items():
d[k] = v
if k == "extra_args":
# Pull out these fields and leave the rest
extracted_common, d[k] = extract_fields_from_arg_list(
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"},
v.strip()
)
else:
d[k] = v
for k, v in self.scenarios[name].items():
if k == "extra_args":
# Pull out these fields and leave the rest
extracted_testsuite, v = extract_fields_from_arg_list(
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"},
v.strip()
)
if k in d:
if isinstance(d[k], str):
# By default, we just concatenate string values of keys
@ -125,6 +171,40 @@ class TwisterConfigParser:
else:
d[k] = v
# Compile conf files in to a single list. The order to apply them is:
# (1) CONF_FILEs extracted from common['extra_args']
# (2) common['extra_conf_files']
# (3) CONF_FILES extracted from scenarios[name]['extra_args']
# (4) scenarios[name]['extra_conf_files']
d["extra_conf_files"] = \
extracted_common.get("CONF_FILE", []) + \
self.common.get("extra_conf_files", []) + \
extracted_testsuite.get("CONF_FILE", []) + \
self.scenarios[name].get("extra_conf_files", [])
# Repeat the above for overlay confs and DTC overlay files
d["extra_overlay_confs"] = \
extracted_common.get("OVERLAY_CONFIG", []) + \
self.common.get("extra_overlay_confs", []) + \
extracted_testsuite.get("OVERLAY_CONFIG", []) + \
self.scenarios[name].get("extra_overlay_confs", [])
d["extra_dtc_overlay_files"] = \
extracted_common.get("DTC_OVERLAY_FILE", []) + \
self.common.get("extra_dtc_overlay_files", []) + \
extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
self.scenarios[name].get("extra_dtc_overlay_files", [])
if any({len(x) > 0 for x in extracted_common.values()}) or \
any({len(x) > 0 for x in extracted_testsuite.values()}):
warnings.warn(
"Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
"in extra_args. This feature is deprecated and will soon "
"result in an error. Use extra_conf_files, extra_overlay_confs "
"or extra_dtc_overlay_files YAML fields instead",
DeprecationWarning
)
for k, kinfo in self.testsuite_valid_keys.items():
if k not in d:
if "required" in kinfo:

62
scripts/pylib/twister/twisterlib/runner.py
View file

@ -814,43 +814,49 @@ class ProjectBuilder(FilterBuilder):
)
sys.stdout.flush()
def cmake(self):
@staticmethod
def cmake_assemble_args(args, handler, extra_conf_files, extra_overlay_confs,
extra_dtc_overlay_files, cmake_extra_args,
build_dir):
if handler.ready:
args.extend(handler.args)
instance = self.instance
args = self.testsuite.extra_args[:]
if extra_conf_files:
args.append(f"CONF_FILE=\"{';'.join(extra_conf_files)}\"")
if instance.handler.ready:
args += instance.handler.args
if extra_dtc_overlay_files:
args.append(f"DTC_OVERLAY_FILE=\"{';'.join(extra_dtc_overlay_files)}\"")
# merge overlay files into one variable
# overlays with prefixes won't be merged but pass to cmake as they are
def extract_overlays(args):
re_overlay = re.compile(r'^\s*OVERLAY_CONFIG=(.*)')
other_args = []
overlays = []
for arg in args:
match = re_overlay.search(arg)
if match:
overlays.append(match.group(1).strip('\'"'))
else:
other_args.append(arg)
overlays = extra_overlay_confs.copy()
args[:] = other_args
return overlays
overlays = extract_overlays(args)
if os.path.exists(os.path.join(instance.build_dir,
"twister", "testsuite_extra.conf")):
overlays.append(os.path.join(instance.build_dir,
"twister", "testsuite_extra.conf"))
additional_overlay_path = os.path.join(
build_dir, "twister", "testsuite_extra.conf"
)
if os.path.exists(additional_overlay_path):
overlays.append(additional_overlay_path)
if overlays:
args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays)))
args_expanded = ["-D{}".format(a.replace('"', '\"')) for a in self.options.extra_args]
args_expanded = args_expanded + ["-D{}".format(a.replace('"', '')) for a in args]
res = self.run_cmake(args_expanded)
# Build the final argument list
args_expanded = ["-D{}".format(a.replace('"', '\"')) for a in cmake_extra_args]
args_expanded.extend(["-D{}".format(a.replace('"', '')) for a in args])
return args_expanded
def cmake(self):
args = self.cmake_assemble_args(
self.testsuite.extra_args.copy(), # extra_args from YAML
self.instance.handler,
self.testsuite.extra_conf_files,
self.testsuite.extra_overlay_confs,
self.testsuite.extra_dtc_overlay_files,
self.options.extra_args, # CMake extra args
self.instance.build_dir,
)
res = self.run_cmake(args)
return res
def build(self):

2
scripts/pylib/twister/twisterlib/testplan.py
View file

@ -448,7 +448,7 @@ class TestPlan:
self.testsuites[suite.name] = suite
except Exception as e:
logger.error("%s: can't load (skipping): %s" % (suite_path, e))
logger.error(f"{suite_path}: can't load (skipping): {e!r}")
self.load_errors += 1
return len(self.testsuites)

30
scripts/schemas/twister/testsuite-schema.yaml
View file

@ -32,6 +32,21 @@ mapping:
"extra_args":
type: str
required: false
"extra_conf_files":
type: seq
required: false
sequence:
- type: str
"extra_overlay_confs":
type: seq
required: false
sequence:
- type: str
"extra_dtc_overlay_files":
type: seq
required: false
sequence:
- type: str
"extra_sections":
type: str
required: false
@ -201,6 +216,21 @@ mapping:
required: false
sequence:
- type: str
"extra_conf_files":
type: seq
required: false
sequence:
- type: str
"extra_overlay_confs":
type: seq
required: false
sequence:
- type: str
"extra_dtc_overlay_files":
type: seq
required: false
sequence:
- type: str
"extra_sections":
type: str
required: false

27
scripts/tests/twister/test_data/testsuites/tests/test_config/test_data.yaml Normal file
View file

@ -0,0 +1,27 @@
common:
extra_args: >
CONF_FILE=conf1;conf2 DTC_OVERLAY_FILE=overlay1;overlay2
OVERLAY_CONFIG=oc1.conf UNRELATED1=abc
extra_conf_files:
- "conf3"
- "conf4"
extra_overlay_confs:
- "oc2.conf"
extra_dtc_overlay_files:
- "overlay3"
- "overlay4"
tests:
test_config.main:
extra_args: >
CONF_FILE=conf5;conf6 DTC_OVERLAY_FILE=overlay5;overlay6
OVERLAY_CONFIG=oc3.conf UNRELATED2=xyz
extra_conf_files:
- "conf7"
- "conf8"
extra_overlay_confs:
- "oc4.conf"
extra_dtc_overlay_files:
- "overlay7"
- "overlay8"
extra_configs:
- CONFIG_FOO=y

46
scripts/tests/twister/test_runner.py Normal file
View file

@ -0,0 +1,46 @@
#!/usr/bin/env python3
# Copyright (c) 2023 Google LLC
#
# SPDX-License-Identifier: Apache-2.0
"""
Tests for runner.py classes
"""
import mock
import os
import sys
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
from twisterlib.runner import ProjectBuilder
@mock.patch("os.path.exists")
def test_projectbuilder_cmake_assemble_args(m):
# Causes the additional_overlay_path to be appended
m.return_value = True
class MockHandler:
pass
handler = MockHandler()
handler.args = ["handler_arg1", "handler_arg2"]
handler.ready = True
assert(ProjectBuilder.cmake_assemble_args(
["basearg1"],
handler,
["a.conf;b.conf", "c.conf"],
["extra_overlay.conf"],
["x.overlay;y.overlay", "z.overlay"],
["cmake1=foo", "cmake2=bar"],
"/builddir/",
) == [
"-Dcmake1=foo", "-Dcmake2=bar",
"-Dbasearg1",
"-Dhandler_arg1", "-Dhandler_arg2",
"-DCONF_FILE=a.conf;b.conf;c.conf",
"-DDTC_OVERLAY_FILE=x.overlay;y.overlay;z.overlay",
"-DOVERLAY_CONFIG=extra_overlay.conf "
"/builddir/twister/testsuite_extra.conf",
])

14
scripts/tests/twister/test_testplan_class.py
View file

@ -35,7 +35,8 @@ def test_testplan_add_testsuites(class_testplan):
'test_a.check_1',
'test_a.check_2',
'test_d.check_1',
'sample_test.app']
'sample_test.app',
'test_config.main']
testsuite_list = []
for key in sorted(class_testplan.testsuites.keys()):
testsuite_list.append(os.path.basename(os.path.normpath(key)))
@ -75,11 +76,8 @@ def test_get_all_testsuites(class_env, all_testsuites_dict):
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
'test_c.check_2', 'test_d.check_1.unit_1a',
'test_d.check_1.unit_1b']
tests = plan.get_all_tests()
result = [c for c in tests]
assert len(plan.get_all_tests()) == len(expected_tests)
assert sorted(result) == sorted(expected_tests)
'test_d.check_1.unit_1b', 'test_config.main']
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
def test_get_platforms(class_env, platforms_list):
""" Testing get_platforms function of TestPlan class in Twister """
@ -279,7 +277,8 @@ QUARANTINE_PLATFORM = {
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3'
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
}
QUARANTINE_MULTIFILES = {
@ -330,6 +329,7 @@ def test_quarantine(class_testplan, platforms_list, test_data,
assert instance.status == 'filtered'
assert instance.reason == "Not under quarantine"
else:
print(testname)
if testname in expected_val:
assert instance.status == 'filtered'
assert instance.reason == "Quarantine: " + expected_val[testname]

32
scripts/tests/twister/test_twister.py
View file

@ -10,6 +10,8 @@ import os
import sys
import pytest
from pathlib import Path
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
@ -45,3 +47,33 @@ def test_incorrect_schema(filename, schema, test_data):
with pytest.raises(Exception) as exception:
scl.yaml_load_verify(filename, schema)
assert str(exception.value) == "Schema validation failed"
def test_testsuite_config_files():
""" Test to validate conf and overlay files are extracted properly """
filename = Path(ZEPHYR_BASE) / "scripts/tests/twister/test_data/testsuites/tests/test_config/test_data.yaml"
schema = scl.yaml_load(Path(ZEPHYR_BASE) / "scripts/schemas/twister/testsuite-schema.yaml")
data = TwisterConfigParser(filename, schema)
data.load()
# Load and validate the specific scenario from testcases.yaml
scenario = data.get_scenario("test_config.main")
assert scenario
# CONF_FILE, DTC_OVERLAY_FILE, OVERLAY_CONFIG fields should be stripped out
# of extra_args. Other fields should remain untouched.
assert scenario["extra_args"] == ["UNRELATED1=abc", "UNRELATED2=xyz"]
# Check that all conf files have been assembled in the correct order
assert ";".join(scenario["extra_conf_files"]) == \
"conf1;conf2;conf3;conf4;conf5;conf6;conf7;conf8"
# Check that all DTC overlay files have been assembled in the correct order
assert ";".join(scenario["extra_dtc_overlay_files"]) == \
"overlay1;overlay2;overlay3;overlay4;overlay5;overlay6;overlay7;overlay8"
# Check that all overlay conf files have been assembled in the correct order
assert scenario["extra_overlay_confs"] == \
["oc1.conf", "oc2.conf", "oc3.conf", "oc4.conf"]
# Check extra kconfig statements, too
assert scenario["extra_configs"] == ["CONFIG_FOO=y"]

16
tests/cmake/overlays/var_expansions/testcase.yaml
View file

@ -4,11 +4,15 @@ common:
platform_allow: native_posix
tests:
cmake.overlays.var_expansions.CONF_FILE:
extra_args:
CONF_FILE=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf;prj.conf
extra_conf_files:
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
- prj.conf
cmake.overlays.var_expansions.OVERLAY_CONFIG:
extra_args:
OVERLAY_CONFIG=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
extra_overlay_confs:
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-overlay.conf
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-overlay.conf
cmake.overlays.var_expansions.DTC_OVERLAY_FILE:
extra_args:
DTC_OVERLAY_FILE=${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-board.overlay;${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-board.overlay
extra_dtc_overlay_files:
- ${ZEPHYR_MY_MODULE_NAME_MODULE_DIR}/zephyr/my_module-board.overlay
- ${ZEPHYR_MY_EXTRA_MODULE_NAME_MODULE_DIR}/zephyr/my_extra_module-board.overlay