sanitycheck: add extra_configs for testing with multiple values
Support new keywords in testcase.yaml that would allow us to inject configuration options to be merged with default configuration instead of having to provide a prj.conf for each variant of the test which is very difficult to keep in sync. Sanitycheck script will create an overlay file that is merged during the build process. This is now done using the extra_configs option which is a yaml list of option with the values, for example: extra_configs: - CONFIG_XXXX=y - CONFIG_YYYY=y With this option we can have multiple tests that for example run on hardware with different values. This type of testing is good on HW but it does not make sense to be built in normal sanitycheck operation because it will be just rebuilding the same code with different values. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif
Anas Nashif
parent
95a8b20c9f
commit
fa695d2b9e
2 changed files with 124 additions and 17 deletions
|
|
@ -10,6 +10,67 @@
|
|||
#
|
||||
type: map
|
||||
mapping:
|
||||
"common":
|
||||
type: map
|
||||
required: no
|
||||
mapping:
|
||||
"arch_exclude":
|
||||
type: str
|
||||
required: no
|
||||
"arch_whitelist":
|
||||
type: str
|
||||
required: no
|
||||
"build_only":
|
||||
type: bool
|
||||
required: no
|
||||
"build_on_all":
|
||||
type: bool
|
||||
required: no
|
||||
"depends_on":
|
||||
type: str
|
||||
required: no
|
||||
"extra_args":
|
||||
type: str
|
||||
required: no
|
||||
"extra_sections":
|
||||
type: str
|
||||
required: no
|
||||
"filter":
|
||||
type: str
|
||||
required: no
|
||||
"min_ram":
|
||||
type: int
|
||||
required: no
|
||||
"min_flash":
|
||||
type: int
|
||||
required: no
|
||||
"platform_exclude":
|
||||
type: str
|
||||
required: no
|
||||
"platform_whitelist":
|
||||
type: str
|
||||
required: no
|
||||
"tags":
|
||||
type: str
|
||||
required: yes
|
||||
"timeout":
|
||||
type: int
|
||||
required: no
|
||||
"toolchain_exclude":
|
||||
type: str
|
||||
required: no
|
||||
"toolchain_whitelist":
|
||||
type: str
|
||||
required: no
|
||||
"type":
|
||||
type: str
|
||||
enum: [ 'unit' ]
|
||||
"skip":
|
||||
type: bool
|
||||
required: no
|
||||
"slow":
|
||||
type: bool
|
||||
required: no
|
||||
# The sample descriptor, if present
|
||||
"sample":
|
||||
type: map
|
||||
|
|
@ -61,6 +122,11 @@ mapping:
|
|||
"extra_args":
|
||||
type: str
|
||||
required: no
|
||||
"extra_configs":
|
||||
type: seq
|
||||
required: no
|
||||
sequence:
|
||||
- type: str
|
||||
"extra_sections":
|
||||
type: str
|
||||
required: no
|
||||
|
|
@ -81,7 +147,7 @@ mapping:
|
|||
required: no
|
||||
"tags":
|
||||
type: str
|
||||
required: yes
|
||||
required: no
|
||||
"timeout":
|
||||
type: int
|
||||
required: no
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ test case>/<block>.
|
|||
|
||||
Each test block in the testcase meta data can define the following key/value pairs:
|
||||
|
||||
tags: <list of tags> (required)
|
||||
tags: <list of tags> (required)
|
||||
A set of string tags for the testcase. Usually pertains to
|
||||
functional domains but can be anything. Command line invocations
|
||||
of this script can filter the set of tests to run based on tag.
|
||||
|
|
@ -809,8 +809,13 @@ class MakeGenerator:
|
|||
by execute() will be keyed by its .name field.
|
||||
"""
|
||||
args = ti.test.extra_args[:]
|
||||
args.extend(["ARCH=%s" % ti.platform.arch,
|
||||
"BOARD=%s" % ti.platform.name])
|
||||
arg_list = [
|
||||
"ARCH=%s" % ti.platform.arch,
|
||||
"BOARD=%s" % ti.platform.name]
|
||||
if len(ti.test.extra_configs) > 0:
|
||||
arg_list.append("OVERLAY_CONFIG=%s" % os.path.join(ti.outdir, "overlay.conf"))
|
||||
|
||||
args.extend(arg_list)
|
||||
args.extend(extra_args)
|
||||
if (ti.platform.qemu_support and (not ti.build_only) and
|
||||
(not build_only) and (enable_slow or not ti.test.slow)):
|
||||
|
|
@ -919,9 +924,10 @@ arch_valid_keys = {"name" : {"type" : "str", "required" : True},
|
|||
platform_valid_keys = {"qemu_support" : {"type" : "bool", "default" : False},
|
||||
"supported_toolchains" : {"type" : "list", "default" : []}}
|
||||
|
||||
testcase_valid_keys = {"tags" : {"type" : "set", "required" : True},
|
||||
testcase_valid_keys = {"tags" : {"type" : "set", "required" : False},
|
||||
"type" : {"type" : "str", "default": "integration"},
|
||||
"extra_args" : {"type" : "list"},
|
||||
"extra_configs" : {"type" : "list"},
|
||||
"build_only" : {"type" : "bool", "default" : False},
|
||||
"build_on_all" : {"type" : "bool", "default" : False},
|
||||
"skip" : {"type" : "bool", "default" : False},
|
||||
|
|
@ -953,7 +959,6 @@ class SanityConfigParser:
|
|||
self.cp = cp
|
||||
|
||||
def _cast_value(self, value, typestr):
|
||||
|
||||
if type(value) is str:
|
||||
v = value.strip()
|
||||
if typestr == "str":
|
||||
|
|
@ -968,7 +973,9 @@ class SanityConfigParser:
|
|||
elif typestr == "bool":
|
||||
return value
|
||||
|
||||
elif typestr.startswith("list"):
|
||||
elif typestr.startswith("list") and type(value) is list:
|
||||
return value
|
||||
elif typestr.startswith("list") and type(value) is str:
|
||||
vs = v.split()
|
||||
if len(typestr) > 4 and typestr[4] == ":":
|
||||
return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
|
||||
|
|
@ -985,7 +992,7 @@ class SanityConfigParser:
|
|||
else:
|
||||
raise ConfigurationError(self.filename, "unknown type '%s'" % value)
|
||||
|
||||
def section(self,name):
|
||||
def section(self, name):
|
||||
for s in self.sections():
|
||||
if name in s:
|
||||
return s.get(name, {})
|
||||
|
|
@ -996,7 +1003,7 @@ class SanityConfigParser:
|
|||
@return a list of string section names"""
|
||||
return self.cp['tests']
|
||||
|
||||
def get_section(self, section, valid_keys):
|
||||
def get_section(self, section, valid_keys, common):
|
||||
"""Get a dictionary representing the keys/values within a section
|
||||
|
||||
@param section The section in the .yaml file to retrieve data from
|
||||
|
|
@ -1022,13 +1029,19 @@ class SanityConfigParser:
|
|||
"""
|
||||
|
||||
d = {}
|
||||
for k, v in common.items():
|
||||
d[k] = v
|
||||
for k, v in self.section(section).items():
|
||||
if k not in valid_keys:
|
||||
raise ConfigurationError(self.filename,
|
||||
"Unknown config key '%s' in definition for '%s'"
|
||||
% (k, section))
|
||||
d[k] = v
|
||||
|
||||
if k in d:
|
||||
if type(d[k]) is str:
|
||||
d[k] += " " + v
|
||||
else:
|
||||
d[k] = v
|
||||
for k, kinfo in valid_keys.items():
|
||||
if k not in d:
|
||||
if "required" in kinfo:
|
||||
|
|
@ -1148,6 +1161,7 @@ class TestCase:
|
|||
self.type = tc_dict["type"]
|
||||
self.tags = tc_dict["tags"]
|
||||
self.extra_args = tc_dict["extra_args"]
|
||||
self.extra_configs = tc_dict["extra_configs"]
|
||||
self.arch_whitelist = tc_dict["arch_whitelist"]
|
||||
self.arch_exclude = tc_dict["arch_exclude"]
|
||||
self.skip = tc_dict["skip"]
|
||||
|
|
@ -1170,6 +1184,7 @@ class TestCase:
|
|||
self.defconfig = {}
|
||||
self.yamlfile = yamlfile
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return self.name
|
||||
|
||||
|
|
@ -1191,6 +1206,18 @@ class TestInstance:
|
|||
self.outdir = os.path.join(base_outdir, platform.name, test.path)
|
||||
self.build_only = build_only or test.build_only
|
||||
|
||||
def create_overlay(self):
|
||||
if len(self.test.extra_configs) > 0:
|
||||
file = os.path.join(self.outdir, "overlay.conf")
|
||||
os.makedirs(self.outdir, exist_ok=True)
|
||||
f = open(file, "w")
|
||||
content = ""
|
||||
for c in self.test.extra_configs:
|
||||
content += c
|
||||
f.write(content)
|
||||
f.close()
|
||||
|
||||
|
||||
def calculate_sizes(self):
|
||||
"""Get the RAM/ROM sizes of a test case.
|
||||
|
||||
|
|
@ -1269,9 +1296,13 @@ class TestSuite:
|
|||
|
||||
workdir = os.path.relpath(dirpath, testcase_root)
|
||||
|
||||
common = {}
|
||||
if 'common' in cp.cp:
|
||||
common = cp.cp['common']
|
||||
|
||||
for section in cp.sections():
|
||||
name = list(section.keys())[0]
|
||||
tc_dict = cp.get_section(name, testcase_valid_keys)
|
||||
tc_dict = cp.get_section(name, testcase_valid_keys, common)
|
||||
tc = TestCase(testcase_root, workdir, name, tc_dict,
|
||||
yaml_path)
|
||||
|
||||
|
|
@ -1332,15 +1363,26 @@ class TestSuite:
|
|||
myp = p
|
||||
break
|
||||
instance = TestInstance(self.testcases[name], myp, self.outdir)
|
||||
instance.create_overlay()
|
||||
instance_list.append(instance)
|
||||
self.add_instances(instance_list)
|
||||
|
||||
def apply_filters(self, args, toolchain):
|
||||
|
||||
def apply_filters(self, platform_filter, arch_filter, tag_filter, exclude_tag,
|
||||
config_filter, testcase_filter, last_failed, all_plats,
|
||||
platform_limit, toolchain, extra_args, enable_ccache):
|
||||
instances = []
|
||||
discards = {}
|
||||
platform_filter = args.platform
|
||||
last_failed = args.only_failed
|
||||
testcase_filter = args.test
|
||||
arch_filter = args.arch
|
||||
tag_filter = args.tag
|
||||
exclude_tag = args.exclude_tag
|
||||
config_filter = args.config
|
||||
platform_limit = args.platform_limit
|
||||
extra_args = args.extra_args
|
||||
enable_ccache = args.ccache
|
||||
all_plats = args.all
|
||||
|
||||
verbose("platform filter: " + str(platform_filter))
|
||||
verbose(" arch_filter: " + str(arch_filter))
|
||||
verbose(" tag_filter: " + str(tag_filter))
|
||||
|
|
@ -1569,6 +1611,7 @@ class TestSuite:
|
|||
tc.tc_filter)
|
||||
continue
|
||||
|
||||
instance.create_overlay()
|
||||
instance_list.append(instance)
|
||||
|
||||
if not instance_list:
|
||||
|
|
@ -2061,9 +2104,7 @@ def main():
|
|||
if args.load_tests:
|
||||
ts.load_from_file(args.load_tests)
|
||||
else:
|
||||
discards = ts.apply_filters(args.platform, args.arch, args.tag, args.exclude_tag, args.config,
|
||||
args.test, args.only_failed, args.all,
|
||||
args.platform_limit, toolchain, args.extra_args, args.ccache)
|
||||
discards = ts.apply_filters(args, toolchain)
|
||||
|
||||
if args.discard_report:
|
||||
ts.discard_report(args.discard_report)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue