michaelh/zephyr
1
0
Fork 0
Code Releases Activity

sanitycheck: support testcases in yaml instead of ini

Browse source 
This commit changes the syntax of the testcase files and changes the
behaviour and configuration of the sanitycheck script.

To avoid having multiple files with different syntax for boards,
samples, tests; this change unifies the syntax and uses YAML instead of
INI.

We maintain the current keywords used in the old syntax and maintain the
flexibility of adding tests with different configuration by using YAML
list configuration. On top of that, the following features are added:

- We now scan for board configurations in the boards directory and look
for a YAML file describing a board and how it should be tested. This
eliminates the need for listing boards per architecture in a special ini
file under scripts/.

- We define hardware information charachterstics in the board YAML file
that helps identifying if a certain test should run on that board or
not. For example, we can specify the available RAM in the board and
filter tests that would require more RAM than the board can handle.

- Boards can be set as default for testing meaning that we always run a
test case (build and run of possible) when sanitycheck is called without
any arguments. Previously this was done only by selecting the first
board defined for a specific architecture.

- Tests can be configured to run on all possible boards, this is to make
sure we always build some basic tests for all boards to catch issues
with the core kernel features.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2017-04-04 18:47:49 -04:00 • committed by Anas Nashif
commit a792a3d410
9 changed files with 207 additions and 229 deletions
Download patch file Download diff file Expand all files Collapse all files

37
scripts/ini2yaml.py Executable file
View file

@ -0,0 +1,37 @@
#!/usr/bin/env python
import ConfigParser, os
import yaml
import sys
sample = False
in_file = sys.argv[1]
if sys.argv[2] == 'sample':
sample = True
out_file = os.path.join(os.path.dirname(in_file), sys.argv[2] + ".yaml")
config = ConfigParser.ConfigParser()
config.readfp(open(sys.argv[1]))
y = {'tests': 'tests'}
tests = []
for section in config.sections():
tc = {}
for opt in config.options(section):
value = config.get(section, opt)
if value in ['false', 'true']:
tc[opt] = True if value == 'true' else False
else:
tc[opt] = value
test = { section : tc}
tests.append(test)
y['tests'] = tests
if sample:
y['sample'] = { 'name': "TBD", 'description': "TBD" }
with open(out_file, "w") as f:
yaml.dump(y, f, width=50, indent=4, default_flow_style=False)

4
scripts/sanity_chk/arches/arc.ini
View file

@ -1,4 +0,0 @@
[arch]
name = arc
platforms = arduino_101_sss quark_se_c1000_ss_devboard em_starterkit panther_ss
supported_toolchains = issm zephyr

17
scripts/sanity_chk/arches/arm.ini
View file

@ -1,17 +0,0 @@
[arch]
name = arm
platforms = qemu_cortex_m3 frdm_k64f arduino_due nucleo_f103rb stm32_mini_a15
olimexino_stm32 96b_nitrogen nrf52_pca10040 hexiwear_k64
nrf51_pca10028 nucleo_f401re 96b_carbon nrf51_blenano
arduino_101_ble cc3200_launchxl quark_se_c1000_ble bbc_microbit
v2m_beetle nucleo_l476rg nrf52840_pca10056 nucleo_f411re
stm3210c_eval nucleo_f334r8 stm32373c_eval mps2_an385 frdm_kw41z
sam_e70_xplained curie_ble nrf52_blenano2 hexiwear_kw40z
cc3220sf_launchxl frdm_kl25z disco_l475_iot1 nucleo_l432kc
nucleo_f413zh stm32l496g_disco stm32f4_disco 96b_carbon_nrf51
stm32f469i_disco nucleo_f412zg sam4s_xplained olimex_stm32_e407
supported_toolchains = zephyr gccarmemb
[qemu_cortex_m3]
qemu_support = true

7
scripts/sanity_chk/arches/nios2.ini
View file

@ -1,7 +0,0 @@
[arch]
name = nios2
platforms = qemu_nios2 altera_max10
supported_toolchains = zephyr
[qemu_nios2]
qemu_support = true

8
scripts/sanity_chk/arches/riscv32.ini
View file

@ -1,8 +0,0 @@
[arch]
name = riscv32
platforms = qemu_riscv32 zedboard_pulpino
supported_toolchains = zephyr
[qemu_riscv32]
qemu_support = true

7
scripts/sanity_chk/arches/unit.ini
View file

@ -1,7 +0,0 @@
[arch]
name = unit
platforms = unit_testing
supported_toolchains = zephyr
[unit_testing]
qemu_support = false

20
scripts/sanity_chk/arches/x86.ini
View file

@ -1,20 +0,0 @@
[arch]
name = x86
platforms = qemu_x86_iamcu arduino_101 qemu_x86 minnowboard galileo quark_d2000_crb quark_se_c1000_devboard
tinytile panther arduino_101_mcuboot
supported_toolchains = zephyr
[qemu_x86]
qemu_support = true
[qemu_x86_iamcu]
qemu_support = true
[quark_d2000_crb]
supported_toolchains = issm zephyr
[arduino_101]
supported_toolchains = issm zephyr
[quark_se_c1000_devboard]
supported_toolchains = issm zephyr

33
scripts/sanity_chk/arches/xtensa.ini
View file

@ -1,33 +0,0 @@
[arch]
name = xtensa
platforms = qemu_xtensa xt-sim xt-sim_XRC_FUSION_AON_ALL_LM
xt-sim_hifi2_std xt-sim_hifi3_bd5 xt-sim_D_233L
xt-sim_D_212GP xt-sim_D_108mini
supported_toolchains = xcc
[qemu_xtensa]
qemu_support = true
supported_toolchains = zephyr
[xt-sim]
qemu_support = true
[xt-sim_XRC_FUSION_AON_ALL_LM]
qemu_support = true
[xt-sim_hifi2_std]
qemu_support = true
[xt-sim_hifi3_bd5]
qemu_support = true
[xt-sim_D_233L]
qemu_support = true
[xt-sim_D_212GP]
qemu_support = true
[xt-sim_D_108mini]
qemu_support = true

303
scripts/sanitycheck
View file

@ -1,4 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# vim: set syntax=python ts=4 :
"""Zephyr Sanity Tests """Zephyr Sanity Tests
This script scans for the set of unit test applications in the git This script scans for the set of unit test applications in the git
@ -7,59 +8,74 @@ build each test case on one platform per architecture, using a precedence
list defined in an architecture configuration file, and if possible list defined in an architecture configuration file, and if possible
run the tests in the QEMU emulator. run the tests in the QEMU emulator.
Test cases are detected by the presence of a 'testcase.ini' file in Test cases are detected by the presence of a 'testcase.yaml' or a sample.yaml
the application's project directory. This file may contain one or files in the application's project directory. This file may contain one or more
more blocks, each identifying a test scenario. The title of the block blocks, each identifying a test scenario. The title of the block is a name for
is a name for the test case, which only needs to be unique for the the test case, which only needs to be unique for the test cases specified in
test cases specified in that testcase.ini file. The full canonical that testcase meta-data. The full canonical name for each test case is <path to
name for each test case is <path to test case>/<ini block>. test case>/<block>.
Each testcase.ini block can define the following key/value pairs: Each test block in the testcase meta data can define the following key/value pairs:
tags = <list of tags> (required) tags: <list of tags> (required)
A set of string tags for the testcase. Usually pertains to A set of string tags for the testcase. Usually pertains to
functional domains but can be anything. Command line invocations functional domains but can be anything. Command line invocations
of this script can filter the set of tests to run based on tag. of this script can filter the set of tests to run based on tag.
skip = <True|False> (default False) skip: <True|False> (default False)
skip testcase unconditionally. This can be used for broken tests. skip testcase unconditionally. This can be used for broken tests.
slow = <True|False> (default False) slow: <True|False> (default False)
Don't run this test case unless --enable-slow was passed in on the Don't run this test case unless --enable-slow was passed in on the
command line. Intended for time-consuming test cases that are only command line. Intended for time-consuming test cases that are only
run under certain circumstances, like daily builds. These test cases run under certain circumstances, like daily builds. These test cases
are still compiled. are still compiled.
extra_args = <list of extra arguments> extra_args: <list of extra arguments>
Extra arguments to pass to Make when building or running the Extra arguments to pass to Make when building or running the
test case. test case.
build_only = <True|False> (default False) build_only: <True|False> (default False)
If true, don't try to run the test under QEMU even if the If true, don't try to run the test under QEMU even if the
selected platform supports it. selected platform supports it.
timeout = <number of seconds> build_on_all: <True|False> (default False)
If true, attempt to build test on all available platforms.
depends_on: <list of features>
A board or platform can announce what features it supports, this option
will enable the test only those platforms that provide this feature.
min_ram: <integer>
minimum amount of RAM needed for this test to build and run. This is
compared with information provided by the board metadata.
min_flash: <integer>
minimum amount of ROM needed for this test to build and run. This is
compared with information provided by the board metadata.
timeout: <number of seconds>
Length of time to run test in QEMU before automatically killing it. Length of time to run test in QEMU before automatically killing it.
Default to 60 seconds. Default to 60 seconds.
arch_whitelist = <list of arches, such as x86, arm, arc> arch_whitelist: <list of arches, such as x86, arm, arc>
Set of architectures that this test case should only be run for. Set of architectures that this test case should only be run for.
arch_exclude = <list of arches, such as x86, arm, arc> arch_exclude: <list of arches, such as x86, arm, arc>
Set of architectures that this test case should not run on. Set of architectures that this test case should not run on.
platform_whitelist = <list of platforms> platform_whitelist: <list of platforms>
Set of platforms that this test case should only be run for. Set of platforms that this test case should only be run for.
platform_exclude = <list of platforms> platform_exclude: <list of platforms>
Set of platforms that this test case should not run on. Set of platforms that this test case should not run on.
extra_sections = <list of extra binary sections> extra_sections: <list of extra binary sections>
When computing sizes, sanitycheck will report errors if it finds When computing sizes, sanitycheck will report errors if it finds
extra, unexpected sections in the Zephyr binary unless they are named extra, unexpected sections in the Zephyr binary unless they are named
here. They will not be included in the size calculation. here. They will not be included in the size calculation.
filter = <expression> filter: <expression>
Filter whether the testcase should be run by evaluating an expression Filter whether the testcase should be run by evaluating an expression
against an environment containing the following values: against an environment containing the following values:
@ -121,28 +137,10 @@ Each testcase.ini block can define the following key/value pairs:
Would match it. Would match it.
Architectures and platforms are defined in an architecture configuration The set of test cases that actually run depends on directives in the testcase
file which are stored by default in scripts/sanity_chk/arches/. These filed and options passed in on the command line. If there is any confusion,
each define an [arch] block with the following key/value pairs: running with -v or --discard-report can help show why particular test cases
were skipped.
name = <arch name>
The name of the arch. Example: x86
platforms = <list of supported platforms in order of precedence>
List of supported platforms for this arch. The ordering here
is used to select a default platform to build for that arch.
For every platform defined, there must be a corresponding block for it
in the arch configuration file. This block can be empty if there are
no special definitions for that arch. Options are:
qemu_support = <True|False> (default False)
Indicates whether binaries for this platform can run under QEMU
The set of test cases that actually run depends on directives in the
testcase and architecture .ini file and options passed in on the command
line. If there is any confusion, running with -v or --discard-report
can help show why particular test cases were skipped.
Metrics (such as pass/fail state and binary size) for the last code Metrics (such as pass/fail state and binary size) for the last code
release are stored in scripts/sanity_chk/sanity_last_release.csv. release are stored in scripts/sanity_chk/sanity_last_release.csv.
@ -176,6 +174,7 @@ import xml.etree.ElementTree as ET
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
from collections import OrderedDict from collections import OrderedDict
from itertools import islice from itertools import islice
import yaml
if "ZEPHYR_BASE" not in os.environ: if "ZEPHYR_BASE" not in os.environ:
sys.stderr.write("$ZEPHYR_BASE environment variable undefined.\n") sys.stderr.write("$ZEPHYR_BASE environment variable undefined.\n")
@ -803,7 +802,7 @@ class MakeGenerator:
by execute() will be keyed by its .name field. by execute() will be keyed by its .name field.
""" """
args = ti.test.extra_args[:] args = ti.test.extra_args[:]
args.extend(["ARCH=%s" % ti.platform.arch.name, args.extend(["ARCH=%s" % ti.platform.arch,
"BOARD=%s" % ti.platform.name]) "BOARD=%s" % ti.platform.name])
args.extend(extra_args) args.extend(extra_args)
if (ti.platform.qemu_support and (not ti.build_only) and if (ti.platform.qemu_support and (not ti.build_only) and
@ -916,9 +915,13 @@ testcase_valid_keys = {"tags" : {"type" : "set", "required" : True},
"type" : {"type" : "str", "default": "integration"}, "type" : {"type" : "str", "default": "integration"},
"extra_args" : {"type" : "list"}, "extra_args" : {"type" : "list"},
"build_only" : {"type" : "bool", "default" : False}, "build_only" : {"type" : "bool", "default" : False},
"build_on_all" : {"type" : "bool", "default" : False},
"skip" : {"type" : "bool", "default" : False}, "skip" : {"type" : "bool", "default" : False},
"slow" : {"type" : "bool", "default" : False}, "slow" : {"type" : "bool", "default" : False},
"timeout" : {"type" : "int", "default" : 60}, "timeout" : {"type" : "int", "default" : 60},
"min_ram" : {"type" : "int", "default" : 8},
"depends_on": {"type" : "set"},
"min_flash" : {"type" : "int", "default" : 32},
"arch_whitelist" : {"type" : "set"}, "arch_whitelist" : {"type" : "set"},
"arch_exclude" : {"type" : "set"}, "arch_exclude" : {"type" : "set"},
"extra_sections" : {"type" : "list", "default" : []}, "extra_sections" : {"type" : "list", "default" : []},
@ -928,37 +931,33 @@ testcase_valid_keys = {"tags" : {"type" : "set", "required" : True},
class SanityConfigParser: class SanityConfigParser:
"""Class to read architecture and test case .ini files with semantic checking """Class to read test case files with semantic checking
""" """
def __init__(self, filename): def __init__(self, filename):
"""Instantiate a new SanityConfigParser object """Instantiate a new SanityConfigParser object
@param filename Source .ini file to read @param filename Source .yaml file to read
""" """
cp = configparser.SafeConfigParser() with open(filename, 'r') as stream:
cp.readfp(open(filename)) cp = yaml.load(stream)
self.filename = filename self.filename = filename
self.cp = cp self.cp = cp
def _cast_value(self, value, typestr): def _cast_value(self, value, typestr):
v = value.strip()
if type(value) is str:
v = value.strip()
if typestr == "str": if typestr == "str":
return v return v
elif typestr == "float": elif typestr == "float":
return float(v) return float(value)
elif typestr == "int": elif typestr == "int":
return int(v) return int(value)
elif typestr == "bool": elif typestr == "bool":
v = v.lower() return value
if v == "true" or v == "1":
return True
elif v == "" or v == "false" or v == "0":
return False
raise ConfigurationError(self.filename,
"bad value for boolean: '%s'" % value)
elif typestr.startswith("list"): elif typestr.startswith("list"):
vs = v.split() vs = v.split()
@ -977,20 +976,24 @@ class SanityConfigParser:
else: else:
raise ConfigurationError(self.filename, "unknown type '%s'" % value) raise ConfigurationError(self.filename, "unknown type '%s'" % value)
def section(self,name):
for s in self.sections():
if name in s:
return s.get(name, {})
def sections(self): def sections(self):
"""Get the set of sections within the .ini file """Get the set of test sections within the .yaml file
@return a list of string section names""" @return a list of string section names"""
return self.cp.sections() return self.cp['tests']
def get_section(self, section, valid_keys): def get_section(self, section, valid_keys):
"""Get a dictionary representing the keys/values within a section """Get a dictionary representing the keys/values within a section
@param section The section in the .ini file to retrieve data from @param section The section in the .yaml file to retrieve data from
@param valid_keys A dictionary representing the intended semantics @param valid_keys A dictionary representing the intended semantics
for this section. Each key in this dictionary is a key that could for this section. Each key in this dictionary is a key that could
be specified, if a key is given in the .ini file which isn't in be specified, if a key is given in the .yaml file which isn't in
here, it will generate an error. Each value in this dictionary here, it will generate an error. Each value in this dictionary
is another dictionary containing metadata: is another dictionary containing metadata:
@ -1010,17 +1013,11 @@ class SanityConfigParser:
""" """
d = {} d = {}
cp = self.cp for k, v in self.section(section).items():
if not cp.has_section(section):
# Just fill it with defaults
cp.add_section(section)
for k, v in cp.items(section):
if k not in valid_keys: if k not in valid_keys:
raise ConfigurationError(self.filename, raise ConfigurationError(self.filename,
"Unknown config key '%s' in definition for '%s'" "Unknown config key '%s' in definition for '%s'"
% (k, section)) % (k, section))
d[k] = v d[k] = v
for k, kinfo in valid_keys.items(): for k, kinfo in valid_keys.items():
@ -1055,7 +1052,7 @@ class Platform:
"""Class representing metadata for a particular platform """Class representing metadata for a particular platform
Maps directly to BOARD when building""" Maps directly to BOARD when building"""
def __init__(self, arch, name, plat_dict): def __init__(self, cfile):
"""Constructor. """Constructor.
@param arch Architecture object for this platform @param arch Architecture object for this platform
@ -1064,41 +1061,40 @@ class Platform:
in the architecture configuration file which has lots of metadata. in the architecture configuration file which has lots of metadata.
See the Architecture class. See the Architecture class.
""" """
self.name = name scp = SanityConfigParser(cfile)
self.qemu_support = plat_dict["qemu_support"] cp = scp.cp
self.arch = arch
self.supported_toolchains = arch.supported_toolchains self.name = cp['identifier']
if plat_dict["supported_toolchains"]: # if no RAM size is specified by the board, take a default of 128K
self.supported_toolchains = plat_dict["supported_toolchains"] self.ram = cp.get("ram", 128)
# Gets populated in a separate step testing = cp.get("testing", {})
self.ignore_tags = testing.get("ignore_tags", [])
self.default = testing.get("default", False)
# if no flash size is specified by the board, take a default of 512K
self.flash = cp.get("flash", 512)
self.supported = set(cp.get("supported", []))
self.qemu_support = True if cp.get('type', "na") == 'qemu' else False
self.arch = cp['arch']
self.supported_toolchains = cp.get("toolchain", [])
self.defconfig = None self.defconfig = None
pass pass
def __repr__(self): def __repr__(self):
return "<%s on %s>" % (self.name, self.arch.name) return "<%s on %s>" % (self.name, self.arch)
class Architecture: class Architecture:
"""Class representing metadata for a particular architecture """Class representing metadata for a particular architecture
""" """
def __init__(self, cfile): def __init__(self, name, platforms):
"""Architecture constructor """Architecture constructor
@param cfile Path to Architecture configuration file, which gives @param cfile Path to Architecture configuration file, which gives
info about the arch and all the platforms for it info about the arch and all the platforms for it
""" """
cp = SanityConfigParser(cfile) self.platforms = platforms
self.platforms = []
arch = cp.get_section("arch", arch_valid_keys) self.name = name
self.name = arch["name"]
self.supported_toolchains = arch["supported_toolchains"]
for plat_name in arch["platforms"]:
verbose("Platform: %s" % plat_name)
plat_dict = cp.get_section(plat_name, platform_valid_keys)
self.platforms.append(Platform(self, plat_name, plat_dict))
def __repr__(self): def __repr__(self):
return "<arch %s>" % self.name return "<arch %s>" % self.name
@ -1110,12 +1106,12 @@ class TestCase:
def __init__(self, testcase_root, workdir, name, tc_dict, inifile): def __init__(self, testcase_root, workdir, name, tc_dict, inifile):
"""TestCase constructor. """TestCase constructor.
This gets called by TestSuite as it finds and reads testcase.ini files. This gets called by TestSuite as it finds and reads test yaml files.
Multiple TestCase instances may be generated from a single testcase.ini, Multiple TestCase instances may be generated from a single testcase.yaml,
each one corresponds to a section within that file. each one corresponds to a section within that file.
We need to have a unique name for every single test case. Since We need to have a unique name for every single test case. Since
a testcase.ini can define multiple tests, the canonical name for a testcase.yaml can define multiple tests, the canonical name for
the test case is <workdir>/<name>. the test case is <workdir>/<name>.
@param testcase_root Absolute path to the root directory where @param testcase_root Absolute path to the root directory where
@ -1126,9 +1122,9 @@ class TestCase:
in the test case configuration file. For many test cases that just in the test case configuration file. For many test cases that just
define one test, can be anything and is usually "test". This is define one test, can be anything and is usually "test". This is
really only used to distinguish between different cases when really only used to distinguish between different cases when
the testcase.ini defines multiple tests the testcase.yaml defines multiple tests
@param tc_dict Dictionary with section values for this test case @param tc_dict Dictionary with section values for this test case
from the testcase.ini file from the testcase.yaml file
""" """
self.code_location = os.path.join(testcase_root, workdir) self.code_location = os.path.join(testcase_root, workdir)
self.type = tc_dict["type"] self.type = tc_dict["type"]
@ -1142,7 +1138,11 @@ class TestCase:
self.tc_filter = tc_dict["filter"] self.tc_filter = tc_dict["filter"]
self.timeout = tc_dict["timeout"] self.timeout = tc_dict["timeout"]
self.build_only = tc_dict["build_only"] self.build_only = tc_dict["build_only"]
self.build_on_all = tc_dict["build_on_all"]
self.slow = tc_dict["slow"] self.slow = tc_dict["slow"]
self.min_ram = tc_dict["min_ram"]
self.depends_on = tc_dict["depends_on"]
self.min_flash = tc_dict["min_flash"]
self.extra_sections = tc_dict["extra_sections"] self.extra_sections = tc_dict["extra_sections"]
self.path = os.path.join(os.path.basename(os.path.abspath(testcase_root)), self.path = os.path.join(os.path.basename(os.path.abspath(testcase_root)),
workdir, name) workdir, name)
@ -1231,41 +1231,40 @@ class TestSuite:
for dirpath, dirnames, filenames in os.walk(testcase_root, for dirpath, dirnames, filenames in os.walk(testcase_root,
topdown=True): topdown=True):
verbose("scanning %s" % dirpath) verbose("scanning %s" % dirpath)
if "testcase.ini" in filenames: if "sample.yaml" in filenames or "testcase.yaml" in filenames:
verbose("Found test case in " + dirpath) verbose("Found possible test case in " + dirpath)
dirnames[:] = [] dirnames[:] = []
ini_path = os.path.join(dirpath, "testcase.ini") if "sample.yaml" in filenames:
cp = SanityConfigParser(ini_path) yaml_path = os.path.join(dirpath, "sample.yaml")
else:
yaml_path = os.path.join(dirpath, "testcase.yaml")
cp = SanityConfigParser(yaml_path)
workdir = os.path.relpath(dirpath, testcase_root) workdir = os.path.relpath(dirpath, testcase_root)
for section in cp.sections(): for section in cp.sections():
tc_dict = cp.get_section(section, testcase_valid_keys) name = list(section.keys())[0]
tc = TestCase(testcase_root, workdir, section, tc_dict, tc_dict = cp.get_section(name, testcase_valid_keys)
ini_path) tc = TestCase(testcase_root, workdir, name, tc_dict,
yaml_path)
self.testcases[tc.name] = tc self.testcases[tc.name] = tc
debug("Reading architecture configuration files under %s..." % arch_root) debug("Reading platform configuration files under %s..." % arch_root)
for dirpath, dirnames, filenames in os.walk(arch_root): for dirpath, dirnames, filenames in os.walk(arch_root):
for filename in filenames: for filename in filenames:
if filename.endswith(".ini"): if filename.endswith(".yaml"):
fn = os.path.join(dirpath, filename) fn = os.path.join(dirpath, filename)
verbose("Found arch configuration " + fn) verbose("Found plaform configuration " + fn)
arch = Architecture(fn) platform = Platform(fn)
self.arches[arch.name] = arch self.platforms.append(platform)
self.platforms.extend(arch.platforms)
arches = []
for p in self.platforms:
arches.append(p.arch)
for a in list(set(arches)):
aplatforms = [ p for p in self.platforms if p.arch == a ]
arch = Architecture(a, aplatforms)
self.arches[a] = arch
# Build up a list of boards based on the presence of
# boards/*/*_defconfig files. We want to make sure that the arch.ini
# files are not missing any boards
all_plats = [plat.name for plat in self.platforms]
for dirpath, dirnames, filenames in os.walk(os.path.join(ZEPHYR_BASE,
"boards")):
for filename in filenames:
if filename.endswith("_defconfig"):
board_name = filename.replace("_defconfig", "")
if board_name not in all_plats:
error("Platform '%s' not specified in any arch .ini file and will not be tested"
% board_name)
self.instances = {} self.instances = {}
def get_last_failed(self): def get_last_failed(self):
@ -1311,13 +1310,15 @@ class TestSuite:
dlist = {} dlist = {}
for tc_name, tc in self.testcases.items(): for tc_name, tc in self.testcases.items():
for arch_name, arch in self.arches.items(): for arch_name, arch in self.arches.items():
instance_list = []
for plat in arch.platforms: for plat in arch.platforms:
instance = TestInstance(tc, plat, self.outdir) instance = TestInstance(tc, plat, self.outdir)
if (arch_name == "unit") != (tc.type == "unit"): if (arch_name == "unit") != (tc.type == "unit"):
continue continue
if tc.build_on_all:
platform_filter = []
if tc.skip: if tc.skip:
continue continue
@ -1348,12 +1349,24 @@ class TestSuite:
if platform_filter and plat.name not in platform_filter: if platform_filter and plat.name not in platform_filter:
continue continue
if plat.ram <= tc.min_ram:
continue
if set(plat.ignore_tags) & tc.tags:
continue
if not tc.depends_on.issubset(set(plat.supported)):
continue
if plat.flash < tc.min_flash:
continue
if tc.platform_whitelist and plat.name not in tc.platform_whitelist: if tc.platform_whitelist and plat.name not in tc.platform_whitelist:
continue continue
if tc.tc_filter and (plat in arch.platforms[:platform_limit] or all_plats or platform_filter): if tc.tc_filter and (plat.default or all_plats or platform_filter):
args = tc.extra_args[:] args = tc.extra_args[:]
args.extend(["ARCH=" + plat.arch.name, args.extend(["ARCH=" + plat.arch,
"BOARD=" + plat.name, "config-sanitycheck"]) "BOARD=" + plat.name, "config-sanitycheck"])
args.extend(extra_args) args.extend(extra_args)
# FIXME would be nice to use a common outdir for this so that # FIXME would be nice to use a common outdir for this so that
@ -1400,6 +1413,9 @@ class TestSuite:
discards[instance] = "Skip filter" discards[instance] = "Skip filter"
continue continue
if tc.build_on_all:
platform_filter = []
if tag_filter and not tc.tags.intersection(tag_filter): if tag_filter and not tc.tags.intersection(tag_filter):
discards[instance] = "Command line testcase tag filter" discards[instance] = "Command line testcase tag filter"
continue continue
@ -1444,6 +1460,22 @@ class TestSuite:
discards[instance] = "Not supported by the toolchain" discards[instance] = "Not supported by the toolchain"
continue continue
if plat.ram <= tc.min_ram:
discards[instance] = "Not enough RAM"
continue
if not tc.depends_on.issubset(set(plat.supported)):
discards[instance] = "No hardware support"
continue
if plat.flash< tc.min_flash:
discards[instance] = "Not enough FLASH"
continue
if set(plat.ignore_tags) & tc.tags:
discards[instance] = "Excluded tags per platform"
continue
defconfig = {"ARCH" : arch.name, "PLATFORM" : plat.name} defconfig = {"ARCH" : arch.name, "PLATFORM" : plat.name}
defconfig.update(os.environ) defconfig.update(os.environ)
for p, tdefconfig in tc.defconfig.items(): for p, tdefconfig in tc.defconfig.items():
@ -1468,10 +1500,15 @@ class TestSuite:
# Every platform in this arch was rejected already # Every platform in this arch was rejected already
continue continue
if default_platforms: if default_platforms and not tc.build_on_all:
self.add_instances(instance_list[:platform_limit]) if not tc.platform_whitelist:
for instance in instance_list[platform_limit:]: instances = list(filter(lambda tc: tc.platform.default, instance_list))
discards[instance] = "Not in first %d platform(s) for arch" % platform_limit self.add_instances(instances)
else:
self.add_instances(instance_list[:platform_limit])
for instance in list(filter(lambda tc: not tc.platform.default, instance_list)):
discards[instance] = "Not a default test platform"
else: else:
self.add_instances(instance_list) self.add_instances(instance_list)
self.discards = discards self.discards = discards
@ -1516,7 +1553,7 @@ class TestSuite:
cw.writeheader() cw.writeheader()
for instance, reason in self.discards.items(): for instance, reason in self.discards.items():
rowdict = {"test" : i.test.name, rowdict = {"test" : i.test.name,
"arch" : i.platform.arch.name, "arch" : i.platform.arch,
"platform" : i.platform.name, "platform" : i.platform.name,
"reason" : reason} "reason" : reason}
cw.writerow(rowdict) cw.writerow(rowdict)
@ -1636,7 +1673,7 @@ class TestSuite:
for name, goal in self.goals.items(): for name, goal in self.goals.items():
i = self.instances[name] i = self.instances[name]
rowdict = {"test" : i.test.name, rowdict = {"test" : i.test.name,
"arch" : i.platform.arch.name, "arch" : i.platform.arch,
"platform" : i.platform.name, "platform" : i.platform.name,
"extra_args" : " ".join(i.test.extra_args), "extra_args" : " ".join(i.test.extra_args),
"qemu" : i.platform.qemu_support} "qemu" : i.platform.qemu_support}
@ -1661,9 +1698,9 @@ def parse_arguments():
parser.add_argument("-p", "--platform", action="append", parser.add_argument("-p", "--platform", action="append",
help="Platform filter for testing. This option may be used multiple " help="Platform filter for testing. This option may be used multiple "
"times. Testcases will only be built/run on the platforms " "times. Testcases will only be built/run on the platforms "
"specified. If this option is not used, then N platforms will " "specified. If this option is not used, then platforms marked "
"automatically be chosen from each arch to build and test, " "as default in the platform metadata file will be chosen "
"where N is provided by the --platform-limit option.") "to build and test. ")
parser.add_argument("-L", "--platform-limit", action="store", type=int, parser.add_argument("-L", "--platform-limit", action="store", type=int,
metavar="N", default=1, metavar="N", default=1,
help="Controls what platforms are tested if --platform or " help="Controls what platforms are tested if --platform or "
@ -1769,7 +1806,7 @@ def parse_arguments():
"called multiple times. Defaults to the 'samples' and " "called multiple times. Defaults to the 'samples' and "
"'tests' directories in the Zephyr tree.") "'tests' directories in the Zephyr tree.")
parser.add_argument("-A", "--arch-root", parser.add_argument("-A", "--arch-root",
default="%s/scripts/sanity_chk/arches" % ZEPHYR_BASE, default="%s/boards" % ZEPHYR_BASE,
help="Directory to search for arch configuration files. All .ini " help="Directory to search for arch configuration files. All .ini "
"files in the directory will be processed.") "files in the directory will be processed.")
parser.add_argument("-z", "--size", action="append", parser.add_argument("-z", "--size", action="append",