83fc06a8fe
A complete overhaul of the sanitycheck script and how we build and run tests. This new version of sanitycheck uses python for job distribution and drop use of Make. In addition to the move to python threading library, the following has been changed: - All handlers now run in parallel, meaning that any simulator will run in parallel and when testing on multiple devices (using --device-testing) the tests are run in parallel. - Lexicial filtering (using the filter keyword in yaml files) is now evaluated at runtime and is no long being pre-processed. This will allow us to immediately start executing tests and skip the wait time that was needed for filtering. - Device testing now supports multiple devices connected at the same time and is managed using a hardware map that needs to be generated and maintained for every test environment. (using --generate-hardware-map option). - Reports are not long stored in the Zephyr tree and instead stored in the output directory where all build artifacts are generated. - Each tested target now has a junit report in the output directory. - Recording option for performance data and other metrics is now available. This will allow us to record the output from the console and store the data for later processing. For example benchmark data can be captured and uploaded to a tracking server. - Test configurations (or instances) are no longer being sorted, this will help with balancing the load when we run sanitycheck on multiple hosts (as we do in CI). And many other cleanups and improvements... Signed-off-by: Anas Nashif <anas.nashif@intel.com>
225 lines
5.4 KiB
YAML
225 lines
5.4 KiB
YAML
#
|
|
# Schema to validate a YAML file describing a Zephyr test platform
|
|
#
|
|
# We load this with pykwalify
|
|
# (http://pykwalify.readthedocs.io/en/unstable/validation-rules.html),
|
|
# a YAML structure validator, to validate the YAML files that describe
|
|
# Zephyr test platforms
|
|
#
|
|
# The original spec comes from Zephyr's sanitycheck script
|
|
#
|
|
type: map
|
|
mapping:
|
|
"common":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"arch_exclude":
|
|
type: str
|
|
required: no
|
|
"arch_whitelist":
|
|
type: str
|
|
required: no
|
|
"build_only":
|
|
type: bool
|
|
required: no
|
|
"build_on_all":
|
|
type: bool
|
|
required: no
|
|
"depends_on":
|
|
type: str
|
|
required: no
|
|
"extra_args":
|
|
type: str
|
|
required: no
|
|
"extra_sections":
|
|
type: str
|
|
required: no
|
|
"filter":
|
|
type: str
|
|
required: no
|
|
"harness":
|
|
type: str
|
|
required: no
|
|
"harness_config":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"type":
|
|
type: str
|
|
required: no
|
|
"fixture":
|
|
type: str
|
|
required: no
|
|
"ordered":
|
|
type: bool
|
|
required: no
|
|
"repeat":
|
|
type: int
|
|
required: no
|
|
"regex":
|
|
type: seq
|
|
required: no
|
|
sequence:
|
|
- type: str
|
|
"record":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"regex":
|
|
type: str
|
|
required: no
|
|
"min_ram":
|
|
type: int
|
|
required: no
|
|
"min_flash":
|
|
type: int
|
|
required: no
|
|
"platform_exclude":
|
|
type: str
|
|
required: no
|
|
"platform_whitelist":
|
|
type: str
|
|
required: no
|
|
"tags":
|
|
type: str
|
|
required: no
|
|
"timeout":
|
|
type: int
|
|
required: no
|
|
"toolchain_exclude":
|
|
type: str
|
|
required: no
|
|
"toolchain_whitelist":
|
|
type: str
|
|
required: no
|
|
"type":
|
|
type: str
|
|
enum: [ 'unit' ]
|
|
"skip":
|
|
type: bool
|
|
required: no
|
|
"slow":
|
|
type: bool
|
|
required: no
|
|
# The sample descriptor, if present
|
|
"sample":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"name":
|
|
type: str
|
|
required: yes
|
|
"description":
|
|
type: str
|
|
required: no
|
|
# The list of testcases -- IDK why this is a sequence of
|
|
# maps maps, shall just be a sequence of maps
|
|
# maybe it is just an artifact?
|
|
"tests":
|
|
type: map
|
|
matching-rule: "any"
|
|
mapping:
|
|
# The key for the testname is any, so
|
|
# regex;(([a-zA-Z0-9_]+)) for this to work, note below we
|
|
# make it required: no
|
|
regex;(([a-zA-Z0-9_]+)):
|
|
type: map
|
|
# has to be not-required, otherwise the parser gets
|
|
# confused and things it never found it
|
|
required: no
|
|
mapping:
|
|
"arch_exclude":
|
|
type: str
|
|
required: no
|
|
"arch_whitelist":
|
|
type: str
|
|
required: no
|
|
"build_only":
|
|
type: bool
|
|
required: no
|
|
"build_on_all":
|
|
type: bool
|
|
required: no
|
|
"depends_on":
|
|
type: str
|
|
required: no
|
|
"extra_args":
|
|
type: str
|
|
required: no
|
|
"extra_configs":
|
|
type: seq
|
|
required: no
|
|
sequence:
|
|
- type: str
|
|
"extra_sections":
|
|
type: str
|
|
required: no
|
|
"filter":
|
|
type: str
|
|
required: no
|
|
"harness":
|
|
type: str
|
|
required: no
|
|
"harness_config":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"type":
|
|
type: str
|
|
required: no
|
|
"fixture":
|
|
type: str
|
|
required: no
|
|
"ordered":
|
|
type: bool
|
|
required: no
|
|
"repeat":
|
|
type: int
|
|
required: no
|
|
"regex":
|
|
type: seq
|
|
required: no
|
|
sequence:
|
|
- type: str
|
|
"record":
|
|
type: map
|
|
required: no
|
|
mapping:
|
|
"regex":
|
|
type: str
|
|
required: no
|
|
"min_ram":
|
|
type: int
|
|
required: no
|
|
"min_flash":
|
|
type: int
|
|
required: no
|
|
"platform_exclude":
|
|
type: str
|
|
required: no
|
|
"platform_whitelist":
|
|
type: str
|
|
required: no
|
|
"tags":
|
|
type: str
|
|
required: no
|
|
"timeout":
|
|
type: int
|
|
required: no
|
|
"toolchain_exclude":
|
|
type: str
|
|
required: no
|
|
"toolchain_whitelist":
|
|
type: str
|
|
required: no
|
|
"type":
|
|
type: str
|
|
enum: [ 'unit' ]
|
|
"skip":
|
|
type: bool
|
|
required: no
|
|
"slow":
|
|
type: bool
|
|
required: no
|