michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: rename TestCase -> TestSuite

Browse source 
TestCase in the old terminology is the test application, but we have
just freed up TestSuite, so use TestSuite here to signify the test
application which consists of many test scenarios and testcases.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-03-23 14:07:54 -04:00
commit 15bc98eb50
4 changed files with 124 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

10
scripts/pylib/twister/harness.py
View file

@ -40,10 +40,10 @@ class Harness:
self.run_id_exists = False self.run_id_exists = False
def configure(self, instance): def configure(self, instance):
config = instance.testcase.harness_config config = instance.testsuite.harness_config
self.id = instance.testcase.id self.id = instance.testsuite.id
self.run_id = instance.run_id self.run_id = instance.run_id
if "ignore_faults" in instance.testcase.tags: if "ignore_faults" in instance.testsuite.tags:
self.fail_on_fault = False self.fail_on_fault = False
if config: if config:
@ -143,11 +143,11 @@ class Pytest(Harness):
def configure(self, instance): def configure(self, instance):
super(Pytest, self).configure(instance) super(Pytest, self).configure(instance)
self.running_dir = instance.build_dir self.running_dir = instance.build_dir
self.source_dir = instance.testcase.source_dir self.source_dir = instance.testsuite.source_dir
self.pytest_root = 'pytest' self.pytest_root = 'pytest'
self.pytest_args = [] self.pytest_args = []
self.is_pytest = True self.is_pytest = True
config = instance.testcase.harness_config config = instance.testsuite.harness_config
if config: if config:
self.pytest_root = config.get('pytest_root', 'pytest') self.pytest_root = config.get('pytest_root', 'pytest')

202
scripts/pylib/twister/twisterlib.py
View file

@ -424,8 +424,8 @@ class Handler:
self.name = instance.name self.name = instance.name
self.instance = instance self.instance = instance
self.timeout = math.ceil(instance.testcase.timeout * instance.platform.timeout_multiplier) self.timeout = math.ceil(instance.testsuite.timeout * instance.platform.timeout_multiplier)
self.sourcedir = instance.testcase.source_dir self.sourcedir = instance.testsuite.source_dir
self.build_dir = instance.build_dir self.build_dir = instance.build_dir
self.log = os.path.join(self.build_dir, "handler.log") self.log = os.path.join(self.build_dir, "handler.log")
self.returncode = 0 self.returncode = 0
@ -477,7 +477,7 @@ class Handler:
add information about next testcases, which were not be add information about next testcases, which were not be
performed due to this error. performed due to this error.
""" """
for c in self.instance.testcase.cases: for c in self.instance.testsuite.cases:
if c not in harness.tests: if c not in harness.tests:
harness.tests[c] = "BLOCK" harness.tests[c] = "BLOCK"
@ -616,7 +616,7 @@ class BinaryHandler(Handler):
def handle(self): def handle(self):
harness_name = self.instance.testcase.harness.capitalize() harness_name = self.instance.testsuite.harness.capitalize()
harness_import = HarnessImporter(harness_name) harness_import = HarnessImporter(harness_name)
harness = harness_import.instance harness = harness_import.instance
harness.configure(self.instance) harness.configure(self.instance)
@ -775,7 +775,7 @@ class DeviceHandler(Handler):
def device_is_available(self, instance): def device_is_available(self, instance):
device = instance.platform.name device = instance.platform.name
fixture = instance.testcase.harness_config.get("fixture") fixture = instance.testsuite.harness_config.get("fixture")
for d in self.testplan.duts: for d in self.testplan.duts:
if fixture and fixture not in d.fixtures: if fixture and fixture not in d.fixtures:
continue continue
@ -928,7 +928,7 @@ class DeviceHandler(Handler):
ser.flush() ser.flush()
harness_name = self.instance.testcase.harness.capitalize() harness_name = self.instance.testsuite.harness.capitalize()
harness_import = HarnessImporter(harness_name) harness_import = HarnessImporter(harness_name)
harness = harness_import.instance harness = harness_import.instance
harness.configure(self.instance) harness.configure(self.instance)
@ -1003,7 +1003,7 @@ class DeviceHandler(Handler):
# empty dictionary results, in order to include it into final report, # empty dictionary results, in order to include it into final report,
# so fill the results as BLOCK # so fill the results as BLOCK
if self.instance.results == {}: if self.instance.results == {}:
for k in self.instance.testcase.cases: for k in self.instance.testsuite.cases:
self.instance.results[k] = 'BLOCK' self.instance.results[k] = 'BLOCK'
if harness.state: if harness.state:
@ -1043,7 +1043,7 @@ class QEMUHandler(Handler):
self.pid_fn = os.path.join(instance.build_dir, "qemu.pid") self.pid_fn = os.path.join(instance.build_dir, "qemu.pid")
if "ignore_qemu_crash" in instance.testcase.tags: if "ignore_qemu_crash" in instance.testsuite.tags:
self.ignore_qemu_crash = True self.ignore_qemu_crash = True
self.ignore_unexpected_eof = True self.ignore_unexpected_eof = True
else: else:
@ -1216,7 +1216,7 @@ class QEMUHandler(Handler):
self.log_fn = self.log self.log_fn = self.log
harness_import = HarnessImporter(self.instance.testcase.harness.capitalize()) harness_import = HarnessImporter(self.instance.testsuite.harness.capitalize())
harness = harness_import.instance harness = harness_import.instance
harness.configure(self.instance) harness.configure(self.instance)
@ -1700,7 +1700,7 @@ class DisablePyTestCollectionMixin(object):
class ScanPathResult: class ScanPathResult:
"""Result of the TestCase.scan_path function call. """Result of the TestSuite.scan_path function call.
Attributes: Attributes:
matches A list of test cases matches A list of test cases
@ -1744,23 +1744,23 @@ class ScanPathResult:
sorted(other.ztest_suite_names))) sorted(other.ztest_suite_names)))
class TestCase(DisablePyTestCollectionMixin): class TestSuite(DisablePyTestCollectionMixin):
"""Class representing a test application """Class representing a test application
""" """
def __init__(self, testcase_root, workdir, name): def __init__(self, testsuite_root, workdir, name):
"""TestCase constructor. """TestSuite constructor.
This gets called by TestPlan as it finds and reads test yaml files. This gets called by TestPlan as it finds and reads test yaml files.
Multiple TestCase instances may be generated from a single testcase.yaml, Multiple TestSuite instances may be generated from a single testcase.yaml,
each one corresponds to an entry within that file. each one corresponds to an entry within that file.
We need to have a unique name for every single test case. Since We need to have a unique name for every single test case. Since
a testcase.yaml can define multiple tests, the canonical name for a testcase.yaml can define multiple tests, the canonical name for
the test case is <workdir>/<name>. the test case is <workdir>/<name>.
@param testcase_root os.path.abspath() of one of the --testcase-root @param testsuite_root os.path.abspath() of one of the --testcase-root
@param workdir Sub-directory of testcase_root where the @param workdir Sub-directory of testsuite_root where the
.yaml test configuration file was found .yaml test configuration file was found
@param name Name of this test case, corresponding to the entry name @param name Name of this test case, corresponding to the entry name
in the test case configuration file. For many test cases that just in the test case configuration file. For many test cases that just
@ -1773,7 +1773,7 @@ class TestCase(DisablePyTestCollectionMixin):
self.source_dir = "" self.source_dir = ""
self.yamlfile = "" self.yamlfile = ""
self.cases = [] self.cases = []
self.name = self.get_unique(testcase_root, workdir, name) self.name = self.get_unique(testsuite_root, workdir, name)
self.id = name self.id = name
self.type = None self.type = None
@ -1802,13 +1802,13 @@ class TestCase(DisablePyTestCollectionMixin):
self.ztest_suite_names = [] self.ztest_suite_names = []
@staticmethod @staticmethod
def get_unique(testcase_root, workdir, name): def get_unique(testsuite_root, workdir, name):
canonical_testcase_root = os.path.realpath(testcase_root) canonical_testsuite_root = os.path.realpath(testsuite_root)
if Path(canonical_zephyr_base) in Path(canonical_testcase_root).parents: if Path(canonical_zephyr_base) in Path(canonical_testsuite_root).parents:
# This is in ZEPHYR_BASE, so include path in name for uniqueness # This is in ZEPHYR_BASE, so include path in name for uniqueness
# FIXME: We should not depend on path of test for unique names. # FIXME: We should not depend on path of test for unique names.
relative_tc_root = os.path.relpath(canonical_testcase_root, relative_tc_root = os.path.relpath(canonical_testsuite_root,
start=canonical_zephyr_base) start=canonical_zephyr_base)
else: else:
relative_tc_root = "" relative_tc_root = ""
@ -1817,7 +1817,7 @@ class TestCase(DisablePyTestCollectionMixin):
unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name)) unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name))
check = name.split(".") check = name.split(".")
if len(check) < 2: if len(check) < 2:
raise TwisterException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \ raise TwisterException(f"""bad test name '{name}' in {testsuite_root}/{workdir}. \
Tests should reference the category and subsystem with a dot as a separator. Tests should reference the category and subsystem with a dot as a separator.
""" """
) )
@ -2095,17 +2095,17 @@ Tests should reference the category and subsystem with a dot as a separator.
class TestInstance(DisablePyTestCollectionMixin): class TestInstance(DisablePyTestCollectionMixin):
"""Class representing the execution of a particular TestCase on a platform """Class representing the execution of a particular TestSuite on a platform
@param test The TestCase object we want to build/execute @param test The TestSuite object we want to build/execute
@param platform Platform object that we want to build and run against @param platform Platform object that we want to build and run against
@param base_outdir Base directory for all test results. The actual @param base_outdir Base directory for all test results. The actual
out directory used is <outdir>/<platform>/<test case name> out directory used is <outdir>/<platform>/<test case name>
""" """
def __init__(self, testcase, platform, outdir): def __init__(self, testsuite, platform, outdir):
self.testcase = testcase self.testsuite = testsuite
self.platform = platform self.platform = platform
self.status = None self.status = None
@ -2114,9 +2114,9 @@ class TestInstance(DisablePyTestCollectionMixin):
self.handler = None self.handler = None
self.outdir = outdir self.outdir = outdir
self.name = os.path.join(platform.name, testcase.name) self.name = os.path.join(platform.name, testsuite.name)
self.run_id = self._get_run_id() self.run_id = self._get_run_id()
self.build_dir = os.path.join(outdir, platform.name, testcase.name) self.build_dir = os.path.join(outdir, platform.name, testsuite.name)
self.run = False self.run = False
@ -2144,18 +2144,18 @@ class TestInstance(DisablePyTestCollectionMixin):
@staticmethod @staticmethod
def testcase_runnable(testcase, fixtures): def testsuite_runnable(testsuite, fixtures):
can_run = False can_run = False
# console harness allows us to run the test and capture data. # console harness allows us to run the test and capture data.
if testcase.harness in [ 'console', 'ztest', 'pytest']: if testsuite.harness in [ 'console', 'ztest', 'pytest']:
can_run = True can_run = True
# if we have a fixture that is also being supplied on the # if we have a fixture that is also being supplied on the
# command-line, then we need to run the test, not just build it. # command-line, then we need to run the test, not just build it.
fixture = testcase.harness_config.get('fixture') fixture = testsuite.harness_config.get('fixture')
if fixture: if fixture:
can_run = (fixture in fixtures) can_run = (fixture in fixtures)
elif testcase.harness: elif testsuite.harness:
can_run = False can_run = False
else: else:
can_run = True can_run = True
@ -2172,15 +2172,15 @@ class TestInstance(DisablePyTestCollectionMixin):
return False return False
# we asked for build-only on the command line # we asked for build-only on the command line
if self.testcase.build_only: if self.testsuite.build_only:
return False return False
# Do not run slow tests: # Do not run slow tests:
skip_slow = self.testcase.slow and not enable_slow skip_slow = self.testsuite.slow and not enable_slow
if skip_slow: if skip_slow:
return False return False
target_ready = bool(self.testcase.type == "unit" or \ target_ready = bool(self.testsuite.type == "unit" or \
self.platform.type == "native" or \ self.platform.type == "native" or \
self.platform.simulation in ["mdb-nsim", "nsim", "renode", "qemu", "tsim", "armfvp", "xt-sim"] or \ self.platform.simulation in ["mdb-nsim", "nsim", "renode", "qemu", "tsim", "armfvp", "xt-sim"] or \
filter == 'runnable') filter == 'runnable')
@ -2201,9 +2201,9 @@ class TestInstance(DisablePyTestCollectionMixin):
if not find_executable("tsim-leon3"): if not find_executable("tsim-leon3"):
target_ready = False target_ready = False
testcase_runnable = self.testcase_runnable(self.testcase, fixtures) testsuite_runnable = self.testsuite_runnable(self.testsuite, fixtures)
return testcase_runnable and target_ready return testsuite_runnable and target_ready
def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable_coverage=False, coverage_platform=[]): def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable_coverage=False, coverage_platform=[]):
# Create this in a "twister/" subdirectory otherwise this # Create this in a "twister/" subdirectory otherwise this
@ -2214,8 +2214,8 @@ class TestInstance(DisablePyTestCollectionMixin):
content = "" content = ""
if self.testcase.extra_configs: if self.testsuite.extra_configs:
content = "\n".join(self.testcase.extra_configs) content = "\n".join(self.testsuite.extra_configs)
if enable_coverage: if enable_coverage:
if platform.name in coverage_platform: if platform.name in coverage_platform:
@ -2232,7 +2232,7 @@ class TestInstance(DisablePyTestCollectionMixin):
if content: if content:
os.makedirs(subdir, exist_ok=True) os.makedirs(subdir, exist_ok=True)
file = os.path.join(subdir, "testcase_extra.conf") file = os.path.join(subdir, "testsuite_extra.conf")
with open(file, "w") as f: with open(file, "w") as f:
f.write(content) f.write(content)
@ -2252,7 +2252,7 @@ class TestInstance(DisablePyTestCollectionMixin):
if len(fns) != 1: if len(fns) != 1:
raise BuildError("Missing/multiple output ELF binary") raise BuildError("Missing/multiple output ELF binary")
return SizeCalculator(fns[0], self.testcase.extra_sections) return SizeCalculator(fns[0], self.testsuite.extra_sections)
def fill_results_by_status(self): def fill_results_by_status(self):
"""Fills results according to self.status """Fills results according to self.status
@ -2274,14 +2274,14 @@ class TestInstance(DisablePyTestCollectionMixin):
self.results[k] = status_to_verdict[self.status] self.results[k] = status_to_verdict[self.status]
def __repr__(self): def __repr__(self):
return "<TestCase %s on %s>" % (self.testcase.name, self.platform.name) return "<TestSuite %s on %s>" % (self.testsuite.name, self.platform.name)
class CMake(): class CMake():
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
def __init__(self, testcase, platform, source_dir, build_dir): def __init__(self, testsuite, platform, source_dir, build_dir):
self.cwd = None self.cwd = None
self.capture_output = True self.capture_output = True
@ -2290,7 +2290,7 @@ class CMake():
self.cmake_cache = {} self.cmake_cache = {}
self.instance = None self.instance = None
self.testcase = testcase self.testsuite = testsuite
self.platform = platform self.platform = platform
self.source_dir = source_dir self.source_dir = source_dir
self.build_dir = build_dir self.build_dir = build_dir
@ -2475,8 +2475,8 @@ class CMake():
class FilterBuilder(CMake): class FilterBuilder(CMake):
def __init__(self, testcase, platform, source_dir, build_dir): def __init__(self, testsuite, platform, source_dir, build_dir):
super().__init__(testcase, platform, source_dir, build_dir) super().__init__(testsuite, platform, source_dir, build_dir)
self.log = "config-twister.log" self.log = "config-twister.log"
@ -2520,24 +2520,24 @@ class FilterBuilder(CMake):
filter_data.update(self.cmake_cache) filter_data.update(self.cmake_cache)
edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle") edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle")
if self.testcase and self.testcase.tc_filter: if self.testsuite and self.testsuite.tc_filter:
try: try:
if os.path.exists(edt_pickle): if os.path.exists(edt_pickle):
with open(edt_pickle, 'rb') as f: with open(edt_pickle, 'rb') as f:
edt = pickle.load(f) edt = pickle.load(f)
else: else:
edt = None edt = None
res = expr_parser.parse(self.testcase.tc_filter, filter_data, edt) res = expr_parser.parse(self.testsuite.tc_filter, filter_data, edt)
except (ValueError, SyntaxError) as se: except (ValueError, SyntaxError) as se:
sys.stderr.write( sys.stderr.write(
"Failed processing %s\n" % self.testcase.yamlfile) "Failed processing %s\n" % self.testsuite.yamlfile)
raise se raise se
if not res: if not res:
return {os.path.join(self.platform.name, self.testcase.name): True} return {os.path.join(self.platform.name, self.testsuite.name): True}
else: else:
return {os.path.join(self.platform.name, self.testcase.name): False} return {os.path.join(self.platform.name, self.testsuite.name): False}
else: else:
self.platform.filter_data = filter_data self.platform.filter_data = filter_data
return filter_data return filter_data
@ -2546,7 +2546,7 @@ class FilterBuilder(CMake):
class ProjectBuilder(FilterBuilder): class ProjectBuilder(FilterBuilder):
def __init__(self, tplan, instance, **kwargs): def __init__(self, tplan, instance, **kwargs):
super().__init__(instance.testcase, instance.platform, instance.testcase.source_dir, instance.build_dir) super().__init__(instance.testsuite, instance.platform, instance.testsuite.source_dir, instance.build_dir)
self.log = "build.log" self.log = "build.log"
self.instance = instance self.instance = instance
@ -2615,7 +2615,7 @@ class ProjectBuilder(FilterBuilder):
instance.handler = QEMUHandler(instance, "qemu") instance.handler = QEMUHandler(instance, "qemu")
args.append("QEMU_PIPE=%s" % instance.handler.get_fifo()) args.append("QEMU_PIPE=%s" % instance.handler.get_fifo())
instance.handler.call_make_run = True instance.handler.call_make_run = True
elif instance.testcase.type == "unit": elif instance.testsuite.type == "unit":
instance.handler = BinaryHandler(instance, "unit") instance.handler = BinaryHandler(instance, "unit")
instance.handler.binary = os.path.join(instance.build_dir, "testbinary") instance.handler.binary = os.path.join(instance.build_dir, "testbinary")
if self.coverage: if self.coverage:
@ -2685,7 +2685,7 @@ class ProjectBuilder(FilterBuilder):
self.instance.status = "filtered" self.instance.status = "filtered"
self.instance.reason = "runtime filter" self.instance.reason = "runtime filter"
results.skipped_runtime += 1 results.skipped_runtime += 1
for case in self.instance.testcase.cases: for case in self.instance.testsuite.cases:
self.instance.results.update({case: 'SKIP'}) self.instance.results.update({case: 'SKIP'})
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
else: else:
@ -2825,7 +2825,7 @@ class ProjectBuilder(FilterBuilder):
logger.error( logger.error(
"{:<25} {:<50} {}FAILED{}: {}".format( "{:<25} {:<50} {}FAILED{}: {}".format(
instance.platform.name, instance.platform.name,
instance.testcase.name, instance.testsuite.name,
Fore.RED, Fore.RED,
Fore.RESET, Fore.RESET,
instance.reason)) instance.reason))
@ -2834,7 +2834,7 @@ class ProjectBuilder(FilterBuilder):
elif instance.status in ["skipped", "filtered"]: elif instance.status in ["skipped", "filtered"]:
status = Fore.YELLOW + "SKIPPED" + Fore.RESET status = Fore.YELLOW + "SKIPPED" + Fore.RESET
results.skipped_configs += 1 results.skipped_configs += 1
results.skipped_cases += len(instance.testcase.cases) results.skipped_cases += len(instance.testsuite.cases)
elif instance.status == "passed": elif instance.status == "passed":
status = Fore.GREEN + "PASSED" + Fore.RESET status = Fore.GREEN + "PASSED" + Fore.RESET
results.passed += 1 results.passed += 1
@ -2866,7 +2866,7 @@ class ProjectBuilder(FilterBuilder):
logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format( logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format(
results.done + results.skipped_filter, total_tests_width, total_to_do , instance.platform.name, results.done + results.skipped_filter, total_tests_width, total_to_do , instance.platform.name,
instance.testcase.name, status, more_info)) instance.testsuite.name, status, more_info))
if instance.status in ["error", "failed", "timeout"]: if instance.status in ["error", "failed", "timeout"]:
self.log_info_file(self.inline_logs) self.log_info_file(self.inline_logs)
@ -2894,7 +2894,7 @@ class ProjectBuilder(FilterBuilder):
def cmake(self): def cmake(self):
instance = self.instance instance = self.instance
args = self.testcase.extra_args[:] args = self.testsuite.extra_args[:]
args += self.extra_args args += self.extra_args
if instance.handler: if instance.handler:
@ -2918,9 +2918,9 @@ class ProjectBuilder(FilterBuilder):
overlays = extract_overlays(args) overlays = extract_overlays(args)
if os.path.exists(os.path.join(instance.build_dir, if os.path.exists(os.path.join(instance.build_dir,
"twister", "testcase_extra.conf")): "twister", "testsuite_extra.conf")):
overlays.append(os.path.join(instance.build_dir, overlays.append(os.path.join(instance.build_dir,
"twister", "testcase_extra.conf")) "twister", "testsuite_extra.conf"))
if overlays: if overlays:
args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays))) args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays)))
@ -2979,12 +2979,12 @@ class TestPlan(DisablePyTestCollectionMixin):
tc_schema = scl.yaml_load( tc_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE, os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "testcase-schema.yaml")) "scripts", "schemas", "twister", "testsuite-schema.yaml"))
quarantine_schema = scl.yaml_load( quarantine_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE, os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "quarantine-schema.yaml")) "scripts", "schemas", "twister", "quarantine-schema.yaml"))
testcase_valid_keys = {"tags": {"type": "set", "required": False}, testsuite_valid_keys = {"tags": {"type": "set", "required": False},
"type": {"type": "str", "default": "integration"}, "type": {"type": "str", "default": "integration"},
"extra_args": {"type": "list"}, "extra_args": {"type": "list"},
"extra_configs": {"type": "list"}, "extra_configs": {"type": "list"},
@ -3012,11 +3012,11 @@ class TestPlan(DisablePyTestCollectionMixin):
} }
SAMPLE_FILENAME = 'sample.yaml' SAMPLE_FILENAME = 'sample.yaml'
TESTCASE_FILENAME = 'testcase.yaml' TESTSUITE_FILENAME = 'testcase.yaml'
def __init__(self, board_root_list=[], testcase_roots=[], outdir=None): def __init__(self, board_root_list=[], testsuite_roots=[], outdir=None):
self.roots = testcase_roots self.roots = testsuite_roots
if not isinstance(board_root_list, list): if not isinstance(board_root_list, list):
self.board_roots = [board_root_list] self.board_roots = [board_root_list]
else: else:
@ -3051,7 +3051,7 @@ class TestPlan(DisablePyTestCollectionMixin):
self.seed = 0 self.seed = 0
# Keep track of which test cases we've filtered out and why # Keep track of which test cases we've filtered out and why
self.testcases = {} self.testsuites = {}
self.quarantine = {} self.quarantine = {}
self.platforms = [] self.platforms = []
self.platform_names = [] self.platform_names = []
@ -3109,7 +3109,7 @@ class TestPlan(DisablePyTestCollectionMixin):
def update_counting(self, results=None): def update_counting(self, results=None):
for instance in self.instances.values(): for instance in self.instances.values():
results.cases += len(instance.testcase.cases) results.cases += len(instance.testsuite.cases)
if instance.status == 'filtered': if instance.status == 'filtered':
results.skipped_filter += 1 results.skipped_filter += 1
results.skipped_configs += 1 results.skipped_configs += 1
@ -3136,7 +3136,7 @@ class TestPlan(DisablePyTestCollectionMixin):
saved_metrics[(ts_name, ts_platform)] = d saved_metrics[(ts_name, ts_platform)] = d
for instance in self.instances.values(): for instance in self.instances.values():
mkey = (instance.testcase.name, instance.platform.name) mkey = (instance.testsuite.name, instance.platform.name)
if mkey not in saved_metrics: if mkey not in saved_metrics:
continue continue
sm = saved_metrics[mkey] sm = saved_metrics[mkey]
@ -3174,7 +3174,7 @@ class TestPlan(DisablePyTestCollectionMixin):
continue continue
logger.info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format( logger.info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format(
i.platform.name, i.testcase.name, Fore.YELLOW, i.platform.name, i.testsuite.name, Fore.YELLOW,
"INFO" if all_deltas else "WARNING", Fore.RESET, "INFO" if all_deltas else "WARNING", Fore.RESET,
metric, delta, value, percentage)) metric, delta, value, percentage))
warnings += 1 warnings += 1
@ -3303,7 +3303,7 @@ class TestPlan(DisablePyTestCollectionMixin):
def get_all_tests(self): def get_all_tests(self):
tests = [] tests = []
for _, tc in self.testcases.items(): for _, tc in self.testsuites.items():
for case in tc.cases: for case in tc.cases:
tests.append(case) tests.append(case)
@ -3325,7 +3325,7 @@ class TestPlan(DisablePyTestCollectionMixin):
return toolchain return toolchain
def add_testcases(self, testcase_filter=[]): def add_testsuites(self, testsuite_filter=[]):
for root in self.roots: for root in self.roots:
root = os.path.abspath(root) root = os.path.abspath(root)
@ -3334,8 +3334,8 @@ class TestPlan(DisablePyTestCollectionMixin):
for dirpath, _, filenames in os.walk(root, topdown=True): for dirpath, _, filenames in os.walk(root, topdown=True):
if self.SAMPLE_FILENAME in filenames: if self.SAMPLE_FILENAME in filenames:
filename = self.SAMPLE_FILENAME filename = self.SAMPLE_FILENAME
elif self.TESTCASE_FILENAME in filenames: elif self.TESTSUITE_FILENAME in filenames:
filename = self.TESTCASE_FILENAME filename = self.TESTSUITE_FILENAME
else: else:
continue continue
@ -3351,9 +3351,9 @@ class TestPlan(DisablePyTestCollectionMixin):
workdir = os.path.relpath(tc_path, root) workdir = os.path.relpath(tc_path, root)
for name in parsed_data.tests.keys(): for name in parsed_data.tests.keys():
tc = TestCase(root, workdir, name) tc = TestSuite(root, workdir, name)
tc_dict = parsed_data.get_test(name, self.testcase_valid_keys) tc_dict = parsed_data.get_test(name, self.testsuite_valid_keys)
tc.source_dir = tc_path tc.source_dir = tc_path
tc.yamlfile = tc_path tc.yamlfile = tc_path
@ -3388,16 +3388,16 @@ class TestPlan(DisablePyTestCollectionMixin):
tc.parse_subcases(tc_path) tc.parse_subcases(tc_path)
if testcase_filter: if testsuite_filter:
if tc.name and tc.name in testcase_filter: if tc.name and tc.name in testsuite_filter:
self.testcases[tc.name] = tc self.testsuites[tc.name] = tc
else: else:
self.testcases[tc.name] = tc self.testsuites[tc.name] = tc
except Exception as e: except Exception as e:
logger.error("%s: can't load (skipping): %s" % (tc_path, e)) logger.error("%s: can't load (skipping): %s" % (tc_path, e))
self.load_errors += 1 self.load_errors += 1
return len(self.testcases) return len(self.testsuites)
def get_platform(self, name): def get_platform(self, name):
selected_platform = None selected_platform = None
@ -3446,7 +3446,7 @@ class TestPlan(DisablePyTestCollectionMixin):
platform = self.get_platform(ts["platform"]) platform = self.get_platform(ts["platform"])
if filter_platform and platform.name not in filter_platform: if filter_platform and platform.name not in filter_platform:
continue continue
instance = TestInstance(self.testcases[testsuite], platform, self.outdir) instance = TestInstance(self.testsuites[testsuite], platform, self.outdir)
if ts.get("run_id"): if ts.get("run_id"):
instance.run_id = ts.get("run_id") instance.run_id = ts.get("run_id")
if self.device_testing: if self.device_testing:
@ -3484,7 +3484,7 @@ class TestPlan(DisablePyTestCollectionMixin):
discards = {} discards = {}
platform_filter = kwargs.get('platform') platform_filter = kwargs.get('platform')
exclude_platform = kwargs.get('exclude_platform', []) exclude_platform = kwargs.get('exclude_platform', [])
testcase_filter = kwargs.get('run_individual_tests', []) testsuite_filter = kwargs.get('run_individual_tests', [])
arch_filter = kwargs.get('arch') arch_filter = kwargs.get('arch')
tag_filter = kwargs.get('tag') tag_filter = kwargs.get('tag')
exclude_tag = kwargs.get('exclude_tag') exclude_tag = kwargs.get('exclude_tag')
@ -3526,9 +3526,9 @@ class TestPlan(DisablePyTestCollectionMixin):
else: else:
platforms = self.platforms platforms = self.platforms
logger.info("Building initial testcase list...") logger.info("Building initial testsuite list...")
for tc_name, tc in self.testcases.items(): for tc_name, tc in self.testsuites.items():
if tc.build_on_all and not platform_filter: if tc.build_on_all and not platform_filter:
platform_scope = self.platforms platform_scope = self.platforms
@ -3554,7 +3554,7 @@ class TestPlan(DisablePyTestCollectionMixin):
platform_scope = list(filter(lambda item: item.name in tc.platform_allow, \ platform_scope = list(filter(lambda item: item.name in tc.platform_allow, \
self.platforms)) self.platforms))
# list of instances per testcase, aka configurations. # list of instances per testsuite, aka configurations.
instance_list = [] instance_list = []
for plat in platform_scope: for plat in platform_scope:
instance = TestInstance(tc, plat, self.outdir) instance = TestInstance(tc, plat, self.outdir)
@ -3599,16 +3599,16 @@ class TestPlan(DisablePyTestCollectionMixin):
discards[instance] = discards.get(instance, "Skip filter") discards[instance] = discards.get(instance, "Skip filter")
if tag_filter and not tc.tags.intersection(tag_filter): if tag_filter and not tc.tags.intersection(tag_filter):
discards[instance] = discards.get(instance, "Command line testcase tag filter") discards[instance] = discards.get(instance, "Command line testsuite tag filter")
if exclude_tag and tc.tags.intersection(exclude_tag): if exclude_tag and tc.tags.intersection(exclude_tag):
discards[instance] = discards.get(instance, "Command line testcase exclude filter") discards[instance] = discards.get(instance, "Command line testsuite exclude filter")
if testcase_filter and tc_name not in testcase_filter: if testsuite_filter and tc_name not in testsuite_filter:
discards[instance] = discards.get(instance, "Testcase name filter") discards[instance] = discards.get(instance, "TestSuite name filter")
if arch_filter and plat.arch not in arch_filter: if arch_filter and plat.arch not in arch_filter:
discards[instance] = discards.get(instance, "Command line testcase arch filter") discards[instance] = discards.get(instance, "Command line testsuite arch filter")
if not force_platform: if not force_platform:
@ -3628,10 +3628,10 @@ class TestPlan(DisablePyTestCollectionMixin):
discards[instance] = discards.get(instance, "Command line platform filter") discards[instance] = discards.get(instance, "Command line platform filter")
if tc.platform_allow and plat.name not in tc.platform_allow: if tc.platform_allow and plat.name not in tc.platform_allow:
discards[instance] = discards.get(instance, "Not in testcase platform allow list") discards[instance] = discards.get(instance, "Not in testsuite platform allow list")
if tc.toolchain_allow and toolchain not in tc.toolchain_allow: if tc.toolchain_allow and toolchain not in tc.toolchain_allow:
discards[instance] = discards.get(instance, "Not in testcase toolchain allow list") discards[instance] = discards.get(instance, "Not in testsuite toolchain allow list")
if not plat.env_satisfied: if not plat.env_satisfied:
discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env))) discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env)))
@ -3660,7 +3660,7 @@ class TestPlan(DisablePyTestCollectionMixin):
discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)") discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
test_configuration = ".".join([instance.platform.name, test_configuration = ".".join([instance.platform.name,
instance.testcase.id]) instance.testsuite.id])
# skip quarantined tests # skip quarantined tests
if test_configuration in self.quarantine and not self.quarantine_verify: if test_configuration in self.quarantine and not self.quarantine_verify:
discards[instance] = discards.get(instance, discards[instance] = discards.get(instance,
@ -3673,7 +3673,7 @@ class TestPlan(DisablePyTestCollectionMixin):
# needs to be added. # needs to be added.
instance_list.append(instance) instance_list.append(instance)
# no configurations, so jump to next testcase # no configurations, so jump to next testsuite
if not instance_list: if not instance_list:
continue continue
@ -3713,7 +3713,7 @@ class TestPlan(DisablePyTestCollectionMixin):
for instance in self.discards: for instance in self.discards:
instance.reason = self.discards[instance] instance.reason = self.discards[instance]
# If integration mode is on all skips on integration_platforms are treated as errors. # If integration mode is on all skips on integration_platforms are treated as errors.
if self.integration and instance.platform.name in instance.testcase.integration_platforms \ if self.integration and instance.platform.name in instance.testsuite.integration_platforms \
and "Quarantine" not in instance.reason: and "Quarantine" not in instance.reason:
instance.status = "error" instance.status = "error"
instance.reason += " but is one of the integration platforms" instance.reason += " but is one of the integration platforms"
@ -3951,9 +3951,9 @@ class TestPlan(DisablePyTestCollectionMixin):
suites = json_data.get("testsuites", []) suites = json_data.get("testsuites", [])
# remove existing testcases that were re-run # remove existing testsuites that were re-run
for i in self.instances.values(): for instance in self.instances.values():
suites = list(filter(lambda d: d['name'] != i.testcase.name, suites)) suites = list(filter(lambda d: d['name'] != instance.testsuite.name, suites))
for instance in self.instances.values(): for instance in self.instances.values():
suite = {} suite = {}
@ -3965,7 +3965,7 @@ class TestPlan(DisablePyTestCollectionMixin):
ram_size = instance.metrics.get ("ram_size", 0) ram_size = instance.metrics.get ("ram_size", 0)
rom_size = instance.metrics.get("rom_size",0) rom_size = instance.metrics.get("rom_size",0)
suite = { suite = {
"name": instance.testcase.name, "name": instance.testsuite.name,
"arch": instance.platform.arch, "arch": instance.platform.arch,
"platform": instance.platform.name, "platform": instance.platform.name,
} }
@ -4020,9 +4020,9 @@ class TestPlan(DisablePyTestCollectionMixin):
with open(filename, "wt") as json_file: with open(filename, "wt") as json_file:
json.dump(report, json_file, indent=4, separators=(',',':')) json.dump(report, json_file, indent=4, separators=(',',':'))
def get_testcase(self, identifier): def get_testsuite(self, identifier):
results = [] results = []
for _, tc in self.testcases.items(): for _, tc in self.testsuites.items():
for case in tc.cases: for case in tc.cases:
if case == identifier: if case == identifier:
results.append(tc) results.append(tc)

0
scripts/schemas/twister/testcase-schema.yaml → scripts/schemas/twister/testsuite-schema.yaml
View file

36
scripts/twister
View file

@ -16,19 +16,19 @@ Test cases are detected by the presence of a 'testcase.yaml' or a sample.yaml
files in the application's project directory. This file may contain one or more files in the application's project directory. This file may contain one or more
blocks, each identifying a test scenario. The title of the block is a name for blocks, each identifying a test scenario. The title of the block is a name for
the test case, which only needs to be unique for the test cases specified in the test case, which only needs to be unique for the test cases specified in
that testcase meta-data. The full canonical name for each test case is <path to that testsuite meta-data. The full canonical name for each test case is <path to
test case>/<block>. test case>/<block>.
Each test block in the testcase meta data can define the following key/value Each test block in the testsuite meta data can define the following key/value
pairs: pairs:
tags: <list of tags> (required) tags: <list of tags> (required)
A set of string tags for the testcase. Usually pertains to A set of string tags for the testsuite. Usually pertains to
functional domains but can be anything. Command line invocations functional domains but can be anything. Command line invocations
of this script can filter the set of tests to run based on tag. of this script can filter the set of tests to run based on tag.
skip: <True|False> (default False) skip: <True|False> (default False)
skip testcase unconditionally. This can be used for broken tests. skip testsuite unconditionally. This can be used for broken tests.
slow: <True|False> (default False) slow: <True|False> (default False)
Don't build or run this test case unless --enable-slow was passed Don't build or run this test case unless --enable-slow was passed
@ -88,7 +88,7 @@ pairs:
here. They will not be included in the size calculation. here. They will not be included in the size calculation.
filter: <expression> filter: <expression>
Filter whether the testcase should be run by evaluating an expression Filter whether the testsuite should be run by evaluating an expression
against an environment containing the following values: against an environment containing the following values:
{ ARCH : <architecture>, { ARCH : <architecture>,
@ -151,7 +151,7 @@ pairs:
Would match it. Would match it.
The set of test cases that actually run depends on directives in the testcase The set of test cases that actually run depends on directives in the testsuite
files and options passed in on the command line. If there is any confusion, files and options passed in on the command line. If there is any confusion,
running with -v or examining the test plan report (testplan.json) running with -v or examining the test plan report (testplan.json)
can help show why particular test cases were skipped. can help show why particular test cases were skipped.
@ -269,7 +269,7 @@ Artificially long but functional example:
help="Load list of tests and platforms to be run from file.") help="Load list of tests and platforms to be run from file.")
case_select.add_argument( case_select.add_argument(
"-T", "--testcase-root", action="append", default=[], "-T", "--testsuite-root", action="append", default=[],
help="Base directory to recursively search for test cases. All " help="Base directory to recursively search for test cases. All "
"testcase.yaml files under here will be processed. May be " "testcase.yaml files under here will be processed. May be "
"called multiple times. Defaults to the 'samples/' and " "called multiple times. Defaults to the 'samples/' and "
@ -588,7 +588,7 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
parser.add_argument( parser.add_argument(
"-p", "--platform", action="append", "-p", "--platform", action="append",
help="Platform filter for testing. This option may be used multiple " help="Platform filter for testing. This option may be used multiple "
"times. Testcases will only be built/run on the platforms " "times. Test suites will only be built/run on the platforms "
"specified. If this option is not used, then platforms marked " "specified. If this option is not used, then platforms marked "
"as default in the platform metadata file will be chosen " "as default in the platform metadata file will be chosen "
"to build and test. ") "to build and test. ")
@ -775,8 +775,8 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
logger.error("west-flash requires device-testing to be enabled") logger.error("west-flash requires device-testing to be enabled")
sys.exit(1) sys.exit(1)
if not options.testcase_root: if not options.testsuite_root:
options.testcase_root = [os.path.join(ZEPHYR_BASE, "tests"), options.testsuite_root = [os.path.join(ZEPHYR_BASE, "tests"),
os.path.join(ZEPHYR_BASE, "samples")] os.path.join(ZEPHYR_BASE, "samples")]
if options.show_footprint or options.compare_report: if options.show_footprint or options.compare_report:
@ -891,7 +891,7 @@ def main():
logger.error("You have provided a wrong subset value: %s." % options.subset) logger.error("You have provided a wrong subset value: %s." % options.subset)
return return
tplan = TestPlan(options.board_root, options.testcase_root, options.outdir) tplan = TestPlan(options.board_root, options.testsuite_root, options.outdir)
# Check version of zephyr repo # Check version of zephyr repo
tplan.check_zephyr_version() tplan.check_zephyr_version()
@ -953,7 +953,7 @@ def main():
if options.test: if options.test:
run_individual_tests = options.test run_individual_tests = options.test
num = tplan.add_testcases(testcase_filter=run_individual_tests) num = tplan.add_testsuites(testsuite_filter=run_individual_tests)
if num == 0: if num == 0:
logger.error("No test cases found at the specified location...") logger.error("No test cases found at the specified location...")
sys.exit(1) sys.exit(1)
@ -1000,7 +1000,7 @@ def main():
if options.list_tags: if options.list_tags:
tags = set() tags = set()
for _, tc in tplan.testcases.items(): for _, tc in tplan.testsuites.items():
tags = tags.union(tc.tags) tags = tags.union(tc.tags)
for t in tags: for t in tags:
@ -1020,7 +1020,7 @@ def main():
print("Tests with duplicate identifiers:") print("Tests with duplicate identifiers:")
for dupe in dupes: for dupe in dupes:
print("- {}".format(dupe)) print("- {}".format(dupe))
for dc in tplan.get_testcase(dupe): for dc in tplan.get_testsuite(dupe):
print(" - {}".format(dc)) print(" - {}".format(dc))
else: else:
print("No duplicates found.") print("No duplicates found.")
@ -1141,7 +1141,7 @@ def main():
count = 0 count = 0
for i in inst.values(): for i in inst.values():
for c in i.testcase.cases: for c in i.testsuite.cases:
print(f"- {c}") print(f"- {c}")
count += 1 count += 1
@ -1160,7 +1160,7 @@ def main():
logger.debug( logger.debug(
"{:<25} {:<50} {}SKIPPED{}: {}".format( "{:<25} {:<50} {}SKIPPED{}: {}".format(
i.platform.name, i.platform.name,
i.testcase.name, i.testsuite.name,
Fore.YELLOW, Fore.YELLOW,
Fore.RESET, Fore.RESET,
reason)) reason))
@ -1169,7 +1169,7 @@ def main():
all_tests = tplan.get_all_tests() all_tests = tplan.get_all_tests()
to_be_run = set() to_be_run = set()
for i, p in tplan.instances.items(): for i, p in tplan.instances.items():
to_be_run.update(p.testcase.cases) to_be_run.update(p.testsuite.cases)
if all_tests - to_be_run: if all_tests - to_be_run:
print("Tests that never build or run:") print("Tests that never build or run:")
@ -1233,7 +1233,7 @@ def main():
return return
logger.info("%d test scenarios (%d configurations) selected, %d configurations discarded due to filters." % logger.info("%d test scenarios (%d configurations) selected, %d configurations discarded due to filters." %
(len(tplan.testcases), len(tplan.instances), len(discards))) (len(tplan.testsuites), len(tplan.instances), len(discards)))
if options.device_testing and not options.build_only: if options.device_testing and not options.build_only:
print("\nDevice testing on:") print("\nDevice testing on:")