twister: rework how we discard tests

Add a filter type to distinguish between testsuite filters that are
maintained in the testcase.yaml file and those provided on the command
line.

The issue was that when something is excluded deliberately on the command
line twister would report an error if that filter would filter out an
integration platform. We do not want that, because the filtering is
being done by the caller knowing that some tests would be not run, even
if those are integration platforms.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-05-14 09:56:47 -04:00
commit 87be2f591f
3 changed files with 81 additions and 69 deletions

View file

@ -2108,7 +2108,9 @@ class TestInstance(DisablePyTestCollectionMixin):
self.platform = platform
self.status = None
self.filters = []
self.reason = "Unknown"
self.filter_type = None
self.metrics = dict()
self.handler = None
self.outdir = outdir
@ -2136,6 +2138,13 @@ class TestInstance(DisablePyTestCollectionMixin):
hash_object.update(random_str)
return hash_object.hexdigest()
def add_filter(self, reason, filter_type):
self.filters.append({'type': filter_type, 'reason': reason })
self.status = "filtered"
self.reason = reason
self.filter_type = filter_type
def add_missing_testscases(self, status, reason=None):
for case in self.testcases:
if not case.status:
@ -2999,6 +3008,17 @@ class ProjectBuilder(FilterBuilder):
instance.metrics["handler_time"] = instance.execution_time
class Filters:
# filters provided on command line by the user/tester
CMD_LINE = 'command line filter'
# filters in the testsuite yaml definition
TESTSUITE = 'testsuite filter'
# filters realted to platform definition
PLATFORM = 'Platform related filter'
# in case a testcase was quarantined.
QUARENTINE = 'Quarantine filter'
class TestPlan(DisablePyTestCollectionMixin):
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
@ -3087,7 +3107,6 @@ class TestPlan(DisablePyTestCollectionMixin):
self.filtered_platforms = []
self.default_platforms = []
self.outdir = os.path.abspath(outdir)
self.discards = {}
self.load_errors = 0
self.instances = dict()
@ -3546,7 +3565,6 @@ class TestPlan(DisablePyTestCollectionMixin):
toolchain = self.get_toolchain()
discards = {}
platform_filter = kwargs.get('platform')
exclude_platform = kwargs.get('exclude_platform', [])
testsuite_filter = kwargs.get('run_individual_tests', [])
@ -3640,7 +3658,7 @@ class TestPlan(DisablePyTestCollectionMixin):
instance.run = True
if not force_platform and plat.name in exclude_platform:
discards[instance] = discards.get(instance, "Platform is excluded on command line.")
instance.add_filter("Platform is excluded on command line.", Filters.CMD_LINE)
if (plat.arch == "unit") != (ts.type == "unit"):
# Discard silently
@ -3648,90 +3666,89 @@ class TestPlan(DisablePyTestCollectionMixin):
if ts.modules and self.modules:
if not set(ts.modules).issubset(set(self.modules)):
discards[instance] = discards.get(instance, f"one or more required module not available: {','.join(ts.modules)}")
instance.add_filter(f"one or more required modules not available: {','.join(ts.modules)}", Filters.TESTSUITE)
if runnable and not instance.run:
discards[instance] = discards.get(instance, "Not runnable on device")
instance.add_filter("Not runnable on device", Filters.PLATFORM)
if self.integration and ts.integration_platforms and plat.name not in ts.integration_platforms:
discards[instance] = discards.get(instance, "Not part of integration platforms")
instance.add_filter("Not part of integration platforms", Filters.TESTSUITE)
if ts.skip:
discards[instance] = discards.get(instance, "Skip filter")
instance.add_filter("Skip filter", Filters.TESTSUITE)
if tag_filter and not ts.tags.intersection(tag_filter):
discards[instance] = discards.get(instance, "Command line testsuite tag filter")
instance.add_filter("Command line testsuite tag filter", Filters.CMD_LINE)
if exclude_tag and ts.tags.intersection(exclude_tag):
discards[instance] = discards.get(instance, "Command line testsuite exclude filter")
instance.add_filter("Command line testsuite exclude filter", Filters.CMD_LINE)
if testsuite_filter and ts_name not in testsuite_filter:
discards[instance] = discards.get(instance, "TestSuite name filter")
instance.add_filter("TestSuite name filter", Filters.CMD_LINE)
if arch_filter and plat.arch not in arch_filter:
discards[instance] = discards.get(instance, "Command line testsuite arch filter")
instance.add_filter("Command line testsuite arch filter", Filters.CMD_LINE)
if not force_platform:
if ts.arch_allow and plat.arch not in ts.arch_allow:
discards[instance] = discards.get(instance, "Not in test case arch allow list")
instance.add_filter("Not in test case arch allow list", Filters.TESTSUITE)
if ts.arch_exclude and plat.arch in ts.arch_exclude:
discards[instance] = discards.get(instance, "In test case arch exclude")
instance.add_filter("In test case arch exclude", Filters.TESTSUITE)
if ts.platform_exclude and plat.name in ts.platform_exclude:
discards[instance] = discards.get(instance, "In test case platform exclude")
instance.add_filter("In test case platform exclude", Filters.TESTSUITE)
if ts.toolchain_exclude and toolchain in ts.toolchain_exclude:
discards[instance] = discards.get(instance, "In test case toolchain exclude")
instance.add_filter("In test case toolchain exclude", Filters.TESTSUITE)
if platform_filter and plat.name not in platform_filter:
discards[instance] = discards.get(instance, "Command line platform filter")
instance.add_filter("Command line platform filter", Filters.CMD_LINE)
if ts.platform_allow and plat.name not in ts.platform_allow:
discards[instance] = discards.get(instance, "Not in testsuite platform allow list")
instance.add_filter("Not in testsuite platform allow list", Filters.TESTSUITE)
if ts.platform_type and plat.type not in ts.platform_type:
discards[instance] = discards.get(instance, "Not in testsuite platform type list")
instance.add_filter("Not in testsuite platform type list", Filters.TESTSUITE)
if ts.toolchain_allow and toolchain not in ts.toolchain_allow:
discards[instance] = discards.get(instance, "Not in testsuite toolchain allow list")
instance.add_filter("Not in testsuite toolchain allow list", Filters.TESTSUITE)
if not plat.env_satisfied:
discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env)))
instance.add_filter("Environment ({}) not satisfied".format(", ".join(plat.env)), Filters.PLATFORM)
if not force_toolchain \
and toolchain and (toolchain not in plat.supported_toolchains) \
and "host" not in plat.supported_toolchains \
and ts.type != 'unit':
discards[instance] = discards.get(instance, "Not supported by the toolchain")
instance.add_filter("Not supported by the toolchain", Filters.PLATFORM)
if plat.ram < ts.min_ram:
discards[instance] = discards.get(instance, "Not enough RAM")
instance.add_filter("Not enough RAM", Filters.PLATFORM)
if ts.depends_on:
dep_intersection = ts.depends_on.intersection(set(plat.supported))
if dep_intersection != set(ts.depends_on):
discards[instance] = discards.get(instance, "No hardware support")
instance.add_filter("No hardware support", Filters.PLATFORM)
if plat.flash < ts.min_flash:
discards[instance] = discards.get(instance, "Not enough FLASH")
instance.add_filter("Not enough FLASH", Filters.PLATFORM)
if set(plat.ignore_tags) & ts.tags:
discards[instance] = discards.get(instance, "Excluded tags per platform (exclude_tags)")
instance.add_filter("Excluded tags per platform (exclude_tags)", Filters.PLATFORM)
if plat.only_tags and not set(plat.only_tags) & ts.tags:
discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
instance.add_filter("Excluded tags per platform (only_tags)", Filters.PLATFORM)
test_configuration = ".".join([instance.platform.name,
instance.testsuite.id])
# skip quarantined tests
if test_configuration in self.quarantine and not self.quarantine_verify:
discards[instance] = discards.get(instance,
f"Quarantine: {self.quarantine[test_configuration]}")
instance.add_filter(f"Quarantine: {self.quarantine[test_configuration]}", Filters.QUARENTINE)
# run only quarantined test to verify their statuses (skip everything else)
if self.quarantine_verify and test_configuration not in self.quarantine:
discards[instance] = discards.get(instance, "Not under quarantine")
instance.add_filter("Not under quarantine", Filters.QUARENTINE)
# if nothing stopped us until now, it means this configuration
# needs to be added.
@ -3763,41 +3780,33 @@ class TestPlan(DisablePyTestCollectionMixin):
elif emulation_platforms:
self.add_instances(instance_list)
for instance in list(filter(lambda inst: not inst.platform.simulation != 'na', instance_list)):
discards[instance] = discards.get(instance, "Not an emulated platform")
instance.add_filter("Not an emulated platform", Filters.PLATFORM)
else:
self.add_instances(instance_list)
for _, case in self.instances.items():
case.create_overlay(case.platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
self.discards = discards
self.selected_platforms = set(p.platform.name for p in self.instances.values())
remove_from_discards = [] # configurations to be removed from discards.
for instance in self.discards:
instance.reason = self.discards[instance]
filtered_instances = list(filter(lambda item: item.status == "filtered", self.instances.values()))
for filtered_instance in filtered_instances:
# If integration mode is on all skips on integration_platforms are treated as errors.
if self.integration and instance.platform.name in instance.testsuite.integration_platforms \
and "Quarantine" not in instance.reason:
instance.status = "error"
instance.reason += " but is one of the integration platforms"
self.instances[instance.name] = instance
# Such configuration has to be removed from discards to make sure it won't get skipped
remove_from_discards.append(instance)
else:
instance.status = "filtered"
if self.integration and filtered_instance.platform.name in filtered_instance.testsuite.integration_platforms \
and "Quarantine" not in filtered_instance.reason:
# Do not treat this as error if filter type is command line
filters = {t['type'] for t in filtered_instance.filters}
if Filters.CMD_LINE in filters:
continue
filtered_instance.status = "error"
filtered_instance.reason += " but is one of the integration platforms"
self.instances[filtered_instance.name] = filtered_instance
instance.add_missing_testscases(instance.status)
# Remove from discards configurations that must not be discarded
# (e.g. integration_platforms when --integration was used)
for instance in remove_from_discards:
del self.discards[instance]
filtered_instance.add_missing_testscases(filtered_instance.status)
self.filtered_platforms = set(p.platform.name for p in self.instances.values()
if p.status != "skipped" )
def add_instances(self, instance_list):
for instance in instance_list:
self.instances[instance.name] = instance

View file

@ -102,12 +102,11 @@ TESTDATA_PART1 = [
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
""" Testing apply_filters function of TestPlan class in Twister
Part 1: Response of apply_filters function (discard dictionary) have
Part 1: Response of apply_filters function have
appropriate values according to the filters
"""
if tc_attribute is None and plat_attribute is None:
class_testplan.apply_filters()
assert not class_testplan.discards
class_testplan.platforms = platforms_list
class_testplan.platform_names = [p.name for p in platforms_list]
@ -160,8 +159,9 @@ def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
platform=['demo_board_2'])
for x in [expected_discards]:
assert x in class_testplan.discards.values()
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
for d in filtered_instances:
assert d.reason == expected_discards
TESTDATA_PART2 = [
("runnable", "True", "Not runnable on device"),
@ -193,9 +193,9 @@ def test_apply_filters_part2(class_testplan, all_testsuites_dict,
]
}
class_testplan.apply_filters(**kwargs)
assert class_testplan.discards
for d in class_testplan.discards.values():
assert d == expected_discards
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
for d in filtered_instances:
assert d.reason == expected_discards
TESTDATA_PART3 = [
@ -223,7 +223,9 @@ def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list
testcase.min_flash = tc_min_flash
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
platform=['demo_board_2'])
assert not class_testplan.discards
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
assert not filtered_instances
def test_add_instances(test_data, class_testplan, all_testsuites_dict, platforms_list):
""" Testing add_instances() function of TestPlan class in Twister

View file

@ -1164,22 +1164,23 @@ def main():
print(f"Tests found: {count}")
return
if VERBOSE > 1 and tplan.discards:
if VERBOSE > 1:
# if we are using command line platform filter, no need to list every
# other platform as excluded, we know that already.
# Show only the discards that apply to the selected platforms on the
# command line
for i, reason in tplan.discards.items():
if options.platform and i.platform.name not in options.platform:
continue
logger.debug(
"{:<25} {:<50} {}SKIPPED{}: {}".format(
i.platform.name,
i.testsuite.name,
Fore.YELLOW,
Fore.RESET,
reason))
for i in tplan.instances.values():
if i.status == "filtered":
if options.platform and i.platform.name not in options.platform:
continue
logger.debug(
"{:<25} {:<50} {}SKIPPED{}: {}".format(
i.platform.name,
i.testsuite.name,
Fore.YELLOW,
Fore.RESET,
i.reason))
if options.report_excluded:
all_tests = tplan.get_all_tests()