twister: rework how we discard tests
Add a filter type to distinguish between testsuite filters that are maintained in the testcase.yaml file and those provided on the command line. The issue was that when something is excluded deliberately on the command line twister would report an error if that filter would filter out an integration platform. We do not want that, because the filtering is being done by the caller knowing that some tests would be not run, even if those are integration platforms. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif
parent
157869338f
commit
87be2f591f
3 changed files with 81 additions and 69 deletions
|
|
@ -2108,7 +2108,9 @@ class TestInstance(DisablePyTestCollectionMixin):
|
|||
self.platform = platform
|
||||
|
||||
self.status = None
|
||||
self.filters = []
|
||||
self.reason = "Unknown"
|
||||
self.filter_type = None
|
||||
self.metrics = dict()
|
||||
self.handler = None
|
||||
self.outdir = outdir
|
||||
|
|
@ -2136,6 +2138,13 @@ class TestInstance(DisablePyTestCollectionMixin):
|
|||
hash_object.update(random_str)
|
||||
return hash_object.hexdigest()
|
||||
|
||||
def add_filter(self, reason, filter_type):
|
||||
self.filters.append({'type': filter_type, 'reason': reason })
|
||||
self.status = "filtered"
|
||||
self.reason = reason
|
||||
self.filter_type = filter_type
|
||||
|
||||
|
||||
def add_missing_testscases(self, status, reason=None):
|
||||
for case in self.testcases:
|
||||
if not case.status:
|
||||
|
|
@ -2999,6 +3008,17 @@ class ProjectBuilder(FilterBuilder):
|
|||
|
||||
instance.metrics["handler_time"] = instance.execution_time
|
||||
|
||||
class Filters:
|
||||
# filters provided on command line by the user/tester
|
||||
CMD_LINE = 'command line filter'
|
||||
# filters in the testsuite yaml definition
|
||||
TESTSUITE = 'testsuite filter'
|
||||
# filters realted to platform definition
|
||||
PLATFORM = 'Platform related filter'
|
||||
# in case a testcase was quarantined.
|
||||
QUARENTINE = 'Quarantine filter'
|
||||
|
||||
|
||||
class TestPlan(DisablePyTestCollectionMixin):
|
||||
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
|
|
@ -3087,7 +3107,6 @@ class TestPlan(DisablePyTestCollectionMixin):
|
|||
self.filtered_platforms = []
|
||||
self.default_platforms = []
|
||||
self.outdir = os.path.abspath(outdir)
|
||||
self.discards = {}
|
||||
self.load_errors = 0
|
||||
self.instances = dict()
|
||||
|
||||
|
|
@ -3546,7 +3565,6 @@ class TestPlan(DisablePyTestCollectionMixin):
|
|||
|
||||
toolchain = self.get_toolchain()
|
||||
|
||||
discards = {}
|
||||
platform_filter = kwargs.get('platform')
|
||||
exclude_platform = kwargs.get('exclude_platform', [])
|
||||
testsuite_filter = kwargs.get('run_individual_tests', [])
|
||||
|
|
@ -3640,7 +3658,7 @@ class TestPlan(DisablePyTestCollectionMixin):
|
|||
instance.run = True
|
||||
|
||||
if not force_platform and plat.name in exclude_platform:
|
||||
discards[instance] = discards.get(instance, "Platform is excluded on command line.")
|
||||
instance.add_filter("Platform is excluded on command line.", Filters.CMD_LINE)
|
||||
|
||||
if (plat.arch == "unit") != (ts.type == "unit"):
|
||||
# Discard silently
|
||||
|
|
@ -3648,90 +3666,89 @@ class TestPlan(DisablePyTestCollectionMixin):
|
|||
|
||||
if ts.modules and self.modules:
|
||||
if not set(ts.modules).issubset(set(self.modules)):
|
||||
discards[instance] = discards.get(instance, f"one or more required module not available: {','.join(ts.modules)}")
|
||||
instance.add_filter(f"one or more required modules not available: {','.join(ts.modules)}", Filters.TESTSUITE)
|
||||
|
||||
if runnable and not instance.run:
|
||||
discards[instance] = discards.get(instance, "Not runnable on device")
|
||||
instance.add_filter("Not runnable on device", Filters.PLATFORM)
|
||||
|
||||
if self.integration and ts.integration_platforms and plat.name not in ts.integration_platforms:
|
||||
discards[instance] = discards.get(instance, "Not part of integration platforms")
|
||||
instance.add_filter("Not part of integration platforms", Filters.TESTSUITE)
|
||||
|
||||
if ts.skip:
|
||||
discards[instance] = discards.get(instance, "Skip filter")
|
||||
instance.add_filter("Skip filter", Filters.TESTSUITE)
|
||||
|
||||
if tag_filter and not ts.tags.intersection(tag_filter):
|
||||
discards[instance] = discards.get(instance, "Command line testsuite tag filter")
|
||||
instance.add_filter("Command line testsuite tag filter", Filters.CMD_LINE)
|
||||
|
||||
if exclude_tag and ts.tags.intersection(exclude_tag):
|
||||
discards[instance] = discards.get(instance, "Command line testsuite exclude filter")
|
||||
instance.add_filter("Command line testsuite exclude filter", Filters.CMD_LINE)
|
||||
|
||||
if testsuite_filter and ts_name not in testsuite_filter:
|
||||
discards[instance] = discards.get(instance, "TestSuite name filter")
|
||||
instance.add_filter("TestSuite name filter", Filters.CMD_LINE)
|
||||
|
||||
if arch_filter and plat.arch not in arch_filter:
|
||||
discards[instance] = discards.get(instance, "Command line testsuite arch filter")
|
||||
instance.add_filter("Command line testsuite arch filter", Filters.CMD_LINE)
|
||||
|
||||
if not force_platform:
|
||||
|
||||
if ts.arch_allow and plat.arch not in ts.arch_allow:
|
||||
discards[instance] = discards.get(instance, "Not in test case arch allow list")
|
||||
instance.add_filter("Not in test case arch allow list", Filters.TESTSUITE)
|
||||
|
||||
if ts.arch_exclude and plat.arch in ts.arch_exclude:
|
||||
discards[instance] = discards.get(instance, "In test case arch exclude")
|
||||
instance.add_filter("In test case arch exclude", Filters.TESTSUITE)
|
||||
|
||||
if ts.platform_exclude and plat.name in ts.platform_exclude:
|
||||
discards[instance] = discards.get(instance, "In test case platform exclude")
|
||||
instance.add_filter("In test case platform exclude", Filters.TESTSUITE)
|
||||
|
||||
if ts.toolchain_exclude and toolchain in ts.toolchain_exclude:
|
||||
discards[instance] = discards.get(instance, "In test case toolchain exclude")
|
||||
instance.add_filter("In test case toolchain exclude", Filters.TESTSUITE)
|
||||
|
||||
if platform_filter and plat.name not in platform_filter:
|
||||
discards[instance] = discards.get(instance, "Command line platform filter")
|
||||
instance.add_filter("Command line platform filter", Filters.CMD_LINE)
|
||||
|
||||
if ts.platform_allow and plat.name not in ts.platform_allow:
|
||||
discards[instance] = discards.get(instance, "Not in testsuite platform allow list")
|
||||
instance.add_filter("Not in testsuite platform allow list", Filters.TESTSUITE)
|
||||
|
||||
if ts.platform_type and plat.type not in ts.platform_type:
|
||||
discards[instance] = discards.get(instance, "Not in testsuite platform type list")
|
||||
instance.add_filter("Not in testsuite platform type list", Filters.TESTSUITE)
|
||||
|
||||
if ts.toolchain_allow and toolchain not in ts.toolchain_allow:
|
||||
discards[instance] = discards.get(instance, "Not in testsuite toolchain allow list")
|
||||
instance.add_filter("Not in testsuite toolchain allow list", Filters.TESTSUITE)
|
||||
|
||||
if not plat.env_satisfied:
|
||||
discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env)))
|
||||
instance.add_filter("Environment ({}) not satisfied".format(", ".join(plat.env)), Filters.PLATFORM)
|
||||
|
||||
if not force_toolchain \
|
||||
and toolchain and (toolchain not in plat.supported_toolchains) \
|
||||
and "host" not in plat.supported_toolchains \
|
||||
and ts.type != 'unit':
|
||||
discards[instance] = discards.get(instance, "Not supported by the toolchain")
|
||||
instance.add_filter("Not supported by the toolchain", Filters.PLATFORM)
|
||||
|
||||
if plat.ram < ts.min_ram:
|
||||
discards[instance] = discards.get(instance, "Not enough RAM")
|
||||
instance.add_filter("Not enough RAM", Filters.PLATFORM)
|
||||
|
||||
if ts.depends_on:
|
||||
dep_intersection = ts.depends_on.intersection(set(plat.supported))
|
||||
if dep_intersection != set(ts.depends_on):
|
||||
discards[instance] = discards.get(instance, "No hardware support")
|
||||
instance.add_filter("No hardware support", Filters.PLATFORM)
|
||||
|
||||
if plat.flash < ts.min_flash:
|
||||
discards[instance] = discards.get(instance, "Not enough FLASH")
|
||||
instance.add_filter("Not enough FLASH", Filters.PLATFORM)
|
||||
|
||||
if set(plat.ignore_tags) & ts.tags:
|
||||
discards[instance] = discards.get(instance, "Excluded tags per platform (exclude_tags)")
|
||||
instance.add_filter("Excluded tags per platform (exclude_tags)", Filters.PLATFORM)
|
||||
|
||||
if plat.only_tags and not set(plat.only_tags) & ts.tags:
|
||||
discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
|
||||
instance.add_filter("Excluded tags per platform (only_tags)", Filters.PLATFORM)
|
||||
|
||||
test_configuration = ".".join([instance.platform.name,
|
||||
instance.testsuite.id])
|
||||
# skip quarantined tests
|
||||
if test_configuration in self.quarantine and not self.quarantine_verify:
|
||||
discards[instance] = discards.get(instance,
|
||||
f"Quarantine: {self.quarantine[test_configuration]}")
|
||||
instance.add_filter(f"Quarantine: {self.quarantine[test_configuration]}", Filters.QUARENTINE)
|
||||
# run only quarantined test to verify their statuses (skip everything else)
|
||||
if self.quarantine_verify and test_configuration not in self.quarantine:
|
||||
discards[instance] = discards.get(instance, "Not under quarantine")
|
||||
instance.add_filter("Not under quarantine", Filters.QUARENTINE)
|
||||
|
||||
# if nothing stopped us until now, it means this configuration
|
||||
# needs to be added.
|
||||
|
|
@ -3763,41 +3780,33 @@ class TestPlan(DisablePyTestCollectionMixin):
|
|||
elif emulation_platforms:
|
||||
self.add_instances(instance_list)
|
||||
for instance in list(filter(lambda inst: not inst.platform.simulation != 'na', instance_list)):
|
||||
discards[instance] = discards.get(instance, "Not an emulated platform")
|
||||
instance.add_filter("Not an emulated platform", Filters.PLATFORM)
|
||||
else:
|
||||
self.add_instances(instance_list)
|
||||
|
||||
for _, case in self.instances.items():
|
||||
case.create_overlay(case.platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
|
||||
|
||||
self.discards = discards
|
||||
self.selected_platforms = set(p.platform.name for p in self.instances.values())
|
||||
|
||||
remove_from_discards = [] # configurations to be removed from discards.
|
||||
for instance in self.discards:
|
||||
instance.reason = self.discards[instance]
|
||||
filtered_instances = list(filter(lambda item: item.status == "filtered", self.instances.values()))
|
||||
for filtered_instance in filtered_instances:
|
||||
# If integration mode is on all skips on integration_platforms are treated as errors.
|
||||
if self.integration and instance.platform.name in instance.testsuite.integration_platforms \
|
||||
and "Quarantine" not in instance.reason:
|
||||
instance.status = "error"
|
||||
instance.reason += " but is one of the integration platforms"
|
||||
self.instances[instance.name] = instance
|
||||
# Such configuration has to be removed from discards to make sure it won't get skipped
|
||||
remove_from_discards.append(instance)
|
||||
else:
|
||||
instance.status = "filtered"
|
||||
if self.integration and filtered_instance.platform.name in filtered_instance.testsuite.integration_platforms \
|
||||
and "Quarantine" not in filtered_instance.reason:
|
||||
# Do not treat this as error if filter type is command line
|
||||
filters = {t['type'] for t in filtered_instance.filters}
|
||||
if Filters.CMD_LINE in filters:
|
||||
continue
|
||||
filtered_instance.status = "error"
|
||||
filtered_instance.reason += " but is one of the integration platforms"
|
||||
self.instances[filtered_instance.name] = filtered_instance
|
||||
|
||||
instance.add_missing_testscases(instance.status)
|
||||
|
||||
# Remove from discards configurations that must not be discarded
|
||||
# (e.g. integration_platforms when --integration was used)
|
||||
for instance in remove_from_discards:
|
||||
del self.discards[instance]
|
||||
filtered_instance.add_missing_testscases(filtered_instance.status)
|
||||
|
||||
self.filtered_platforms = set(p.platform.name for p in self.instances.values()
|
||||
if p.status != "skipped" )
|
||||
|
||||
|
||||
def add_instances(self, instance_list):
|
||||
for instance in instance_list:
|
||||
self.instances[instance.name] = instance
|
||||
|
|
|
|||
|
|
@ -102,12 +102,11 @@ TESTDATA_PART1 = [
|
|||
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
|
||||
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
|
||||
""" Testing apply_filters function of TestPlan class in Twister
|
||||
Part 1: Response of apply_filters function (discard dictionary) have
|
||||
Part 1: Response of apply_filters function have
|
||||
appropriate values according to the filters
|
||||
"""
|
||||
if tc_attribute is None and plat_attribute is None:
|
||||
class_testplan.apply_filters()
|
||||
assert not class_testplan.discards
|
||||
|
||||
class_testplan.platforms = platforms_list
|
||||
class_testplan.platform_names = [p.name for p in platforms_list]
|
||||
|
|
@ -160,8 +159,9 @@ def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list
|
|||
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
||||
platform=['demo_board_2'])
|
||||
|
||||
for x in [expected_discards]:
|
||||
assert x in class_testplan.discards.values()
|
||||
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
||||
for d in filtered_instances:
|
||||
assert d.reason == expected_discards
|
||||
|
||||
TESTDATA_PART2 = [
|
||||
("runnable", "True", "Not runnable on device"),
|
||||
|
|
@ -193,9 +193,9 @@ def test_apply_filters_part2(class_testplan, all_testsuites_dict,
|
|||
]
|
||||
}
|
||||
class_testplan.apply_filters(**kwargs)
|
||||
assert class_testplan.discards
|
||||
for d in class_testplan.discards.values():
|
||||
assert d == expected_discards
|
||||
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
||||
for d in filtered_instances:
|
||||
assert d.reason == expected_discards
|
||||
|
||||
|
||||
TESTDATA_PART3 = [
|
||||
|
|
@ -223,7 +223,9 @@ def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list
|
|||
testcase.min_flash = tc_min_flash
|
||||
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
||||
platform=['demo_board_2'])
|
||||
assert not class_testplan.discards
|
||||
|
||||
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
||||
assert not filtered_instances
|
||||
|
||||
def test_add_instances(test_data, class_testplan, all_testsuites_dict, platforms_list):
|
||||
""" Testing add_instances() function of TestPlan class in Twister
|
||||
|
|
|
|||
|
|
@ -1164,22 +1164,23 @@ def main():
|
|||
print(f"Tests found: {count}")
|
||||
return
|
||||
|
||||
if VERBOSE > 1 and tplan.discards:
|
||||
if VERBOSE > 1:
|
||||
# if we are using command line platform filter, no need to list every
|
||||
# other platform as excluded, we know that already.
|
||||
# Show only the discards that apply to the selected platforms on the
|
||||
# command line
|
||||
|
||||
for i, reason in tplan.discards.items():
|
||||
if options.platform and i.platform.name not in options.platform:
|
||||
continue
|
||||
logger.debug(
|
||||
"{:<25} {:<50} {}SKIPPED{}: {}".format(
|
||||
i.platform.name,
|
||||
i.testsuite.name,
|
||||
Fore.YELLOW,
|
||||
Fore.RESET,
|
||||
reason))
|
||||
for i in tplan.instances.values():
|
||||
if i.status == "filtered":
|
||||
if options.platform and i.platform.name not in options.platform:
|
||||
continue
|
||||
logger.debug(
|
||||
"{:<25} {:<50} {}SKIPPED{}: {}".format(
|
||||
i.platform.name,
|
||||
i.testsuite.name,
|
||||
Fore.YELLOW,
|
||||
Fore.RESET,
|
||||
i.reason))
|
||||
|
||||
if options.report_excluded:
|
||||
all_tests = tplan.get_all_tests()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue