michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: Add quarantine feature

Browse source 
Adds feature allowing to use yaml file with dictionaries defining
tests to be quarantined (extra arg "--quarantine-list FILENAME").
The dictionaries are validated according to the proper schema
and loaded.

A flat list is created containing quarantined configurations
(configuration = platform + scenario). Configurations under quarantine
are skipped and get "Quarantine" as a reason in the results reports.

A "comment" can be added to a quarantine entry in the quarantine yaml
with more details (e.g. issue #) and it will be also added to
the report.

The status of tests under quarantine can be verify if
`--quarantine-verify` is used in addition to
"--quarantine-list FILENAME". Using these args will make twister skip
all tests which are not on the quarantine list.

Signed-off-by: Maciej Perkowski <Maciej.Perkowski@nordicsemi.no>
This commit is contained in:
Maciej Perkowski 2021-03-11 13:18:33 +01:00 • committed by Anas Nashif
commit 9c6dfce0c0
3 changed files with 97 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

43
scripts/pylib/twister/twisterlib.py
View file

@ -2547,6 +2547,9 @@ class TestSuite(DisablePyTestCollectionMixin):
tc_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "testcase-schema.yaml"))
quarantine_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "quarantine-schema.yaml"))
testcase_valid_keys = {"tags": {"type": "set", "required": False},
"type": {"type": "str", "default": "integration"},
@ -2609,9 +2612,11 @@ class TestSuite(DisablePyTestCollectionMixin):
self.generator_cmd = None
self.warnings_as_errors = True
self.overflow_as_errors = False
self.quarantine_verify = False
# Keep track of which test cases we've filtered out and why
self.testcases = {}
self.quarantine = {}
self.platforms = []
self.selected_platforms = []
self.filtered_platforms = []
@ -2960,6 +2965,34 @@ class TestSuite(DisablePyTestCollectionMixin):
break
return selected_platform
def load_quarantine(self, file):
"""
Loads quarantine list from the given yaml file. Creates a dictionary
of all tests configurations (platform + scenario: comment) that shall be
skipped due to quarantine
"""
# Load yaml into quarantine_yaml
quarantine_yaml = scl.yaml_load_verify(file, self.quarantine_schema)
# Create quarantine_list with a product of the listed
# platforms and scenarios for each entry in quarantine yaml
quarantine_list = []
for quar_dict in quarantine_yaml:
if quar_dict['platforms'][0] == "all":
plat = [p.name for p in self.platforms]
else:
plat = quar_dict['platforms']
comment = quar_dict.get('comment', "NA")
quarantine_list.append([{".".join([p, s]): comment}
for p in plat for s in quar_dict['scenarios']])
# Flatten the quarantine_list
quarantine_list = [it for sublist in quarantine_list for it in sublist]
# Change quarantine_list into a dictionary
for d in quarantine_list:
self.quarantine.update(d)
def load_from_file(self, file, filter_status=[], filter_platform=[]):
try:
with open(file, "r") as fp:
@ -3167,6 +3200,16 @@ class TestSuite(DisablePyTestCollectionMixin):
if plat.only_tags and not set(plat.only_tags) & tc.tags:
discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
test_configuration = ".".join([instance.platform.name,
instance.testcase.id])
# skip quarantined tests
if test_configuration in self.quarantine and not self.quarantine_verify:
discards[instance] = discards.get(instance,
f"Quarantine: {self.quarantine[test_configuration]}")
# run only quarantined test to verify their statuses (skip everything else)
if self.quarantine_verify and test_configuration not in self.quarantine:
discards[instance] = discards.get(instance, "Not under quarantine")
# if nothing stopped us until now, it means this configuration
# needs to be added.
instance_list.append(instance)