twister: Add quarantine feature

Adds feature allowing to use yaml file with dictionaries defining
tests to be quarantined (extra arg "--quarantine-list FILENAME").
The dictionaries are validated according to the proper schema
and loaded.

A flat list is created containing quarantined configurations
(configuration = platform + scenario). Configurations under quarantine
are skipped and get "Quarantine" as a reason in the results reports.

A "comment" can be added to a quarantine entry in the quarantine yaml
with more details (e.g. issue #) and it will be also added to
the report.

The status of tests under quarantine can be verify if
`--quarantine-verify` is used in addition to
"--quarantine-list FILENAME". Using these args will make twister skip
all tests which are not on the quarantine list.

Signed-off-by: Maciej Perkowski <Maciej.Perkowski@nordicsemi.no>

scripts/pylib/twister/twisterlib.py

@@ -2547,6 +2547,9 @@ class TestSuite(DisablePyTestCollectionMixin):
     tc_schema = scl.yaml_load(
         os.path.join(ZEPHYR_BASE,
                      "scripts", "schemas", "twister", "testcase-schema.yaml"))
+    quarantine_schema = scl.yaml_load(
+        os.path.join(ZEPHYR_BASE,
+                     "scripts", "schemas", "twister", "quarantine-schema.yaml"))
 
     testcase_valid_keys = {"tags": {"type": "set", "required": False},
                        "type": {"type": "str", "default": "integration"},
@@ -2609,9 +2612,11 @@ class TestSuite(DisablePyTestCollectionMixin):
         self.generator_cmd = None
         self.warnings_as_errors = True
         self.overflow_as_errors = False
+        self.quarantine_verify = False
 
         # Keep track of which test cases we've filtered out and why
         self.testcases = {}
+        self.quarantine = {}
         self.platforms = []
         self.selected_platforms = []
         self.filtered_platforms = []
@@ -2960,6 +2965,34 @@ class TestSuite(DisablePyTestCollectionMixin):
                 break
         return selected_platform
 
+    def load_quarantine(self, file):
+        """
+        Loads quarantine list from the given yaml file. Creates a dictionary
+        of all tests configurations (platform + scenario: comment) that shall be
+        skipped due to quarantine
+        """
+
+        # Load yaml into quarantine_yaml
+        quarantine_yaml = scl.yaml_load_verify(file, self.quarantine_schema)
+
+        # Create quarantine_list with a product of the listed
+        # platforms and scenarios for each entry in quarantine yaml
+        quarantine_list = []
+        for quar_dict in quarantine_yaml:
+            if quar_dict['platforms'][0] == "all":
+                plat = [p.name for p in self.platforms]
+            else:
+                plat = quar_dict['platforms']
+            comment = quar_dict.get('comment', "NA")
+            quarantine_list.append([{".".join([p, s]): comment}
+                                   for p in plat for s in quar_dict['scenarios']])
+
+        # Flatten the quarantine_list
+        quarantine_list = [it for sublist in quarantine_list for it in sublist]
+        # Change quarantine_list into a dictionary
+        for d in quarantine_list:
+            self.quarantine.update(d)
+
     def load_from_file(self, file, filter_status=[], filter_platform=[]):
         try:
             with open(file, "r") as fp:
@@ -3167,6 +3200,16 @@ class TestSuite(DisablePyTestCollectionMixin):
                 if plat.only_tags and not set(plat.only_tags) & tc.tags:
                     discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
 
+                test_configuration = ".".join([instance.platform.name,
+                                               instance.testcase.id])
+                # skip quarantined tests
+                if test_configuration in self.quarantine and not self.quarantine_verify:
+                    discards[instance] = discards.get(instance,
+                                                      f"Quarantine: {self.quarantine[test_configuration]}")
+                # run only quarantined test to verify their statuses (skip everything else)
+                if self.quarantine_verify and test_configuration not in self.quarantine:
+                    discards[instance] = discards.get(instance, "Not under quarantine")
+
                 # if nothing stopped us until now, it means this configuration
                 # needs to be added.
                 instance_list.append(instance)

scripts/schemas/twister/quarantine-schema.yaml

@@ -0,0 +1,31 @@
+#
+# Schema to validate a YAML file providing the list of configurations
+# under quarantine
+#
+# We load this with pykwalify
+# (http://pykwalify.readthedocs.io/en/unstable/validation-rules.html),
+# a YAML structure validator, to validate the YAML files that provide
+# a list of configurations (scenarios + platforms) under quarantine
+#
+type: seq
+matching: all
+sequence:
+  - type: map
+    required: yes
+    matching: all
+    mapping:
+      "scenarios":
+        type: seq
+        required: true
+        sequence:
+          - type: str
+          - unique: True
+      "platforms":
+        required: true
+        type: seq
+        sequence:
+          - type: str
+          - unique: True
+      "comment":
+        type: str
+        required: false

scripts/twister

@@ -464,6 +464,20 @@ Artificially long but functional example:
         action="store",
         help="Load list of tests and platforms to be run from file.")
 
+    parser.add_argument(
+        "--quarantine-list",
+        metavar="FILENAME",
+        help="Load list of test scenarios under quarantine. The entries in "
+             "the file need to correspond to the test scenarios names as in"
+             "corresponding tests .yaml files. These scenarios"
+             "will be skipped with quarantine as the reason")
+
+    parser.add_argument(
+        "--quarantine-verify",
+        action="store_true",
+        help="Use the list of test scenarios under quarantine and run them"
+             "to verify their current status")
+
     case_select.add_argument(
         "-E",
         "--save-tests",
@@ -1021,6 +1035,15 @@ def main():
     else:
         last_run = os.path.join(options.outdir, "twister.csv")
 
+    if options.quarantine_list:
+        suite.load_quarantine(options.quarantine_list)
+
+    if options.quarantine_verify:
+        if not options.quarantine_list:
+            logger.error("No quarantine list given to be verified")
+            sys.exit(1)
+        suite.quarantine_verify = options.quarantine_verify
+
     if options.only_failed:
         suite.load_from_file(last_run, filter_status=['skipped', 'passed'])
         suite.selected_platforms = set(p.platform.name for p in suite.instances.values())
@@ -1040,9 +1063,6 @@ def main():
         suite.load_from_file(last_run, filter_status=['skipped', 'error'],
                              filter_platform=connected_list)
         suite.selected_platforms = set(p.platform.name for p in suite.instances.values())
-
-
-
     else:
         discards = suite.apply_filters(
             enable_slow=options.enable_slow,