twister: Add quarantine feature

doc/guides/test/twister.rst

@@ -634,3 +634,40 @@ using an external J-Link probe.  The "probe_id" keyword overrides the
       product: DAPLink CMSIS-DAP
       runner: jlink
       serial: null
+
+Quarantine
+++++++++++
+
+Twister allows using user-defined yaml files defining the list of tests to be put
+under quarantine. Such tests will be skipped and marked accordingly in the output
+reports. This feature is especially useful when running larger test suits, where
+a failure of one test can affect the execution of other tests (e.g. putting the
+physical board in a corrupted state).
+
+To use the quarantine feature one has to add the argument
+``--quarantine-list <PATH_TO_QUARANTINE_YAML>`` to a twister call.
+The current status of tests on the quarantine list can also be verified by adding
+``--quarantine-verify`` to the above argument. This will make twister skip all tests
+which are not on the given list.
+
+A quarantine yaml has to be a sequence of dictionaries. Each dictionary has to have
+"scenarios" and "platforms" entries listing combinations of scenarios and platforms
+to put under quarantine. In addition, an optional entry "comment" can be used, where
+some more details can be given (e.g. link to a reported issue). These comments will also
+be added to the output reports.
+
+An example of entries in a quarantine yaml::
+
+    - scenarios:
+        - sample.basic.helloworld
+      platforms:
+        - all
+      comment: "Link to the issue: https://github.com/zephyrproject-rtos/zephyr/pull/33287"
+
+    - scenarios:
+        - kernel.common
+        - kernel.common.misra
+        - kernel.common.nano64
+      platforms:
+        - qemu_cortex_m3
+        - native_posix

scripts/pylib/twister/twisterlib.py

@@ -2547,6 +2547,9 @@ class TestSuite(DisablePyTestCollectionMixin):
     tc_schema = scl.yaml_load(
         os.path.join(ZEPHYR_BASE,
                      "scripts", "schemas", "twister", "testcase-schema.yaml"))
+    quarantine_schema = scl.yaml_load(
+        os.path.join(ZEPHYR_BASE,
+                     "scripts", "schemas", "twister", "quarantine-schema.yaml"))
 
     testcase_valid_keys = {"tags": {"type": "set", "required": False},
                        "type": {"type": "str", "default": "integration"},
@@ -2609,9 +2612,11 @@ def __init__(self, board_root_list=[], testcase_roots=[], outdir=None):
         self.generator_cmd = None
         self.warnings_as_errors = True
         self.overflow_as_errors = False
+        self.quarantine_verify = False
 
         # Keep track of which test cases we've filtered out and why
         self.testcases = {}
+        self.quarantine = {}
         self.platforms = []
         self.selected_platforms = []
         self.filtered_platforms = []
@@ -2960,6 +2965,34 @@ def get_platform(self, name):
                 break
         return selected_platform
 
+    def load_quarantine(self, file):
+        """
+        Loads quarantine list from the given yaml file. Creates a dictionary
+        of all tests configurations (platform + scenario: comment) that shall be
+        skipped due to quarantine
+        """
+
+        # Load yaml into quarantine_yaml
+        quarantine_yaml = scl.yaml_load_verify(file, self.quarantine_schema)
+
+        # Create quarantine_list with a product of the listed
+        # platforms and scenarios for each entry in quarantine yaml
+        quarantine_list = []
+        for quar_dict in quarantine_yaml:
+            if quar_dict['platforms'][0] == "all":
+                plat = [p.name for p in self.platforms]
+            else:
+                plat = quar_dict['platforms']
+            comment = quar_dict.get('comment', "NA")
+            quarantine_list.append([{".".join([p, s]): comment}
+                                   for p in plat for s in quar_dict['scenarios']])
+
+        # Flatten the quarantine_list
+        quarantine_list = [it for sublist in quarantine_list for it in sublist]
+        # Change quarantine_list into a dictionary
+        for d in quarantine_list:
+            self.quarantine.update(d)
+
     def load_from_file(self, file, filter_status=[], filter_platform=[]):
         try:
             with open(file, "r") as fp:
@@ -3155,6 +3188,16 @@ def apply_filters(self, **kwargs):
                 if plat.only_tags and not set(plat.only_tags) & tc.tags:
                     discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
 
+                test_configuration = ".".join([instance.platform.name,
+                                               instance.testcase.id])
+                # skip quarantined tests
+                if test_configuration in self.quarantine and not self.quarantine_verify:
+                    discards[instance] = discards.get(instance,
+                                                      f"Quarantine: {self.quarantine[test_configuration]}")
+                # run only quarantined test to verify their statuses (skip everything else)
+                if self.quarantine_verify and test_configuration not in self.quarantine:
+                    discards[instance] = discards.get(instance, "Not under quarantine")
+
                 # if nothing stopped us until now, it means this configuration
                 # needs to be added.
                 instance_list.append(instance)

scripts/schemas/twister/quarantine-schema.yaml

@@ -0,0 +1,31 @@
+#
+# Schema to validate a YAML file providing the list of configurations
+# under quarantine
+#
+# We load this with pykwalify
+# (http://pykwalify.readthedocs.io/en/unstable/validation-rules.html),
+# a YAML structure validator, to validate the YAML files that provide
+# a list of configurations (scenarios + platforms) under quarantine
+#
+type: seq
+matching: all
+sequence:
+  - type: map
+    required: yes
+    matching: all
+    mapping:
+      "scenarios":
+        type: seq
+        required: true
+        sequence:
+          - type: str
+          - unique: True
+      "platforms":
+        required: true
+        type: seq
+        sequence:
+          - type: str
+          - unique: True
+      "comment":
+        type: str
+        required: false

scripts/twister

@@ -464,6 +464,20 @@ Artificially long but functional example:
         action="store",
         help="Load list of tests and platforms to be run from file.")
 
+    parser.add_argument(
+        "--quarantine-list",
+        metavar="FILENAME",
+        help="Load list of test scenarios under quarantine. The entries in "
+             "the file need to correspond to the test scenarios names as in"
+             "corresponding tests .yaml files. These scenarios"
+             "will be skipped with quarantine as the reason")
+
+    parser.add_argument(
+        "--quarantine-verify",
+        action="store_true",
+        help="Use the list of test scenarios under quarantine and run them"
+             "to verify their current status")
+
     case_select.add_argument(
         "-E",
         "--save-tests",
@@ -1021,6 +1035,15 @@ def main():
     else:
         last_run = os.path.join(options.outdir, "twister.csv")
 
+    if options.quarantine_list:
+        suite.load_quarantine(options.quarantine_list)
+
+    if options.quarantine_verify:
+        if not options.quarantine_list:
+            logger.error("No quarantine list given to be verified")
+            sys.exit(1)
+        suite.quarantine_verify = options.quarantine_verify
+
     if options.only_failed:
         suite.load_from_file(last_run, filter_status=['skipped', 'passed'])
         suite.selected_platforms = set(p.platform.name for p in suite.instances.values())
@@ -1040,9 +1063,6 @@ def main():
         suite.load_from_file(last_run, filter_status=['skipped', 'error'],
                              filter_platform=connected_list)
         suite.selected_platforms = set(p.platform.name for p in suite.instances.values())
-
-
-
     else:
         discards = suite.apply_filters(
             enable_slow=options.enable_slow,