[run-webkit-tests] Route media tests to a mutually exclusive group

https://bugs.webkit.org/show_bug.cgi?id=273797
rdar://127629496

Reviewed by Sam Sneddon.

We have evidence that media tests can interfere with one another. We should run media
tests in a dedicated worker so that these tests never interfere with other media tests.

* Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
(LayoutTestRunner.run_tests): Use groups as defined by the current port. Dispatch grouped shards
first, so that we aren't waiting on groups at the end of the test run.
(TestShard.__init__): Support empty test list.
* Tools/Scripts/webkitpy/port/base.py:
(Port.sharding_groups): Return all groups and their callback definitions.
(Port.group_for_shard): Given a shard, return the group that shard is part of.
* Tools/Scripts/webkitpy/port/darwin.py:
(DarwinPort.sharding_groups): Add dedicated 'media' and 'mediacapture' groups for all shards
containing a media or mediacapture tests.
* Tools/Scripts/webkitpy/port/darwin_testcase.py:
(DarwinTest.test_lanes):
* Tools/Scripts/webkitpy/port/port_testcase.py:
(PortTestCase.test_lanes):

Canonical link: https://commits.webkit.org/279079@main

Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py

@@ -165,19 +165,38 @@ def run_tests(self, expectations, test_inputs, num_workers, retrying, device_typ
             LayoutTestRunner.instance = self
             with TaskPool(
                 workers=num_workers,
+                mutually_exclusive_groups=list(self._port.sharding_groups().keys()),
                 setup=setup_shard, setupkwargs=dict(
                     port=self._port,
                     devices=devices,
                     results_directory=self._results_directory,
                     retrying=self._retrying,
                 ), teardown=teardown_shard,
-                enter_grace_period=300, exit_grace_period=30
+                enter_grace_period=300, exit_grace_period=30,
             ) as pool:
+                was_sent = set()
+
+                # Dispatch shards from groups first, so we start dedicated groups running before all our other shards
                 for shard in all_shards:
+                    group = self._port.group_for_shard(shard)
+                    if not group:
+                        continue
+
+                    was_sent.add(shard.name)
                     pool.do(
                         run_shard, shard,
                         callback=lambda value: self._annotate_results_with_additional_failures(value),
+                        group=group,
                     )
+
+                for shard in all_shards:
+                    if shard.name in was_sent:
+                        continue
+                    pool.do(
+                        run_shard, shard,
+                        callback=lambda value: self._annotate_results_with_additional_failures(value),
+                    )
+
                 pool.wait()
 
         except TestRunInterruptedException as e:
@@ -510,7 +529,7 @@ class TestShard(object):
     def __init__(self, name, test_inputs):
         self.name = name
         self.test_inputs = test_inputs
-        self.needs_servers = test_inputs[0].needs_servers
+        self.needs_servers = test_inputs[0].needs_servers if len(test_inputs) > 0 else False
 
     def shorten(self, string):
         if not string:

Tools/Scripts/webkitpy/port/base.py

@@ -261,6 +261,15 @@ def all_baseline_search_paths(self, device_type=None):
 
         return paths
 
+    def sharding_groups(self):
+        return {}
+
+    def group_for_shard(self, shard):
+        for group, filter_fn in self.sharding_groups().items():
+            if filter_fn(shard):
+                return group
+        return None
+
     def check_build(self):
         """This routine is used to ensure that the build is up to date
         and all the needed binaries are present."""

Tools/Scripts/webkitpy/port/darwin.py

@@ -53,6 +53,11 @@ def __init__(self, host, port_name, **kwargs):
             # with MallocStackLogging enabled.
             self.set_option_default("batch_size", 1000)
 
+    def sharding_groups(self):
+        return {
+            'media': lambda shard: 'media' in shard.name or 'webaudio' in shard.name,
+        }
+
     def default_timeout_ms(self):
         if self.get_option('guard_malloc'):
             return 350 * 1000

Tools/Scripts/webkitpy/port/darwin_testcase.py

@@ -30,6 +30,7 @@
 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2, MockProcess, ScriptError
 from webkitpy.common.system.systemhost_mock import MockSystemHost
 from webkitpy.common.version_name_map import VersionNameMap
+from webkitpy.layout_tests.controllers.layout_test_runner import TestShard
 
 from webkitcorepy import OutputCapture
 
@@ -40,6 +41,17 @@ def assert_skipped_file_search_paths(self, port_name, expected_paths, use_webkit
         port = self.make_port(port_name=port_name, options=MockOptions(webkit_test_runner=use_webkit2))
         self.assertEqual(port._skipped_file_search_paths(), expected_paths)
 
+    def test_sharding_groups(self):
+        port = self.make_port()
+        self.assertEqual(sorted(port.sharding_groups().keys()), ['media'])
+        self.assertEqual('media', port.group_for_shard(TestShard('media/something', [])))
+        self.assertEqual('media', port.group_for_shard(TestShard('webaudio/something', [])))
+        self.assertEqual('media', port.group_for_shard(TestShard('fast/media/something', [])))
+        self.assertEqual('media', port.group_for_shard(TestShard('media-session/something', [])))
+        self.assertEqual('media', port.group_for_shard(TestShard('imported/mediacapture-fromelement/something', [])))
+        self.assertEqual('media', port.group_for_shard(TestShard('something/media', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('fast/something', [])))
+
     def test_default_timeout_ms(self):
         super(DarwinTest, self).test_default_timeout_ms()
         self.assertEqual(self.make_port(options=MockOptions(guard_malloc=True)).default_timeout_ms(), 350000)

Tools/Scripts/webkitpy/port/port_testcase.py

@@ -48,6 +48,7 @@
 from webkitpy.port.driver import DriverOutput
 from webkitpy.port.image_diff import ImageDiffer, ImageDiffResult
 from webkitpy.port.server_process_mock import MockServerProcess
+from webkitpy.layout_tests.controllers.layout_test_runner import TestShard
 from webkitpy.layout_tests.servers import http_server_base
 from webkitpy.tool.mocktool import MockOptions
 
@@ -127,6 +128,17 @@ def make_port(self, host=None, port_name=None, options=None, os_name=None, os_ve
         port._config.build_directory = lambda configuration, for_host=False: '/mock-build'
         return port
 
+    def test_sharding_groups(self):
+        port = self.make_port()
+        self.assertEqual(sorted(port.sharding_groups().keys()), [])
+        self.assertEqual(None, port.group_for_shard(TestShard('media/something', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('webaudio/something', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('fast/media/something', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('media-session/something', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('imported/mediacapture-fromelement/something', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('something/media', [])))
+        self.assertEqual(None, port.group_for_shard(TestShard('fast/something', [])))
+
     def test_default_timeout_ms(self):
         self.assertEqual(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 30000)
         self.assertEqual(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 30000)