[Merge-Queue] Prioritize oldest builds first

https://bugs.webkit.org/show_bug.cgi?id=239094
<rdar://problem/91595070 >]

Reviewed by Stephanie Lewis.

* Tools/CISupport/ews-build/loadConfig.py:
(loadBuilderConfig):
(async_sort): Added.
(prioritizeBuilders): Prioritize the builder with the oldest request.
* Tools/CISupport/ews-build/loadConfig_unittest.py:
(TestPrioritizeBuilders):
(TestPrioritizeBuilders.MockBuilder):
(TestPrioritizeBuilders.test_builders_over_testers):
(TestPrioritizeBuilders.test_starvation):

Canonical link: https://commits.webkit.org/249569@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@292785 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/CISupport/ews-build/loadConfig.py

@@ -31,6 +31,8 @@
 from buildbot.worker import Worker
 from buildbot.util import identifiers as buildbot_identifiers
 from buildbot.changes.filter import ChangeFilter
+from datetime import datetime, timezone
+from twisted.internet import defer
 
 from factories import (APITestsFactory, BindingsFactory, BuildFactory, CommitQueueFactory, Factory, GTKBuildFactory,
                        GTKTestsFactory, JSCBuildFactory, JSCBuildAndTestsFactory, JSCTestsFactory, MergeQueueFactory, StressTestFactory,
@@ -113,9 +115,33 @@ def filter_fn(change, schedulerName=schedulerName):
     c['schedulers'].append(forceScheduler)
 
 
+# Copied from https://github.com/buildbot/buildbot/blob/master/master/buildbot/util/async_sort.py
+@defer.inlineCallbacks
+def async_sort(l, key, max_parallel=10):
+    sem = defer.DeferredSemaphore(max_parallel)
+    try:
+        keys = yield defer.gatherResults([sem.run(key, i) for i in l])
+    except defer.FirstError as e:
+        raise e.subFailure.value
+
+    keys = {id(l[i]): v for i, v in enumerate(keys)}
+    l.sort(key=lambda x: keys[id(x)])
+
+
 def prioritizeBuilders(buildmaster, builders):
-    # Prioritize builder queues over tester queues
-    builders.sort(key=lambda b: 'build' in b.name.lower(), reverse=True)
+    # Prioritize builder queues over tester queues.
+    # Otherwise, prioritize older requests.
+    # Inspired by https://docs.buildbot.net/latest/manual/customization.html#builder-priority-functions
+    @defer.inlineCallbacks
+    def key(b):
+        request_time = yield b.getOldestRequestTime()
+        return (
+            'build' not in b.name.lower(),
+            bool(b.building) or bool(b.old_building),
+            request_time or datetime.now(timezone.utc),
+        )
+
+    async_sort(builders, key)
     return builders
 
 

Tools/CISupport/ews-build/loadConfig_unittest.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2021 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2022 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -27,6 +27,9 @@
 import os
 import unittest
 
+from datetime import datetime, timedelta, timezone
+from twisted.internet import defer
+
 import loadConfig
 
 
@@ -246,5 +249,41 @@ def test_success(self):
         loadConfig.checkWorkersAndBuildersForConsistency({}, [self.ews101, {'name': 'ews102', 'platform': 'mac-sierra'}], [self.WK2Builder])
 
 
+class TestPrioritizeBuilders(unittest.TestCase):
+    class MockBuilder(object):
+        def __init__(self, name, building=False, oldestRequestTime=None):
+            self.name = name
+            self.building = building
+            self.old_building = False
+            self._oldestRequestTime = oldestRequestTime or datetime.now(timezone.utc)
+
+        def getOldestRequestTime(self):
+            return self._oldestRequestTime
+
+    def test_builders_over_testers(self):
+        builders = [
+            self.MockBuilder('macOS-BigSur-Debug-Build-EWS'),
+            self.MockBuilder('macOS-BigSur-Debug-WK1-Tests-EWS'),
+            self.MockBuilder('macOS-BigSur-Release-Build-EWS'),
+        ]
+        sorted_builders = loadConfig.prioritizeBuilders(None, builders)
+        self.assertEqual(
+            ['macOS-BigSur-Debug-Build-EWS', 'macOS-BigSur-Release-Build-EWS', 'macOS-BigSur-Debug-WK1-Tests-EWS'],
+            [builder.name for builder in sorted_builders],
+        )
+
+    def test_starvation(self):
+        builders = [
+            self.MockBuilder('Commit-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=30)),
+            self.MockBuilder('Merge-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=10)),
+            self.MockBuilder('Unsafe-Merge-Queue', oldestRequestTime=datetime.now(timezone.utc) - timedelta(seconds=60)),
+        ]
+        sorted_builders = loadConfig.prioritizeBuilders(None, builders)
+        self.assertEqual(
+            ['Unsafe-Merge-Queue', 'Commit-Queue', 'Merge-Queue'],
+            [builder.name for builder in sorted_builders],
+        )
+
+
 if __name__ == '__main__':
     unittest.main()

Tools/ChangeLog

@@ -1,3 +1,21 @@
+2022-04-12  Jonathan Bedard  <jbedard@apple.com>
+
+        [Merge-Queue] Prioritize oldest builds first
+        https://bugs.webkit.org/show_bug.cgi?id=239094
+        <rdar://problem/91595070 >]
+
+        Reviewed by Stephanie Lewis.
+
+        * CISupport/ews-build/loadConfig.py:
+        (loadBuilderConfig):
+        (async_sort): Added.
+        (prioritizeBuilders): Prioritize the builder with the oldest request.
+        * CISupport/ews-build/loadConfig_unittest.py:
+        (TestPrioritizeBuilders):
+        (TestPrioritizeBuilders.MockBuilder):
+        (TestPrioritizeBuilders.test_builders_over_testers):
+        (TestPrioritizeBuilders.test_starvation):
+
 2022-04-12  Angelos Oikonomopoulos  <angelos@igalia.com>
 
         [JSC] Report test flakiness to resultsdb