Merge pull request #7204 from p12tic/worker-state-race

worker: Fix race condition in worker state handling

master/buildbot/worker/base.py

@@ -27,6 +27,7 @@
 from buildbot.process.properties import Properties
 from buildbot.util import Notifier
 from buildbot.util import bytes2unicode
+from buildbot.util import deferwaiter
 from buildbot.util import service
 
 
@@ -56,6 +57,10 @@ class AbstractWorker(service.BuildbotService):
     # override to True if isCompatibleWithBuild may return False
     builds_may_be_incompatible = False
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._deferwaiter = deferwaiter.DeferWaiter()
+
     def checkConfig(self, name, password, max_builds=None,
                     notify_on_missing=None,
                     missing_timeout=None,
@@ -365,6 +370,7 @@ def stopService(self):
         yield self.disconnect()
         yield self.waitForCompleteShutdown()
 
+        yield self._deferwaiter.wait()
         yield super().stopService()
 
     def isCompatibleWithBuild(self, build_props):
@@ -664,10 +670,18 @@ def maybeShutdown(self):
         d.addErrback(log.err, 'error while shutting down worker')
 
     def _update_paused(self):
-        self.master.data.updates.set_worker_paused(self.workerid, self._paused, self._pause_reason)
+        self._deferwaiter.add(
+            self.master.data.updates.set_worker_paused(
+                self.workerid,
+                self._paused,
+                self._pause_reason
+            )
+        )
 
     def _update_graceful(self):
-        self.master.data.updates.set_worker_graceful(self.workerid, self._graceful)
+        self._deferwaiter.add(
+            self.master.data.updates.set_worker_graceful(self.workerid, self._graceful)
+        )
 
     def pause(self, reason):
         """Stop running new builds on the worker."""

master/buildbot/worker/latent.py

@@ -28,7 +28,6 @@
 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
 from buildbot.interfaces import LatentWorkerSubstantiatiationCancelled
 from buildbot.util import Notifier
-from buildbot.util import deferwaiter
 from buildbot.worker.base import AbstractWorker
 
 
@@ -162,7 +161,6 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._substantiation_notifier = Notifier()
         self._start_stop_lock = defer.DeferredLock()
-        self._deferwaiter = deferwaiter.DeferWaiter()
         self._check_instance_timer = None
 
     def checkConfig(self, name, password,