Merge pull request #1411 from tardyp/t3024

serialize remoteCommand

master/buildbot/process/remotecommand.py

@@ -64,6 +64,7 @@ def __init__(self, remote_command, args, ignore_updates=False,
         self.builder_name = None
         self.commandID = None
         self.deferred = None
+        self.loglock = defer.DeferredLock()
 
     def __repr__(self):
         return "<RemoteCommand '%s' at %d>" % (self.remote_command, id(self))
@@ -211,6 +212,7 @@ def _unwrap(self, log):
             return log.unwrap()
         return log
 
+    @util.deferredLocked('loglock')
     @defer.inlineCallbacks
     def addStdout(self, data):
         if self.collectStdout:
@@ -219,6 +221,7 @@ def addStdout(self, data):
             log_ = yield self._unwrap(self.logs[self.stdioLogName])
             log_.addStdout(data)
 
+    @util.deferredLocked('loglock')
     @defer.inlineCallbacks
     def addStderr(self, data):
         if self.collectStderr:
@@ -227,12 +230,14 @@ def addStderr(self, data):
             log_ = yield self._unwrap(self.logs[self.stdioLogName])
             log_.addStderr(data)
 
+    @util.deferredLocked('loglock')
     @defer.inlineCallbacks
     def addHeader(self, data):
         if self.stdioLogName is not None and self.stdioLogName in self.logs:
             log_ = yield self._unwrap(self.logs[self.stdioLogName])
             log_.addHeader(data)
 
+    @util.deferredLocked('loglock')
     @defer.inlineCallbacks
     def addToLog(self, logname, data):
         # Activate delayed logs on first data.
@@ -283,6 +288,7 @@ def remoteUpdate(self, update):
                     self.updates[k] = []
                 self.updates[k].append(update[k])
 
+    @util.deferredLocked('loglock')
     @defer.inlineCallbacks
     def remoteComplete(self, maybeFailure):
         if self._startTime and self._remoteElapsed:

master/buildbot/test/integration/test_customservices.py

@@ -24,30 +24,10 @@
 
 class CustomServiceMaster(RunMasterBase):
 
-    @defer.inlineCallbacks
-    def doForceBuild(self):
-
-        # force a build, and wait until it is finished
-        d = defer.Deferred()
-        consumer = yield self.master.mq.startConsuming(
-            lambda e, data: d.callback(data),
-            ('builds', None, 'finished'))
-
-        # use data api to force a build
-        yield self.master.data.control("force", {}, ("forceschedulers", "force"))
-
-        # wait until we receive the build finished event
-        build = yield d
-        consumer.stopConsuming()
-
-        # enrich the build result, with the step results
-        build["steps"] = yield self.master.data.get(("builds", build['buildid'], "steps"))
-        defer.returnValue(build)
-
     @defer.inlineCallbacks
     def test_customService(self):
 
-        build = yield self.doForceBuild()
+        build = yield self.doForceBuild(wantSteps=True)
 
         self.assertEqual(build['steps'][0]['state_string'], 'num reconfig: 1')
 
@@ -59,7 +39,7 @@ def test_customService(self):
         # are reconfigured as expected
         yield self.master.reconfig()
 
-        build = yield self.doForceBuild()
+        build = yield self.doForceBuild(wantSteps=True)
 
         self.assertEqual(myService.num_reconfig, 2)
         self.assertEqual(build['steps'][0]['state_string'], 'num reconfig: 2')

master/buildbot/test/integration/test_setproperyfromcommand.py

@@ -0,0 +1,67 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.test.util.integration import RunMasterBase
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.internet import task
+
+# This integration test helps reproduce http://trac.buildbot.net/ticket/3024
+# we make sure that we can reconfigure the master while build is running
+
+
+class SetProperyFromCommand(RunMasterBase):
+
+    @defer.inlineCallbacks
+    def test_setProp(self):
+        oldNewLog = self.master.data.updates.newLog
+
+        @defer.inlineCallbacks
+        def newLog(*arg, **kw):
+            # Simulate db delay. We usually don't test race conditions
+            # with delays, but in integrations test, that would be pretty tricky
+            yield task.deferLater(reactor, .1, lambda: None)
+            res = yield oldNewLog(*arg, **kw)
+            defer.returnValue(res)
+        self.master.data.updates.newLog = newLog
+        build = yield self.doForceBuild(wantProperties=True)
+
+        self.assertEqual(build['properties']['test'], (u'foo', u'SetPropertyFromCommand Step'))
+
+
+# master configuration
+
+num_reconfig = 0
+
+
+def masterConfig():
+    global num_reconfig
+    num_reconfig += 1
+    c = {}
+    from buildbot.plugins import schedulers, steps, util
+
+    c['schedulers'] = [
+        schedulers.ForceScheduler(
+            name="force",
+            builderNames=["testy"])]
+
+    f = util.BuildFactory()
+    f.addStep(steps.SetPropertyFromCommand(property="test", command=["echo", "foo"]))
+    c['builders'] = [
+        util.BuilderConfig(name="testy",
+                           slavenames=["local1"],
+                           factory=f)]
+
+    return c

master/buildbot/test/util/integration.py

@@ -85,3 +85,26 @@ def tearDown(self):
 
         # (trial will verify all reactor-based timers have been cleared, etc.)
         self.tearDownDirs()
+
+    @defer.inlineCallbacks
+    def doForceBuild(self, wantSteps=False, wantProperties=False):
+
+        # force a build, and wait until it is finished
+        d = defer.Deferred()
+        consumer = yield self.master.mq.startConsuming(
+            lambda e, data: d.callback(data),
+            ('builds', None, 'finished'))
+
+        # use data api to force a build
+        yield self.master.data.control("force", {}, ("forceschedulers", "force"))
+
+        # wait until we receive the build finished event
+        build = yield d
+        consumer.stopConsuming()
+
+        # enrich the build result, with the step results
+        if wantSteps:
+            build["steps"] = yield self.master.data.get(("builds", build['buildid'], "steps"))
+        if wantProperties:
+            build["properties"] = yield self.master.data.get(("builds", build['buildid'], "properties"))
+        defer.returnValue(build)