Merge branch 'master' into nine

master/buildbot/clients/tryclient.py

@@ -242,7 +242,7 @@ def readPatch(self, res, patchlevel):
         self.patch = (patchlevel, mpatch)
 
     def getPatch(self, res):
-        d = self.dovc(["diff", "-du"])
+        d = self.dovc(["diff"])
         d.addCallback(self.readPatch, self.patchlevel)
         return d
 

master/buildbot/process/botmaster.py

@@ -363,9 +363,7 @@ def stopService(self):
         # quiesce.  First, let the parent stopService succeed between
         # activities; then the loop will stop calling itself, since
         # self.running is false.
-        yield self.activity_lock.acquire()
-        yield service.Service.stopService(self)
-        yield self.activity_lock.release()
+        yield self.activity_lock.run(service.Service.stopService, self)
 
         # now let any outstanding calls to maybeStartBuildsOn to finish, so
         # they don't get interrupted in mid-stride.  This tends to be

master/buildbot/process/buildstep.py

@@ -50,6 +50,7 @@ def __init__(self, remote_command, args, ignore_updates=False,
         self.collectStdout = collectStdout
         self.collectStderr = collectStderr
         self.stdout = ''
+        self.stderr = ''
 
         self._startTime = None
         self._remoteElapsed = None
@@ -690,26 +691,29 @@ def failed(self, why):
         try:
             if self.progress:
                 self.progress.finish()
-            self.addHTMLLog("err.html", formatFailure(why))
-            self.addCompleteLog("err.text", why.getTraceback())
+            try:
+                self.addCompleteLog("err.text", why.getTraceback())
+                self.addHTMLLog("err.html", formatFailure(why))
+            except Exception:
+                log.err(Failure(), "error while formatting exceptions")
+
             # could use why.getDetailedTraceback() for more information
             self.step_status.setText([self.name, "exception"])
             self.step_status.setText2([self.name])
             self.step_status.stepFinished(EXCEPTION)
 
             hidden = self._maybeEvaluate(self.hideStepIf, EXCEPTION, self)
             self.step_status.setHidden(hidden)
-        except:
-            log.msg("exception during failure processing")
-            log.err()
+        except Exception:
+            log.err(Failure(), "exception during failure processing")
             # the progress stuff may still be whacked (the StepStatus may
             # think that it is still running), but the build overall will now
             # finish
+
         try:
             self.releaseLocks()
-        except:
-            log.msg("exception while releasing locks")
-            log.err()
+        except Exception:
+            log.err(Failure(), "exception while releasing locks")
 
         log.msg("BuildStep.failed now firing callback")
         self.deferred.callback(EXCEPTION)

master/buildbot/schedulers/base.py

@@ -202,26 +202,19 @@ def _changeCallback(self, key, msg, fileIsImportant, change_filter,
 
         # use change_consumption_lock to ensure the service does not stop
         # while this change is being processed
-        d = self._change_consumption_lock.acquire()
-        d.addCallback(lambda _ : self.gotChange(change, important))
-        def release(x):
-            self._change_consumption_lock.release()
-        d.addBoth(release)
+        d = self._change_consumption_lock.run(self.gotChange, change, important)
         d.addErrback(log.err, 'while processing change')
 
     def _stopConsumingChanges(self):
         # (note: called automatically in stopService)
 
         # acquire the lock change consumption lock to ensure that any change
         # consumption is complete before we are done stopping consumption
-        d = self._change_consumption_lock.acquire()
-        def stop(x):
+        def stop():
             if self._change_consumer:
                 self._change_consumer.stopConsuming()
                 self._change_consumer = None
-            self._change_consumption_lock.release()
-        d.addBoth(stop)
-        return d
+        return self._change_consumption_lock.run(stop)
 
     def gotChange(self, change, important):
         """

master/buildbot/schedulers/timed.py

@@ -88,15 +88,13 @@ def stopService(self):
         # shut down any pending actuation, and ensure that we wait for any
         # current actuation to complete by acquiring the lock.  This ensures
         # that no build will be scheduled after stopService is complete.
-        d = self.actuationLock.acquire()
-        def stop_actuating(_):
+        def stop_actuating():
             self.actuateOk = False
             self.actuateAt = None
             if self.actuateAtTimer:
                 self.actuateAtTimer.cancel()
             self.actuateAtTimer = None
-        d.addCallback(stop_actuating)
-        d.addCallback(lambda _ : self.actuationLock.release())
+        d = self.actuationLock.run(stop_actuating)
 
         # and chain to the parent class
         d.addCallback(lambda _ : base.BaseScheduler.stopService(self))
@@ -137,14 +135,7 @@ def scheduleNextBuild(self):
 
         @returns: Deferred
         """
-        d = self.actuationLock.acquire()
-        d.addCallback(lambda _ : self._scheduleNextBuild_locked())
-        # always release the lock
-        def release(x):
-            self.actuationLock.release()
-            return x
-        d.addBoth(release)
-        return d
+        return self.actuationLock.run(self._scheduleNextBuild_locked)
 
     ## utilities
 
@@ -181,10 +172,8 @@ def _actuate(self):
         self.actuateAtTimer = None
         self.lastActuated = self.actuateAt
 
-        d = self.actuationLock.acquire()
-
         @defer.inlineCallbacks
-        def set_state_and_start(_):
+        def set_state_and_start():
             # bail out if we shouldn't be actuating anymore
             if not self.actuateOk:
                 return
@@ -198,12 +187,7 @@ def set_state_and_start(_):
 
             # schedule the next build (noting the lock is already held)
             yield self._scheduleNextBuild_locked()
-        d.addCallback(set_state_and_start)
-
-        def unlock(x):
-            self.actuationLock.release()
-            return x
-        d.addBoth(unlock)
+        d = self.actuationLock.run(set_state_and_start)
 
         # this function can't return a deferred, so handle any failures via
         # log.err

master/buildbot/status/status_push.py

@@ -378,10 +378,16 @@ def popChunk(self):
 
         while True:
             items = self.queue.popChunk(chunkSize)
+            newitems = []
+            for item in items:
+                if hasattr(item, 'asDict'):
+                    newitems.append(item.asDict())
+                else:
+                    newitems.append(item)
             if self.debug:
-                packets = json.dumps(items, indent=2, sort_keys=True)
+                packets = json.dumps(newitems, indent=2, sort_keys=True)
             else:
-                packets = json.dumps(items, separators=(',',':'))
+                packets = json.dumps(newitems, separators=(',',':'))
             params = {'packets': packets}
             params.update(self.extra_post_params)
             data = urllib.urlencode(params)

master/buildbot/status/web/files/default.css

@@ -566,6 +566,10 @@ span.header {
 	display: none;
 }
 
+pre {
+	white-space: pre-wrap;
+}
+
 /* change comments (use regular colors here) */
 pre.comments>a:link,pre.comments>a:visited {
 	color: blue;

master/buildbot/steps/python_twisted.py

@@ -191,13 +191,14 @@ class Trial(ShellCommand):
     logfiles = {"test.log": "_trial_temp/test.log"}
     # we use test.log to track Progress at the end of __init__()
 
-    renderables = ['tests']
+    renderables = ['tests', 'jobs']
     flunkOnFailure = True
     python = None
     trial = "trial"
     trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer
     # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead
     trialArgs = []
+    jobs = None
     testpath = UNSPECIFIED # required (but can be None)
     testChanges = False # TODO: needs better name
     recurse = False
@@ -209,7 +210,7 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None,
                  testpath=UNSPECIFIED,
                  tests=None, testChanges=None,
                  recurse=None, randomly=None,
-                 trialMode=None, trialArgs=None,
+                 trialMode=None, trialArgs=None, jobs=None,
                  **kwargs):
         """
         @type  testpath: string
@@ -245,6 +246,11 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None,
         @param trialArgs: a list of arguments to pass to trial, available to
                           turn on any extra flags you like. Defaults to [].
 
+        @type jobs: integer
+        @param jobs: integer to be used as trial -j/--jobs option (for
+                     running tests on several workers).  Only supported
+                     since Twisted-12.3.0.
+
         @type  tests: list of strings
         @param tests: a list of test modules to run, like
                       ['twisted.test.test_defer', 'twisted.test.test_process'].
@@ -310,6 +316,8 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None,
             self.trialMode = trialMode
         if trialArgs is not None:
             self.trialArgs = trialArgs
+        if jobs is not None:
+            self.jobs = jobs
 
         if testpath is not UNSPECIFIED:
             self.testpath = testpath
@@ -361,8 +369,6 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None,
 
         # this counter will feed Progress along the 'test cases' metric
         self.addLogObserver('stdio', TrialTestCaseCounter())
-        # this one just measures bytes of output in _trial_temp/test.log
-        self.addLogObserver('test.log', OutputProgressObserver('test.log'))
 
     def setupEnvironment(self, cmd):
         ShellCommand.setupEnvironment(self, cmd)
@@ -381,6 +387,25 @@ def setupEnvironment(self, cmd):
                 e['PYTHONPATH'] = ppath
 
     def start(self):
+        # choose progressMetrics and logfiles based on whether trial is being
+        # run with multiple workers or not.
+        output_observer = OutputProgressObserver('test.log')
+
+        if self.jobs is not None:
+            self.jobs = int(self.jobs)
+            self.command.append("--jobs=%d" % self.jobs)
+
+            # using -j/--jobs flag produces more than one test log.
+            self.logfiles = {}
+            for i in xrange(self.jobs):
+                self.logfiles['test.%d.log' % i] = '_trial_temp/%d/test.log' % i
+                self.logfiles['err.%d.log' % i] = '_trial_temp/%d/err.log' % i
+                self.logfiles['out.%d.log' % i] = '_trial_temp/%d/out.log' % i
+                self.addLogObserver('test.%d.log' % i, output_observer)
+        else:
+            # this one just measures bytes of output in _trial_temp/test.log
+            self.addLogObserver('test.log', output_observer)
+
         # now that self.build.allFiles() is nailed down, finish building the
         # command
         if self.testChanges:

master/buildbot/steps/vstudio.py

@@ -353,12 +353,18 @@ class VC9(VC8):
 
 VS2008 = VC9
 
-# VC10 doesn't looks like it needs extra stuff.
+# VC10 doesn't look like it needs extra stuff.
 class VC10(VC9):
     default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 10.0'
 
 VS2010 = VC10
 
+# VC11 doesn't look like it needs extra stuff.
+class VC11(VC10):
+    default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 11.0'
+
+VS2012 = VC11
+
 class MsBuild(VisualStudio):
     platform = None
 

master/buildbot/test/integration/test_master.py

@@ -58,6 +58,11 @@ def do_test_master(self):
         self.failIf(mock_reactor.stop.called,
             "startService tried to stop the reactor; check logs")
 
+        # hang out for a fraction of a second, to let startup processes run
+        d = defer.Deferred()
+        reactor.callLater(0.01, d.callback, None)
+        yield d
+
         # stop the service
         yield m.stopService()
 

master/buildbot/test/unit/test_steps_python_twisted.py

@@ -133,4 +133,62 @@ def testProperties(self):
         self.expectOutcome(result=SUCCESS, status_text=['2 tests', 'passed'])
         return self.runStep()
 
+    def test_run_jobs(self):
+        """
+        The C{jobs} kwarg should correspond to trial's -j option (
+        included since Twisted 12.3.0), and make corresponding changes to
+        logfiles.
+        """
+        self.setupStep(python_twisted.Trial(workdir='build',
+                                    tests = 'testname',
+                                    testpath = None,
+                                    jobs=2))
+
+        self.expectCommands(
+            ExpectShell(workdir='build',
+                        command=['trial', '--reporter=bwverbose', '--jobs=2',
+                                 'testname'],
+                        usePTY="slave-config",
+                        logfiles={
+                            'test.0.log': '_trial_temp/0/test.log',
+                            'err.0.log': '_trial_temp/0/err.log',
+                            'out.0.log': '_trial_temp/0/out.log',
+                            'test.1.log': '_trial_temp/1/test.log',
+                            'err.1.log': '_trial_temp/1/err.log',
+                            'out.1.log': '_trial_temp/1/out.log',
+                        })
+            + ExpectShell.log('stdio', stdout="Ran 1 tests\n")
+            + 0
+        )
+        self.expectOutcome(result=SUCCESS, status_text=['1 test', 'passed'])
+        return self.runStep()
+
+    def test_run_jobsProperties(self):
+        """
+        C{jobs} should accept Properties
+        """
+        self.setupStep(python_twisted.Trial(workdir='build',
+                                     tests = 'testname',
+                                     jobs=Property('jobs_count'),
+                                     testpath=None))
+        self.properties.setProperty('jobs_count', '2', 'Test')
+
+        self.expectCommands(
+            ExpectShell(workdir='build',
+                        command=['trial', '--reporter=bwverbose', '--jobs=2',
+                                 'testname'],
+                        usePTY="slave-config",
+                        logfiles={
+                            'test.0.log': '_trial_temp/0/test.log',
+                            'err.0.log': '_trial_temp/0/err.log',
+                            'out.0.log': '_trial_temp/0/out.log',
+                            'test.1.log': '_trial_temp/1/test.log',
+                            'err.1.log': '_trial_temp/1/err.log',
+                            'out.1.log': '_trial_temp/1/out.log',
+                        })
+            + ExpectShell.log('stdio', stdout="Ran 1 tests\n")
+            + 0
+        )
+        self.expectOutcome(result=SUCCESS, status_text=['1 test', 'passed'])
+        return self.runStep()
 

master/buildbot/test/unit/test_steps_vstudio.py

@@ -744,6 +744,30 @@ def test_installdir(self):
         return self.runStep()
 
 
+class TestVC11(VC8ExpectedEnvMixin, steps.BuildStepMixin, unittest.TestCase):
+
+    def setUp(self):
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_installdir(self):
+        self.setupStep(vstudio.VC11(projectfile='pf', config='cfg',
+                                   project='pj'))
+        self.expectCommands(
+            ExpectShell(workdir='wkdir', usePTY='slave-config',
+                        command=['devenv.com', 'pf', '/Rebuild',
+                                 'cfg', '/Project', 'pj' ],
+                        env=self.getExpectedEnv(
+                            r'C:\Program Files\Microsoft Visual Studio 11.0'))
+            + 0
+        )
+        self.expectOutcome(result=SUCCESS,
+                status_text=["compile", "0 projects", "0 files"])
+        return self.runStep()
+
+
 class TestMsBuild(steps.BuildStepMixin, unittest.TestCase):
 
     def setUp(self):
@@ -796,3 +820,5 @@ def test_vs2008(self):
     def test_vs2010(self):
         self.assertIdentical(vstudio.VS2010, vstudio.VC10)
 
+    def test_vs2012(self):
+        self.assertIdentical(vstudio.VS2012, vstudio.VC11)