Merge branch 'buck-buildrequest-waited_for' of git://github.com/bukzo…

…r/buildbot into nine

(and a bunch of other stuff from master)

master/buildbot/data/masters.py

@@ -108,7 +108,7 @@ def expireMasters(self, _reactor):
         too_old = epoch2datetime(_reactor.seconds() - 60*EXPIRE_MINUTES)
         masters = yield self.master.db.masters.getMasters()
         for m in masters:
-            if m['last_active'] >= too_old:
+            if m['last_active'] is not None and m['last_active'] >= too_old:
                 continue
 
             # mark the master inactive, and send a message on its behalf

master/buildbot/db/buildrequests.py

@@ -265,4 +265,5 @@ def mkdt(epoch):
                 buildername=row.buildername, priority=row.priority,
                 claimed=claimed, claimed_at=claimed_at, mine=mine,
                 complete=bool(row.complete), results=row.results,
-                submitted_at=submitted_at, complete_at=complete_at)
+                submitted_at=submitted_at, complete_at=complete_at,
+				waited_for=bool(row.waited_for))

master/buildbot/db/migrate/versions/033_buildrequests_waited_for.py

@@ -0,0 +1,12 @@
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    buildrequests_table = sa.Table('buildrequests', metadata, autoload=True)
+
+	# boolean indicating whether there is a step blocking, waiting for this request to complete
+    waited_for = sa.Column('waited_for', sa.SmallInteger,
+        server_default=sa.DefaultClause("0"))
+    waited_for.create(buildrequests_table)

master/buildbot/db/model.py

@@ -45,7 +45,7 @@ class Model(base.DBConnectorComponent):
     # * dates are stored as unix timestamps (UTC-ish epoch time)
     #
     # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres;
-    #   use sa.Integer instead
+    #   use sa.SmallInteger instead
 
     # build requests
 
@@ -73,6 +73,10 @@ class Model(base.DBConnectorComponent):
 
         # time the buildrequest was completed, or NULL
         sa.Column('complete_at', sa.Integer),
+
+        # boolean indicating whether there is a step blocking, waiting for this request to complete
+        sa.Column('waited_for', sa.SmallInteger,
+            server_default=sa.DefaultClause("0")),
     )
 
     # Each row in this table represents a claimed build request, where the

master/buildbot/process/buildrequest.py

@@ -105,6 +105,7 @@ def _make_br(cls, brid, brdict, master):
         dt = brdict['submitted_at']
         buildrequest.submittedAt = dt and calendar.timegm(dt.utctimetuple())
         buildrequest.master = master
+        buildrequest.waitedFor = brdict['waited_for']
 
         # fetch the buildset to get the reason
         buildset = yield master.db.buildsets.getBuildset(brdict['buildsetid'])

master/buildbot/process/buildrequestdistributor.py

@@ -245,6 +245,8 @@ def _getNextUnclaimedBuildRequest(self):
                 if nextBreq not in breqs:
                     nextBreq = None
             except Exception:
+                log.err(Failure(),
+                        "from _getNextUnclaimedBuildRequest for builder '%s'" % (self.bldr,))
                 nextBreq = None
         else:
             # otherwise just return the first build

master/buildbot/process/buildstep.py

@@ -366,6 +366,8 @@ def __init__(self, workdir, command, env=None,
                  user=None):
 
         self.command = command # stash .command, set it later
+        self.fake_command = [w[2] if (isinstance(w, tuple) and len(w) == 3 and w[0] =='obfuscated')
+                             else w for w in self.command]
         if env is not None:
             # avoid mutating the original master.cfg dictionary. Each
             # ShellCommand gets its own copy, any start() methods won't be
@@ -401,13 +403,13 @@ def _start(self):
                 self.step.slaveVersionIsOlderThan("shell", "2.16")):
             m = "slave does not support the 'user' parameter"
             raise BuildSlaveTooOldError(m)
-        what = "command '%s' in dir '%s'" % (self.args['command'],
+        what = "command '%s' in dir '%s'" % (self.fake_command,
                                              self.args['workdir'])
         log.msg(what)
         return RemoteCommand._start(self)
 
     def __repr__(self):
-        return "<RemoteShellCommand '%s'>" % repr(self.command)
+        return "<RemoteShellCommand '%s'>" % repr(self.fake_command)
 
 class _BuildStepFactory(util.ComparableMixin):
     """

master/buildbot/steps/python.py

@@ -71,7 +71,10 @@ class PyFlakes(ShellCommand):
     description = ["running", "pyflakes"]
     descriptionDone = ["pyflakes"]
     flunkOnFailure = False
-    flunkingIssues = ["undefined"] # any pyflakes lines like this cause FAILURE
+    # any pyflakes lines like this cause FAILURE
+    flunkingIssues = ["undefined"]
+    # we need a separate variable for syntax errors
+    hasSyntaxError = False
 
     MESSAGES = ("unused", "undefined", "redefs", "import*", "misc")
 
@@ -110,22 +113,32 @@ def createSummary(self, log):
                 m = "undefined"
             elif line.find("redefinition of unused") != -1:
                 m = "redefs"
+            elif line.find("invalid syntax") != -1:
+                self.hasSyntaxError = True
+                # we can do this, because if a syntax error occurs
+                # the output will only contain the info about it, nothing else
+                m = "misc"
             else:
                 m = "misc"
             summaries[m].append(line)
             counts[m] += 1
 
         self.descriptionDone = self.descriptionDone[:]
-        for m in self.MESSAGES:
-            if counts[m]:
-                self.descriptionDone.append("%s=%d" % (m, counts[m]))
-                self.addCompleteLog(m, "".join(summaries[m]))
-            self.setProperty("pyflakes-%s" % m, counts[m], "pyflakes")
-        self.setProperty("pyflakes-total", sum(counts.values()), "pyflakes")
 
+        # we log 'misc' as syntax-error
+        if self.hasSyntaxError:
+            self.addCompleteLog("syntax-error", "".join(summaries['misc']))
+        else:
+            for m in self.MESSAGES:
+                if counts[m]:
+                    self.descriptionDone.append("%s=%d" % (m, counts[m]))
+                    self.addCompleteLog(m, "".join(summaries[m]))
+                self.setProperty("pyflakes-%s" % m, counts[m], "pyflakes")
+            self.setProperty("pyflakes-total", sum(counts.values()),
+                             "pyflakes")
 
     def evaluateCommand(self, cmd):
-        if cmd.didFail():
+        if cmd.didFail() or self.hasSyntaxError:
             return FAILURE
         for m in self.flunkingIssues:
             if self.getProperty("pyflakes-%s" % m):

master/buildbot/steps/source/base.py

@@ -14,10 +14,15 @@
 # Copyright Buildbot Team Members
 
 
+import os
+import StringIO
+
 from twisted.python import log
 from buildbot.process.buildstep import LoggingBuildStep
 from buildbot.status.builder import SKIPPED, FAILURE
 from buildbot.steps.slave import CompositeStepMixin
+from buildbot.steps.transfer import _FileReader
+from buildbot.process import buildstep
 
 class Source(LoggingBuildStep, CompositeStepMixin):
     """This is a base class to generate a source tree in the buildslave.
@@ -104,6 +109,7 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
         self.logEnviron = logEnviron
         self.env = env
         self.timeout = timeout
+        self.retry = retry
 
         descriptions_for_mode = {
             "clobber": "checkout",
@@ -179,6 +185,76 @@ def computeSourceRevision(self, changes):
         self.checkoutDelay value."""
         return None
 
+    def applyPatch(self, patchlevel):
+        patch_command = ['patch', '-p%s' % patchlevel, '--remove-empty-files',
+                         '--force', '--forward', '-i', '.buildbot-diff']
+        cmd = buildstep.RemoteShellCommand(self.workdir,
+                                           patch_command,
+                                           env=self.env,
+                                           logEnviron=self.logEnviron)
+
+        cmd.useLog(self.stdio_log, False)
+        d = self.runCommand(cmd)
+        def evaluateCommand(_):
+            if cmd.didFail():
+                raise buildstep.BuildStepFailed()
+            return cmd.rc
+
+        d.addCallback(evaluateCommand)
+        return d
+
+    def patch(self, _, patch):
+        patchlevel = patch[0]
+        diff = patch[1]
+        root = None
+        if len(patch) >= 3:
+            root = patch[2]
+
+        if (root and
+            os.path.abspath(os.path.join(self.workdir, root)
+                            ).startswith(os.path.abspath(self.workdir))):
+            self.workdir = os.path.join(self.workdir, root)
+
+        def _downloadFile(buf, filename):
+            filereader = _FileReader(StringIO.StringIO(buf))
+            args = {
+                'slavedest': filename,
+                'maxsize': None,
+                'reader': filereader,
+                'blocksize': 16*1024,
+                'workdir': self.workdir,
+                'mode' : None
+                }
+            cmd = buildstep.RemoteCommand('downloadFile', args)
+            cmd.useLog(self.stdio_log, False)
+            log.msg("Downloading file: %s" % (filename))
+            d = self.runCommand(cmd)
+            def evaluateCommand(_):
+                if cmd.didFail():
+                    raise buildstep.BuildStepFailed()
+                return cmd.rc
+
+            d.addCallback(evaluateCommand)
+            return d
+
+        d = _downloadFile(diff, ".buildbot-diff")
+        d.addCallback(lambda _ : _downloadFile("patched\n", ".buildbot-patched"))
+        d.addCallback(lambda _: self.applyPatch(patchlevel))
+        cmd = buildstep.RemoteCommand('rmdir', {'dir': os.path.join(self.workdir, ".buildbot-diff"),
+                                                'logEnviron':self.logEnviron})
+        cmd.useLog(self.stdio_log, False)
+        d.addCallback(lambda _: self.runCommand(cmd))
+        def evaluateCommand(cmd):
+            if cmd.didFail():
+                raise buildstep.BuildStepFailed()
+            return cmd.rc
+        d.addCallback(lambda _: evaluateCommand(cmd))
+        return d
+
+    def sourcedirIsPatched(self):
+        d = self.pathExists(self.build.path_module.join(self.workdir, '.buildbot-patched'))
+        return d
+
     def start(self):
         if self.notReally:
             log.msg("faking %s checkout/update" % self.name)

master/buildbot/steps/source/bzr.py

@@ -16,7 +16,7 @@
 import os
 
 from twisted.python import log
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 
 from buildbot.process import buildstep
 from buildbot.steps.source.base import Source
@@ -67,33 +67,38 @@ def checkInstall(bzrInstalled):
             if not bzrInstalled:
                 raise BuildSlaveTooOldError("bzr is not installed on slave")
             return 0
-
         d.addCallback(checkInstall)
+
+        d.addCallback(lambda _: self.sourcedirIsPatched())
+        def checkPatched(patched):
+            if patched:
+                return self._dovccmd(['clean-tree', '--ignored', '--force'])
+            else:
+                return 0
+        d.addCallback(checkPatched)
+
         if self.mode == 'full':
             d.addCallback(lambda _: self.full())
         elif self.mode == 'incremental':
             d.addCallback(lambda _: self.incremental())
 
+        if patch:
+            d.addCallback(self.patch, patch)
         d.addCallback(self.parseGotRevision)
         d.addCallback(self.finish)
         d.addErrback(self.failed)
         return d
 
+    @defer.inlineCallbacks
     def incremental(self):
-        d = self._sourcedirIsUpdatable()
-        def _cmd(updatable):
-            if updatable:
-                command = ['update']
-            else:
-                command = ['checkout', self.repourl, '.']
-
+        updatable = yield self._sourcedirIsUpdatable()
+        if updatable:
+            command = ['update']
             if self.revision:
                 command.extend(['-r', self.revision])
-            return command
-
-        d.addCallback(_cmd)
-        d.addCallback(self._dovccmd)
-        return d
+            yield self._dovccmd(command)
+        else:
+            yield self._doFull()
 
     @defer.inlineCallbacks
     def full(self):
@@ -116,7 +121,7 @@ def full(self):
         else:
             raise ValueError("Unknown method, check your configuration")
 
-    def clobber(self):
+    def _clobber(self):
         cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
                                                 'logEnviron': self.logEnviron,})
         cmd.useLog(self.stdio_log, False)
@@ -126,6 +131,10 @@ def checkRemoval(res):
                 raise RuntimeError("Failed to delete directory")
             return res
         d.addCallback(lambda _: checkRemoval(cmd.rc))
+        return d
+
+    def clobber(self):
+        d = self._clobber()
         d.addCallback(lambda _: self._doFull())
         return d
 
@@ -166,7 +175,29 @@ def _doFull(self):
         command = ['checkout', self.repourl, '.']
         if self.revision:
             command.extend(['-r', self.revision])
-        d = self._dovccmd(command)
+
+        if self.retry:
+            abandonOnFailure = (self.retry[1] <= 0)
+        else:
+            abandonOnFailure = True
+        d = self._dovccmd(command, abandonOnFailure=abandonOnFailure)
+        def _retry(res):
+            if self.stopped or res == 0:
+                return res
+            delay, repeats = self.retry
+            if repeats > 0:
+                log.msg("Checkout failed, trying %d more times after %d seconds" 
+                    % (repeats, delay))
+                self.retry = (delay, repeats-1)
+                df = defer.Deferred()
+                df.addCallback(lambda _: self._clobber())
+                df.addCallback(lambda _: self._doFull())
+                reactor.callLater(delay, df.callback, None)
+                return df
+            return res
+
+        if self.retry:
+            d.addCallback(_retry)
         return d
 
     def finish(self, res):