Merge branch 'master' of github.com:buildbot/buildbot

* 'master' of github.com:buildbot/buildbot:
  Exit with error code 1 if sendchange fails.
  Check that we have a remote before starting the next step, or that we have a slave reference before releasing locks.
  Fix up some queries for large numbers of build requests or builds.
  Remove unused import
  Follow-up to #797: Don't set now=True so buildbot doesn't think builds have to be started right away.

master/buildbot/db/connector.py

@@ -826,12 +826,14 @@ def claim_buildrequests(self, now, master_name, master_incarnation, brids,
                                    now, master_name, master_incarnation, brids)
     def _txn_claim_buildrequests(self, t, now, master_name, master_incarnation,
                                  brids):
-        q = self.quoteq("UPDATE buildrequests"
-                        " SET claimed_at = ?,"
-                        "     claimed_by_name = ?, claimed_by_incarnation = ?"
-                        " WHERE id IN " + self.parmlist(len(brids)))
-        qargs = [now, master_name, master_incarnation] + list(brids)
-        t.execute(q, qargs)
+        while brids:
+            batch, brids = brids[:100], brids[100:]
+            q = self.quoteq("UPDATE buildrequests"
+                            " SET claimed_at = ?,"
+                            "     claimed_by_name = ?, claimed_by_incarnation = ?"
+                            " WHERE id IN " + self.parmlist(len(batch)))
+            qargs = [now, master_name, master_incarnation] + list(batch)
+            t.execute(q, qargs)
 
     def build_started(self, brid, buildnumber):
         return self.runInteractionNow(self._txn_build_started, brid, buildnumber)
@@ -848,10 +850,12 @@ def builds_finished(self, bids):
         return self.runInteractionNow(self._txn_build_finished, bids)
     def _txn_build_finished(self, t, bids):
         now = self._getCurrentTime()
-        q = self.quoteq("UPDATE builds SET finish_time = ?"
-                        " WHERE id IN " + self.parmlist(len(bids)))
-        qargs = [now] + list(bids)
-        t.execute(q, qargs)
+        while bids:
+            batch, bids = bids[:100], bids[100:]
+            q = self.quoteq("UPDATE builds SET finish_time = ?"
+                            " WHERE id IN " + self.parmlist(len(batch)))
+            qargs = [now] + list(batch)
+            t.execute(q, qargs)
 
     def get_build_info(self, bid):
         return self.runInteractionNow(self._txn_get_build_info, bid)
@@ -878,11 +882,13 @@ def resubmit_buildrequests(self, brids):
     def _txn_resubmit_buildreqs(self, t, brids):
         # the interrupted build that gets resubmitted will still have the
         # same submitted_at value, so it should be re-started first
-        q = self.quoteq("UPDATE buildrequests"
-                        " SET claimed_at=0,"
-                        "     claimed_by_name=NULL, claimed_by_incarnation=NULL"
-                        " WHERE id IN " + self.parmlist(len(brids)))
-        t.execute(q, brids)
+        while brids:
+            batch, brids = brids[:100], brids[100:]
+            q = self.quoteq("UPDATE buildrequests"
+                            " SET claimed_at=0,"
+                            "     claimed_by_name=NULL, claimed_by_incarnation=NULL"
+                            " WHERE id IN " + self.parmlist(len(batch)))
+            t.execute(q, batch)
         self.notify("add-buildrequest", *brids)
 
     def retire_buildrequests(self, brids, results):
@@ -891,20 +897,23 @@ def _txn_retire_buildreqs(self, t, brids, results):
         now = self._getCurrentTime()
         #q = self.db.quoteq("DELETE FROM buildrequests WHERE id IN "
         #                   + self.db.parmlist(len(brids)))
-        q = self.quoteq("UPDATE buildrequests"
-                        " SET complete=1, results=?, complete_at=?"
-                        " WHERE id IN " + self.parmlist(len(brids)))
-        t.execute(q, [results, now]+brids)
-        # now, does this cause any buildsets to complete?
-        q = self.quoteq("SELECT bs.id"
-                        " FROM buildsets AS bs, buildrequests AS br"
-                        " WHERE br.buildsetid=bs.id AND bs.complete=0"
-                        "  AND br.id in "
-                        + self.parmlist(len(brids)))
-        t.execute(q, brids)
-        bsids = [bsid for (bsid,) in t.fetchall()]
-        for bsid in bsids:
-            self._check_buildset(t, bsid, now)
+        while brids:
+            batch, brids = brids[:100], brids[100:]
+
+            q = self.quoteq("UPDATE buildrequests"
+                            " SET complete=1, results=?, complete_at=?"
+                            " WHERE id IN " + self.parmlist(len(batch)))
+            t.execute(q, [results, now]+batch)
+            # now, does this cause any buildsets to complete?
+            q = self.quoteq("SELECT bs.id"
+                            " FROM buildsets AS bs, buildrequests AS br"
+                            " WHERE br.buildsetid=bs.id AND bs.complete=0"
+                            "  AND br.id in "
+                            + self.parmlist(len(batch)))
+            t.execute(q, batch)
+            bsids = [bsid for (bsid,) in t.fetchall()]
+            for bsid in bsids:
+                self._check_buildset(t, bsid, now)
         self.notify("retire-buildrequest", *brids)
         self.notify("modify-buildset", *bsids)
 
@@ -917,27 +926,30 @@ def _txn_cancel_buildrequest(self, t, brids):
         # buildset will appear complete and SUCCESS-ful). But we haven't
         # thought it through enough to be sure. So for now, "cancel" means
         # "mark as complete and FAILURE".
-        if True:
-            now = self._getCurrentTime()
-            q = self.quoteq("UPDATE buildrequests"
-                            " SET complete=1, results=?, complete_at=?"
-                            " WHERE id IN " + self.parmlist(len(brids)))
-            t.execute(q, [FAILURE, now]+brids)
-        else:
-            q = self.quoteq("DELETE FROM buildrequests"
-                            " WHERE id IN " + self.parmlist(len(brids)))
-            t.execute(q, brids)
-
-        # now, does this cause any buildsets to complete?
-        q = self.quoteq("SELECT bs.id"
-                        " FROM buildsets AS bs, buildrequests AS br"
-                        " WHERE br.buildsetid=bs.id AND bs.complete=0"
-                        "  AND br.id in "
-                        + self.parmlist(len(brids)))
-        t.execute(q, brids)
-        bsids = [bsid for (bsid,) in t.fetchall()]
-        for bsid in bsids:
-            self._check_buildset(t, bsid, now)
+        while brids:
+            batch, brids = brids[:100], brids[100:]
+
+            if True:
+                now = self._getCurrentTime()
+                q = self.quoteq("UPDATE buildrequests"
+                                " SET complete=1, results=?, complete_at=?"
+                                " WHERE id IN " + self.parmlist(len(batch)))
+                t.execute(q, [FAILURE, now]+batch)
+            else:
+                q = self.quoteq("DELETE FROM buildrequests"
+                                " WHERE id IN " + self.parmlist(len(batch)))
+                t.execute(q, batch)
+
+            # now, does this cause any buildsets to complete?
+            q = self.quoteq("SELECT bs.id"
+                            " FROM buildsets AS bs, buildrequests AS br"
+                            " WHERE br.buildsetid=bs.id AND bs.complete=0"
+                            "  AND br.id in "
+                            + self.parmlist(len(batch)))
+            t.execute(q, batch)
+            bsids = [bsid for (bsid,) in t.fetchall()]
+            for bsid in bsids:
+                self._check_buildset(t, bsid, now)
 
         self.notify("cancel-buildrequest", *brids)
         self.notify("modify-buildset", *bsids)

master/buildbot/process/base.py

@@ -339,6 +339,8 @@ def getNextStep(self):
             return None
         if not self.steps:
             return None
+        if not self.remote:
+            return None
         if self.terminate:
             while True:
                 s = self.steps.pop(0)

master/buildbot/process/builder.py

@@ -928,7 +928,8 @@ def buildFinished(self, build, sb, bids):
             brids = [br.id for br in build.requests]
             self.db.retire_buildrequests(brids, results)
 
-        sb.slave.releaseLocks()
+        if sb.slave:
+            sb.slave.releaseLocks()
 
         self.triggerNewBuildCheck()
 

master/buildbot/scripts/runner.py

@@ -841,9 +841,14 @@ def sendchange(config, runReactor=False):
     d = s.send(branch, revision, comments, files, category=category, when=when,
                properties=properties, repository=repository, project=project)
     if runReactor:
-        d.addCallbacks(s.printSuccess, s.printFailure)
+        status = [True]
+        def failed(res):
+            status[0] = False
+            s.printFailure(res)
+        d.addCallbacks(s.printSuccess, failed)
         d.addBoth(s.stop)
         s.run()
+        return status[0]
     return d
 
 
@@ -1120,7 +1125,8 @@ def run():
         from buildbot.scripts.reconfig import Reconfigurator
         Reconfigurator().run(so)
     elif command == "sendchange":
-        sendchange(so, True)
+        if not sendchange(so, True):
+            sys.exit(1)
     elif command == "debugclient":
         debugclient(so)
     elif command == "statuslog":

master/buildbot/status/web/builder.py

@@ -152,7 +152,7 @@ def force(self, req, auth_ok=False):
         try:
             c = interfaces.IControl(self.getBuildmaster(req))
             bc = c.getBuilder(self.builder_status.getName())
-            bc.submitBuildRequest(s, r, properties, now=True)
+            bc.submitBuildRequest(s, r, properties)
         except interfaces.NoSlaveError:
             # TODO: tell the web user that their request could not be
             # honored

master/buildbot/test/unit/test_schedulers_timed_Nightly.py

@@ -1,7 +1,6 @@
 import time
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 from buildbot.schedulers import timed
 from buildbot.changes.manager import ChangeManager