Merge branch 'master' of git@github.com:djmitche/buildbot

buildbot/db.py

@@ -347,7 +347,7 @@ def get_sqlite_dbapi_name():
         # don't use built-in sqlite3 on 2.5 -- it has *bad* bugs
         if sys.version_info >= (2,6):
             import sqlite3
-            sqlite_dbapi_name = "pysqlite2.dbapi2"
+            sqlite_dbapi_name = "sqlite3"
         else:
             raise
     return sqlite_dbapi_name

buildbot/master.py

@@ -11,18 +11,17 @@
 from twisted.internet import defer, reactor
 from twisted.spread import pb
 from twisted.cred import portal, checkers
-from twisted.application import service, strports, internet
+from twisted.application import service, strports
+from twisted.application.internet import TimerService
 
 import buildbot
 # sibling imports
 from buildbot.util import now, safeTranslate
 from buildbot.pbutil import NewCredPerspective
 from buildbot.process.builder import Builder, IDLE
-from buildbot.buildrequest import BuildRequest
 from buildbot.status.builder import Status, BuildSetStatus
 from buildbot.changes.changes import Change
 from buildbot.changes.manager import ChangeManager
-from buildbot.sourcestamp import SourceStamp
 from buildbot.buildslave import BuildSlave
 from buildbot import interfaces, locks
 from buildbot.process.properties import Properties
@@ -383,6 +382,8 @@ def _avatarAttached(self, p, mind):
 #   TCPServer(self.site)
 #   UNIXServer(ResourcePublisher(self.site))
 
+class _Unset: pass  # marker
+
 class BuildMaster(service.MultiService):
     debug = 0
     manhole = None
@@ -437,6 +438,7 @@ def __init__(self, basedir, configFileName="master.cfg", db=None):
 
         self.db = None
         self.db_url = None
+        self.db_poll_interval = _Unset
         if db:
             self.loadDatabase(db)
 
@@ -542,6 +544,7 @@ def loadConfig(self, f, check_synchronously_only=False):
 
             # optional
             db_url = config.get("db_url", "sqlite:///state.sqlite")
+            db_poll_interval = config.get("db_poll_interval", None)
             debugPassword = config.get('debugPassword')
             manhole = config.get('manhole')
             status = config.get('status', [])
@@ -629,6 +632,10 @@ def loadConfig(self, f, check_synchronously_only=False):
             raise KeyError("c['interlocks'] is no longer accepted")
         assert self.db_url is None or db_url == self.db_url, \
                 "Cannot change db_url after master has started"
+        assert db_poll_interval is None or isinstance(db_poll_interval, int), \
+               "db_poll_interval must be an integer: seconds between polls"
+        assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \
+               "Cannot change db_poll_interval after master has started"
 
         assert isinstance(change_sources, (list, tuple))
         for s in change_sources:
@@ -785,7 +792,8 @@ def loadConfig(self, f, check_synchronously_only=False):
         self.buildHorizon = buildHorizon
 
         # Set up the database
-        d.addCallback(lambda res: self.loadConfig_Database(db_url))
+        d.addCallback(lambda res:
+                      self.loadConfig_Database(db_url, db_poll_interval))
 
         # self.slaves: Disconnect any that were attached and removed from the
         # list. Update self.checker with the new list of passwords, including
@@ -851,7 +859,7 @@ def _done(res):
         d.addErrback(log.err)
         return d
 
-    def loadDatabase(self, db_spec):
+    def loadDatabase(self, db_spec, db_poll_interval=None):
         if self.db:
             return
         self.db = open_db(db_spec)
@@ -868,15 +876,26 @@ def loadDatabase(self, db_spec):
 
         self.scheduler_manager = sm
         sm.setServiceParent(self)
-        # it'd be nice if TimerService let us set now=False
-        t = internet.TimerService(30, sm.trigger)
-        t.setServiceParent(self)
-        # should we try to remove this? Periodic() requires at least one kick
 
-    def loadConfig_Database(self, db_url):
+        # Set db_poll_interval (perhaps to 30 seconds) if you are using
+        # multiple buildmasters that share a common database, such that the
+        # masters need to discover what each other is doing by polling the
+        # database. TODO: this will be replaced by the DBNotificationServer.
+        if db_poll_interval:
+            # it'd be nice if TimerService let us set now=False
+            t1 = TimerService(db_poll_interval, sm.trigger)
+            t1.setServiceParent(self)
+            t2 = TimerService(db_poll_interval, self.botmaster.loop.trigger)
+            t2.setServiceParent(self)
+        # adding schedulers (like when loadConfig happens) will trigger the
+        # scheduler loop at least once, which we need to jump-start things
+        # like Periodic.
+
+    def loadConfig_Database(self, db_url, db_poll_interval):
         self.db_url = db_url
+        self.db_poll_interval = db_poll_interval
         db_spec = DB.from_url(db_url, self.basedir)
-        self.loadDatabase(db_spec)
+        self.loadDatabase(db_spec, db_poll_interval)
 
     def loadConfig_Slaves(self, new_slaves):
         # set up the Checker with the names and passwords of all valid bots

buildbot/process/builder.py

@@ -7,10 +7,9 @@
 from twisted.application import service, internet
 from twisted.internet import defer
 
-from buildbot import interfaces, util, buildrequest
+from buildbot import interfaces, util
 from buildbot.status.progress import Expectations
 from buildbot.status.builder import RETRY
-from buildbot.process import base
 from buildbot.process.properties import Properties
 from buildbot.eventual import eventually
 

buildbot/process/subunitlogobserver.py

@@ -1,11 +1,8 @@
 # -*- test-case-name: buildbot.test.test_buildstep -*-
 
 from unittest import TestResult
-
-from buildbot.steps.shell import ShellCommand
 from buildbot.process import buildstep
 
-
 class DiscardStream:
     """A trivial thunk used to discard passthrough content."""
 

buildbot/schedulers/manager.py

@@ -39,6 +39,7 @@
 from twisted.python import log
 from buildbot import loop
 from buildbot.util import defaultdict
+from buildbot.eventual import eventually
 
 class SchedulerManager(loop.MultiServiceLoop):
     def __init__(self, master, db, change_svc):
@@ -94,6 +95,7 @@ def _attach(ign):
             for s in list(self):
                 if s.upstream_name:
                     self.upstream_subscribers[s.upstream_name].append(s)
+            eventually(self.trigger)
         d.addCallback(_attach)
         d.addErrback(log.err)
         return d

buildbot/schedulers/timed.py

@@ -99,13 +99,14 @@ def _run(self, t):
         if last_build is None:
             self.start_HEAD_build(t)
             self.update_last_build(t, now)
-            return None
+            last_build = now
         when = last_build + self.periodicBuildTimer
         if when < now:
             self.start_HEAD_build(t)
             self.update_last_build(t, now)
-            return None
-        return when - now + 1.0
+            last_build = now
+            when = now + self.periodicBuildTimer
+        return when + 1.0
 
 
 class Nightly(_Base, ClassifierMixin, TimedBuildMixin):

buildbot/status/web/buildstatus.py

@@ -33,8 +33,9 @@ def content(self, request, ctx):
 
         # Display each step, starting by the last one.
         for i in range(len(build.getSteps()) - 1, -1, -1):
-            if build.getSteps()[i].getText():
-                rows.append(IBox(build.getSteps()[i]).getBox(request).td(align="center"))
+            step = build.getSteps()[i]
+            if step.isStarted() and step.getText():
+                rows.append(IBox(step).getBox(request).td(align="center"))
 
         # Display the bottom box with the build number in it.
         ctx['build'] = IBox(build).getBox(request).td(align="center")

buildbot/test/runs/test_db.py

@@ -48,6 +48,7 @@
 from buildbot.changes.manager import ChangeManager
 from buildbot.test.pollmixin import PollMixin
 from buildbot.test.runutils import RunMixin
+from buildbot.eventual import flushEventualQueue
 
 class ShouldFailMixin:
 
@@ -285,6 +286,7 @@ def _check4(ign):
             self.failUnlessEqual(all[0].name, "one")
             self.failUnlessEqual(all[1].builderNames, ["builder-two-other"])
         d.addCallback(_check4)
+        d.addCallback(flushEventualQueue)
         return d
 
     def stall(self, res, timeout):

buildbot/test/runutils.py

@@ -73,15 +73,15 @@ class MasterMixin:
     master = None
     basedir = None
     slave_basedir = None
-    def create_master(self):
+    def create_master(self, **kwargs):
         assert not self.master, "you called create_master twice"
         # probably because you subclassed RunMixin instead of MasterMixin
         self.slaves = {}
         if self.basedir is None:
             self.basedir = self.mktemp()
         basedir = self.basedir
         os.makedirs(basedir)
-        self.master = master.BuildMaster(basedir)
+        self.master = master.BuildMaster(basedir, **kwargs)
         spec = db.DB.from_url("sqlite:///state.sqlite", basedir=basedir)
         db.create_db(spec)
         self.master.loadDatabase(spec)