Merge branch 'master' into nine

* master: (94 commits)
  relnotes for pull 436
  changes to the style guide (language clean up and rst tweaks.)
  Changed "Annotate" to "Responsible Users"
  Fix ticket #268.
  indentation typo fixed
  master list feature added to hgbuildbot documentation
  hook a single master or a list of masters to mercurial
  Added check for ints for caches and added a test.
  Added check for ints for caches and added a test.
  remove accidentally duplicated test
  - Added check and tests for int values for caches
  Test that RemoteShellCommand's constructor can run with trivial arguments.
  Tests for master-side source steps timeout.
  Correct typo in parameter to RemoteCommand
  don't expect interrupting a command to return a signal on windows
  fix tests for homedir searching on win32
  fix double-slashifying in fake expanduser
  fix win32 test error in test_scripts_runner.TestCreateMasterOptions.test_db_basedir
  use an older version of sqlite to generate the databases, for compatibility with slaves
  unpickle v0.8.2 pickles too
  ...

Conflicts:
	master/buildbot/test/unit/test_process_builder.py

master/MANIFEST.in

@@ -1,4 +1,4 @@
-include MANIFEST.in README NEWS CREDITS COPYING UPGRADING
+include MANIFEST.in README CREDITS COPYING UPGRADING
 
 include docs/examples/*.cfg
 include docs/conf.py

master/buildbot/buildslave.py

@@ -754,6 +754,7 @@ class AbstractLatentBuildSlave(AbstractBuildSlave):
     substantiated = False
     substantiation_deferred = None
     substantiation_build = None
+    insubstantiating = False
     build_wait_timer = None
     _shutdown_callback_handle = None
 
@@ -824,7 +825,7 @@ def clean_up(failure):
         return d
 
     def attached(self, bot):
-        if self.substantiation_deferred is None:
+        if self.substantiation_deferred is None and self.build_wait_timeout >= 0:
             msg = 'Slave %s received connection while not trying to ' \
                     'substantiate.  Disconnecting.' % (self.slavename,)
             log.msg(msg)
@@ -866,6 +867,11 @@ def _substantiation_failed(self, failure):
         subject = "Buildbot: buildslave %s never substantiated" % self.slavename
         return self._mail_missing_message(subject, text)
 
+    def canStartBuild(self):
+        if self.insubstantiating:
+            return False
+        return AbstractBuildSlave.canStartBuild(self)
+
     def buildStarted(self, sb):
         assert self.substantiated
         self._clearBuildWaitTimer()
@@ -876,7 +882,10 @@ def buildFinished(self, sb):
 
         self.building.remove(sb.builder_name)
         if not self.building:
-            self._setBuildWaitTimer()
+            if self.build_wait_timeout == 0:
+                self.insubstantiate()
+            else:
+                self._setBuildWaitTimer()
 
     def _clearBuildWaitTimer(self):
         if self.build_wait_timer is not None:
@@ -886,10 +895,14 @@ def _clearBuildWaitTimer(self):
 
     def _setBuildWaitTimer(self):
         self._clearBuildWaitTimer()
+        if self.build_wait_timeout < 0:
+            return
         self.build_wait_timer = reactor.callLater(
             self.build_wait_timeout, self._soft_disconnect)
 
+    @defer.inlineCallbacks
     def insubstantiate(self, fast=False):
+        self.insubstantiating = True
         self._clearBuildWaitTimer()
         d = self.stop_instance(fast)
         if self._shutdown_callback_handle is not None:
@@ -898,9 +911,13 @@ def insubstantiate(self, fast=False):
             reactor.removeSystemEventTrigger(handle)
         self.substantiated = False
         self.building.clear() # just to be sure
-        return d
+        yield d
+        self.insubstantiating = False
 
     def _soft_disconnect(self, fast=False):
+        if not self.build_wait_timeout < 0:
+            return AbstractBuildSlave.disconnect(self)
+
         d = AbstractBuildSlave.disconnect(self)
         if self.slave is not None:
             # this could be called when the slave needs to shut down, such as

master/buildbot/changes/hgbuildbot.py

@@ -47,8 +47,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
     # read config parameters
     baseurl = ui.config('hgbuildbot', 'baseurl',
                             ui.config('web', 'baseurl', ''))
-    master = ui.config('hgbuildbot', 'master')
-    if master:
+    masters = ui.configlist('hgbuildbot', 'master')
+    if masters:
         branchtype = ui.config('hgbuildbot', 'branchtype', 'inrepo')
         branch = ui.config('hgbuildbot', 'branch')
         fork = ui.configbool('hgbuildbot', 'fork', False)
@@ -88,11 +88,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
         auth = 'change:changepw'
     auth = auth.split(':', 1)
 
-    s = sendchange.Sender(master, auth=auth)
-    d = defer.Deferred()
-    reactor.callLater(0, d.callback, None)
     # process changesets
-    def _send(res, c):
+    def _send(res, s, c):
         if not fork:
             ui.status("rev %s sent\n" % c['revision'])
         return s.send(c['branch'], c['revision'], c['comments'],
@@ -110,30 +107,35 @@ def _send(res, c):
     repository = strip(repo.root, stripcount)
     repository = baseurl + repository
 
-    for rev in xrange(start, end):
-        # send changeset
-        node = repo.changelog.node(rev)
-        manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node)
-        parents = filter(lambda p: not p == nullid, repo.changelog.parents(node))
-        if branchtype == 'inrepo':
-            branch = extra['branch']
-        is_merge = len(parents) > 1
-        # merges don't always contain files, but at least one file is required by buildbot
-        if is_merge and not files:
-            files = ["merge"]
-        properties = {'is_merge': is_merge}
-        if branch:
-            branch = fromlocal(branch)
-        change = {
-            'master': master,
-            'username': fromlocal(user),
-            'revision': hex(node),
-            'comments': fromlocal(desc),
-            'files': files,
-            'branch': branch,
-            'properties':properties
-        }
-        d.addCallback(_send, change)
+    for master in masters:
+        s = sendchange.Sender(master, auth=auth)
+        d = defer.Deferred()
+        reactor.callLater(0, d.callback, None)
+
+        for rev in xrange(start, end):
+            # send changeset
+            node = repo.changelog.node(rev)
+            manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node)
+            parents = filter(lambda p: not p == nullid, repo.changelog.parents(node))
+            if branchtype == 'inrepo':
+                branch = extra['branch']
+            is_merge = len(parents) > 1
+            # merges don't always contain files, but at least one file is required by buildbot
+            if is_merge and not files:
+                files = ["merge"]
+            properties = {'is_merge': is_merge}
+            if branch:
+                branch = fromlocal(branch)
+            change = {
+                'master': master,
+                'username': fromlocal(user),
+                'revision': hex(node),
+                'comments': fromlocal(desc),
+                'files': files,
+                'branch': branch,
+                'properties':properties
+            }
+            d.addCallback(_send, s, change)
 
     def _printSuccess(res):
         ui.status(s.getSuccessString(res) + '\n')

master/buildbot/config.py

@@ -15,9 +15,10 @@
 
 from __future__ import with_statement
 
-import re
 import os
+import re
 import sys
+import warnings
 from buildbot.util import safeTranslate
 from buildbot import interfaces
 from buildbot import locks
@@ -379,6 +380,11 @@ def load_caches(self, filename, config_dict, errors):
             if not isinstance(caches, dict):
                 errors.addError("c['caches'] must be a dictionary")
             else:
+                valPairs = caches.items()
+                for (x, y) in valPairs:
+                  if (not isinstance(y, int)):
+                     errors.addError(
+                     "value for cache size '%s' must be an integer" % x)
                 self.caches.update(caches)
 
         if 'buildCacheSize' in config_dict:
@@ -443,6 +449,12 @@ def mapper(b):
             errors.addError("c['builders'] must be a list of builder configs")
             return
 
+        for builder in builders:
+            if os.path.isabs(builder.builddir):
+                warnings.warn("Absolute path '%s' for builder may cause "
+                        "mayhem.  Perhaps you meant to specify slavebuilddir "
+                        "instead.")
+
         self.builders = builders
 
 
@@ -566,15 +578,6 @@ def check_lock(l):
                 for l in b.locks:
                     check_lock(l)
 
-            # factories don't necessarily need to implement a .steps attribute
-            # but in practice most do, so we'll check that if it exists
-            if not hasattr(b.factory, 'steps'):
-                continue
-            for s in b.factory.steps:
-                for l in s[1].get('locks', []):
-                    check_lock(l)
-
-
     def check_builders(self, errors):
         # look both for duplicate builder names, and for builders pointing
         # to unknown slaves

master/buildbot/db/changes.py

@@ -196,9 +196,12 @@ def thd(conn):
             for table_name in ('scheduler_changes', 'sourcestamp_changes',
                                'change_files', 'change_properties', 'changes',
                                'change_users'):
-                table = self.db.model.metadata.tables[table_name]
-                conn.execute(
-                    table.delete(table.c.changeid.in_(ids_to_delete)))
+                remaining = ids_to_delete[:]
+                while remaining:
+                    batch, remaining = remaining[:100], remaining[100:]
+                    table = self.db.model.metadata.tables[table_name]
+                    conn.execute(
+                        table.delete(table.c.changeid.in_(batch)))
         return self.db.pool.do(thd)
 
     def _chdict_from_change_row_thd(self, conn, ch_row):

master/buildbot/interfaces.py

@@ -220,10 +220,6 @@ def getBuilderNames():
         @returns: list of names via Deferred"""
     def isFinished():
         pass
-    def waitUntilSuccess():
-        """Return a Deferred that fires (with this IBuildSetStatus object)
-        when the outcome of the BuildSet is known, i.e., upon the first
-        failure, or after all builds complete successfully."""
     def waitUntilFinished():
         """Return a Deferred that fires (with this IBuildSetStatus object)
         when all builds have finished."""
@@ -1204,3 +1200,8 @@ def render(value):
 
 class IScheduler(Interface):
     pass
+
+class IBuildStepFactory(Interface):
+    def buildStep():
+        """
+        """