Merge branch 'master' into nine

Conflicts:
	master/buildbot/schedulers/base.py
	master/buildbot/test/unit/test_steps_trigger.py

master/buildbot/__init__.py

@@ -36,7 +36,7 @@
     VERSION_MATCH = re.compile(r'\d+\.\d+\.\d+(\w|-)*')
 
     try:
-        dir  = os.path.dirname(os.path.abspath(__file__))
+        dir = os.path.dirname(os.path.abspath(__file__))
         p = Popen(['git', 'describe', '--tags', '--always'], cwd=dir, stdout=PIPE, stderr=PIPE)
         out = p.communicate()[0]
 

master/buildbot/buildslave.py

@@ -962,7 +962,7 @@ def _soft_disconnect(self, fast=False):
             # here, we just do what should be appropriate for the first case,
             # and put our heads in the sand for the second, at least for now.
             # The best solution to the odd situation is removing it as a
-            # possibilty: make the master in charge of connecting to the
+            # possibility: make the master in charge of connecting to the
             # slave, rather than vice versa. TODO.
             yield defer.DeferredList([
                 AbstractBuildSlave.disconnect(self),

master/buildbot/config.py

@@ -285,7 +285,7 @@ def copy_str_param(name, alt_key=None):
             error("codebaseGenerator must be a callable accepting a dict and returning a str")
         else:
             self.codebaseGenerator = codebaseGenerator
-            
+
         prioritizeBuilders = config_dict.get('prioritizeBuilders')
         if prioritizeBuilders is not None and not callable(prioritizeBuilders):
             error("prioritizeBuilders must be a callable")
@@ -388,8 +388,8 @@ def load_caches(self, filename, config_dict):
             else:
                 valPairs = caches.items()
                 for (x, y) in valPairs:
-                  if (not isinstance(y, int)):
-                     error("value for cache size '%s' must be an integer" % x)
+                    if not isinstance(y, int):
+                        error("value for cache size '%s' must be an integer" % x)
                 self.caches.update(caches)
 
         if 'buildCacheSize' in config_dict:

master/buildbot/libvirtbuildslave.py

@@ -34,7 +34,7 @@ class WorkQueue(object):
     I am a class that turns parallel access into serial access.
 
     I exist because we want to run libvirt access in threads as we don't
-    trust calls not to block, but under load libvirt doesnt seem to like
+    trust calls not to block, but under load libvirt doesn't seem to like
     this kind of threaded use.
     """
 
@@ -126,7 +126,7 @@ def __init__(self, uri):
 
     @defer.inlineCallbacks
     def lookupByName(self, name):
-        """ I lookup an existing prefined domain """
+        """ I lookup an existing predefined domain """
         res = yield queue.executeInThread(self.connection.lookupByName, name)
         defer.returnValue(self.DomainClass(self, res))
 
@@ -182,11 +182,11 @@ def _find_existing_instance(self):
 
         domains = yield self.connection.all()
         for d in domains:
-           name = yield d.name()
-           if name.startswith(self.name):
-               self.domain = d
-               self.substantiated = True
-               break
+            name = yield d.name()
+            if name.startswith(self.name):
+                self.domain = d
+                self.substantiated = True
+                break
 
         self.ready = True
 
@@ -243,8 +243,8 @@ def start_instance(self, build):
         in the list of defined virtual machines and start that.
         """
         if self.domain is not None:
-             log.msg("Cannot start_instance '%s' as already active" % self.name)
-             defer.returnValue(False)
+            log.msg("Cannot start_instance '%s' as already active" % self.name)
+            defer.returnValue(False)
 
         yield self._prepare_base_image()
 
@@ -272,8 +272,8 @@ def stop_instance(self, fast=False):
         """
         log.msg("Attempting to stop '%s'" % self.name)
         if self.domain is None:
-             log.msg("I don't think that domain is even running, aborting")
-             return defer.succeed(None)
+            log.msg("I don't think that domain is even running, aborting")
+            return defer.succeed(None)
 
         domain = self.domain
         self.domain = None

master/buildbot/locks.py

@@ -219,12 +219,15 @@ class LockAccess(util.ComparableMixin):
     """
 
     compare_attrs = ['lockid', 'mode']
-    def __init__(self, lockid, mode):
+    def __init__(self, lockid, mode, _skipChecks=False):
         self.lockid = lockid
         self.mode = mode
 
-        assert isinstance(lockid, (MasterLock, SlaveLock))
-        assert mode in ['counting', 'exclusive']
+        if not _skipChecks:
+            # these checks fail with mock < 0.8.0 when lockid is a Mock
+            # TODO: remove this in Buildbot-0.9.0+
+            assert isinstance(lockid, (MasterLock, SlaveLock))
+            assert mode in ['counting', 'exclusive']
 
 
 class BaseLockId(util.ComparableMixin):

master/buildbot/process/builder.py

@@ -513,7 +513,7 @@ def setExpectations(self, progress):
             # the first time we get a good build, create our Expectations
             # based upon its results
             self.expectations = Expectations(progress)
-        log.msg("new expectations: %s seconds" % \
+        log.msg("new expectations: %s seconds" %
                 self.expectations.expectedBuildTime())
 
     # Build Creation

master/buildbot/process/buildrequestdistributor.py

@@ -361,37 +361,37 @@ def remove(x):
             return x
         d.addErrback(log.err, "while strting builds on %s" % (new_builders,))
 
-    @defer.inlineCallbacks
     def _maybeStartBuildsOn(self, new_builders):
         new_builders = set(new_builders)
         existing_pending = set(self._pending_builders)
 
         # if we won't add any builders, there's nothing to do
         if new_builders < existing_pending:
-            return
-
-        # reset the list of pending builders; this is async, so begin
-        # by grabbing a lock
-        yield self.pending_builders_lock.acquire()
-
-        try:
-            # re-fetch existing_pending, in case it has changed while acquiring
-            # the lock
-            existing_pending = set(self._pending_builders)
+            return defer.succeed(None)
 
-            # then sort the new, expanded set of builders
-            self._pending_builders = \
-                yield self._sortBuilders(list(existing_pending | new_builders))
-
-            # start the activity loop, if we aren't already working on that.
-            if not self.active:
-                self._activityLoop()
-        except Exception:
-            log.err(Failure(),
-                    "while attempting to start builds on %s" % self.name)
+        # reset the list of pending builders
+        @defer.inlineCallbacks
+        def resetPendingBuildersList(new_builders):
+            try:
+                # re-fetch existing_pending, in case it has changed 
+                # while acquiring the lock
+                existing_pending = set(self._pending_builders)
+
+                # then sort the new, expanded set of builders
+                self._pending_builders = \
+                    yield self._sortBuilders(
+                            list(existing_pending | new_builders))
+
+                # start the activity loop, if we aren't already
+                # working on that.
+                if not self.active:
+                    self._activityLoop()
+            except Exception:
+                log.err(Failure(),
+                        "while attempting to start builds on %s" % self.name)
 
-        # release the lock unconditionally
-        self.pending_builders_lock.release()
+        return self.pending_builders_lock.run(
+                resetPendingBuildersList, new_builders)
 
     @defer.inlineCallbacks
     def _defaultSorter(self, master, builders):

master/buildbot/scripts/base.py

@@ -30,6 +30,20 @@ def isBuildmasterDir(dir):
         contents = f.read()
     return "Application('buildmaster')" in contents
 
+def getConfigFileWithFallback(basedir, defaultName='master.cfg'):
+    configFile = os.path.abspath(os.path.join(basedir, defaultName))
+    if os.path.exists(configFile):
+        return configFile
+    # execute the .tac file to see if its configfile location exists
+    tacFile = os.path.join(basedir, 'buildbot.tac')
+    if os.path.exists(tacFile):
+        # don't mess with the global namespace
+        tacGlobals = {}
+        execfile(tacFile, tacGlobals)
+        return tacGlobals["configfile"]
+    # No config file found; return default location and fail elsewhere
+    return configFile
+
 class SubcommandOptions(usage.Options):
     # subclasses should set this to a list-of-lists in order to source the
     # .buildbot/options file.  Note that this *only* works with optParameters,

master/buildbot/scripts/checkconfig.py

@@ -16,12 +16,12 @@
 import sys
 import os
 from buildbot import config
+from buildbot.scripts.base import getConfigFileWithFallback
 
 class ConfigLoader(object):
-    def __init__(self, basedir=os.getcwd(), configFileName="master.cfg"):
+    def __init__(self, basedir=os.getcwd(), configFileName='master.cfg'):
         self.basedir = os.path.abspath(basedir)
-        self.configFileName = os.path.abspath(
-                                os.path.join(basedir, configFileName))
+        self.configFileName = getConfigFileWithFallback(basedir, configFileName)
 
     def load(self, quiet=False):
         try:

master/buildbot/scripts/runner.py

@@ -27,8 +27,8 @@
 
 from buildbot.scripts import base
 
-# Note that the terms 'options' and 'config' are used intechangeably here - in
-# fact, they are intercanged several times.  Caveat legator.
+# Note that the terms 'options' and 'config' are used interchangeably here - in
+# fact, they are interchanged several times.  Caveat legator.
 
 def validate_master_option(master):
     """Validate master (-m, --master) command line option.

master/buildbot/scripts/upgrade_master.py

@@ -45,13 +45,13 @@ def checkBasedir(config):
 
     return True
 
-def loadConfig(config):
+def loadConfig(config, configFileName='master.cfg'):
     if not config['quiet']:
-        print "checking master.cfg"
+        print "checking %s" % configFileName
 
     try:
         master_cfg = config_module.MasterConfig.loadConfig(
-                                            config['basedir'], 'master.cfg')
+                                            config['basedir'], configFileName)
     except config_module.ConfigErrors, e:
         print "Errors loading configuration:"
         for msg in e.errors:
@@ -160,7 +160,8 @@ def upgradeMaster(config, _noMonkey=False):
 
     os.chdir(config['basedir'])
 
-    master_cfg = loadConfig(config)
+    configFile = base.getConfigFileWithFallback(config['basedir'])
+    master_cfg = loadConfig(config, configFile)
     if not master_cfg:
         defer.returnValue(1)
         return

master/buildbot/scripts/user.py

@@ -41,6 +41,8 @@ def user(config):
             user['identifier'] = sorted(user.values())[0]
 
     uc = usersclient.UsersClient(master, username, passwd, port)
-    yield uc.send(op, bb_username, bb_password, ids, info)
+    output = yield uc.send(op, bb_username, bb_password, ids, info)
+    if output:
+        print output
 
     defer.returnValue(0)

master/buildbot/status/web/baseweb.py

@@ -44,6 +44,9 @@
 from buildbot.status.web.root import RootPage
 from buildbot.status.web.users import UsersResource
 from buildbot.status.web.change_hook import ChangeHookResource
+from twisted.cred.portal import IRealm, Portal
+from twisted.web import resource, guard
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
 
 # this class contains the WebStatus class.  Basic utilities are in base.py,
 # and specific pages are each in their own module.
@@ -149,7 +152,8 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
                  order_console_by_time=False, changecommentlink=None,
                  revlink=None, projects=None, repositories=None,
                  authz=None, logRotateLength=None, maxRotatedFiles=None,
-                 change_hook_dialects = {}, provide_feeds=None, jinja_loaders=None):
+                 change_hook_dialects = {}, provide_feeds=None, jinja_loaders=None,
+                 change_hook_auth=None):
         """Run a web server that provides Buildbot status.
 
         @type  http_port: int or L{twisted.application.strports} string
@@ -314,6 +318,12 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
 
         self.authz = authz
 
+        # check for correctness of HTTP auth parameters
+        if change_hook_auth is not None:
+            if not isinstance(change_hook_auth, tuple) or len(change_hook_auth) != 2:
+                config.error("Invalid credentials for change_hook auth")
+        self.change_hook_auth = change_hook_auth
+
         self.orderConsoleByTime = order_console_by_time
 
         # If we were given a site object, go ahead and use it. (if not, we add one later)
@@ -345,7 +355,10 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
         self.change_hook_dialects = {}
         if change_hook_dialects:
             self.change_hook_dialects = change_hook_dialects
-            self.putChild("change_hook", ChangeHookResource(dialects = self.change_hook_dialects))
+            resource_obj = ChangeHookResource(dialects=self.change_hook_dialects)
+            if self.change_hook_auth is not None:
+                resource_obj = self.setupProtectedResource(resource_obj)
+            self.putChild("change_hook", resource_obj)
 
         # Set default feeds
         if provide_feeds is None:
@@ -355,6 +368,27 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
 
         self.jinja_loaders = jinja_loaders
 
+    def setupProtectedResource(self, resource_obj):
+        class SimpleRealm(object):
+            """
+            A realm which gives out L{ChangeHookResource} instances for authenticated
+            users.
+            """
+            implements(IRealm)
+
+            def requestAvatar(self, avatarId, mind, *interfaces):
+                if resource.IResource in interfaces:
+                    return (resource.IResource, resource_obj, lambda: None)
+                raise NotImplementedError()
+
+        login, password = self.change_hook_auth
+        checker = InMemoryUsernamePasswordDatabaseDontUse()
+        checker.addUser(login, password)
+        portal = Portal(SimpleRealm(), [checker])
+        credentialFactory = guard.BasicCredentialFactory('Protected area')
+        wrapper = guard.HTTPAuthSessionWrapper(portal, [credentialFactory])
+        return wrapper
+
     def setupUsualPages(self, numbuilds, num_events, num_events_max):
         #self.putChild("", IndexOrWaterfallRedirection())
         self.putChild("waterfall", WaterfallStatusResource(num_events=num_events,