Merge branch 'master' into nine

Conflicts:
	master/buildbot/changes/gitpoller.py
	master/buildbot/schedulers/base.py
	master/buildbot/test/fake/fakemaster.py
	master/buildbot/test/unit/test_changes_gitpoller.py
	master/buildbot/test/util/changesource.py

master/buildbot/buildslave.py

@@ -141,7 +141,7 @@ def locksAvailable(self):
         if not self.locks:
             return True
         for lock, access in self.locks:
-            if not lock.isAvailable(access):
+            if not lock.isAvailable(self, access):
                 return False
         return True
 

master/buildbot/changes/svnpoller.py

@@ -35,12 +35,16 @@ def split_file_alwaystrunk(path):
     return (None, path)
 
 def split_file_branches(path):
-    # turn trunk/subdir/file.c into (None, "subdir/file.c")
-    # and branches/1.5.x/subdir/file.c into ("branches/1.5.x", "subdir/file.c")
+    # turn "trunk/subdir/file.c" into (None, "subdir/file.c")
+    # and "trunk/subdir/" into (None, "subdir/")
+    # and "trunk/" into (None, "")
+    # and "branches/1.5.x/subdir/file.c" into ("branches/1.5.x", "subdir/file.c")
+    # and "branches/1.5.x/subdir/" into ("branches/1.5.x", "subdir/")
+    # and "branches/1.5.x/" into ("branches/1.5.x", "")
     pieces = path.split('/')
-    if pieces[0] == 'trunk':
+    if len(pieces) > 1 and pieces[0] == 'trunk':
         return (None, '/'.join(pieces[1:]))
-    elif pieces[0] == 'branches':
+    elif len(pieces) > 2 and pieces[0] == 'branches':
         return ('/'.join(pieces[0:2]), '/'.join(pieces[2:]))
     else:
         return None
@@ -310,30 +314,43 @@ def create_changes(self, new_logentries):
                 continue
 
             for p in pathlist.getElementsByTagName("path"):
+                kind = p.getAttribute("kind")
                 action = p.getAttribute("action")
                 path = "".join([t.data for t in p.childNodes])
                 path = path.encode("ascii")
                 if path.startswith("/"):
                     path = path[1:]
+                if kind == "dir" and not path.endswith("/"):
+                    path += "/"
                 where = self._transform_path(path)
 
                 # if 'where' is None, the file was outside any project that
                 # we care about and we should ignore it
                 if where:
                     branch, filename = where
                     if not branch in branches:
-                        branches[branch] = { 'files': []}
-                    branches[branch]['files'].append(filename)
+                        branches[branch] = { 'files': [], 'number_of_directories': 0}
+                    if filename == "":
+                        # root directory of branch
+                        branches[branch]['files'].append(filename)
+                        branches[branch]['number_of_directories'] += 1
+                    elif filename.endswith("/"):
+                        # subdirectory of branch
+                        branches[branch]['files'].append(filename[:-1])
+                        branches[branch]['number_of_directories'] += 1
+                    else:
+                        branches[branch]['files'].append(filename)
 
                     if not branches[branch].has_key('action'):
                         branches[branch]['action'] = action
 
             for branch in branches.keys():
                 action = branches[branch]['action']
                 files  = branches[branch]['files']
+                number_of_directories_changed = branches[branch]['number_of_directories']
                 number_of_files_changed = len(files)
 
-                if action == u'D' and number_of_files_changed == 1 and files[0] == '':
+                if action == u'D' and number_of_directories_changed == 1 and number_of_files_changed == 1 and files[0] == '':
                     log.msg("Ignoring deletion of branch '%s'" % branch)
                 else:
                     chdict = dict(

master/buildbot/locks.py

@@ -29,17 +29,16 @@ class BaseLock:
     Class handling claiming and releasing of L{self}, and keeping track of
     current and waiting owners.
 
-    @note: Ideally, we'd like to maintain FIFO order. The place to do that
-           would be the L{isAvailable()} function. However, this function is
-           called by builds/steps both for the first time, and after waking
-           them up by L{self} from the L{self.waiting} queue. There is
-           currently no way of distinguishing between them.
+    We maintain the wait queue in FIFO order, and ensure that counting waiters
+    in the queue behind exclusive waiters cannot acquire the lock. This ensures
+    that exclusive waiters are not starved.
     """
     description = "<BaseLock>"
 
     def __init__(self, name, maxCount=1):
         self.name = name          # Name of the lock
-        self.waiting = []         # Current queue, tuples (LockAccess, deferred)
+        self.waiting = []         # Current queue, tuples (waiter, LockAccess,
+                                  #                        deferred)
         self.owners = []          # Current owners, tuples (owner, LockAccess)
         self.maxCount = maxCount  # maximal number of counting owners
 
@@ -67,26 +66,38 @@ def _getOwnersCount(self):
         return num_excl, num_counting
 
 
-    def isAvailable(self, access):
+    def isAvailable(self, requester, access):
         """ Return a boolean whether the lock is available for claiming """
-        debuglog("%s isAvailable(%s): self.owners=%r"
-                                            % (self, access, self.owners))
+        debuglog("%s isAvailable(%s, %s): self.owners=%r"
+                                      % (self, requester, access, self.owners))
         num_excl, num_counting = self._getOwnersCount()
+
+        # Find all waiters ahead of the requester in the wait queue
+        for idx, waiter in enumerate(self.waiting):
+            if waiter[0] == requester:
+                w_index = idx
+                break
+        else:
+            w_index = len(self.waiting) 
+        ahead = self.waiting[:w_index]
+
         if access.mode == 'counting':
             # Wants counting access
-            return num_excl == 0 and num_counting < self.maxCount
+            return num_excl == 0 and num_counting + len(ahead) < self.maxCount \
+                    and all([w[1].mode == 'counting' for w in ahead])
         else:
             # Wants exclusive access
-            return num_excl == 0 and num_counting == 0
+            return num_excl == 0 and num_counting == 0 and len(ahead) == 0
 
     def claim(self, owner, access):
         """ Claim the lock (lock must be available) """
         debuglog("%s claim(%s, %s)" % (self, owner, access.mode))
         assert owner is not None
-        assert self.isAvailable(access), "ask for isAvailable() first"
+        assert self.isAvailable(owner, access), "ask for isAvailable() first"
 
         assert isinstance(access, LockAccess)
         assert access.mode in ['counting', 'exclusive']
+        self.waiting = [w for w in self.waiting if w[0] != owner]
         self.owners.append((owner, access))
         debuglog(" %s is claimed '%s'" % (self, access.mode))
 
@@ -109,22 +120,24 @@ def release(self, owner, access):
         # After an exclusive access, we may need to wake up several waiting.
         # Break out of the loop when the first waiting client should not be awakened.
         num_excl, num_counting = self._getOwnersCount()
-        while len(self.waiting) > 0:
-            access, d = self.waiting[0]
-            if access.mode == 'counting':
+        for i, (w_owner, w_access, d) in enumerate(self.waiting):
+            if w_access.mode == 'counting':
                 if num_excl > 0 or num_counting == self.maxCount:
                     break
                 else:
                     num_counting = num_counting + 1
             else:
-                # access.mode == 'exclusive'
+                # w_access.mode == 'exclusive'
                 if num_excl > 0 or num_counting > 0:
                     break
                 else:
                     num_excl = num_excl + 1
 
-            del self.waiting[0]
-            reactor.callLater(0, d.callback, self)
+            # If the waiter has a deferred, wake it up and clear the deferred
+            # from the wait queue entry to indicate that it has been woken.
+            if d:
+                self.waiting[i] = (w_owner, w_access, None)
+                reactor.callLater(0, d.callback, self)
 
         # notify any listeners
         self.release_subs.deliver()
@@ -138,17 +151,23 @@ def waitUntilMaybeAvailable(self, owner, access):
         """
         debuglog("%s waitUntilAvailable(%s)" % (self, owner))
         assert isinstance(access, LockAccess)
-        if self.isAvailable(access):
+        if self.isAvailable(owner, access):
             return defer.succeed(self)
         d = defer.Deferred()
-        self.waiting.append((access, d))
+
+        # Are we already in the wait queue?
+        w = [i for i, w in enumerate(self.waiting) if w[0] == owner]
+        if w:
+            self.waiting[w[0]] = (owner, access, d)
+        else:
+            self.waiting.append((owner, access, d))
         return d
 
     def stopWaitingUntilAvailable(self, owner, access, d):
         debuglog("%s stopWaitingUntilAvailable(%s)" % (self, owner))
         assert isinstance(access, LockAccess)
-        assert (access, d) in self.waiting
-        self.waiting.remove( (access, d) )
+        assert (owner, access, d) in self.waiting
+        self.waiting = [w for w in self.waiting if w[0] != owner]
 
     def isOwner(self, owner, access):
         return (owner, access) in self.owners

master/buildbot/process/build.py

@@ -280,7 +280,7 @@ def acquireLocks(self, res=None):
             return defer.succeed(None)
         log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks))
         for lock, access in self.locks:
-            if not lock.isAvailable(access):
+            if not lock.isAvailable(self, access):
                 log.msg("Build %s waiting for lock %s" % (self, lock))
                 d = lock.waitUntilMaybeAvailable(self, access)
                 d.addCallback(self.acquireLocks)

master/buildbot/process/builder.py

@@ -89,6 +89,7 @@ def reconfigService(self, new_config):
 
         self.config = builder_config
 
+        self.builder_status.setCategory(builder_config.category)
         self.builder_status.setSlavenames(self.config.slavenames)
         self.builder_status.setCacheSize(new_config.caches['Builds'])
 

master/buildbot/process/buildstep.py

@@ -545,7 +545,7 @@ def acquireLocks(self, res=None):
             return defer.succeed(None)
         log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks))
         for lock, access in self.locks:
-            if not lock.isAvailable(access):
+            if not lock.isAvailable(self, access):
                 self.step_status.setWaitingForLocks(True)
                 log.msg("step %s waiting for lock %s" % (self, lock))
                 d = lock.waitUntilMaybeAvailable(self, access)

master/buildbot/process/properties.py

@@ -424,11 +424,12 @@ def __init__(self, fmtstring, *args, **kwargs):
         self.fmtstring = fmtstring 
         self.args = args
         self.kwargs = kwargs 
+        if self.args and self.kwargs:
+            config.error("Interpolate takes either positional or keyword "
+                         "substitutions, not both.")
         if not self.args:
             self.interpolations = {}
             self._parse(fmtstring)
-        if self.args and self.kwargs:
-            raise ValueError('Interpolate takes either positional or keyword substitutions, not both.')
 
     @staticmethod
     def _parse_prop(arg):

master/buildbot/schedulers/base.py

@@ -13,6 +13,7 @@
 #
 # Copyright Buildbot Team Members
 
+import logging
 from zope.interface import implements
 from twisted.python import failure, log
 from twisted.application import service
@@ -21,8 +22,9 @@
 from buildbot.util import ComparableMixin
 from buildbot.changes import changes
 from buildbot import config, interfaces
+from buildbot.util.state import StateMixin
 
-class BaseScheduler(service.MultiService, ComparableMixin):
+class BaseScheduler(service.MultiService, ComparableMixin, StateMixin):
     """
     Base class for all schedulers; this provides the equipment to manage
     reconfigurations and to handle basic scheduler state.  It also provides
@@ -105,7 +107,6 @@ def __init__(self, name, builderNames, properties,
         # internal variables
         self._change_consumer = None
         self._change_consumption_lock = defer.DeferredLock()
-        self._objectid = None
 
     ## service handling
 
@@ -120,46 +121,6 @@ def stopService(self):
         d.addCallback(lambda _ : service.MultiService.stopService(self))
         return d
 
-    ## state management
-
-    @defer.inlineCallbacks
-    def getState(self, *args, **kwargs):
-        """
-        For use by subclasses; get a named state value from the scheduler's
-        state, defaulting to DEFAULT.
-
-        @param name: name of the value to retrieve
-        @param default: (optional) value to return if C{name} is not present
-        @returns: state value via a Deferred
-        @raises KeyError: if C{name} is not present and no default is given
-        @raises TypeError: if JSON parsing fails
-        """
-        # get the objectid, if not known
-        if self._objectid is None:
-            self._objectid = yield self.master.db.state.getObjectId(self.name,
-                                                    self.__class__.__name__)
-
-        rv = yield self.master.db.state.getState(self._objectid, *args,
-                                                                    **kwargs)
-        defer.returnValue(rv)
-
-    @defer.inlineCallbacks
-    def setState(self, key, value):
-        """
-        For use by subclasses; set a named state value in the scheduler's
-        persistent state.  Note that value must be json-able.
-
-        @param name: the name of the value to change
-        @param value: the value to set - must be a JSONable object
-        @param returns: Deferred
-        @raises TypeError: if JSONification fails
-        """
-        # get the objectid, if not known
-        if self._objectid is None:
-            self._objectid = yield self.master.db.state.getObjectId(self.name,
-                                                self.__class__.__name__)
-
-        yield self.master.db.state.setState(self._objectid, key, value)
 
     ## status queries
 
@@ -223,7 +184,9 @@ def _changeCallback(self, key, msg, fileIsImportant, change_filter,
         if change_filter and not change_filter.filter_change(change):
             return
         if change.codebase not in self.codebases:
-            log.msg('change contains codebase %s that is not processed by this scheduler' % change.codebase)
+            log.msg('change contains codebase %s that is not processed by'
+                ' scheduler %s' % (change.codebase, self.name),
+                logLevel=logging.DEBUG)
             return
         if fileIsImportant:
             try:
@@ -384,17 +347,10 @@ def addBuildsetForSourceStampSetDetails(self, reason, sourcestamps, properties):
              # sourcestamp attributes for this codebase.
             ss.update(sourcestamps.get(codebase,{}))
 
-            # at least repository must be set, this is normaly forced except when
-            # codebases is not explicitly set in configuration file.
-            ss_repository = ss.get('repository')
-            if not ss_repository:
-                config.error("The codebases argument is not set but still receiving " +
-                             "non empty codebase values")
-
             # add sourcestamp to the new setid
             yield self.master.db.sourcestamps.addSourceStamp(
                         codebase=codebase,
-                        repository=ss_repository,
+                        repository=ss.get('repository'),
                         branch=ss.get('branch', None),
                         revision=ss.get('revision', None),
                         project=ss.get('project', ''),

master/buildbot/status/builder.py

@@ -295,6 +295,10 @@ def getLastFinishedBuild(self):
             b = self.getBuild(-2)
         return b
 
+    def setCategory(self, category):
+        # used during reconfig
+        self.category = category
+
     def getCategory(self):
         return self.category
 

master/buildbot/status/web/authz.py

@@ -40,12 +40,14 @@ def __init__(self,
             default_action=False,
             auth=None,
             useHttpHeader=False,
+            httpLoginUrl=False,
             **kwargs):
         self.auth = auth
         if auth:
             assert IAuth.providedBy(auth)
 
         self.useHttpHeader = useHttpHeader
+        self.httpLoginUrl = httpLoginUrl
 
         self.config = dict( (a, default_action) for a in self.knownActions )
         for act in self.knownActions:
@@ -65,7 +67,7 @@ def session(self, request):
 
     def authenticated(self, request):
         if self.useHttpHeader:
-            return request.getUser() != None
+            return request.getUser() != ''
         return self.session(request) != None
 
     def getUserInfo(self, user):

master/buildbot/status/web/build.py

@@ -67,10 +67,8 @@ def performAction(self, req):
                     msg += "could not get builder control"
             else:
                 tup = yield bc.rebuildBuild(b, reason, extraProperties)
-                # check that (bsid, brids) were properly stored
-                if not (isinstance(tup, tuple) and 
-                        isinstance(tup[0], int) and
-                        isinstance(tup[1], dict)):
+                # rebuildBuild returns None on error (?!)
+                if not tup:
                     msg = "rebuilding a build failed "+ str(tup)
             # we're at
             # http://localhost:8080/builders/NAME/builds/5/rebuild?[args]