Merge branch 'master' into nine

* master: (40 commits)
  update docs on buildbot upgrade-master and --db option
  Rename FakeRenderable to ConstantRenderable.
  Rename a locally define FakeRenderable
  Add some docstrings to GitPoller.
  Update tests covering the irc bot.
  Detect private messages when the nickname has uppercase letters.
  Get irc contacts with case insensitive name.
  React to the irc bot nickname instead of the generic name "buildbot".
  Use repo forall instead of find to rm git locks
  Round the DevDetails instead of DevComment if there are some details.
  Do not round the last DevSlave we didn't the first eigther.
  last should be set in the loop it is used.
  Remove left "ear" of Offline Legend in Console View.
  tabs to spaces
  Add Mock LogObserver
  pyflakes
  Add a 'jinja_loaders' parameter to WebStatus for customizers who want to use additional Jina2 loaders
  poller-web-hook: Add a test of all allowed pollers.
  Update RpmBuild
  Nested Mock.revoveDone within Mock.start
  ...

Conflicts:
	master/buildbot/test/unit/test_schedulers_triggerable.py

master/buildbot/changes/base.py

@@ -35,36 +35,61 @@ class PollingChangeSource(ChangeSource):
     Utility subclass for ChangeSources that use some kind of periodic polling
     operation.  Subclasses should define C{poll} and set C{self.pollInterval}.
     The rest is taken care of.
+
+    Any subclass will be available via the "poller" webhook.
     """
 
     pollInterval = 60
     "time (in seconds) between calls to C{poll}"
 
     _loop = None
 
+    def __init__(self, name=None, pollInterval=60*10):
+        if name:
+            self.setName(name)
+        self.pollInterval = pollInterval
+
+        self.doPoll = util.misc.SerializedInvocation(self.doPoll)
+
+    def doPoll(self):
+        """
+        This is the method that is called by LoopingCall to actually poll.
+        It may also be called by change hooks to request a poll.
+        It is serialiazed - if you call it while a poll is in progress
+        then the 2nd invocation won't start until the 1st has finished.
+        """
+        d = defer.maybeDeferred(self.poll)
+        d.addErrback(log.err, 'while polling for changes')
+        return d
+
     def poll(self):
         """
         Perform the polling operation, and return a deferred that will fire
         when the operation is complete.  Failures will be logged, but the
         method will be called again after C{pollInterval} seconds.
         """
 
+    def startLoop(self):
+        self._loop = task.LoopingCall(self.doPoll)
+        self._loop.start(self.pollInterval, now=False)
+
+    def stopLoop(self):
+        if self._loop and self._loop.running:
+            self._loop.stop()
+            self._loop = None
+
     def startService(self):
         ChangeSource.startService(self)
-        def do_poll():
-            d = defer.maybeDeferred(self.poll)
-            d.addErrback(log.err, 'while polling for changes')
-            return d
-
-        # delay starting the loop until the reactor is running, and do not
-        # run it immediately - if services are still starting up, they may
-        # miss an initial flood of changes
-        def start_loop():
-            self._loop = task.LoopingCall(do_poll)
-            self._loop.start(self.pollInterval, now=False)
-        reactor.callWhenRunning(start_loop)
+
+        # delay starting doing anything until the reactor is running - if
+        # services are still starting up, they may miss an initial flood of
+        # changes
+        if self.pollInterval:
+            reactor.callWhenRunning(self.startLoop)
+        else:
+            reactor.callWhenRunning(self.doPoll)
 
     def stopService(self):
-        if self._loop and self._loop.running:
-            self._loop.stop()
+        self.stopLoop()
         return ChangeSource.stopService(self)
+

master/buildbot/changes/bonsaipoller.py

@@ -206,14 +206,16 @@ class BonsaiPoller(base.PollingChangeSource):
                      "module", "branch", "cvsroot"]
 
     def __init__(self, bonsaiURL, module, branch, tree="default",
-                 cvsroot="/cvsroot", pollInterval=30, project=''):
+                 cvsroot="/cvsroot", pollInterval=30, project='', name=None):
+
+        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)
+
         self.bonsaiURL = bonsaiURL
         self.module = module
         self.branch = branch
         self.tree = tree
         self.cvsroot = cvsroot
         self.repository = module != 'all' and module or ''
-        self.pollInterval = pollInterval
         self.lastChange = time.time()
         self.lastPoll = time.time()
 

master/buildbot/changes/gitpoller.py

@@ -35,15 +35,18 @@ def __init__(self, repourl, branch='master',
                  gitbin='git', usetimestamps=True,
                  category=None, project=None,
                  pollinterval=-2, fetch_refspec=None,
-                 encoding='utf-8'):
+                 encoding='utf-8', name=None):
+
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
+
+        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)
+
         if project is None: project = ''
 
         self.repourl = repourl
         self.branch = branch
-        self.pollInterval = pollInterval
         self.fetch_refspec = fetch_refspec
         self.encoding = encoding
         self.lastChange = time.time()
@@ -93,7 +96,7 @@ def make_dir(_):
         def git_init(_):
             log.msg('gitpoller: initializing working dir from %s' % self.repourl)
             d = utils.getProcessOutputAndValue(self.gitbin,
-                    ['init', self.workdir], env=os.environ)
+                    ['init', '--bare', self.workdir], env=os.environ)
             d.addCallback(self._convert_nonzero_to_failure)
             d.addErrback(self._stop_on_failure)
             return d
@@ -218,6 +221,7 @@ def process(git_output):
         return d
 
     def _get_changes(self):
+        """Fetch changes from remote repository."""
         log.msg('gitpoller: polling git repo at %s' % self.repourl)
 
         self.lastPoll = time.time()
@@ -238,6 +242,13 @@ def _get_changes(self):
 
     @defer.inlineCallbacks
     def _process_changes(self, unused_output):
+        """
+        Read changes since last change.
+
+        - Read list of commit hashes.
+        - Extract details from each commit.
+        - Add changes to database.
+        """
         # get the change list
         revListArgs = ['log', '%s..origin/%s' % (self.branch, self.branch), r'--format=%H']
         self.changeCount = 0
@@ -291,6 +302,7 @@ def _process_changes_failure(self, f):
         return None
 
     def _catch_up(self, res):
+        """Update repository to record last seen change."""
         if self.changeCount == 0:
             log.msg('gitpoller: no changes, no catch_up')
             return

master/buildbot/changes/p4poller.py

@@ -66,11 +66,14 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
                  p4base='//', p4bin='p4',
                  split_file=lambda branchfile: (None, branchfile),
                  pollInterval=60 * 10, histmax=None, pollinterval=-2,
-                 encoding='utf8', project=None):
+                 encoding='utf8', project=None, name=None):
+
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
 
+        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)
+
         if project is None:
             project = ''
 
@@ -80,7 +83,6 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
         self.p4base = p4base
         self.p4bin = p4bin
         self.split_file = split_file
-        self.pollInterval = pollInterval
         self.encoding = encoding
         self.project = util.ascii2unicode(project)
 

master/buildbot/changes/svnpoller.py

@@ -66,11 +66,14 @@ def __init__(self, svnurl, split_file=None,
                  pollInterval=10*60, histmax=100,
                  svnbin='svn', revlinktmpl='', category=None, 
                  project='', cachepath=None, pollinterval=-2,
-                 extra_args=None):
+                 extra_args=None, name=None):
+
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
 
+        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)
+
         if svnurl.endswith("/"):
             svnurl = svnurl[:-1] # strip the trailing slash
         self.svnurl = svnurl
@@ -85,7 +88,6 @@ def __init__(self, svnurl, split_file=None,
                                          # required for ssh-agent auth
 
         self.svnbin = svnbin
-        self.pollInterval = pollInterval
         self.histmax = histmax
         self._prefix = None
         self.category = util.ascii2unicode(category)

master/buildbot/process/build.py

@@ -97,6 +97,9 @@ def getSourceStamp(self, codebase=''):
                 return source
         return None
 
+    def getAllSourceStamps(self):
+        return list(self.sources)
+
     def allChanges(self):
         for s in self.sources:
             for c in s.changes:

master/buildbot/schedulers/triggerable.py

@@ -17,20 +17,22 @@
 from twisted.internet import defer
 from buildbot.schedulers import base
 from buildbot.process.properties import Properties
+from buildbot import config
 
 class Triggerable(base.BaseScheduler):
 
     compare_attrs = base.BaseScheduler.compare_attrs
 
-    def __init__(self, name, builderNames, properties={}):
-        base.BaseScheduler.__init__(self, name, builderNames, properties)
+    def __init__(self, name, builderNames, properties={}, **kwargs):
+        base.BaseScheduler.__init__(self, name, builderNames, properties,
+                                    **kwargs)
         self._waiters = {}
         self._buildset_complete_consumer = None
         self.reason = "Triggerable(%s)" % name
 
-    def trigger(self, ss_setid, set_props=None):
-        """Trigger this scheduler with the given sourcestampset ID. Returns a
-        deferred that will fire when the buildset is finished."""
+    def trigger(self, sourcestamps = None, set_props=None):
+        """Trigger this scheduler with the optional given list of sourcestamps
+        Returns a deferred that will fire when the buildset is finished."""
         # properties for this buildset are composed of our own properties,
         # potentially overridden by anything from the triggering build
         props = Properties()
@@ -41,11 +43,7 @@ def trigger(self, ss_setid, set_props=None):
         # note that this does not use the buildset subscriptions mechanism, as
         # the duration of interest to the caller is bounded by the lifetime of
         # this process.
-        if ss_setid:
-            d = self.addBuildsetForSourceStamp(reason=self.reason, setid=ss_setid,
-                    properties=props)
-        else:
-            d = self.addBuildsetForLatest(reason=self.reason, properties=props)
+        d = self._addBuildsetForTrigger(self.reason, sourcestamps, props)
         def setup_waiter((bsid,brids)):
             d = defer.Deferred()
             self._waiters[bsid] = (d, brids)
@@ -69,6 +67,50 @@ def stopService(self):
 
         return base.BaseScheduler.stopService(self)
 
+    @defer.inlineCallbacks
+    def _addBuildsetForTrigger(self, reason, sourcestamps,  properties):
+        if sourcestamps is None:
+            sourcestamps = {}
+
+        # Define new setid for this set of triggering sourcestamps
+        new_setid = yield self.master.db.sourcestampsets.addSourceStampSet()
+
+        # Merge codebases with the passed list of sourcestamps
+        # This results in a new sourcestamp for each codebase
+        for codebase in self.codebases:
+            ss = self.codebases[codebase].copy()
+             # apply info from passed sourcestamps onto the configured default
+             # sourcestamp attributes for this codebase.
+            ss.update(sourcestamps.get(codebase,{}))
+
+            # at least repository must be set, this is normaly forced except when 
+            # codebases is not explicitly set in configuration file.
+            ss_repository = ss.get('repository')
+            if not ss_repository:
+                config.error("The codebases argument is not set but still receiving " +
+                             "non empty codebase values")
+
+            # add sourcestamp to the new setid
+            yield self.master.db.sourcestamps.addSourceStamp(
+                        codebase=codebase,
+                        repository=ss_repository,
+                        branch=ss.get('branch', None),
+                        revision=ss.get('revision', None),
+                        project=ss.get('project', ''),
+                        changeids=[c['number'] for c in getattr(ss, 'changes', [])],
+                        patch_body=ss.get('patch_body', None),
+                        patch_level=ss.get('patch_level', None),
+                        patch_author=ss.get('patch_author', None),
+                        patch_comment=ss.get('patch_comment', None),
+                        sourcestampsetid=new_setid)
+
+        bsid,brids = yield self.addBuildsetForSourceStamp(
+                                setid=new_setid, reason=reason,
+                                properties=properties)
+
+        defer.returnValue((bsid,brids))
+
+
     def _updateWaiters(self):
         if self._waiters and not self._buildset_complete_consumer:
             self._buildset_complete_consumer = self.master.mq.startConsuming(

master/buildbot/sourcestamp.py

@@ -252,8 +252,18 @@ def asDict(self):
         result = {}
         # Constant
         result['revision'] = self.revision
+
         # TODO(maruel): Make the patch content a suburl.
         result['hasPatch'] = self.patch is not None
+        if self.patch:
+            result['patch_level'] = self.patch[0]
+            result['patch_body'] = self.patch[1]
+            if len(self.patch) > 2:
+                result['patch_subdir'] = self.patch[2]
+            if self.patch_info:
+                result['patch_author'] = self.patch_info[0]
+                result['patch_comment'] = self.patch_info[1]
+
         result['branch'] = self.branch
         result['changes'] = [c.asDict() for c in getattr(self, 'changes', [])]
         result['project'] = self.project

master/buildbot/status/web/base.py

@@ -481,7 +481,7 @@ def map_branches(branches):
 # jinja utilities
 
 def createJinjaEnv(revlink=None, changecommentlink=None,
-                     repositories=None, projects=None):
+                     repositories=None, projects=None, jinja_loaders=None):
     ''' Create a jinja environment changecommentlink is used to
         render HTML in the WebStatus and for mail changes
 
@@ -503,10 +503,12 @@ def createJinjaEnv(revlink=None, changecommentlink=None,
     # See http://buildbot.net/trac/ticket/658
     assert not hasattr(sys, "frozen"), 'Frozen config not supported with jinja (yet)'
 
-    default_loader = jinja2.PackageLoader('buildbot.status.web', 'templates')
-    root = os.path.join(os.getcwd(), 'templates')
-    loader = jinja2.ChoiceLoader([jinja2.FileSystemLoader(root),
-                                  default_loader])
+    all_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'templates'))]
+    if jinja_loaders:
+        all_loaders.extend(jinja_loaders)
+    all_loaders.append(jinja2.PackageLoader('buildbot.status.web', 'templates'))
+    loader = jinja2.ChoiceLoader(all_loaders)
+
     env = jinja2.Environment(loader=loader,
                              extensions=['jinja2.ext.i18n'],
                              trim_blocks=True,

master/buildbot/status/web/baseweb.py

@@ -148,7 +148,7 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
                  order_console_by_time=False, changecommentlink=None,
                  revlink=None, projects=None, repositories=None,
                  authz=None, logRotateLength=None, maxRotatedFiles=None,
-                 change_hook_dialects = {}, provide_feeds=None):
+                 change_hook_dialects = {}, provide_feeds=None, jinja_loaders=None):
         """Run a web server that provides Buildbot status.
 
         @type  http_port: int or L{twisted.application.strports} string
@@ -266,6 +266,10 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
                               Otherwise, a dictionary of strings of
                               the type of feeds provided.  Current
                               possibilities are "atom", "json", and "rss"
+
+        @type  jinja_loaders: None or list
+        @param jinja_loaders: If not empty, a list of additional Jinja2 loader
+                              objects to search for templates.
         """
 
         service.MultiService.__init__(self)
@@ -344,6 +348,8 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
         else:
             self.provide_feeds = provide_feeds
 
+        self.jinja_loaders = jinja_loaders
+
     def setupUsualPages(self, numbuilds, num_events, num_events_max):
         #self.putChild("", IndexOrWaterfallRedirection())
         self.putChild("waterfall", WaterfallStatusResource(num_events=num_events,
@@ -402,7 +408,7 @@ def either(a,b): # a if a else b for py2.4
         else:
             revlink = self.master.config.revlink
         self.templates = createJinjaEnv(revlink, self.changecommentlink,
-                                        self.repositories, self.projects)
+                                        self.repositories, self.projects, self.jinja_loaders)
 
         if not self.site: