Merge remote-tracking branch 'upstream/master'

common/pylintrc

@@ -45,13 +45,13 @@ load-plugins=
 # R0915: Too many statements (%s/%s)*
 # R0921: Abstract class not referenced*
 # R0922: Abstract class is only
-# R0924: Badly implemented %s, implements %s but not %s*
 # W0122: Use of the exec statement*
 # W0142: Used * or ** magic
 # W0402: Uses of a deprecated module %r
 # W0511: FIXME or XXX is detected
 # W0601: Global variable %r undefined at the module level
 # W0613: Unused argument %r
+# W0633: Attempting to unpack a non-sequence
 # E1103: %s %r has no %r member (but some types could not be inferred)
 
 disable=
@@ -72,7 +72,6 @@ disable=
   R0915,
   R0921,
   R0922,
-  R0924,
   W0402,
   W0511,
   W0601,
@@ -132,13 +131,15 @@ disable=
   W0622,
   W0631,
   W0632,
+  W0633,
   W0701,
   W0702,
   W0703,
   W0710,
   W1001,
   W1402,
-  W1501
+  W1501,
+  superfluous-parens
 
 [REPORTS]
 

master/buildbot/buildslave/base.py

@@ -178,7 +178,22 @@ def _lockReleased(self):
             return  # oh well..
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
-    def _saveSlaveInfoDict(self, slaveinfo):
+    def _applySlaveInfo(self, info):
+        if not info:
+            return
+
+        self.slave_status.setAdmin(info.get("admin"))
+        self.slave_status.setHost(info.get("host"))
+        self.slave_status.setAccessURI(info.get("access_uri"))
+        self.slave_status.setVersion(info.get("version"))
+
+    def _saveSlaveInfoDict(self):
+        slaveinfo = {
+            'admin': self.slave_status.getAdmin(),
+            'host': self.slave_status.getHost(),
+            'access_uri': self.slave_status.getAccessURI(),
+            'version': self.slave_status.getVersion(),
+        }
         return self.master.db.buildslaves.updateBuildslave(
             name=self.slavename,
             slaveinfo=slaveinfo,
@@ -192,16 +207,10 @@ def applyInfo(buildslave):
             if buildslave is None:
                 return
 
-            self.updateSlaveInfo(**buildslave['slaveinfo'])
+            self._applySlaveInfo(buildslave.get('slaveinfo'))
 
         return d
 
-    def updateSlaveInfo(self, **kwargs):
-        self.slave_status.updateInfo(**kwargs)
-
-    def getSlaveInfo(self, key, default=None):
-        return self.slave_status.getInfo(key, default)
-
     def setServiceParent(self, parent):
         # botmaster needs to set before setServiceParent which calls startService
         self.botmaster = parent
@@ -211,7 +220,6 @@ def setServiceParent(self, parent):
     def startService(self):
         self.updateLocks()
         self.startMissingTimer()
-        self.slave_status.addInfoWatcher(self._saveSlaveInfoDict)
         d = self._getSlaveInfo()
         d.addCallback(lambda _: service.MultiService.startService(self))
         return d
@@ -266,7 +274,6 @@ def reconfigService(self, new_config):
                                                                 new_config)
 
     def stopService(self):
-        self.slave_status.removeInfoWatcher(self._saveSlaveInfoDict)
         if self.registration:
             self.registration.unregister()
             self.registration = None
@@ -471,12 +478,7 @@ def _commands_unavailable(why):
         def _accept_slave(res):
             self.slave_status.setConnected(True)
 
-            self.slave_status.updateInfo(
-                admin=state.get("admin"),
-                host=state.get("host"),
-                access_uri=state.get("access_uri"),
-                version=state.get("version"),
-            )
+            self._applySlaveInfo(state)
 
             self.slave_commands = state.get("slave_commands")
             self.slave_environ = state.get("slave_environ")
@@ -494,6 +496,8 @@ def _accept_slave(res):
             self.stopMissingTimer()
             self.botmaster.master.status.slaveConnected(self.slavename)
 
+        d.addCallback(lambda _: self._saveSlaveInfoDict())
+
         d.addCallback(lambda _: self.updateSlave())
 
         d.addCallback(lambda _:

master/buildbot/changes/base.py

@@ -47,12 +47,16 @@ class PollingChangeSource(ChangeSource):
     pollInterval = 60
     "time (in seconds) between calls to C{poll}"
 
+    pollAtLaunch = False
+    "determines when the first poll occurs. True = immediately on launch, False = wait for one pollInterval."
+
     _loop = None
 
-    def __init__(self, name=None, pollInterval=60 * 10):
+    def __init__(self, name=None, pollInterval=60 * 10, pollAtLaunch=False):
         if name:
             self.setName(name)
         self.pollInterval = pollInterval
+        self.pollAtLaunch = pollAtLaunch
 
         self.doPoll = util.misc.SerializedInvocation(self.doPoll)
 
@@ -76,7 +80,7 @@ def poll(self):
 
     def startLoop(self):
         self._loop = task.LoopingCall(self.doPoll)
-        self._loop.start(self.pollInterval, now=False)
+        self._loop.start(self.pollInterval, now=self.pollAtLaunch)
 
     def stopLoop(self):
         if self._loop and self._loop.running:

master/buildbot/changes/bonsaipoller.py

@@ -221,12 +221,12 @@ def _getRevision(self):
 
 class BonsaiPoller(base.PollingChangeSource):
     compare_attrs = ["bonsaiURL", "pollInterval", "tree",
-                     "module", "branch", "cvsroot"]
+                     "module", "branch", "cvsroot", "pollAtLaunch"]
 
     def __init__(self, bonsaiURL, module, branch, tree="default",
-                 cvsroot="/cvsroot", pollInterval=30, project=''):
+                 cvsroot="/cvsroot", pollInterval=30, project='', pollAtLaunch=False):
 
-        base.PollingChangeSource.__init__(self, name=bonsaiURL, pollInterval=pollInterval)
+        base.PollingChangeSource.__init__(self, name=bonsaiURL, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
 
         self.bonsaiURL = bonsaiURL
         self.module = module

master/buildbot/changes/gerritchangesource.py

@@ -155,7 +155,8 @@ def addChangeFromEvent(self, properties, event):
                 'repository': "ssh://%s@%s:%s/%s" % (
                     self.username, self.gerritserver,
                     self.gerritport, event_change["project"]),
-                'branch': event_change["branch"],
+                'branch': "%s/%s" % (event_change["branch"],
+                                     event_change['number']),
                 'revision': event["patchSet"]["revision"],
                 'revlink': event_change["url"],
                 'comments': event_change["subject"],

master/buildbot/changes/gitpoller.py

@@ -35,21 +35,22 @@ class GitPoller(base.PollingChangeSource, StateMixin):
 
     compare_attrs = ["repourl", "branches", "workdir",
                      "pollInterval", "gitbin", "usetimestamps",
-                     "category", "project"]
+                     "category", "project", "pollAtLaunch"]
 
     def __init__(self, repourl, branches=None, branch=None,
                  workdir=None, pollInterval=10 * 60,
                  gitbin='git', usetimestamps=True,
                  category=None, project=None,
                  pollinterval=-2, fetch_refspec=None,
-                 encoding='utf-8'):
+                 encoding='utf-8', pollAtLaunch=False):
 
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
 
         base.PollingChangeSource.__init__(self, name=repourl,
-                                          pollInterval=pollInterval)
+                                          pollInterval=pollInterval,
+                                          pollAtLaunch=pollAtLaunch)
 
         if project is None:
             project = ''
@@ -98,29 +99,73 @@ def setLastRev(lastRev):
         return d
 
     def describe(self):
-        status = ""
+        str = ('GitPoller watching the remote git repository ' +
+               self.repourl)
+
+        if self.branches:
+            if self.branches is True:
+                str += ', branches: ALL'
+            elif not callable(self.branches):
+                str += ', branches: ' + ', '.join(self.branches)
+
         if not self.master:
-            status = "[STOPPED - check log]"
-        str = ('GitPoller watching the remote git repository %s, branches: %s %s'
-               % (self.repourl, ', '.join(self.branches), status))
+            str += " [STOPPED - check log]"
+
         return str
 
+    def _getBranches(self):
+        d = self._dovccmd('ls-remote', [self.repourl])
+
+        @d.addCallback
+        def parseRemote(rows):
+            branches = []
+            for row in rows.splitlines():
+                if not '\t' in row:
+                    # Not a useful line
+                    continue
+                sha, ref = row.split("\t")
+                branches.append(ref)
+            return branches
+        return d
+
+    def _headsFilter(self, branch):
+        """Filter out remote references that don't begin with 'refs/heads'."""
+        return branch.startswith("refs/heads/")
+
+    def _removeHeads(self, branch):
+        """Remove 'refs/heads/' prefix from remote references."""
+        if branch.startswith("refs/heads/"):
+            branch = branch[11:]
+        return branch
+
+    def _trackerBranch(self, branch):
+        return "refs/buildbot/%s/%s" % (urllib.quote(self.repourl, ''),
+                                        self._removeHeads(branch))
+
     @defer.inlineCallbacks
     def poll(self):
         yield self._dovccmd('init', ['--bare', self.workdir])
 
+        branches = self.branches
+        if branches is True or callable(branches):
+            branches = yield self._getBranches()
+            if callable(self.branches):
+                branches = filter(self.branches, branches)
+            else:
+                branches = filter(self._headsFilter, branches)
+
         refspecs = [
-            '+%s:%s' % (branch, self._localBranch(branch))
-            for branch in self.branches
+            '+%s:%s' % (self._removeHeads(branch), self._trackerBranch(branch))
+            for branch in branches
         ]
         yield self._dovccmd('fetch',
                             [self.repourl] + refspecs, path=self.workdir)
 
         revs = {}
-        for branch in self.branches:
+        for branch in branches:
             try:
-                revs[branch] = rev = yield self._dovccmd('rev-parse',
-                                                         [self._localBranch(branch)], path=self.workdir)
+                revs[branch] = rev = yield self._dovccmd(
+                    'rev-parse', [self._trackerBranch(branch)], path=self.workdir)
                 yield self._process_changes(rev, branch)
             except:
                 log.err(_why="trying to poll branch %s of %s"
@@ -255,6 +300,3 @@ def _convert_nonzero_to_failure(res):
             return stdout.strip()
         d.addCallback(_convert_nonzero_to_failure)
         return d
-
-    def _localBranch(self, branch):
-        return "refs/buildbot/%s/%s" % (urllib.quote(self.repourl, ''), branch)

master/buildbot/changes/hgpoller.py

@@ -33,15 +33,15 @@ class HgPoller(base.PollingChangeSource):
 
     compare_attrs = ["repourl", "branch", "workdir",
                      "pollInterval", "hgpoller", "usetimestamps",
-                     "category", "project"]
+                     "category", "project", "pollAtLaunch"]
 
     db_class_name = 'HgPoller'
 
     def __init__(self, repourl, branch='default',
                  workdir=None, pollInterval=10 * 60,
                  hgbin='hg', usetimestamps=True,
                  category=None, project='', pollinterval=-2,
-                 encoding='utf-8'):
+                 encoding='utf-8', pollAtLaunch=False):
 
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
@@ -50,7 +50,7 @@ def __init__(self, repourl, branch='default',
         self.repourl = repourl
         self.branch = branch
         base.PollingChangeSource.__init__(
-            self, name=repourl, pollInterval=pollInterval)
+            self, name=repourl, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
         self.encoding = encoding
         self.lastChange = time.time()
         self.lastPoll = time.time()

master/buildbot/changes/p4poller.py

@@ -55,7 +55,7 @@ class P4Source(base.PollingChangeSource, util.ComparableMixin):
     them to the change master."""
 
     compare_attrs = ["p4port", "p4user", "p4passwd", "p4base",
-                     "p4bin", "pollInterval"]
+                     "p4bin", "pollInterval", "pollAtLaunch"]
 
     env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER",
                 "P4CHARSET", "PATH"]
@@ -76,13 +76,13 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
                  split_file=lambda branchfile: (None, branchfile),
                  pollInterval=60 * 10, histmax=None, pollinterval=-2,
                  encoding='utf8', project=None, name=None,
-                 server_tz=None):
+                 server_tz=None, pollAtLaunch=False):
 
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
 
-        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)
+        base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
 
         if project is None:
             project = ''

master/buildbot/changes/svnpoller.py

@@ -79,7 +79,7 @@ class SVNPoller(base.PollingChangeSource, util.ComparableMixin):
     compare_attrs = ["svnurl", "split_file",
                      "svnuser", "svnpasswd", "project",
                      "pollInterval", "histmax",
-                     "svnbin", "category", "cachepath"]
+                     "svnbin", "category", "cachepath", "pollAtLaunch"]
 
     parent = None  # filled in when we're added
     last_change = None
@@ -90,13 +90,13 @@ def __init__(self, svnurl, split_file=None,
                  pollInterval=10 * 60, histmax=100,
                  svnbin='svn', revlinktmpl='', category=None,
                  project='', cachepath=None, pollinterval=-2,
-                 extra_args=None):
+                 extra_args=None, pollAtLaunch=False):
 
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
 
-        base.PollingChangeSource.__init__(self, name=svnurl, pollInterval=pollInterval)
+        base.PollingChangeSource.__init__(self, name=svnurl, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch)
 
         if svnurl.endswith("/"):
             svnurl = svnurl[:-1]  # strip the trailing slash