Merge branch 'master' into nine

Conflicts:
	master/buildbot/buildslave/base.py
	master/buildbot/changes/gerritchangesource.py
	master/buildbot/changes/gitpoller.py
	master/buildbot/process/buildstep.py
	master/buildbot/process/logobserver.py
	master/buildbot/process/remotecommand.py
	master/buildbot/status/master.py
	master/buildbot/status/web/baseweb.py
	master/buildbot/status/web/build.py
	master/buildbot/steps/source/base.py
	master/buildbot/test/unit/test_changes_gerritchangesource.py
	master/buildbot/test/unit/test_changes_hgpoller.py
	master/docs/relnotes/index.rst

CONTRIBUTING.md

@@ -5,6 +5,15 @@ Thank you for contributing to Buildbot!
 
 What appears below is just a quick summary.  See http://trac.buildbot.net/wiki/Development for the full story.
 
+Issues, Bugs, Tickets
+---------------------
+
+*We do not use the GitHub Issue Tracker for bugs*
+
+Please file tickets for any bugs you discover at http://trac.buildbot.net.
+
+The GitHub Issue Tracker is enabled only because it is a more usable interface for pull requests.
+
 Patches
 -------
 

common/validate.sh

@@ -100,7 +100,9 @@ trap "rm -f ${tempfile}" 1 2 3 15
 git diff --name-only $REVRANGE | grep '\.py$' | grep -v '\(^master/\(contrib\|docs\)\|/setup\.py\)' > ${tempfile}
 py_files=()
 while read line; do
-    py_files+=($line)
+    if test -f "${line}"; then
+        py_files+=($line)
+    fi
 done < ${tempfile}
 
 echo "${MAGENTA}Validating the following commits:${NORM}"

master/buildbot/buildslave/base.py

@@ -337,6 +337,7 @@ def attached(self, conn):
         self.slave_status.addGracefulWatcher(self._gracefulChanged)
         self.conn = conn
         self._old_builder_list = None  # clear builder list before proceed
+        self.slave_status.addPauseWatcher(self._pauseChanged)
 
         self.slave_status.setConnected(True)
 
@@ -385,6 +386,7 @@ def detached(self):
         self.conn = None
         self._old_builder_list = []
         self.slave_status.removeGracefulWatcher(self._gracefulChanged)
+        self.slave_status.removePauseWatcher(self._pauseChanged)
         self.slave_status.setConnected(False)
         log.msg("BuildSlave.detached(%s)" % self.slavename)
         self.master.status.slaveDisconnected(self.slavename)
@@ -564,6 +566,12 @@ def maybeShutdown(self):
         d = self.shutdown()
         d.addErrback(log.err, 'error while shutting down slave')
 
+    def _pauseChanged(self, paused):
+        if paused is True:
+            self.botmaster.master.status.slavePaused(self.slavename)
+        else:
+            self.botmaster.master.status.slaveUnpaused(self.slavename)
+
     def pause(self):
         """Stop running new builds on the slave."""
         self.slave_status.setPaused(True)
@@ -574,7 +582,7 @@ def unpause(self):
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
     def isPaused(self):
-        return self.paused
+        return self.slave_status.isPaused()
 
 
 class BuildSlave(AbstractBuildSlave):

master/buildbot/changes/gerritchangesource.py

@@ -13,12 +13,12 @@
 #
 # Copyright Buildbot Team Members
 
-from twisted.internet import reactor
-
 from buildbot import util
 from buildbot.changes import base
 from buildbot.util import json
+from collections import MutableMapping
 from twisted.internet import defer
+from twisted.internet import reactor
 from twisted.internet.protocol import ProcessProtocol
 from twisted.python import log
 
@@ -42,7 +42,13 @@ class GerritChangeSource(base.ChangeSource):
     STREAM_BACKOFF_MAX = 60
     "(seconds) maximum time to wait before retrying a failed connection"
 
-    def __init__(self, gerritserver, username, gerritport=29418, identity_file=None, name=None):
+    def __init__(self,
+                 gerritserver,
+                 username,
+                 gerritport=29418,
+                 identity_file=None,
+                 name=None,
+                 handled_events=("patchset-created", "ref-updated")):
         """
         @type  gerritserver: string
         @param gerritserver: the dns or ip that host the gerrit ssh server,
@@ -54,8 +60,10 @@ def __init__(self, gerritserver, username, gerritport=29418, identity_file=None,
         @param username: the username to use to connect to gerrit,
 
         @type  identity_file: string
-        @param identity_file: identity file to for authentication (optional).
+        @param identity_file: identity file to for authentication (optional),
 
+        @type  handled_events: list
+        @param handled_events: event to be handled (optional).
         """
         # TODO: delete API comment when documented
 
@@ -68,6 +76,7 @@ def __init__(self, gerritserver, username, gerritport=29418, identity_file=None,
         self.gerritport = gerritport
         self.username = username
         self.identity_file = identity_file
+        self.handled_events = list(handled_events)
         self.process = None
         self.wantProcess = False
         self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
@@ -83,13 +92,14 @@ def outReceived(self, data):
             """Do line buffering."""
             self.data += data
             lines = self.data.split("\n")
-            self.data = lines.pop(-1)  # last line is either empty or incomplete
+            # last line is either empty or incomplete
+            self.data = lines.pop(-1)
             for line in lines:
-                log.msg("gerrit: %s" % (line,))
+                log.msg("gerrit: %s" % line)
                 yield self.change_source.lineReceived(line)
 
         def errReceived(self, data):
-            log.msg("gerrit stderr: %s" % (data,))
+            log.msg("gerrit stderr: %s" % data)
 
         def processEnded(self, status_object):
             self.change_source.streamProcessStopped()
@@ -98,64 +108,82 @@ def lineReceived(self, line):
         try:
             event = json.loads(line.decode('utf-8'))
         except ValueError:
-            log.msg("bad json line: %s" % (line,))
+            msg = "bad json line: %s"
+            log.msg(msg % line)
             return defer.succeed(None)
 
-        if not(isinstance(event, type({})) and "type" in event):
-            log.msg("no type in event %s" % (line,))
+        if not(isinstance(event, MutableMapping) and "type" in event):
+            msg = "no type in event %s"
+            log.msg(msg % line)
             return defer.succeed(None)
-        func = getattr(self, "eventReceived_" + event["type"].replace("-", "_"), None)
-        if func is None:
-            log.msg("unsupported event %s" % (event["type"],))
+
+        if not (event['type'] in self.handled_events):
+            msg = "the event type '%s' is not setup to handle"
+            log.msg(msg % event['type'])
             return defer.succeed(None)
 
         # flatten the event dictionary, for easy access with WithProperties
         def flatten(properties, base, event):
             for k, v in event.items():
-                if isinstance(v, dict):
-                    flatten(properties, base + "." + k, v)
+                name = "%s.%s" % (base, k)
+                if isinstance(v, MutableMapping):
+                    flatten(properties, name, v)
                 else:  # already there
-                    properties[base + "." + k] = v
+                    properties[name] = v
 
         properties = {}
         flatten(properties, "event", event)
-        return func(properties, event)
+        event_with_change = "change" in event and "patchSet" in event
+        func_name = "eventReceived_%s" % event["type"].replace("-", "_")
+        func = getattr(self, func_name, None)
+        if func is None and event_with_change:
+            return self. addChangeFromEvent(properties, event)
+        elif func is None:
+            log.msg("unsupported event %s" % (event["type"],))
+            return defer.succeed(None)
+        else:
+            return func(properties, event)
 
     def addChange(self, chdict):
         d = self.master.data.updates.addChange(**chdict)
         # eat failures..
         d.addErrback(log.err, 'error adding change from GerritChangeSource')
         return d
 
-    def eventReceived_patchset_created(self, properties, event):
-        change = event["change"]
-        username = change["owner"].get("username", "unknown")
-        return self.addChange(dict(
-            author="%s <%s>" % (change["owner"].get("name", username), change["owner"].get("email", "unknown@example.com")),
-            project=change["project"],
-            repository="ssh://%s@%s:%s/%s" % (
-                self.username, self.gerritserver, self.gerritport, change["project"]),
-            branch=change["branch"] + "/" + change["number"],
-            revision=event["patchSet"]["revision"],
-            revlink=change["url"],
-            comments=change["subject"],
-            files=[u"unknown"],
-            category=event["type"],
-            properties=properties))
+    def addChangeFromEvent(self, properties, event):
+        if "change" in event and "patchSet" in event:
+            event_change = event["change"]
+            username = event_change["owner"].get("username", u"unknown")
+            return self.addChange({
+                'author': "%s <%s>" % (
+                    event_change["owner"].get("name", username),
+                    event_change["owner"].get("email", u'unknown@example.com')),
+                'project': util.ascii2unicode(event_change["project"]),
+                'repository': u"ssh://%s@%s:%s/%s" % (
+                    self.username, self.gerritserver,
+                    self.gerritport, event_change["project"]),
+                'branch': event_change["branch"],
+                'revision': event["patchSet"]["revision"],
+                'revlink': event_change["url"],
+                'comments': event_change["subject"],
+                'files': [u"unknown"],
+                'category': event["type"],
+                'properties': properties})
 
     def eventReceived_ref_updated(self, properties, event):
         ref = event["refUpdate"]
         author = "gerrit"
 
         if "submitter" in event:
-            username = event["submitter"].get("username", "unknown")
-            author = "%s <%s>" % (event["submitter"].get("name", username), event["submitter"].get("email", "unnkown@example.com"))
+            author = "%s <%s>" % (
+                event["submitter"]["name"], event["submitter"]["email"])
 
         return self.addChange(dict(
             author=author,
             project=ref["project"],
             repository="ssh://%s@%s:%s/%s" % (
-                self.username, self.gerritserver, self.gerritport, ref["project"]),
+                self.username, self.gerritserver,
+                self.gerritport, ref["project"]),
             branch=ref["refName"],
             revision=ref["newRev"],
             comments="Gerrit: patchset(s) merged.",
@@ -172,28 +200,34 @@ def streamProcessStopped(self):
             return
 
         now = util.now()
-        if now - self.lastStreamProcessStart < self.STREAM_GOOD_CONNECTION_TIME:
-            # bad startup; start the stream process again after a timeout, and then
-            # increase the timeout
-            log.msg("'gerrit stream-events' failed; restarting after %ds" % round(self.streamProcessTimeout))
-            reactor.callLater(self.streamProcessTimeout, self.startStreamProcess)
+        if now - self.lastStreamProcessStart < \
+           self.STREAM_GOOD_CONNECTION_TIME:
+            # bad startup; start the stream process again after a timeout,
+            # and then increase the timeout
+            log.msg(
+                "'gerrit stream-events' failed; restarting after %ds"
+                % round(self.streamProcessTimeout))
+            reactor.callLater(
+                self.streamProcessTimeout, self.startStreamProcess)
             self.streamProcessTimeout *= self.STREAM_BACKOFF_EXPONENT
             if self.streamProcessTimeout > self.STREAM_BACKOFF_MAX:
                 self.streamProcessTimeout = self.STREAM_BACKOFF_MAX
         else:
-            # good startup, but lost connection; restart immediately, and set the timeout
-            # to its minimum
+            # good startup, but lost connection; restart immediately,
+            # and set the timeout to its minimum
             self.startStreamProcess()
             self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
 
     def startStreamProcess(self):
         log.msg("starting 'gerrit stream-events'")
         self.lastStreamProcessStart = util.now()
-        args = [self.username + "@" + self.gerritserver, "-p", str(self.gerritport)]
+        uri = "%s@%s" % (self.username, self.gerritserver)
+        args = [uri, "-p", str(self.gerritport)]
         if self.identity_file is not None:
             args = args + ['-i', self.identity_file]
-        self.process = reactor.spawnProcess(self.LocalPP(self), "ssh",
-                                            ["ssh"] + args + ["gerrit", "stream-events"])
+        self.process = reactor.spawnProcess(
+            self.LocalPP(self), "ssh",
+            ["ssh"] + args + ["gerrit", "stream-events"])
 
     def activate(self):
         self.wantProcess = True
@@ -203,13 +237,13 @@ def deactivate(self):
         self.wantProcess = False
         if self.process:
             self.process.signalProcess("KILL")
-        # TODO: if this occurs while the process is restarting, some exceptions may
-        # be logged, although things will settle down normally
+        # TODO: if this occurs while the process is restarting, some exceptions
+        # may be logged, although things will settle down normally
 
     def describe(self):
         status = ""
         if not self.process:
             status = "[NOT CONNECTED - check log]"
-        str = ('GerritChangeSource watching the remote Gerrit repository %s@%s %s' %
-               (self.username, self.gerritserver, status))
-        return str
+        msg = ("GerritChangeSource watching the remote "
+               "Gerrit repository %s@%s %s")
+        return msg % (self.username, self.gerritserver, status)

master/buildbot/changes/gitpoller.py

@@ -13,7 +13,9 @@
 #
 # Copyright Buildbot Team Members
 
+import itertools
 import os
+import re
 import urllib
 
 from twisted.internet import defer
@@ -128,9 +130,13 @@ def poll(self):
         self.lastRev.update(revs)
         yield self.setState('lastRev', self.lastRev)
 
+    def _decode(self, git_output):
+        return git_output.decode(self.encoding)
+
     def _get_commit_comments(self, rev):
         args = ['--no-walk', r'--format=%s%n%b', rev, '--']
         d = self._dovccmd('log', args, path=self.workdir)
+        d.addCallback(self._decode)
         return d
 
     def _get_commit_timestamp(self, rev):
@@ -155,13 +161,16 @@ def _get_commit_files(self, rev):
         args = ['--name-only', '--no-walk', r'--format=%n', rev, '--']
         d = self._dovccmd('log', args, path=self.workdir)
 
-        def process(git_output):
-            fileList = git_output.split()
+        def decode_file(file):
+            # git use octal char sequences in quotes when non ASCII
+            match = re.match('^"(.*)"$', file)
+            if match:
+                file = match.groups()[0].decode('string_escape')
+            return self._decode(file)
 
-            # filenames in git are presumably just like POSIX filenames -
-            # encoding-free bytestrings.  In most cases, they'll UTF-8 and
-            # mostly ASCII, so that's a safe assumption here.
-            return [f.decode('utf-8', 'replace') for f in fileList]
+        def process(git_output):
+            fileList = [decode_file(file) for file in itertools.ifilter(lambda s: len(s), git_output.splitlines())]
+            return fileList
         d.addCallback(process)
         return d
 
@@ -170,7 +179,7 @@ def _get_commit_author(self, rev):
         d = self._dovccmd('log', args, path=self.workdir)
 
         def process(git_output):
-            git_output = git_output.decode(self.encoding)
+            git_output = self._decode(git_output)
             if len(git_output) == 0:
                 raise EnvironmentError('could not get commit author for rev')
             return git_output

master/buildbot/changes/hgpoller.py

@@ -97,14 +97,14 @@ def _getRevDetails(self, rev):
         args = ['log', '-r', rev, os.linesep.join((
             '--template={date|hgdate}',
             '{author}',
-            '{files}',
+            "{files % '{file}" + os.pathsep + "'}",
             '{desc|strip}'))]
         # Mercurial fails with status 255 if rev is unknown
         d = utils.getProcessOutput(self.hgbin, args, path=self._absWorkdir(),
                                    env=os.environ, errortoo=False)
 
         def process(output):
-            # fortunately, Mercurial issues all filenames one one line
+            # all file names are on one line
             date, author, files, comments = output.decode(self.encoding, "replace").split(
                 os.linesep, 3)
 
@@ -117,7 +117,7 @@ def process(output):
                     log.msg('hgpoller: caught exception converting output %r '
                             'to timestamp' % date)
                     raise
-            return stamp, author.strip(), files.split(), comments.strip()
+            return stamp, author.strip(), files.split(os.pathsep)[:-1], comments.strip()
 
         d.addCallback(process)
         return d

master/buildbot/process/users/manual.py

@@ -156,6 +156,7 @@ def perspective_commandline(self, op, bb_username, bb_password, ids, info):
                     # when adding, we update the user after the first attr
                     once_through = False
                     for attr in user:
+                        result = None
                         if op == 'update' or once_through:
                             if uid:
                                 result = yield self.master.db.users.updateUser(