Merge branch 'master' into nine

Conflicts:
	.travis.yml
	master/buildbot/buildslave/base.py
	master/buildbot/test/unit/test_status_web_hooks_github.py

.travis.yml

@@ -3,8 +3,10 @@
 
 language: python
 
+# Available Python versions:
+# http://about.travis-ci.org/docs/user/ci-environment/#Python-VM-images
 python:
-  - "2.5"
+  # "2.5" -- not supported by Travis CI anymore
   - "2.6"
   - "2.7"
 
@@ -15,13 +17,6 @@ env:
   - TWISTED=latest SQLALCHEMY=latest SQLALCHEMY_MIGRATE=latest
 
 matrix:
-  exclude:
-    # Disable not supported Twisted versions on Python 2.5
-    - python: "2.5"
-      env: TWISTED=12.2.0 SQLALCHEMY=latest SQLALCHEMY_MIGRATE=0.7.1
-    - python: "2.5"
-      env: TWISTED=13.0.0 SQLALCHEMY=latest SQLALCHEMY_MIGRATE=0.7.1
-
   include:
     # Test different versions on SQLAlchemy
     - python: "2.7"
@@ -71,21 +66,17 @@ install:
     done
   - git describe
 
-  # zope.interface dropped Python 2.5 support in 4.0.0
-  - "[ $TRAVIS_PYTHON_VERSION != '2.5' ] || pip install 'zope.interface<4.0.0'"
-  # Python 2.5 doesn't have multiprocessing natively
-  - "[ $TRAVIS_PYTHON_VERSION != '2.5' ] || pip install multiprocessing"
   - "[ $TWISTED = latest ] || pip install Twisted==$TWISTED"
   - "[ $SQLALCHEMY = latest ] || pip install sqlalchemy==$SQLALCHEMY"
   - "[ $SQLALCHEMY_MIGRATE = latest ] || pip install sqlalchemy-migrate==$SQLALCHEMY_MIGRATE"
   - (cd master; python setup.py develop)
   - (cd slave;  python setup.py develop)
   - (cd www;    python setup.py develop)
   # mock is preinstalled on Travis
-  # txgithub requires Twisted >= 12.3.0, which doesn't work on Python 2.5.
-  - "[ $TRAVIS_PYTHON_VERSION = '2.5' ] || pip install txgithub"
+  # txgithub requires Twisted >= 12.3.0
+  - pip install txgithub
   # txrequests support only Python 2.6 and 2.7.
-  - "[ $TRAVIS_PYTHON_VERSION = '2.5' ] || pip install txrequests"
+  - pip install txrequests
 
   # Determine is current configuration is latest
   - |

master/buildbot/buildslave/base.py

@@ -16,8 +16,8 @@
 
 import time
 
-from email.Message import Message
-from email.Utils import formatdate
+from email.message import Message
+from email.utils import formatdate
 from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.python import log

master/buildbot/changes/gitpoller.py

@@ -131,13 +131,6 @@ def poll(self):
     def _get_commit_comments(self, rev):
         args = ['--no-walk', r'--format=%s%n%b', rev, '--']
         d = self._dovccmd('log', args, path=self.workdir)
-
-        def process(git_output):
-            git_output = git_output.decode(self.encoding)
-            if len(git_output) == 0:
-                raise EnvironmentError('could not get commit comment for rev')
-            return git_output
-        d.addCallback(process)
         return d
 
     def _get_commit_timestamp(self, rev):

master/buildbot/changes/mail.py

@@ -22,10 +22,10 @@
 import time
 
 from email import message_from_file
-from email.Iterators import body_line_iterator
-from email.Utils import mktime_tz
-from email.Utils import parseaddr
-from email.Utils import parsedate_tz
+from email.iterators import body_line_iterator
+from email.utils import mktime_tz
+from email.utils import parseaddr
+from email.utils import parsedate_tz
 
 from buildbot import util
 from buildbot.interfaces import IChangeSource

master/buildbot/db/model.py

@@ -18,17 +18,13 @@
 import migrate.versioning.schema
 import sqlalchemy as sa
 
+from migrate import exceptions
+
 from buildbot.db import base
 from buildbot.db.types.json import JsonObject
 from twisted.python import log
 from twisted.python import util
 
-try:
-    from migrate.versioning import exceptions
-    _hush_pyflakes = exceptions
-except ImportError:
-    from migrate import exceptions
-
 
 class Model(base.DBConnectorComponent):
     #
@@ -746,13 +742,3 @@ def thd(engine):
         # import the prerequisite classes for pickles
         check_sqlalchemy_migrate_version()
         return self.db.pool.do_with_engine(thd)
-
-# migrate has a bug in one of its warnings; this is fixed in version control
-# (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we
-# get realistic tracebacks
-try:
-    import migrate.versioning.exceptions as ex1
-    import migrate.changeset.exceptions as ex2
-    ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning
-except (ImportError, AttributeError):
-    pass

master/buildbot/status/mail.py

@@ -19,11 +19,11 @@
 import urllib
 
 from StringIO import StringIO
-from email.MIMEMultipart import MIMEMultipart
-from email.MIMENonMultipart import MIMENonMultipart
-from email.MIMEText import MIMEText
-from email.Message import Message
-from email.Utils import formatdate
+from email.message import Message
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import formatdate
 
 from twisted.internet import defer
 from twisted.internet import reactor
@@ -45,8 +45,8 @@
 
 # this incantation teaches email to output utf-8 using 7- or 8-bit encoding,
 # although it has no effect before python-2.7.
-from email import Charset
-Charset.add_charset('utf-8', Charset.SHORTEST, None, 'utf-8')
+from email import charset
+charset.add_charset('utf-8', charset.SHORTEST, None, 'utf-8')
 
 from buildbot import config
 from buildbot import interfaces

master/buildbot/status/tinderbox.py

@@ -14,8 +14,8 @@
 # Copyright Buildbot Team Members
 
 
-from email.Message import Message
-from email.Utils import formatdate
+from email.message import Message
+from email.utils import formatdate
 
 from twisted.internet import defer
 from zope.interface import implements

master/buildbot/status/web/hooks/github.py

@@ -69,6 +69,13 @@ def process_change(payload, user, repo, repo_url, project):
             log.msg("Branch `%s' deleted, ignoring" % branch)
         else:
             for commit in payload['commits']:
+                if 'distinct' in commit and not commit['distinct']:
+                    log.msg(
+                        'Commit `%s` is a non-distinct commit, ignoring...' % (
+                            commit['id'])
+                    )
+                    continue
+
                 files = []
                 if 'added' in commit:
                     files.extend(commit['added'])

master/buildbot/steps/maxq.py

@@ -14,11 +14,23 @@
 # Copyright Buildbot Team Members
 
 from buildbot import config
+from buildbot.process import buildstep
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
 from buildbot.steps.shell import ShellCommand
 
 
+class MaxQObserver(buildstep.LogLineObserver):
+
+    def __init__(self):
+        buildstep.LogLineObserver.__init__(self)
+        self.failures = 0
+
+    def outLineReceived(self, line):
+        if line.startswith('TEST FAILURE:'):
+            self.failures += 1
+
+
 class MaxQ(ShellCommand):
     flunkOnFailure = True
     name = "maxq"
@@ -28,10 +40,11 @@ def __init__(self, testdir=None, **kwargs):
             config.error("please pass testdir")
         kwargs['command'] = 'run_maxq.py %s' % (testdir,)
         ShellCommand.__init__(self, **kwargs)
+        self.observer = MaxQObserver()
+        self.addLogObserver('stdio', self.observer)
 
     def commandComplete(self, cmd):
-        output = cmd.logs['stdio'].getText()
-        self.failures = output.count('\nTEST FAILURE:')
+        self.failures = self.observer.failures
 
     def evaluateCommand(self, cmd):
         # treat a nonzero exit status as a failure, if no other failures are

master/buildbot/steps/package/deb/lintian.py

@@ -18,12 +18,24 @@
 """
 
 from buildbot import config
+from buildbot.process import buildstep
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
 from buildbot.status.results import WARNINGS
 from buildbot.steps.shell import ShellCommand
 
 
+class MaxQObserver(buildstep.LogLineObserver):
+
+    def __init__(self):
+        buildstep.LogLineObserver.__init__(self)
+        self.failures = 0
+
+    def outLineReceived(self, line):
+        if line.startswith('TEST FAILURE:'):
+            self.failures += 1
+
+
 class DebLintian(ShellCommand):
     name = "lintian"
     description = ["Lintian running"]

master/buildbot/steps/source/cvs.py

@@ -16,7 +16,7 @@
 import re
 import time
 
-from email.Utils import formatdate
+from email.utils import formatdate
 
 from twisted.internet import defer
 from twisted.internet import reactor

master/buildbot/steps/source/oldsource.py

@@ -18,7 +18,7 @@
 from buildbot.interfaces import IRenderable
 from buildbot.process.buildstep import RemoteCommand
 from buildbot.steps.source.base import Source
-from email.Utils import formatdate
+from email.utils import formatdate
 from twisted.internet import defer
 from twisted.python import log
 from warnings import warn

master/buildbot/steps/source/repo.py

@@ -92,7 +92,7 @@ class Repo(Source):
 
     """ Class for Repo with all the smarts """
     name = 'repo'
-    renderables = ["manifestURL", "manifestFile", "tarball", "jobs",
+    renderables = ["manifestURL", "manifestBranch", "manifestFile", "tarball", "jobs",
                    "syncAllBranches", "updateTarballAge", "manifestOverrideUrl",
                    "repoDownloads"]
 

master/buildbot/test/unit/test_changes_gitpoller.py

@@ -105,18 +105,23 @@ def cb_desired(r):
                 [self.assertIsInstance(e, unicode) for e in r]
         d.addCallback(cb_desired)
         d.addCallback(lambda _: self.assertAllCommandsRan())
+        return d
 
     def test_get_commit_author(self):
         authorStr = 'Sammy Jankis <email@example.com>'
         return self._perform_git_output_test(self.poller._get_commit_author,
                                              ['log', '--no-walk', '--format=%aN <%aE>', self.dummyRevStr, '--'],
                                              authorStr, authorStr)
 
-    def test_get_commit_comments(self):
-        commentStr = 'this is a commit message\n\nthat is multiline'
+    def _test_get_commit_comments(self, commentStr):
         return self._perform_git_output_test(self.poller._get_commit_comments,
                                              ['log', '--no-walk', '--format=%s%n%b', self.dummyRevStr, '--'],
-                                             commentStr, commentStr)
+                                             commentStr, commentStr, emptyRaisesException=False)
+
+    def test_get_commit_comments(self):
+        comments = ['this is a commit message\n\nthat is multiline',
+                    'single line message', '']
+        return defer.DeferredList([self._test_get_commit_comments(commentStr) for commentStr in comments])
 
     def test_get_commit_files(self):
         filesStr = 'file1\nfile2'

master/buildbot/test/unit/test_changes_mail_CVSMaildirSource.py

@@ -17,8 +17,8 @@
 
 from buildbot.changes.mail import CVSMaildirSource
 from email import message_from_string
-from email.Utils import mktime_tz
-from email.Utils import parsedate_tz
+from email.utils import mktime_tz
+from email.utils import parsedate_tz
 
 #
 # Sample message from CVS version 1.11

master/buildbot/test/unit/test_status_mail.py

@@ -665,7 +665,7 @@ def do_test_sendToInterestedUsers(self, lookup=None, extraRecipients=[],
                                       sendToInterestedUsers=True,
                                       exp_called_with=None, exp_TO=None,
                                       exp_CC=None):
-        from email.Message import Message
+        from email.message import Message
         m = Message()
 
         mn = MailNotifier(fromaddr='from@example.org',
@@ -777,7 +777,7 @@ def test_sendToInterestedUsers_False(self):
             exp_TO="marla@mayhem.net")
 
     def test_sendToInterestedUsers_two_builds(self):
-        from email.Message import Message
+        from email.message import Message
         m = Message()
 
         mn = MailNotifier(fromaddr="from@example.org", lookup=None)

master/buildbot/test/util/steps.py

@@ -157,6 +157,10 @@ def addLogObserver(logname, observer):
             observer.step = step
         step.addLogObserver = addLogObserver
 
+        # add any observers defined in the constructor, before this monkey-patch
+        for n, o in step._pendingLogObservers:
+            addLogObserver(n, o)
+
         # set defaults
 
         step.setDefaultWorkdir('wkdir')