Merge branch 'master' into nine

Conflicts:
	.travis.yml
	Makefile
	common/validate.sh
	master/buildbot/db/buildrequests.py
	master/buildbot/process/buildstep.py
	master/buildbot/sourcestamp.py
	master/buildbot/status/web/base.py
	master/buildbot/status/web/builder.py
	master/buildbot/steps/source/cvs.py
	master/buildbot/steps/source/git.py
	master/buildbot/test/unit/test_changes_p4poller.py
	master/buildbot/test/unit/test_db_buildsets.py
	master/buildbot/test/unit/test_process_buildstep.py
	master/buildbot/test/unit/test_sourcestamp.py
	master/buildbot/test/unit/test_status_mail.py

.travis.yml

@@ -82,6 +82,8 @@ install:
   # mock is preinstalled on Travis
   # txgithub requires Twisted >= 12.3.0, which doesn't work on Python 2.5.
   - "[ $TRAVIS_PYTHON_VERSION = '2.5' ] || pip install txgithub"
+  # txrequests support only Python 2.6 and 2.7.
+  - "[ $TRAVIS_PYTHON_VERSION = '2.5' ] || pip install txrequests"
 
   # Determine is current configuration is latest
   - |

common/validate.sh

@@ -9,6 +9,7 @@ _ESC=$'\e'
 GREEN="$_ESC[0;32m"
 MAGENTA="$_ESC[0;35m"
 RED="$_ESC[0;31m"
+LTCYAN="$_ESC[1;36m"
 YELLOW="$_ESC[1;33m"
 NORM="$_ESC[0;0m"
 
@@ -23,7 +24,7 @@ fi
 
 status() {
     echo ""
-    echo "${YELLOW}-- ${*} --${NORM}"
+    echo "${LTCYAN}-- ${*} --${NORM}"
 }
 
 ok=true
@@ -34,6 +35,11 @@ not_ok() {
     problem_summary="$problem_summary"$'\n'"${RED}**${NORM} ${*}"
 }
 
+warning() {
+    echo "${YELLOW}** ${*} **${NORM}"
+    problem_summary="$problem_summary"$'\n'"${YELLOW}**${NORM} ${*} (warning)"
+}
+
 check_tabs() {
     git diff "$REVRANGE" | grep -q $'+.*\t'
 }
@@ -89,11 +95,11 @@ run_tests || not_ok "tests failed"
 
 status "checking formatting"
 check_tabs && not_ok "$REVRANGE adds tabs"
-check_long_lines && not_ok "$REVRANGE adds long lines"
+check_long_lines && warning "$REVRANGE adds long lines"
 check_yield_defer_returnValue && not_ok "$REVRANGE yields defer.returnValue"
 
 status "checking for release notes"
-check_relnotes || not_ok "$REVRANGE does not add release notes"
+check_relnotes || warning "$REVRANGE does not add release notes"
 
 status "running pyflakes"
 sandbox/bin/pyflakes master/buildbot slave/buildslave || not_ok "failed pyflakes"
@@ -107,6 +113,9 @@ if [ $master_res != 0 ] || [ $slave_res != 0 ]; then
     not_ok "failed pylint";
 fi
 
+status "running pep8"
+pep8 --config=common/pep8rc master slave || not_ok "failed pep8"
+
 status "building docs"
 make -C master/docs VERSION=latest clean html || not_ok "docs failed"
 

master/buildbot/buildslave/ec2.py

@@ -194,24 +194,29 @@ def get_image(self):
             options = []
             get_match = re.compile(self.valid_ami_location_regex).match
             for image in self.conn.get_all_images(
-                owners=self.valid_ami_owners):
-                # gather sorting data
+                    owners=self.valid_ami_owners):
+                # Image must be available
+                if image.state != 'available':
+                    continue
+                # Image must match regex
                 match = get_match(image.location)
-                if match:
-                    alpha_sort = int_sort = None
-                    if level < 2:
-                        try:
-                            alpha_sort = match.group(1)
-                        except IndexError:
-                            level = 2
-                        else:
-                            if level == 0:
-                                try:
-                                    int_sort = int(alpha_sort)
-                                except ValueError:
-                                    level = 1
-                    options.append([int_sort, alpha_sort,
-                                    image.location, image.id, image])
+                if not match:
+                    continue
+                # Gather sorting information
+                alpha_sort = int_sort = None
+                if level < 2:
+                    try:
+                        alpha_sort = match.group(1)
+                    except IndexError:
+                        level = 2
+                    else:
+                        if level == 0:
+                            try:
+                                int_sort = int(alpha_sort)
+                            except ValueError:
+                                level = 1
+                options.append([int_sort, alpha_sort,
+                                image.location, image.id, image])
             if level:
                 log.msg('sorting images at level %d' % level)
                 options = [candidate[level:] for candidate in options]

master/buildbot/process/buildstep.py

@@ -79,6 +79,17 @@ class BuildStep(object, properties.PropertiesMixin):
     # properties set on a build step are, by nature, always runtime properties
     set_runtime_properties = True
 
+    renderables = [
+            'haltOnFailure',
+            'flunkOnWarnings',
+            'flunkOnFailure',
+            'warnOnWarnings',
+            'warnOnFailure',
+            'alwaysRun',
+            'doStepIf',
+            'hideStepIf',
+    ]
+
     # 'parms' holds a list of all the parameters we care about, to allow
     # users to instantiate a subclass of BuildStep with a mixture of
     # arguments, some of which are for us, some of which are for the subclass
@@ -308,7 +319,9 @@ def finished(self, results):
                                          ["cancelled"])
                 self.step_status.setText2(["cancelled"])
             else:
-                results = EXCEPTION
+                # leave RETRY as-is, but change anything else to EXCEPTION
+                if results != RETRY:
+                    results = EXCEPTION
                 self.step_status.setText(self.describe(True) +
                                          ["interrupted"])
                 self.step_status.setText2(["interrupted"])

master/buildbot/steps/source/cvs.py

@@ -143,19 +143,20 @@ def copy(self):
                                                 'logEnviron': self.logEnviron})
         cmd.useLog(self.stdio_log, False)
         d = self.runCommand(cmd)        
-        self.workdir = 'source'
+        old_workdir = self.workdir
+        self.workdir = self.srcdir
         d.addCallback(lambda _: self.incremental())
         def copy(_):
-            cmd = remotecommand.RemoteCommand('cpdir',
-                                          {'fromdir': 'source',
-                                           'todir':'build',
-                                           'logEnviron': self.logEnviron,})
+            cmd = remotecommand.RemoteCommand('cpdir', {
+                                            'fromdir': self.srcdir,
+                                            'todir': old_workdir,
+                                            'logEnviron': self.logEnviron,})
             cmd.useLog(self.stdio_log, False)
             d = self.runCommand(cmd)
             return d
         d.addCallback(copy)
         def resetWorkdir(_):
-            self.workdir = 'build'
+            self.workdir = old_workdir
             return 0
         d.addCallback(resetWorkdir)
         return d
@@ -265,6 +266,16 @@ def evaluateCommand(cmd):
         d.addCallback(lambda _: evaluateCommand(cmd))
         return d
 
+    def _cvsEntriesContainStickyDates(self, entries):
+        for line in entries.splitlines():
+            if line == 'D': # the last line contains just a single 'D'
+                pass
+            elif line.split('/')[-1].startswith('D'):
+                # fields are separated by slashes, the last field contains the tag or date
+                # sticky dates start with 'D'
+                return True
+        return False # no sticky dates
+
     @defer.inlineCallbacks
     def _sourcedirIsUpdatable(self):
         myFileWriter = StringFileWriter()
@@ -304,6 +315,19 @@ def _sourcedirIsUpdatable(self):
             defer.returnValue(False)
             return
 
+        # if there are sticky dates (from an earlier build with revision),
+        # we can't update (unless we remove those tags with cvs update -A)
+        myFileWriter.buffer = ""
+        cmd = buildstep.RemoteCommand('uploadFile',
+                dict(slavesrc='Entries', **args),
+                ignore_updates=True)
+        yield self.runCommand(cmd)
+        if cmd.rc is not None and cmd.rc != 0:
+            defer.returnValue(False)
+            return
+        if self._cvsEntriesContainStickyDates(myFileWriter.buffer):
+            defer.returnValue(False)
+
         defer.returnValue(True)
 
     def parseGotRevision(self, res):

master/buildbot/steps/source/git.py

@@ -120,6 +120,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None,
         self.getDescription = getDescription
         self.config = config
         self.supportsBranch = True
+        self.srcdir = 'source'
         Source.__init__(self, **kwargs)
 
         if self.mode not in ['incremental', 'full']:
@@ -244,20 +245,21 @@ def copy(self):
         cmd.useLog(self.stdio_log, False)
         d = self.runCommand(cmd)
 
-        self.workdir = 'source'
+        old_workdir = self.workdir
+        self.workdir = self.srcdir
         d.addCallback(lambda _: self.incremental())
         def copy(_):
             cmd = remotecommand.RemoteCommand('cpdir',
-                                          {'fromdir': 'source',
-                                           'todir':'build',
+                                          {'fromdir': self.srcdir,
+                                           'todir': old_workdir,
                                            'logEnviron': self.logEnviron,
                                            'timeout': self.timeout,})
             cmd.useLog(self.stdio_log, False)
             d = self.runCommand(cmd)
             return d
         d.addCallback(copy)
         def resetWorkdir(_):
-            self.workdir = 'build'
+            self.workdir = old_workdir
             return 0
 
         d.addCallback(resetWorkdir)
@@ -483,7 +485,8 @@ def _sourcedirIsUpdatable(self):
 
     def _updateSubmodule(self, _):
         if self.submodules:
-            return self._dovccmd(['submodule', 'update', '--recursive'])
+            return self._dovccmd(['submodule', 'update',
+                                  '--init', '--recursive'])
         else:
             return defer.succeed(0)
 

master/buildbot/test/unit/test_process_buildstep.py

@@ -18,7 +18,7 @@
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.python import log
-from buildbot.process import buildstep, remotecommand
+from buildbot.process import buildstep, remotecommand, properties
 from buildbot.process.buildstep import regex_log_evaluator
 from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION
 from buildbot.test.fake import fakebuild, remotecommand as fakeremotecommand, slave
@@ -246,6 +246,14 @@ def test_describe_suffix(self):
         self.assertEqual(step2.describe(done=True),
                          [step2.name] + descriptionSuffix)
 
+    @defer.inlineCallbacks
+    def test_step_renders_flunkOnFailure(self):
+        self.setupStep(TestBuildStep.FakeBuildStep(flunkOnFailure=properties.Property('fOF')))
+        self.properties.setProperty('fOF', 'yes', 'test')
+        self.expectOutcome(result=SUCCESS, status_text=["generic"])
+        yield self.runStep()
+        self.assertEqual(self.step.flunkOnFailure, 'yes')
+
 class TestLoggingBuildStep(unittest.TestCase):
 
     def makeRemoteCommand(self, rc, stdout, stderr=''):
@@ -295,7 +303,7 @@ def setUp(self):
     def tearDown(self):
         return self.tearDownBuildStep()
 
-    def test_step_raining_buildstepfailed_in_start(self):
+    def test_step_raising_buildstepfailed_in_start(self):
         self.setupStep(FailingCustomStep())
         self.expectOutcome(result=FAILURE, status_text=["generic"])
         return self.runStep()

master/buildbot/test/unit/test_steps_http.py

@@ -47,6 +47,10 @@ def setUp(self):
         if txrequests is None:
             raise unittest.SkipTest("Need to install txrequests to test http steps")
 
+        # ignore 'http_proxy' environment variable when running tests
+        session = http.getSession()
+        session.trust_env = False
+
         # port 0 means random unused port
         self.listener = reactor.listenTCP(0, Site(TestPage()))
         self.port = self.listener.getHost().port