Merge branch 'master' (early part) into nine

Conflicts:
	master/buildbot/db/buildsets.py
	master/buildbot/master.py
	master/buildbot/steps/shell.py
	master/buildbot/test/unit/test_db_buildsets.py
	master/buildbot/util/__init__.py

+autopep8

master/README

@@ -37,7 +37,7 @@ REQUIREMENTS:
 
   See http://buildbot.net/buildbot/docs/latest/manual/installation.html
 
-  Briefly: python, Twisted, Jinja, simplejson, and SQLite.  Simplejson and
+  Briefly: python, Twisted, Jinja2, simplejson, and SQLite.  Simplejson and
   SQLite are included with recent versions of Python.
 
 

master/buildbot/changes/filter.py

@@ -35,7 +35,8 @@ def __init__(self,
                  # PROJECT_FN returns True when called with the project; repository,
                  # branch, and so on are similar.  Note that the regular expressions
                  # are anchored to the first character of the string.  For convenience,
-                 # a list can also be specified to the singular option (e.g,. PROJETS
+                 # a list can also be specified to the singular option (e.g,.
+                 # PROJETS
                  project=None, project_re=None, project_fn=None,
                  repository=None, repository_re=None, repository_fn=None,
                  branch=NotABranch, branch_re=None, branch_fn=None,
@@ -82,7 +83,7 @@ def filter_change(self, change):
             return False
         for chg_attr, (filt_list, filt_re, filt_fn) in self.checks.items():
             if chg_attr.startswith("prop:"):
-                chg_val = change.properties.get(chg_attr.split(":",1)[1], '')
+                chg_val = change.properties.get(chg_attr.split(":", 1)[1], '')
             else:
                 chg_val = getattr(change, chg_attr, '')
             if filt_list is not None and chg_val not in filt_list:
@@ -127,7 +128,8 @@ def fromSchedulerConstructorArgs(change_filter=None,
                                    "branch or categories")
             return change_filter
         elif branch is not NotABranch or categories:
-            # build a change filter from the deprecated category and branch args
+            # build a change filter from the deprecated category and branch
+            # args
             cfargs = {}
             if branch is not NotABranch:
                 cfargs['branch'] = branch

master/buildbot/changes/gerritchangesource.py

@@ -15,9 +15,8 @@
 
 from buildbot import util
 from buildbot.changes import base
-from buildbot.util import json
 from buildbot.changes.filter import ChangeFilter
-from buildbot.util import NotABranch
+from buildbot.util import json
 from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.internet.protocol import ProcessProtocol
@@ -42,6 +41,7 @@ def __init__(self,
             self.checks["prop:event.change.branch"] = self.checks["branch"]
             del self.checks["branch"]
 
+
 class GerritChangeSource(base.ChangeSource):
 
     """This source will maintain a connection to gerrit ssh server

master/buildbot/db/buildsets.py

@@ -32,6 +32,10 @@ class BsDict(dict):
     pass
 
 
+class BsProps(dict):
+    pass
+
+
 class BuildsetsConnectorComponent(base.DBConnectorComponent):
     # Documentation is in developer/db.rst
 
@@ -114,7 +118,13 @@ def thd(conn):
             transaction.commit()
 
             return (bsid, brids)
-        defer.returnValue((yield self.db.pool.do(thd)))
+
+        bsid, brids = yield self.db.pool.do(thd)
+
+        # Seed the buildset property cache.
+        self.getBuildsetProperties.cache.put(bsid, BsProps(properties))
+
+        defer.returnValue((bsid, brids))
 
     def completeBuildset(self, bsid, results, complete_at=None,
                          _reactor=reactor):
@@ -191,6 +201,7 @@ def thd(conn):
                                   for row in res.fetchall()]))
         return self.db.pool.do(thd)
 
+    @base.cached("BuildsetProperties")
     def getBuildsetProperties(self, bsid):
         def thd(conn):
             bsp_tbl = self.db.model.buildset_properties
@@ -205,7 +216,7 @@ def thd(conn):
                               tuple(properties)))
                 except ValueError:
                     pass
-            return dict(l)
+            return BsProps(l)
         return self.db.pool.do(thd)
 
     def _thd_row2dict(self, conn, row):

master/buildbot/master.py

@@ -51,6 +51,7 @@
 from buildbot.status.master import Status
 from buildbot.status.results import RETRY
 from buildbot.util import ascii2unicode
+from buildbot.util import check_functional_environment
 from buildbot.util import datetime2epoch
 from buildbot.util import service
 from buildbot.util.eventual import eventually
@@ -109,6 +110,9 @@ def __init__(self, basedir, configFileName="master.cfg", umask=None):
         # local cache for this master's object ID
         self._object_id = None
 
+        # Check environment is sensible
+        check_functional_environment(self.config)
+
         # figure out local hostname
         try:
             self.hostname = os.uname()[1]  # only on unix

master/buildbot/steps/http.py

@@ -78,8 +78,6 @@ def __init__(self, url, method, description=None, descriptionDone=None, **kwargs
         for p in HTTPStep.requestsParams:
             v = kwargs.pop(p, None)
             self.__dict__[p] = v
-            if v is not None:
-                self.requestkwargs[p] = v
         if method not in ('POST', 'GET', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'):
             config.error("Wrong method given: '%s' is not known" % method)
         if description is not None:
@@ -96,6 +94,11 @@ def doRequest(self):
         # create a new session if it doesn't exist
         self.session = getSession()
 
+        for p in self.__dict__ and self.requestsParams:
+            v = self.__dict__[p]
+            if v is not None:
+                self.requestkwargs[p] = v
+
         log = self.addLog('log')
 
         # known methods already tested in __init__

master/buildbot/steps/shell.py

@@ -336,14 +336,14 @@ def commandComplete(self, cmd):
             if self.strip:
                 result = result.strip()
             propname = self.property
-            self.setProperty(propname, result, "SetProperty Step")
+            self.setProperty(propname, result, "SetPropertyFromCommand Step")
             self.property_changes[propname] = result
         else:
             new_props = self.extract_fn(cmd.rc,
                                         self.observer.getStdout(),
                                         self.observer.getStderr())
             for k, v in new_props.items():
-                self.setProperty(k, v, "SetProperty Step")
+                self.setProperty(k, v, "SetPropertyFromCommand Step")
             self.property_changes = new_props
 
     def createSummary(self, log):
@@ -521,8 +521,8 @@ def warningLogConsumer(self):
             if (directoryLeaveRe and
                 self.directoryStack and
                     directoryLeaveRe.search(line)):
-                    self.directoryStack.pop()
-                    continue
+                self.directoryStack.pop()
+                continue
 
             match = wre.match(line)
             if match:
@@ -613,7 +613,7 @@ def createSummary(self, log):
 
     def evaluateCommand(self, cmd):
         if (cmd.didFail() or
-           (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount)):
+                (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount)):
             return FAILURE
         if self.warnCount:
             return WARNINGS

master/buildbot/test/fake/fakedb.py

@@ -1198,10 +1198,10 @@ def _row2dict(self, row):
         del row['properties']
         return row
 
-    def getBuildsetProperties(self, bsid):
-        if bsid in self.buildsets:
+    def getBuildsetProperties(self, key, no_cache=False):
+        if key in self.buildsets:
             return defer.succeed(
-                self.buildsets[bsid]['properties'])
+                self.buildsets[key]['properties'])
         else:
             return defer.succeed({})
 

master/buildbot/test/unit/test_buildslave_libvirt.py

@@ -41,6 +41,7 @@ def test_constructor_nolibvirt(self):
         self.assertRaises(config.ConfigErrors, self.ConcreteBuildSlave,
                           'bot', 'pass', None, 'path', 'path')
 
+    @defer.inlineCallbacks
     def test_constructor_minimal(self):
         bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'path', 'otherpath')
         yield bs._find_existing_deferred
@@ -49,7 +50,7 @@ def test_constructor_minimal(self):
         self.assertEqual(bs.connection, self.conn)
         self.assertEqual(bs.image, 'path')
         self.assertEqual(bs.base_image, 'otherpath')
-        self.assertEqual(bs.keepalive_interval, 3600)
+        self.assertEqual(bs.missing_timeout, 1200)
 
     @defer.inlineCallbacks
     def test_find_existing(self):

master/buildbot/test/unit/test_db_buildsets.py

@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 import datetime
+import mock
 
 from buildbot.db import buildsets
 from buildbot.test.fake import fakedb
@@ -72,7 +73,7 @@ def getBuildsets(self, count=None, branch=None, repository=None,
 
     def test_signature_getBuildsetProperties(self):
         @self.assertArgSpecMatches(self.db.buildsets.getBuildsetProperties)
-        def getBuildsetProperties(self, bsid):
+        def getBuildsetProperties(self, key, no_cache=False):
             pass
 
     @defer.inlineCallbacks
@@ -579,3 +580,27 @@ def finish_setup(_):
 
     def tearDown(self):
         return self.tearDownConnectorComponent()
+
+    @defer.inlineCallbacks
+    def test_addBuildset_properties_cache(self):
+        """
+        Test that `addChange` properly seeds the `getChange` cache.
+        """
+
+        # Patchup the buildset properties cache so we can verify that
+        # it got called form `addBuildset`.
+        mockedCachePut = mock.Mock()
+        self.patch(
+            self.db.buildsets.getBuildsetProperties.cache,
+            "put", mockedCachePut)
+
+        # Setup a dummy set of properties to insert with the buildset.
+        props = dict(prop=(['list'], 'test'))
+
+        # Now, call `addBuildset`, and verify that the above properties
+        # were seeed in the `getBuildsetProperties` cache.
+        bsid, _ = yield self.db.buildsets.addBuildset(
+            sourcestamps=[234], reason='because',
+            properties=props, builderNames=['a', 'b'],
+            waited_for=False)
+        mockedCachePut.assert_called_once_with(bsid, props)

master/buildbot/test/unit/test_process_builder.py

@@ -224,6 +224,7 @@ def mkbld(brids):
         self.assertEqual(self.master.data.updates.claimedBuildRequests,
                          set([10, 11, 12, 15]))
 
+    @defer.inlineCallbacks
     def test_canStartBuild(self):
         yield self.makeBuilder()
 

master/buildbot/test/unit/test_steps_http.py

@@ -13,6 +13,7 @@
 #
 # Copyright Buildbot Team Members
 
+from buildbot.process import properties
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
 from buildbot.steps import http
@@ -21,6 +22,7 @@
 from twisted.trial import unittest
 from twisted.web.resource import Resource
 from twisted.web.server import Site
+
 try:
     import txrequests
     assert txrequests
@@ -106,3 +108,12 @@ def test_header(self):
         self.expectLogfile('log', "URL: %s\nStatus: 200\n ------ Content ------\nTrue" % (url, ))
         self.expectOutcome(result=SUCCESS, status_text=["Status", "code:", '200'])
         return self.runStep()
+
+    def test_params_renderable(self):
+        url = self.getURL()
+        self.setupStep(http.GET(url, params=properties.Property("x")))
+        self.properties.setProperty('x', {'param_1': 'param_1', 'param_2': 2}, 'here')
+        self.expectLogfile('log', "URL: %s?param_1=param_1&param_2=2\nStatus: 200\n ------ Content ------\nOK" % (url, ))
+        self.expectLogfile('content', "OK")
+        self.expectOutcome(result=SUCCESS, status_text=["Status", "code:", '200'])
+        return self.runStep()

master/buildbot/test/unit/test_util.py

@@ -14,6 +14,8 @@
 # Copyright Buildbot Team Members
 
 import datetime
+import mock
+import os
 
 from twisted.internet import reactor
 from twisted.internet import task
@@ -266,3 +268,20 @@ def test_sleep(self):
         self.assertFalse(d.called)
         clock.advance(1)
         self.assertTrue(d.called)
+
+
+class FunctionalEnvironment(unittest.TestCase):
+
+    def test_working_locale(self):
+        environ = {'LANG': 'en_GB.UTF-8'}
+        self.patch(os, 'environ', environ)
+        config = mock.Mock()
+        util.check_functional_environment(config)
+        self.assertEqual(config.error.called, False)
+
+    def test_broken_locale(self):
+        environ = {'LANG': 'NINE.UTF-8'}
+        self.patch(os, 'environ', environ)
+        config = mock.Mock()
+        util.check_functional_environment(config)
+        config.error.assert_called()

master/buildbot/util/__init__.py

@@ -16,6 +16,7 @@
 
 import calendar
 import datetime
+import locale
 import re
 import string
 import time
@@ -116,7 +117,7 @@ def getConfigDict(self):
         compare_attrs = []
         reflect.accumulateClassList(self.__class__, 'compare_attrs', compare_attrs)
         return dict([(k, getattr(self, k)) for k in compare_attrs
-                    if hasattr(self, k) and k not in ("passwd", "password")])
+                     if hasattr(self, k) and k not in ("passwd", "password")])
 
 
 def diffSets(old, new):
@@ -265,8 +266,19 @@ def asyncSleep(delay):
     return d
 
 
+def check_functional_environment(config):
+    try:
+        locale.getdefaultlocale()
+    except KeyError:
+        config.error("\n".join([
+            "Your environment has incorrect locale settings. This means python cannot handle strings safely.",
+            "Please check 'LANG', 'LC_CTYPE', 'LC_ALL' and 'LANGUAGE' are either unset or set to a valid locale.",
+        ]))
+
+
 __all__ = [
     'naturalSort', 'now', 'formatInterval', 'ComparableMixin', 'json',
     'safeTranslate', 'none_or_str',
     'NotABranch', 'deferredLocked', 'SerializedInvocation', 'UTC',
-    'diffSets', 'makeList', 'in_reactor', 'string2boolean']
+    'diffSets', 'makeList', 'in_reactor', 'string2boolean',
+    'check_functional_environment']

master/docs/manual/cfg-builders.rst

@@ -73,10 +73,11 @@ Other optional keys may be set on each ``BuilderConfig``:
 ``nextSlave``
     If provided, this is a function that controls which slave will be assigned
     future jobs. The function is passed two arguments, the :class:`Builder`
-    object which is assigning a new job, and a list of :class:`BuildSlave`
-    objects. The function should return one of the :class:`BuildSlave`
+    object which is assigning a new job, and a list of :class:`SlaveBuilder`
+    objects. The function should return one of the :class:`SlaveBuilder`
     objects, or ``None`` if none of the available slaves should be
-    used.
+    used. As an example, for each ``slave`` in the list, ``slave.slave`` will 
+    be a :class:`BuildSlave` object, and ``slave.slave.slavename`` is the slave's name.
     The function can optionally return a Deferred, which should fire with the same results.
 
 ``nextBuild``

master/docs/relnotes/0.8.9.rst

@@ -182,6 +182,8 @@ Features
 
 * Systemd unit files for Buildbot are available in the :bb:src:`contrib/` directory.
 
+* We've added some extra checking to make sure that you have a valid locale before starting buildbot (#2608).
+
 
 Forward Compatibility
 ~~~~~~~~~~~~~~~~~~~~~

master/docs/relnotes/index.rst

@@ -111,6 +111,8 @@ Fixes
 
 * Buildbot is now compatible with SQLAlchemy 0.8 and higher, using the newly-released SQLAlchemy-Migrate.
 
+* The :bb:step:`HTTPStep` step's requeset parameters are now renderable.
+
 Deprecations, Removals, and Non-Compatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~