Merge remote-tracking branch 'upstream/master'

.coveragerc

@@ -1 +1,35 @@
-common/coveragerc
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+    if runtime.platformType  == 'win32'
+
+    # 'pass' generally means 'this won't be called'
+    ^ *pass *$
+
+    # conditionals on twisted versions aren't coverable
+    if twisted.version
+
+include =
+    master/*
+    slave/*
+
+omit =
+    # omit all of our tests
+    */test/*
+    # templates cause coverage errors
+    */templates/*
+    master/buildbot/scripts/buildbot_tac.tmpl

.travis.yml

@@ -18,7 +18,7 @@ env:
 
 matrix:
   include:
-    # Test different versions on SQLAlchemy
+    # Test different versions of SQLAlchemy
     - python: "2.7"
       env: TWISTED=12.0.0 SQLALCHEMY=0.6.0 SQLALCHEMY_MIGRATE=0.7.1
     - python: "2.7"
@@ -42,9 +42,9 @@ matrix:
 
 # Dependencies installation commands
 install:
-  - "[ $TWISTED = latest ] || pip install Twisted==$TWISTED"
-  - "[ $SQLALCHEMY = latest ] || pip install sqlalchemy==$SQLALCHEMY"
-  - "[ $SQLALCHEMY_MIGRATE = latest ] || pip install sqlalchemy-migrate==$SQLALCHEMY_MIGRATE"
+  - "echo 'travis_fold:start:tw'; [ $TWISTED = latest ] || pip install Twisted==$TWISTED; echo 'travis_fold:end:tw'"
+  - "echo 'travis_fold:start:sa'; [ $SQLALCHEMY = latest ] || pip install sqlalchemy==$SQLALCHEMY; echo 'travis_fold:end:sa'"
+  - "echo 'travis_fold:start:sa'; [ $SQLALCHEMY_MIGRATE = latest ] || pip install sqlalchemy-migrate==$SQLALCHEMY_MIGRATE; echo 'travis_fold:end:sam'"
   - (cd master; python setup.py develop)
   - (cd slave;  python setup.py develop)
   # mock is preinstalled on Travis
@@ -53,7 +53,7 @@ install:
   # txrequests support only Python 2.6 and 2.7.
   - pip install txrequests
 
-  # Determine is current configuration is latest
+  # Determine if current configuration is latest
   - |
     if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $TWISTED == latest && \
           $SQLALCHEMY = latest && $SQLALCHEMY_MIGRATE = latest ]]; then
@@ -63,7 +63,8 @@ install:
     fi;
 
   # Run additional tests only in latest configuration
-  - "[ $IS_LATEST = false ] || pip install pylint"
+  # Note pylint version is pinned because newer versions can't import zope.interface - http://www.logilab.org/92792
+  - "[ $IS_LATEST = false ] || pip install pylint==1.1.0"
   - "[ $IS_LATEST = false ] || pip install pyflakes"
   - "[ $IS_LATEST = false ] || pip install sphinx"
   - "[ $IS_LATEST = false ] || pip install pep8"
@@ -74,7 +75,7 @@ script:
   - trial --reporter=text --rterrors buildslave.test
 
   # Run additional tests only in latest configuration
-  - "[ $IS_LATEST = false ] || make pylint"
+  - "[ $IS_LATEST = false ] || make pylint==1.1.0"
   - "[ $IS_LATEST = false ] || make pyflakes"
   - "[ $IS_LATEST = false ] || make docs"
   - "[ $IS_LATEST = false ] || make pep8"

MAINTAINERS.txt

@@ -183,22 +183,22 @@ OpenCSW
 
 Gentoo
     S: Maintained
-    M: Dustin J. Mitchell <dustin@v.igoro.us> (irc:djmitche)
+    M: Dustin J. Mitchell <dustin@buildbot.net> (irc:djmitche)
     U: http://packages.gentoo.org/package/dev-util/buildbot
 
 
 == Other Contact Information ==
 
 Security
     D: Contacts members of this team directly with any security concerns
-    M: Dustin J. Mitchell <dustin@v.igoro.us> (irc:djmitche)
+    M: Dustin J. Mitchell <dustin@buildbot.net> (irc:djmitche)
     M: Amber Yust <ayust@yelp.com> (irc:Aaeriele)
     M: Steve Milner <smilner@redhat.com>
     U: http://trac.buildbot.net/wiki/SecurityPolicy
 
 Metabuildbot Slave Donors
     D: Maintainers of buildslaves for the Metabuildbot
-    M: Dustin J. Mitchell <dustin@v.igoro.us> (irc:djmitche)
+    M: Dustin J. Mitchell <dustin@buildbot.net> (irc:djmitche)
     M: Mozilla <dustin@mozilla.com>
     M: Marc-Antoine Ruel <maruel@chromium.org>
     M: Tom Prince <tom.prince@ualberta.net> (irc:tomprince)
@@ -213,7 +213,7 @@ Committers
     M: Ben Hearsum <bhearsum@mozilla.com>
     M: Brian Warner <warner@lothar.com>
     M: Chris AtLee <catlee@mozilla.com>
-    M: Dustin J. Mitchell <dustin@v.igoro.us> (irc:djmitche)
+    M: Dustin J. Mitchell <dustin@buildbot.net> (irc:djmitche)
     M: Marc-Antoine Ruel <maruel@chromium.org>
     M: Marcus Lindblom <macke@yar.nu>
     M: Tom Prince <tom.prince@ualberta.net> (irc:tomprince)

common/pep8rc

@@ -8,6 +8,8 @@ max-line-length = 100
 # enabled in future.
 #
 # E122 continuation line missing indentation or outdented
+# E123 closing bracket does not match indentation of opening bracket's line
+#      (pep8 seems to misdiagnose this)
 # E126 continuation line over-indented for hanging indent
 # E128 continuation line under-indented for visual indent
 # E211 whitespace before '('
@@ -16,4 +18,4 @@ max-line-length = 100
 # E712 comparison to False should be 'if cond is False:' or 'if not cond:'
 # E721 do not compare types, use 'isinstance()'
 
-ignore = E122,E126,E128,E211,E501,E711,E712,E721
+ignore = E122,E123,E126,E128,E211,E501,E711,E712,E721

common/validate.sh

@@ -72,7 +72,7 @@ fi
 # get a list of changed files, used below; this uses a tempfile to work around
 # shell behavior when piping to 'while'
 tempfile=$(mktemp)
-trap 'rm -f ${tempfile}' 1 2 3 15
+trap 'rm -f ${tempfile}; exit 1' 1 2 3 15
 git diff --name-only $REVRANGE | grep '\.py$' | grep -v '\(^master/docs\|/setup\.py\)' > ${tempfile}
 py_files=()
 while read line; do
@@ -115,7 +115,7 @@ else
     for filename in ${py_files[@]}; do
         LINEWIDTH=$(grep -E "max-line-length" common/pep8rc | sed 's/ //g' | cut -d'=' -f 2)
         # even if we dont enforce errors, if they can be fixed automatically, thats better..
-        IGNORES=E501,W6
+        IGNORES=E123,E501,W6
         # ignore is not None for SQLAlchemy code..
         if [[ "$filename" =~ "/db/" ]]; then
             IGNORES=$IGNORES,E711,E712

master/CREDITS

@@ -254,6 +254,7 @@ Yoz Grahame
 Zandr Milewski
 Zellyn Hunter
 Zooko Wilcox-O'Hearn
+Konstantinos Koukopoulos
 
 Name Unknown:
 adam

master/README

@@ -37,7 +37,7 @@ REQUIREMENTS:
 
   See http://buildbot.net/buildbot/docs/latest/manual/installation.html
 
-  Briefly: python, Twisted, Jinja, simplejson, and SQLite.  Simplejson and
+  Briefly: python, Twisted, Jinja2, simplejson, and SQLite.  Simplejson and
   SQLite are included with recent versions of Python.
 
 

master/buildbot/buildslave/__init__.py

@@ -13,8 +13,12 @@
 #
 # Copyright Buildbot Team Members
 
-from buildbot.buildslave.base import (
-    AbstractBuildSlave, BuildSlave, AbstractLatentBuildSlave)
+from buildbot.buildslave.base import AbstractBuildSlave
+from buildbot.buildslave.base import AbstractLatentBuildSlave
+from buildbot.buildslave.base import BuildSlave
 
 _hush_pyflakes = [
-    AbstractBuildSlave, BuildSlave, AbstractLatentBuildSlave]
+    AbstractBuildSlave,
+    BuildSlave,
+    AbstractLatentBuildSlave,
+]

master/buildbot/buildslave/base.py

@@ -178,22 +178,7 @@ def _lockReleased(self):
             return  # oh well..
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
-    def _applySlaveInfo(self, info):
-        if not info:
-            return
-
-        self.slave_status.setAdmin(info.get("admin"))
-        self.slave_status.setHost(info.get("host"))
-        self.slave_status.setAccessURI(info.get("access_uri"))
-        self.slave_status.setVersion(info.get("version"))
-
-    def _saveSlaveInfoDict(self):
-        slaveinfo = {
-            'admin': self.slave_status.getAdmin(),
-            'host': self.slave_status.getHost(),
-            'access_uri': self.slave_status.getAccessURI(),
-            'version': self.slave_status.getVersion(),
-        }
+    def _saveSlaveInfoDict(self, slaveinfo):
         return self.master.db.buildslaves.updateBuildslave(
             name=self.slavename,
             slaveinfo=slaveinfo,
@@ -207,10 +192,16 @@ def applyInfo(buildslave):
             if buildslave is None:
                 return
 
-            self._applySlaveInfo(buildslave.get('slaveinfo'))
+            self.updateSlaveInfo(**buildslave['slaveinfo'])
 
         return d
 
+    def updateSlaveInfo(self, **kwargs):
+        self.slave_status.updateInfo(**kwargs)
+
+    def getSlaveInfo(self, key, default=None):
+        return self.slave_status.getInfo(key, default)
+
     def setServiceParent(self, parent):
         # botmaster needs to set before setServiceParent which calls startService
         self.botmaster = parent
@@ -220,6 +211,7 @@ def setServiceParent(self, parent):
     def startService(self):
         self.updateLocks()
         self.startMissingTimer()
+        self.slave_status.addInfoWatcher(self._saveSlaveInfoDict)
         d = self._getSlaveInfo()
         d.addCallback(lambda _: service.MultiService.startService(self))
         return d
@@ -274,6 +266,7 @@ def reconfigService(self, new_config):
                                                                 new_config)
 
     def stopService(self):
+        self.slave_status.removeInfoWatcher(self._saveSlaveInfoDict)
         if self.registration:
             self.registration.unregister()
             self.registration = None
@@ -477,7 +470,12 @@ def _commands_unavailable(why):
         def _accept_slave(res):
             self.slave_status.setConnected(True)
 
-            self._applySlaveInfo(state)
+            self.slave_status.updateInfo(
+                admin=state.get("admin"),
+                host=state.get("host"),
+                access_uri=state.get("access_uri"),
+                version=state.get("version"),
+            )
 
             self.slave_commands = state.get("slave_commands")
             self.slave_environ = state.get("slave_environ")
@@ -495,8 +493,6 @@ def _accept_slave(res):
             self.stopMissingTimer()
             self.master.status.slaveConnected(self.slavename)
 
-        d.addCallback(lambda _: self._saveSlaveInfoDict())
-
         d.addCallback(lambda _: self.updateSlave())
 
         d.addCallback(lambda _:

master/buildbot/buildslave/ec2.py

@@ -59,8 +59,8 @@ def __init__(self, name, password, instance_type, ami=None,
                  max_builds=None, notify_on_missing=[], missing_timeout=60 * 20,
                  build_wait_timeout=60 * 10, properties={}, locks=None,
                  spot_instance=False, max_spot_price=1.6, volumes=[],
-                 placement=None, price_multiplier=1.2, retry=1,
-                 retry_price_adjustment=1, product_description='Linux/UNIX'):
+                 placement=None, price_multiplier=1.2, tags={},
+                 retry_price_adjustment=1, product_description='Linux/UNIX'):):
 
         AbstractLatentBuildSlave.__init__(
             self, name, password, max_builds, notify_on_missing,
@@ -143,7 +143,8 @@ def __init__(self, name, password, instance_type, ami=None,
                                                        aws_access_key_id=identifier,
                                                        aws_secret_access_key=secret_identifier)
             else:
-                raise ValueError('The specified region does not exist: {0}'.format(region))
+                raise ValueError(
+                    'The specified region does not exist: {0}'.format(region))
 
         else:
             self.conn = boto.connect_ec2(identifier, secret_identifier)
@@ -205,6 +206,7 @@ def __init__(self, name, password, instance_type, ami=None,
         if elastic_ip is not None:
             elastic_ip = self.conn.get_all_addresses([elastic_ip])[0]
         self.elastic_ip = elastic_ip
+        self.tags = tags
 
     def get_image(self):
         if self.image is not None:
@@ -274,8 +276,11 @@ def _start_instance(self):
             instance_type=self.instance_type, user_data=self.user_data,
             placement=self.placement)
         self.instance = reservation.instances[0]
-        instance_id, image_id, start_time = self._wait_for_instance(reservation)
+        instance_id, image_id, start_time = self._wait_for_instance(
+            reservation)
         if None not in [instance_id, image_id, start_time]:
+            if len(self.tags) > 0:
+                self.conn.create_tags(instance_id, self.tags)
             return [instance_id, image_id, start_time]
         else:
             log.msg('%s %s failed to start instance %s (%s)' %
@@ -332,10 +337,12 @@ def _stop_instance(self, instance, fast):
 
     def _submit_request(self):
         timestamp_yesterday = time.gmtime(int(time.time() - 86400))
-        spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday)
-        spot_prices = self.conn.get_spot_price_history(start_time=spot_history_starttime,
-                                                       product_description=self.product_description,
-                                                       availability_zone=self.placement)
+        spot_history_starttime = time.strftime(
+            '%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday)
+        spot_prices = self.conn.get_spot_price_history(
+            start_time=spot_history_starttime,
+            product_description='Linux/UNIX (Amazon VPC)',
+            availability_zone=self.placement)
         price_sum = 0.0
         price_count = 0
         for price in spot_prices:
@@ -418,7 +425,8 @@ def _wait_for_instance(self, image):
                      self.output.output))
             if self.elastic_ip is not None:
                 self.instance.use_ip(self.elastic_ip)
-            start_time = '%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds)
+            start_time = '%02d:%02d:%02d' % (
+                minutes // 60, minutes % 60, seconds)
             if len(self.volumes) > 0:
                 self._attach_volumes()
             return self.instance.id, image.id, start_time
@@ -432,7 +440,8 @@ def _wait_for_instance(self, image):
     def _wait_for_request(self, reservation):
         duration = 0
         interval = self._poll_resolution
-        requests = self.conn.get_all_spot_instance_requests(request_ids=[reservation.id])
+        requests = self.conn.get_all_spot_instance_requests(
+            request_ids=[reservation.id])
         request = requests[0]
         request_status = request.status.code
         while request_status in SPOT_REQUEST_PENDING_STATES:
@@ -442,7 +451,8 @@ def _wait_for_request(self, reservation):
                 log.msg('%s %s has waited %d minutes for spot request %s' %
                         (self.__class__.__name__, self.slavename, duration // 60,
                          request.id))
-            requests = self.conn.get_all_spot_instance_requests(request_ids=[request.id])
+            requests = self.conn.get_all_spot_instance_requests(
+                request_ids=[request.id])
             request = requests[0]
             request_status = request.status.code
         if request_status == FULFILLED: