Merge branch 'github-master' into upstream-master

Conflicts:
	master/buildbot/steps/http.py

.travis.yml

@@ -18,7 +18,7 @@ env:
 
 matrix:
   include:
-    # Test different versions on SQLAlchemy
+    # Test different versions of SQLAlchemy
     - python: "2.7"
       env: TWISTED=12.0.0 SQLALCHEMY=0.6.0 SQLALCHEMY_MIGRATE=0.7.1
     - python: "2.7"
@@ -53,7 +53,7 @@ install:
   # txrequests support only Python 2.6 and 2.7.
   - pip install txrequests
 
-  # Determine is current configuration is latest
+  # Determine if current configuration is latest
   - |
     if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $TWISTED == latest && \
           $SQLALCHEMY = latest && $SQLALCHEMY_MIGRATE = latest ]]; then

common/pep8rc

@@ -8,6 +8,8 @@ max-line-length = 100
 # enabled in future.
 #
 # E122 continuation line missing indentation or outdented
+# E123 closing bracket does not match indentation of opening bracket's line
+#      (pep8 seems to misdiagnose this)
 # E126 continuation line over-indented for hanging indent
 # E128 continuation line under-indented for visual indent
 # E211 whitespace before '('
@@ -16,4 +18,4 @@ max-line-length = 100
 # E712 comparison to False should be 'if cond is False:' or 'if not cond:'
 # E721 do not compare types, use 'isinstance()'
 
-ignore = E122,E126,E128,E211,E501,E711,E712,E721
+ignore = E122,E123,E126,E128,E211,E501,E711,E712,E721

common/validate.sh

@@ -72,7 +72,7 @@ fi
 # get a list of changed files, used below; this uses a tempfile to work around
 # shell behavior when piping to 'while'
 tempfile=$(mktemp)
-trap 'rm -f ${tempfile}' 1 2 3 15
+trap 'rm -f ${tempfile}; exit 1' 1 2 3 15
 git diff --name-only $REVRANGE | grep '\.py$' | grep -v '\(^master/docs\|/setup\.py\)' > ${tempfile}
 py_files=()
 while read line; do
@@ -115,7 +115,7 @@ else
     for filename in ${py_files[@]}; do
         LINEWIDTH=$(grep -E "max-line-length" common/pep8rc | sed 's/ //g' | cut -d'=' -f 2)
         # even if we dont enforce errors, if they can be fixed automatically, thats better..
-        IGNORES=E501,W6
+        IGNORES=E123,E501,W6
         # ignore is not None for SQLAlchemy code..
         if [[ "$filename" =~ "/db/" ]]; then
             IGNORES=$IGNORES,E711,E712

master/CREDITS

@@ -254,6 +254,7 @@ Yoz Grahame
 Zandr Milewski
 Zellyn Hunter
 Zooko Wilcox-O'Hearn
+Konstantinos Koukopoulos
 
 Name Unknown:
 adam

master/buildbot/buildslave/__init__.py

@@ -13,8 +13,12 @@
 #
 # Copyright Buildbot Team Members
 
-from buildbot.buildslave.base import (
-    AbstractBuildSlave, BuildSlave, AbstractLatentBuildSlave)
+from buildbot.buildslave.base import AbstractBuildSlave
+from buildbot.buildslave.base import AbstractLatentBuildSlave
+from buildbot.buildslave.base import BuildSlave
 
 _hush_pyflakes = [
-    AbstractBuildSlave, BuildSlave, AbstractLatentBuildSlave]
+    AbstractBuildSlave,
+    BuildSlave,
+    AbstractLatentBuildSlave,
+]

master/buildbot/buildslave/base.py

@@ -178,22 +178,7 @@ def _lockReleased(self):
             return  # oh well..
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
-    def _applySlaveInfo(self, info):
-        if not info:
-            return
-
-        self.slave_status.setAdmin(info.get("admin"))
-        self.slave_status.setHost(info.get("host"))
-        self.slave_status.setAccessURI(info.get("access_uri"))
-        self.slave_status.setVersion(info.get("version"))
-
-    def _saveSlaveInfoDict(self):
-        slaveinfo = {
-            'admin': self.slave_status.getAdmin(),
-            'host': self.slave_status.getHost(),
-            'access_uri': self.slave_status.getAccessURI(),
-            'version': self.slave_status.getVersion(),
-        }
+    def _saveSlaveInfoDict(self, slaveinfo):
         return self.master.db.buildslaves.updateBuildslave(
             name=self.slavename,
             slaveinfo=slaveinfo,
@@ -207,10 +192,16 @@ def applyInfo(buildslave):
             if buildslave is None:
                 return
 
-            self._applySlaveInfo(buildslave.get('slaveinfo'))
+            self.updateSlaveInfo(**buildslave['slaveinfo'])
 
         return d
 
+    def updateSlaveInfo(self, **kwargs):
+        self.slave_status.updateInfo(**kwargs)
+
+    def getSlaveInfo(self, key, default=None):
+        return self.slave_status.getInfo(key, default)
+
     def setServiceParent(self, parent):
         # botmaster needs to set before setServiceParent which calls startService
         self.botmaster = parent
@@ -220,6 +211,7 @@ def setServiceParent(self, parent):
     def startService(self):
         self.updateLocks()
         self.startMissingTimer()
+        self.slave_status.addInfoWatcher(self._saveSlaveInfoDict)
         d = self._getSlaveInfo()
         d.addCallback(lambda _: service.MultiService.startService(self))
         return d
@@ -274,6 +266,7 @@ def reconfigService(self, new_config):
                                                                 new_config)
 
     def stopService(self):
+        self.slave_status.removeInfoWatcher(self._saveSlaveInfoDict)
         if self.registration:
             self.registration.unregister()
             self.registration = None
@@ -477,7 +470,12 @@ def _commands_unavailable(why):
         def _accept_slave(res):
             self.slave_status.setConnected(True)
 
-            self._applySlaveInfo(state)
+            self.slave_status.updateInfo(
+                admin=state.get("admin"),
+                host=state.get("host"),
+                access_uri=state.get("access_uri"),
+                version=state.get("version"),
+            )
 
             self.slave_commands = state.get("slave_commands")
             self.slave_environ = state.get("slave_environ")
@@ -495,8 +493,6 @@ def _accept_slave(res):
             self.stopMissingTimer()
             self.master.status.slaveConnected(self.slavename)
 
-        d.addCallback(lambda _: self._saveSlaveInfoDict())
-
         d.addCallback(lambda _: self.updateSlave())
 
         d.addCallback(lambda _:

master/buildbot/buildslave/ec2.py

@@ -59,7 +59,7 @@ def __init__(self, name, password, instance_type, ami=None,
                  max_builds=None, notify_on_missing=[], missing_timeout=60 * 20,
                  build_wait_timeout=60 * 10, properties={}, locks=None,
                  spot_instance=False, max_spot_price=1.6, volumes=[],
-                 placement=None, price_multiplier=1.2):
+                 placement=None, price_multiplier=1.2, tags={}):
 
         AbstractLatentBuildSlave.__init__(
             self, name, password, max_builds, notify_on_missing,
@@ -138,7 +138,8 @@ def __init__(self, name, password, instance_type, ami=None,
                                                        aws_access_key_id=identifier,
                                                        aws_secret_access_key=secret_identifier)
             else:
-                raise ValueError('The specified region does not exist: {0}'.format(region))
+                raise ValueError(
+                    'The specified region does not exist: {0}'.format(region))
 
         else:
             self.conn = boto.connect_ec2(identifier, secret_identifier)
@@ -200,6 +201,7 @@ def __init__(self, name, password, instance_type, ami=None,
         if elastic_ip is not None:
             elastic_ip = self.conn.get_all_addresses([elastic_ip])[0]
         self.elastic_ip = elastic_ip
+        self.tags = tags
 
     def get_image(self):
         if self.image is not None:
@@ -269,8 +271,11 @@ def _start_instance(self):
             instance_type=self.instance_type, user_data=self.user_data,
             placement=self.placement)
         self.instance = reservation.instances[0]
-        instance_id, image_id, start_time = self._wait_for_instance(reservation)
+        instance_id, image_id, start_time = self._wait_for_instance(
+            reservation)
         if None not in [instance_id, image_id, start_time]:
+            if len(self.tags) > 0:
+                self.conn.create_tags(instance_id, self.tags)
             return [instance_id, image_id, start_time]
         else:
             log.msg('%s %s failed to start instance %s (%s)' %
@@ -327,10 +332,12 @@ def _stop_instance(self, instance, fast):
 
     def _request_spot_instance(self):
         timestamp_yesterday = time.gmtime(int(time.time() - 86400))
-        spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday)
-        spot_prices = self.conn.get_spot_price_history(start_time=spot_history_starttime,
-                                                       product_description='Linux/UNIX (Amazon VPC)',
-                                                       availability_zone=self.placement)
+        spot_history_starttime = time.strftime(
+            '%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday)
+        spot_prices = self.conn.get_spot_price_history(
+            start_time=spot_history_starttime,
+            product_description='Linux/UNIX (Amazon VPC)',
+            availability_zone=self.placement)
         price_sum = 0.0
         price_count = 0
         for price in spot_prices:
@@ -350,11 +357,13 @@ def _request_spot_instance(self):
         else:
             log.msg('%s %s requesting spot instance with price %0.2f.' %
                     (self.__class__.__name__, self.slavename, target_price))
-        reservations = self.conn.request_spot_instances(target_price, self.ami, key_name=self.keypair_name,
-                                                        security_groups=[self.security_name],
-                                                        instance_type=self.instance_type,
-                                                        user_data=self.user_data,
-                                                        placement=self.placement)
+        reservations = self.conn.request_spot_instances(
+            target_price, self.ami, key_name=self.keypair_name,
+            security_groups=[
+                self.security_name],
+            instance_type=self.instance_type,
+            user_data=self.user_data,
+            placement=self.placement)
         request = self._wait_for_request(reservations[0])
         instance_id = request.instance_id
         reservations = self.conn.get_all_instances(instance_ids=[instance_id])
@@ -385,7 +394,8 @@ def _wait_for_instance(self, image):
                      self.output.output))
             if self.elastic_ip is not None:
                 self.instance.use_ip(self.elastic_ip)
-            start_time = '%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds)
+            start_time = '%02d:%02d:%02d' % (
+                minutes // 60, minutes % 60, seconds)
             if len(self.volumes) > 0:
                 self._attach_volumes()
             return self.instance.id, image.id, start_time
@@ -397,7 +407,8 @@ def _wait_for_request(self, reservation):
                 (self.__class__.__name__, self.slavename))
         duration = 0
         interval = self._poll_resolution
-        requests = self.conn.get_all_spot_instance_requests(request_ids=[reservation.id])
+        requests = self.conn.get_all_spot_instance_requests(
+            request_ids=[reservation.id])
         request = requests[0]
         request_status = request.status.code
         while request_status in SPOT_REQUEST_PENDING_STATES:
@@ -407,7 +418,8 @@ def _wait_for_request(self, reservation):
                 log.msg('%s %s has waited %d minutes for spot request %s' %
                         (self.__class__.__name__, self.slavename, duration // 60,
                          request.id))
-            requests = self.conn.get_all_spot_instance_requests(request_ids=[request.id])
+            requests = self.conn.get_all_spot_instance_requests(
+                request_ids=[request.id])
             request = requests[0]
             request_status = request.status.code
         if request_status == FULFILLED: