Changes for v2.8.1 release

CHANGES.rst

@@ -1,6 +1,105 @@
 Changelog
 =========
 
+Changes in Apache Libcloud v2.8.1
+---------------------------------
+
+Common
+~~~~~~~
+
+- Fix ``LIBCLOUD_DEBUG_PRETTY_PRINT_RESPONSE`` functionality and make sure it
+  works correctly under Python 3 when ``response.read()`` function returns
+  unicode and not bytes.
+
+  (GITHUB-1430)
+  [Tomaz Muraus]
+
+Compute
+~~~~~~~
+
+- [GCE] Fix ``list_nodes()`` method so it correctly handles pagination
+  and returns all the nodes if there are more than 500 nodes available
+  in total.
+
+  Previously, only first 500 nodes were returned.
+
+  Reported by @TheSushiChef.
+  (GITHUB-1409, GITHUB-1360)
+  [Tomaz Muraus]
+
+- Fix some incorrect type annotations in the base compute API.
+
+  Reported by @dpeschman.
+  (GITHUB-1413)
+  [Tomaz Muraus]
+
+- [OpenStack] Fix error with getting node id in ``_to_floating_ip`` method
+  when region is not called ``nova``.
+  (GITHUB-1411, GITHUB-1412)
+  [Miguel Caballer - @micafer]
+
+- [EC2] Fix ``ex_userdata`` keyword argument in the ``create_node()`` method
+  being ignored / not working correctly.
+
+  NOTE: This regression has been inadvertently introduced in v2.8.0.
+  (GITHUB-1426)
+  [Dan Chaffelson - @Chaffelson]
+
+- [EC2] Update ``create_volume`` method to automatically select first available
+  availability zone if one is not explicitly provided via ``location`` argument.
+  [Tomaz Muraus]
+
+Storage
+~~~~~~~
+
+- [AWS S3] Fix upload object code so uploaded data MD5 checksum check is not
+  performed at the end of the upload when AWS KMS server side encryption is
+  used.
+
+  If AWS KMS server side object encryption is used, ETag header value in the
+  response doesn't contain data MD5 digest so we can't perform a checksum
+  check.
+
+  Reported by Jonathan Harden - @jfharden.
+  (GITHUB-1401, GITHUB-1406)
+  [Tomaz Muraus - @Kami]
+
+- [Google Storage] Fix a bug when uploading an object would fail and result
+  in 401 "invalid signature" error when object mime type contained mixed
+  casing and when S3 Interoperability authentication method was used.
+
+  Reported by Will Abson - wabson.
+  (GITHUB-1417, GITHUB-1418)
+  [Tomaz Muraus]
+
+- Fix ``upload_object_via_stream`` method so "Illegal seek" errors which
+  can arise when calculating iterator content hash are ignored. Those errors
+  likely indicate that the underlying file handle / iterator is a pipe which
+  doesn't support seek and that the error is not fatal and we should still
+  proceed.
+
+  Reported by Per Buer - @perbu.
+
+  (GITHUB-1424, GITHUB-1427)
+  [Tomaz Muraus]
+
+DNS
+~~~
+
+- [Gandi Live] Update the driver and make sure it matches the latest service /
+  API updates.
+  (GITHUB-1416)
+  [Ryan Lee - @zepheiryan]
+
+- [CloudFlare] Fix ``export_zone_to_bind_format`` method.
+
+  Previously it threw an exception, because ``record.extra`` dictionary
+  didn't contain ``priority`` key.
+
+  Reported by James Montgomery - @gh-jamesmontgomery.
+  (GITHUB-1428, GITHUB-1429)
+  [Tomaz Muraus]
+
 Changes in Apache Libcloud v2.8.0
 ---------------------------------
 

contrib/scrape-ec2-sizes.py

@@ -300,7 +300,7 @@ def dump():
     separators = (',', ': ')
 
     print("INSTANCE_TYPES = " + json.dumps(sizes, indent=4, sort_keys=True,
-                                           separators=separators)).replace('null', 'None')
+                                           separators=separators).replace('null', 'None'))
     print("REGION_DETAILS = " + json.dumps(regions, indent=4, sort_keys=True,
                                            separators=separators))
 

docs/conf.py

@@ -73,9 +73,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.8.0'
+version = '2.8.1'
 # The full version, including alpha/beta/rc tags.
-release = '2.8.0'
+release = '2.8.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

libcloud/__init__.py

@@ -46,7 +46,7 @@
     'enable_debug'
 ]
 
-__version__ = '2.8.0'
+__version__ = '2.8.1'
 
 
 def enable_debug(fo):

libcloud/common/gandi_live.py

@@ -139,13 +139,20 @@ def parse_body(self):
         valid_http_codes = [
             httplib.OK,
             httplib.CREATED,
-            httplib.NO_CONTENT
         ]
         if self.status in valid_http_codes:
             if json_error:
                 raise JsonParseError(body, self.status)
             else:
                 return body
+        elif self.status == httplib.NO_CONTENT:
+            # Parse error for empty body is acceptable, but a non-empty body
+            # is not.
+            if len(body) > 0:
+                msg = '"No Content" response contained content'
+                raise GandiLiveBaseError(msg, self.status)
+            else:
+                return {}
         elif self.status == httplib.NOT_FOUND:
             message = self._get_error(body, json_error)
             raise ResourceNotFoundError(message, self.status)

libcloud/compute/base.py

@@ -350,7 +350,7 @@ def __init__(self,
                  disk,  # type: int
                  bandwidth,  # type: Optional[int]
                  price,  # type: float
-                 driver,  # type: Type[NodeDriver]
+                 driver,  # type: NodeDriver
                  extra=None  # type: Optional[dict]
                  ):
         """
@@ -421,7 +421,7 @@ class NodeImage(UuidMixin):
     def __init__(self,
                  id,  # type: str
                  name,  # type: str
-                 driver,  # type: Type[NodeDriver]
+                 driver,  # type: NodeDriver
                  extra=None  # type: Optional[dict]
                  ):
         """
@@ -467,7 +467,7 @@ def __init__(self,
                  id,  # type: str
                  image_id,  # type: str
                  state,  # type: NodeImageMemberState
-                 driver,  # type: Type[NodeDriver]
+                 driver,  # type: NodeDriver
                  created=None,  # type: datetime.datetime
                  extra=None  # type: Optional[dict]
                  ):
@@ -522,7 +522,7 @@ def __init__(self,
                  id,  # type: str
                  name,  # type: str
                  country,  # type: str
-                 driver,  # type: Type[NodeDriver]
+                 driver,  # type: NodeDriver
                  extra=None  # type: Optional[dict]
                  ):
         """
@@ -822,7 +822,7 @@ class NodeDriver(BaseDriver):
     name = None  # type: str
     api_name = None  # type: str
     website = None  # type: str
-    type = None  # type: Provider
+    type = None  # type: Union[Provider,str]
     port = None  # type: int
     features = {'create_node': []}  # type: Dict[str, List[str]]
 

libcloud/compute/drivers/ec2.py

@@ -1954,7 +1954,7 @@ def create_node(self, name, size, image, location=None, auth=None,
             params['KeyName'] = ex_keyname
 
         if ex_userdata:
-            params['UserData'] = base64.b64encode(b('ex_userdata'))\
+            params['UserData'] = base64.b64encode(b(ex_userdata))\
                 .decode('utf-8')
 
         if ex_clienttoken:

libcloud/compute/drivers/gce.py

@@ -21,6 +21,7 @@
 
 import datetime
 import time
+import itertools
 import sys
 
 from libcloud.common.base import LazyObject
@@ -133,7 +134,7 @@ def request(self, *args, **kwargs):
 
         return response
 
-    def request_aggregated_items(self, api_name):
+    def request_aggregated_items(self, api_name, zone=None):
         """
         Perform request(s) to obtain all results from 'api_name'.
 
@@ -145,12 +146,19 @@ def request_aggregated_items(self, api_name):
                   for valid names.
         :type     api_name: ``str``
 
+        :param   zone: Optional zone to use.
+        :type zone: :class:`GCEZone`
+
         :return:  dict in the format of the API response.
                   format: { 'items': {'key': {api_name: []}} }
                   ex: { 'items': {'zones/us-central1-a': {disks: []}} }
         :rtype:   ``dict``
         """
-        request_path = "/aggregated/%s" % api_name
+        if zone:
+            request_path = "/zones/%s/%s" % (zone.name, api_name)
+        else:
+            request_path = "/aggregated/%s" % (api_name)
+
         api_responses = []
 
         params = {'maxResults': 500}
@@ -159,6 +167,15 @@ def request_aggregated_items(self, api_name):
             self.gce_params = params
             response = self.request(request_path, method='GET').object
             if 'items' in response:
+                if zone:
+                    # Special case when we are handling pagination for a
+                    # specific zone
+                    items = response['items']
+                    response['items'] = {
+                        'zones/%s' % (zone): {
+                            api_name: items
+                        }
+                    }
                 api_responses.append(response)
             more_results = 'pageToken' in params
         return self._merge_response_items(api_name, api_responses)
@@ -2569,50 +2586,39 @@ def list_nodes(self, ex_zone=None, ex_use_disk_cache=True):
         :return:  List of Node objects
         :rtype:   ``list`` of :class:`Node`
         """
-        list_nodes = []
         zone = self._set_zone(ex_zone)
-        if zone is None:
-            request = '/aggregated/instances'
-        else:
-            request = '/zones/%s/instances' % (zone.name)
-        response = self.connection.request(request, method='GET').object
+        response = self.connection.request_aggregated_items('instances',
+                                                            zone=zone)
+
+        if not response.get('items', []):
+            return []
+
+        list_nodes = []
+
+        # The aggregated response returns a dict for each zone
+        # Create volume cache now for fast lookups of disk info.
+        self._ex_populate_volume_dict()
+
+        items = response['items'].values()
+        instances = [item.get('instances', []) for item in items]
+        instances = itertools.chain(*instances)
+
+        for instance in instances:
+            try:
+                node = self._to_node(instance,
+                                     use_disk_cache=ex_use_disk_cache)
+            except ResourceNotFoundError:
+                # If a GCE node has been deleted between
+                #   - is was listed by `request('.../instances', 'GET')
+                #   - it is converted by `self._to_node(i)`
+                # `_to_node()` will raise a ResourceNotFoundError.
+                #
+                # Just ignore that node and return the list of the
+                # other nodes.
+                continue
+
+            list_nodes.append(node)
 
-        if 'items' in response:
-            # The aggregated response returns a dict for each zone
-            if zone is None:
-                # Create volume cache now for fast lookups of disk info.
-                self._ex_populate_volume_dict()
-                for v in response['items'].values():
-                    for i in v.get('instances', []):
-                        try:
-                            list_nodes.append(
-                                self._to_node(i,
-                                              use_disk_cache=ex_use_disk_cache)
-                            )
-                        # If a GCE node has been deleted between
-                        #   - is was listed by `request('.../instances', 'GET')
-                        #   - it is converted by `self._to_node(i)`
-                        # `_to_node()` will raise a ResourceNotFoundError.
-                        #
-                        # Just ignore that node and return the list of the
-                        # other nodes.
-                        except ResourceNotFoundError:
-                            pass
-            else:
-                for i in response['items']:
-                    try:
-                        list_nodes.append(
-                            self._to_node(i, use_disk_cache=ex_use_disk_cache)
-                        )
-                    # If a GCE node has been deleted between
-                    #   - is was listed by `request('.../instances', 'GET')
-                    #   - it is converted by `self._to_node(i)`
-                    # `_to_node()` will raise a ResourceNotFoundError.
-                    #
-                    # Just ignore that node and return the list of the
-                    # other nodes.
-                    except ResourceNotFoundError:
-                        pass
         # Clear the volume cache as lookups are complete.
         self._ex_volume_dict = {}
         return list_nodes

libcloud/compute/drivers/openstack.py

@@ -3927,8 +3927,8 @@ def _to_floating_ip(self, obj):
                                            port.extra["mac_address"]}
 
         if 'port_details' in obj and obj['port_details']:
-            if obj['port_details']['device_owner'] in ['compute:nova',
-                                                       'compute:None']:
+            dev_owner = obj['port_details']['device_owner']
+            if dev_owner and dev_owner.startswith("compute:"):
                 instance_id = obj['port_details']['device_id']
 
         ip_address = obj['floating_ip_address']

libcloud/dns/drivers/cloudflare.py

@@ -74,6 +74,7 @@
     'created_on',
     'modified_on',
     'data',
+    'priority'
 }
 
 RECORD_CREATE_ATTRIBUTES = {

libcloud/storage/base.py

@@ -24,6 +24,7 @@
 
 import os.path                          # pylint: disable-msg=W0404
 import hashlib
+import errno
 from os.path import join as pjoin
 
 from libcloud.utils.py3 import httplib
@@ -653,7 +654,20 @@ def _hash_buffered_stream(self, stream, hasher, blocksize=65536):
             # Ensure we start from the begining of a stream in case stream is
             # not at the beginning
             if hasattr(stream, 'seek'):
-                stream.seek(0)
+                try:
+                    stream.seek(0)
+                except OSError as e:
+                    if e.errno != errno.ESPIPE:
+                        # This represents "OSError: [Errno 29] Illegal seek"
+                        # error. This could either mean that the underlying
+                        # handle doesn't support seek operation (e.g. pipe) or
+                        # that the invalid seek position is provided. Sadly
+                        # there is no good robust way to distinghuish that so
+                        # we simply ignore all the "Illeal seek" errors so
+                        # this function works correctly with pipes.
+                        # See https://github.com/apache/libcloud/pull/1427 for
+                        # details
+                        raise e
 
             for chunk in libcloud.utils.files.read_in_chunks(iterator=stream):
                 hasher.update(b(chunk))