From b3d8b15b81a934d9364df168ae5433549a5a880e Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 12:54:47 +0200 Subject: [PATCH 01/23] Add cinder support LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 167 ++++++++++++++++-- .../openstack_v1.1/_v2_0__snapshot.json | 14 ++ .../openstack_v1.1/_v2_0__snapshots.json | 46 +++++ .../openstack_v1.1/_v2_0__volume.json | 18 ++ .../openstack_v1.1/_v2_0__volumes.json | 44 +++++ libcloud/test/compute/test_openstack.py | 78 ++++++++ 6 files changed, 357 insertions(+), 10 deletions(-) create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 211c6514ef..89e49f45b3 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -89,6 +89,12 @@ class OpenStackNetworkConnection(OpenStackBaseConnection): service_region = 'RegionOne' +class OpenStackVolumeConnection(OpenStackBaseConnection): + service_type = 'volume' + service_name = 'cinder' + service_region = 'RegionOne' + + class OpenStackNodeDriver(NodeDriver, OpenStackDriverMixin): """ Base OpenStack node driver. Should not be used directly. @@ -2196,20 +2202,27 @@ def _to_volume(self, api_node): return StorageVolume( id=api_node['id'], - name=api_node['displayName'], + name=api_node.get('name', api_node.get('displayName', None)), size=api_node['size'], state=state, driver=self, extra={ - 'description': api_node['displayDescription'], + 'description': api_node.get('description', + api_node.get('displayDescription', + None)), 'attachments': [att for att in api_node['attachments'] if att], # TODO: remove in 1.18.0 'state': api_node.get('status', None), - 'snapshot_id': api_node.get('snapshotId', None), - 'location': api_node.get('availabilityZone', None), - 'volume_type': api_node.get('volumeType', None), + 'snapshot_id': api_node.get('snapshot_id', + api_node.get('snapshotId', None)), + 'location': api_node.get('availability_zone', + api_node.get('availabilityZone', + None)), + 'volume_type': api_node.get('volume_type', + api_node.get('volumeType', None)), 'metadata': api_node.get('metadata', None), - 'created_at': api_node.get('createdAt', None) + 'created_at': api_node.get('created_at', + api_node.get('createdAt', None)) } ) @@ -2218,10 +2231,13 @@ def _to_snapshot(self, data): data = data['snapshot'] volume_id = data.get('volume_id', data.get('volumeId', None)) - display_name = data.get('display_name', data.get('displayName', None)) + display_name = data.get('name', + data.get('display_name', + data.get('displayName', None))) created_at = data.get('created_at', data.get('createdAt', None)) - description = data.get('display_description', - data.get('displayDescription', None)) + description = data.get('description', + data.get('display_description', + data.get('displayDescription', None))) status = data.get('status', None) extra = {'volume_id': volume_id, @@ -2510,6 +2526,15 @@ def encode_data(self, data): return json.dumps(data) +class OpenStack_2_VolumeConnection(OpenStackVolumeConnection): + responseCls = OpenStack_1_1_Response + accept_format = 'application/json' + default_content_type = 'application/json; charset=UTF-8' + + def encode_data(self, data): + return json.dumps(data) + + class OpenStack_2_PortInterfaceState(Type): """ Standard states of OpenStack_2_PortInterfaceState @@ -2558,6 +2583,14 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): # accessed from there. network_connectionCls = OpenStack_2_NetworkConnection network_connection = None + + # Similarly not all node-related operations are exposed through the + # compute API + # See https://developer.openstack.org/api-ref/compute/ + # For example, volume management are made in the cinder service + volume_connectionCls = OpenStack_2_VolumeConnection + volume_connection = None + type = Provider.OPENSTACK features = {"create_node": ["generates_password"]} @@ -2590,8 +2623,18 @@ def __init__(self, *args, **kwargs): super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) self.image_connection = self.connection + # We run the init once to get the Cinder V2 API connection + # and put that on the object under self.volume_connection. + if original_ex_force_base_url or kwargs.get('ex_force_volume_url'): + kwargs['ex_force_base_url'] = \ + str(kwargs.pop('ex_force_volume_url', + original_ex_force_base_url)) + self.connectionCls = self.volume_connectionCls + super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) + self.volume_connection = self.connection + # We run the init once to get the Neutron V2 API connection - # and put that on the object under self.image_connection. + # and put that on the object under self.network_connection. if original_ex_force_base_url or kwargs.get('ex_force_network_url'): kwargs['ex_force_base_url'] = \ str(kwargs.pop('ex_force_network_url', @@ -2966,6 +3009,110 @@ def ex_get_port(self, port_interface_id): ) return self._to_port(response.object['port']) + def list_volumes(self): + return self._to_volumes( + self.connection.request('/volumes/detail').object) + + def ex_get_volume(self, volumeId): + return self._to_volume( + self.connection.request('/volumes/%s' % volumeId).object) + + def create_volume(self, size, name, location=None, snapshot=None, + ex_volume_type=None): + """ + Create a new volume. + + :param size: Size of volume in gigabytes (required) + :type size: ``int`` + + :param name: Name of the volume to be created + :type name: ``str`` + + :param location: Which data center to create a volume in. If + empty, undefined behavior will be selected. + (optional) + :type location: :class:`.NodeLocation` + + :param snapshot: Snapshot from which to create the new + volume. (optional) + :type snapshot: :class:`.VolumeSnapshot` + + :param ex_volume_type: What kind of volume to create. + (optional) + :type ex_volume_type: ``str`` + + :return: The newly created volume. + :rtype: :class:`StorageVolume` + """ + volume = { + 'name': name, + 'description': name, + 'size': size, + 'metadata': { + 'contents': name, + }, + } + + if ex_volume_type: + volume['volume_type'] = ex_volume_type + + if location: + volume['availability_zone'] = location + + if snapshot: + volume['snapshot_id'] = snapshot.id + + resp = self.connection.request('/volumes', + method='POST', + data={'volume': volume}) + return self._to_volume(resp.object) + + def destroy_volume(self, volume): + return self.connection.request('/volumes/%s' % volume.id, + method='DELETE').success() + + def ex_list_snapshots(self): + return self._to_snapshots( + self.connection.request('/snapshots/detail').object) + + def create_volume_snapshot(self, volume, name=None, ex_description=None, + ex_force=True): + """ + Create snapshot from volume + + :param volume: Instance of `StorageVolume` + :type volume: `StorageVolume` + + :param name: Name of snapshot (optional) + :type name: `str` | `NoneType` + + :param ex_description: Description of the snapshot (optional) + :type ex_description: `str` | `NoneType` + + :param ex_force: Specifies if we create a snapshot that is not in + state `available`. For example `in-use`. Defaults + to True. (optional) + :type ex_force: `bool` + + :rtype: :class:`VolumeSnapshot` + """ + data = {'snapshot': {'volume_id': volume.id, 'force': ex_force}} + + if name is not None: + data['snapshot']['name'] = name + + if ex_description is not None: + data['snapshot']['description'] = ex_description + + return self._to_snapshot(self.connection.request('/snapshots', + method='POST', + data=data).object) + + def destroy_volume_snapshot(self, snapshot): + resp = self.connection.request('/snapshots/%s' % snapshot.id, + method='DELETE') + return resp.status == httplib.ACCEPTED + class OpenStack_1_1_FloatingIpPool(object): """ diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json new file mode 100644 index 0000000000..3b4e846305 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json @@ -0,0 +1,14 @@ +{ + "snapshot": { + "status": "available", + "os-extended-snapshot-attributes:progress": "100%", + "description": "Daily backup", + "created_at": "2013-02-25T04:13:17.07Z", + "metadata": {}, + "volume_id": "5aa119a8-d25b-45a7-8d1b-88e127885635", + "os-extended-snapshot-attributes:project_id": "0c2eba2c5af04d3f9e9d0d410b371fde", + "size": 1, + "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5", + "name": "test" + } +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json new file mode 100644 index 0000000000..763831c7c0 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json @@ -0,0 +1,46 @@ +{ + "snapshots": [ + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "snap-001", + "volume_id": "373f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2012-02-29T03:50:07Z", + "size": 1, + "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2015-11-29T02:25:51.000000", + "size": 1, + "id": "4fbbdccf-e058-6502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot", + "volume_id": "373f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2013-02-29T03:50:07Z", + "size": 1, + "id": "1fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + } + ] +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json new file mode 100644 index 0000000000..46587e69fb --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json @@ -0,0 +1,18 @@ +{ + "volume": { + "status": "available", + "attachments": [], + "availability_zone": "nova", + "bootable": "false", + "os-vol-host-attr:host": "ip-10-168-107-25", + "source_volid": null, + "snapshot_id": null, + "id": "cd76a3a1-c4ce-40f6-9b9f-07a61508938d", + "description": "Super volume.", + "name": "test", + "created_at": "2013-02-25T02:40:21.000000", + "volume_type": "None", + "size": 1, + "metadata": {} + } +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json new file mode 100644 index 0000000000..8415a184df --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json @@ -0,0 +1,44 @@ +{ + "volumes": [ + { + "status": "in-use", + "attachments": [ + { + "server_id": "f4fda93b-06e0-4743-8117-bc8bcecd651b", + "attachment_id": "3b4db356-253d-4fab-bfa0-e3626c0b8405", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "device": "/dev/vdb", + "id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38" + } + ], + "availability_zone": "nova", + "replication_status": "disabled", + "snapshot_id": null, + "id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "size": 2, + "user_id": "32779452fcd34ae1a53a797ac8a1e064", + "metadata": {}, + "description": "", + "name": "test-volume-attachments", + "created_at": "2013-06-24T11:20:13.000000", + "volume_type": "lvmdriver-1" + }, + { + "status": "some-unknown-state", + "migration_status": null, + "attachments": [], + "availability_zone": "nova", + "replication_status": "disabled", + "snapshot_id": "01f48111-7866-4cd2-986a-e92683c4a363", + "id": "cfcec3bc-b736-4db5-9535-4c24112691b5", + "size": 50, + "user_id": "32779452fcd34ae1a53a797ac8a1e064", + "metadata": {}, + "description": "some description", + "name": "test_volume", + "bootable": "false", + "created_at": "2013-06-21T12:39:02.000000", + "volume_type": null + } + ] +} diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index c9be795178..8594a3091e 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -1807,6 +1807,49 @@ def test_attach_port_interface(self): self.assertTrue(ret) + def test_list_volumes(self): + volumes = self.driver.list_volumes() + self.assertEqual(len(volumes), 2) + volume = volumes[0] + + self.assertEqual('6edbc2f4-1507-44f8-ac0d-eed1d2608d38', volume.id) + self.assertEqual('test-volume-attachments', volume.name) + self.assertEqual(StorageVolumeState.INUSE, volume.state) + self.assertEqual(2, volume.size) + self.assertEqual(volume.extra, { + 'description': '', + 'attachments': [{ + "attachment_id": "3b4db356-253d-4fab-bfa0-e3626c0b8405", + "id": '6edbc2f4-1507-44f8-ac0d-eed1d2608d38', + "device": "/dev/vdb", + "server_id": "f4fda93b-06e0-4743-8117-bc8bcecd651b", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + }], + 'snapshot_id': None, + 'state': 'in-use', + 'location': 'nova', + 'volume_type': 'lvmdriver-1', + 'metadata': {}, + 'created_at': '2013-06-24T11:20:13.000000' + }) + + # also test that unknown state resolves to StorageVolumeState.UNKNOWN + volume = volumes[1] + self.assertEqual('cfcec3bc-b736-4db5-9535-4c24112691b5', volume.id) + self.assertEqual('test_volume', volume.name) + self.assertEqual(50, volume.size) + self.assertEqual(StorageVolumeState.UNKNOWN, volume.state) + self.assertEqual(volume.extra, { + 'description': 'some description', + 'attachments': [], + 'snapshot_id': '01f48111-7866-4cd2-986a-e92683c4a363', + 'state': 'some-unknown-state', + 'location': 'nova', + 'volume_type': None, + 'metadata': {}, + 'created_at': '2013-06-21T12:39:02.000000', + }) + class OpenStack_1_1_FactoryMethodTests(OpenStack_1_1_Tests): should_list_locations = False @@ -2291,6 +2334,41 @@ def _v2_1337_v2_0_subnets(self, method, url, body, headers): body = self.fixtures.load('_v2_0__subnets.json') return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + def _v2_1337_volumes_detail(self, method, url, body, headers): + body = self.fixtures.load('_v2_0__volumes.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_volumes(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__volume.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_volumes_cd76a3a1_c4ce_40f6_9b9f_07a61508938d(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('_v2_0__volume.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots_detail(self, method, url, body, headers): + body = self.fixtures.load('_v2_0__snapshots.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__snapshot.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('_v2_0__snapshot.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + + # This exists because the nova compute url in devstack has v2 in there but the v1.1 fixtures # work fine. From 1761c77c836faa0af53f390bc9ae012ab56e653b Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 13:03:38 +0200 Subject: [PATCH 02/23] Add cinder support LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 89e49f45b3..c1d31c401f 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -89,8 +89,8 @@ class OpenStackNetworkConnection(OpenStackBaseConnection): service_region = 'RegionOne' -class OpenStackVolumeConnection(OpenStackBaseConnection): - service_type = 'volume' +class OpenStackVolumeV2Connection(OpenStackBaseConnection): + service_type = 'volumev2' service_name = 'cinder' service_region = 'RegionOne' @@ -2526,7 +2526,7 @@ def encode_data(self, data): return json.dumps(data) -class OpenStack_2_VolumeConnection(OpenStackVolumeConnection): +class OpenStack_2_VolumeV2Connection(OpenStackVolumeV2Connection): responseCls = OpenStack_1_1_Response accept_format = 'application/json' default_content_type = 'application/json; charset=UTF-8' @@ -2588,8 +2588,8 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): # compute API # See https://developer.openstack.org/api-ref/compute/ # For example, volume management are made in the cinder service - volume_connectionCls = OpenStack_2_VolumeConnection - volume_connection = None + volumev2_connectionCls = OpenStack_2_VolumeV2Connection + volumev2_connection = None type = Provider.OPENSTACK @@ -2624,14 +2624,14 @@ def __init__(self, *args, **kwargs): self.image_connection = self.connection # We run the init once to get the Cinder V2 API connection - # and put that on the object under self.volume_connection. + # and put that on the object under self.volumev2_connection. if original_ex_force_base_url or kwargs.get('ex_force_volume_url'): kwargs['ex_force_base_url'] = \ str(kwargs.pop('ex_force_volume_url', original_ex_force_base_url)) - self.connectionCls = self.volume_connectionCls + self.connectionCls = self.volumev2_connectionCls super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) - self.volume_connection = self.connection + self.volumev2_connection = self.connection # We run the init once to get the Neutron V2 API connection # and put that on the object under self.network_connection. From 78e96a51eaca71c0156553de599fbf6b9e1179f6 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 15:19:43 +0200 Subject: [PATCH 03/23] Add cinder support LIBCLOUD-874 --- docs/compute/drivers/openstack.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/compute/drivers/openstack.rst b/docs/compute/drivers/openstack.rst index 887057abfb..24dd839d24 100644 --- a/docs/compute/drivers/openstack.rst +++ b/docs/compute/drivers/openstack.rst @@ -99,6 +99,10 @@ Available arguments: driver obtains API endpoint URL from the server catalog, but if this argument is provided, this step is skipped and the provided value is used directly. Only valid in case of api_version >= 2.0. + * ``ex_force_volume_url`` - Base URL to the OpenStack cinder API endpoint. By default, + driver obtains API endpoint URL from the server catalog, but if this argument + is provided, this step is skipped and the provided value is used directly. Only valid + in case of api_version >= 2.0. Some examples which show how to use this arguments can be found in the section below. From 82b3dd034eb23e2ce7df01f11305a08b52c4773c Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 16:10:39 +0200 Subject: [PATCH 04/23] Add cinder support LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 14 +++---- .../test/common/test_openstack_identity.py | 9 ++-- .../fixtures/openstack/_v2_0__auth.json | 22 ++++++++++ libcloud/test/compute/test_openstack.py | 42 +++++++++++++++++++ 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index c1d31c401f..0e91dcb06c 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -3011,11 +3011,11 @@ def ex_get_port(self, port_interface_id): def list_volumes(self): return self._to_volumes( - self.connection.request('/volumes/detail').object) + self.volumev2_connection.request('/volumes/detail').object) def ex_get_volume(self, volumeId): return self._to_volume( - self.connection.request('/volumes/%s' % volumeId).object) + self.volumev2_connection.request('/volumes/%s' % volumeId).object) def create_volume(self, size, name, location=None, snapshot=None, ex_volume_type=None): @@ -3062,18 +3062,18 @@ def create_volume(self, size, name, location=None, snapshot=None, if snapshot: volume['snapshot_id'] = snapshot.id - resp = self.connection.request('/volumes', + resp = self.volumev2_connection.request('/volumes', method='POST', data={'volume': volume}) return self._to_volume(resp.object) def destroy_volume(self, volume): - return self.connection.request('/volumes/%s' % volume.id, + return self.volumev2_connection.request('/volumes/%s' % volume.id, method='DELETE').success() def ex_list_snapshots(self): return self._to_snapshots( - self.connection.request('/snapshots/detail').object) + self.volumev2_connection.request('/snapshots/detail').object) def create_volume_snapshot(self, volume, name=None, ex_description=None, ex_force=True): @@ -3104,12 +3104,12 @@ def create_volume_snapshot(self, volume, name=None, ex_description=None, if ex_description is not None: data['snapshot']['description'] = ex_description - return self._to_snapshot(self.connection.request('/snapshots', + return self._to_snapshot(self.volumev2_connection.request('/snapshots', method='POST', data=data).object) def destroy_volume_snapshot(self, snapshot): - resp = self.connection.request('/snapshots/%s' % snapshot.id, + resp = self.volumev2_connection.request('/snapshots/%s' % snapshot.id, method='DELETE') return resp.status == httplib.ACCEPTED diff --git a/libcloud/test/common/test_openstack_identity.py b/libcloud/test/common/test_openstack_identity.py index 916193fbc8..64b9eea48d 100644 --- a/libcloud/test/common/test_openstack_identity.py +++ b/libcloud/test/common/test_openstack_identity.py @@ -532,7 +532,7 @@ def test_parsing_auth_v2(self): catalog = OpenStackServiceCatalog(service_catalog=service_catalog, auth_version='2.0') entries = catalog.get_entries() - self.assertEqual(len(entries), 8) + self.assertEqual(len(entries), 9) entry = [e for e in entries if e.service_name == 'cloudServers'][0] self.assertEqual(entry.service_type, 'compute') @@ -599,7 +599,8 @@ def test_get_service_types(self): auth_version='2.0') service_types = catalog.get_service_types() self.assertEqual(service_types, ['compute', 'image', 'network', - 'object-store', 'rax:object-cdn']) + 'object-store', 'rax:object-cdn', + 'volumev2']) service_types = catalog.get_service_types(region='ORD') self.assertEqual(service_types, ['rax:object-cdn']) @@ -613,8 +614,8 @@ def test_get_service_names(self): auth_version='2.0') service_names = catalog.get_service_names() - self.assertEqual(service_names, ['cloudFiles', 'cloudFilesCDN', - 'cloudServers', + self.assertEqual(service_names, ['cinder', 'cloudFiles', + 'cloudFilesCDN', 'cloudServers', 'cloudServersOpenStack', 'cloudServersPreprod', 'glance', diff --git a/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json b/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json index 79c677695f..d2bd3c3743 100644 --- a/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json +++ b/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json @@ -120,6 +120,28 @@ "name": "neutron", "type": "network" }, + { + "endpoints": [ + { + "region": "RegionOne", + "tenantId": "1337", + "publicURL": "https://test_endpoint.com/v2/1337", + "versionInfo": "https://test_endpoint.com/v2/", + "versionList": "https://test_endpoint.com/", + "versionId": "2" + }, + { + "region": "fr1", + "tenantId": "1337", + "publicURL": "https://test_endpoint.com/v2/1337", + "versionInfo": "https://test_endpoint.com/v2/", + "versionList": "https://test_endpoint.com/", + "versionId": "2" + } + ], + "name": "cinder", + "type": "volumev2" + }, { "endpoints": [ { diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index 8594a3091e..6b7eeb8463 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -1565,6 +1565,11 @@ def setUp(self): # normally authentication happens lazily, but we force it here self.driver.network_connection._populate_hosts_and_request_paths() + self.driver_klass.volumev2_connectionCls.conn_class = OpenStack_2_0_MockHttp + self.driver_klass.volumev2_connectionCls.auth_url = "https://auth.api.example.com" + # normally authentication happens lazily, but we force it here + self.driver.volumev2_connection._populate_hosts_and_request_paths() + def test_ex_force_auth_token_passed_to_connection(self): base_url = 'https://servers.api.rackspacecloud.com/v1.1/slug' kwargs = { @@ -1850,6 +1855,43 @@ def test_list_volumes(self): 'created_at': '2013-06-21T12:39:02.000000', }) + def test_create_volume_passes_location_to_request_only_if_not_none(self): + with patch.object(self.driver.volumev2_connection, 'request') as mock_request: + self.driver.create_volume(1, 'test', location='mylocation') + name, args, kwargs = mock_request.mock_calls[0] + self.assertEqual(kwargs["data"]["volume"]["availability_zone"], "mylocation") + + def test_create_volume_does_not_pass_location_to_request_if_none(self): + with patch.object(self.driver.volumev2_connection, 'request') as mock_request: + self.driver.create_volume(1, 'test') + name, args, kwargs = mock_request.mock_calls[0] + self.assertFalse("availability_zone" in kwargs["data"]["volume"]) + + def test_create_volume_passes_volume_type_to_request_only_if_not_none(self): + with patch.object(self.driver.volumev2_connection, 'request') as mock_request: + self.driver.create_volume(1, 'test', ex_volume_type='myvolumetype') + name, args, kwargs = mock_request.mock_calls[0] + self.assertEqual(kwargs["data"]["volume"]["volume_type"], "myvolumetype") + + def test_create_volume_does_not_pass_volume_type_to_request_if_none(self): + with patch.object(self.driver.volumev2_connection, 'request') as mock_request: + self.driver.create_volume(1, 'test') + name, args, kwargs = mock_request.mock_calls[0] + self.assertFalse("volume_type" in kwargs["data"]["volume"]) + + def test_ex_create_snapshot_does_not_post_optional_parameters_if_none(self): + volume = self.driver.list_volumes()[0] + with patch.object(self.driver, '_to_snapshot'): + with patch.object(self.driver.volumev2_connection, 'request') as mock_request: + self.driver.create_volume_snapshot(volume, + name=None, + ex_description=None, + ex_force=True) + + name, args, kwargs = mock_request.mock_calls[0] + self.assertFalse("display_name" in kwargs["data"]["snapshot"]) + self.assertFalse("display_description" in kwargs["data"]["snapshot"]) + class OpenStack_1_1_FactoryMethodTests(OpenStack_1_1_Tests): should_list_locations = False From 557e1e5fa661eb2dedc914913606ea5cdb7fcb90 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 16:20:20 +0200 Subject: [PATCH 05/23] Add cinder support LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 0e91dcb06c..d3b8c0f20d 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -3063,13 +3063,13 @@ def create_volume(self, size, name, location=None, snapshot=None, volume['snapshot_id'] = snapshot.id resp = self.volumev2_connection.request('/volumes', - method='POST', - data={'volume': volume}) + method='POST', + data={'volume': volume}) return self._to_volume(resp.object) def destroy_volume(self, volume): return self.volumev2_connection.request('/volumes/%s' % volume.id, - method='DELETE').success() + method='DELETE').success() def ex_list_snapshots(self): return self._to_snapshots( @@ -3104,13 +3104,13 @@ def create_volume_snapshot(self, volume, name=None, ex_description=None, if ex_description is not None: data['snapshot']['description'] = ex_description - return self._to_snapshot(self.volumev2_connection.request('/snapshots', - method='POST', - data=data).object) + return self._to_snapshot( + self.volumev2_connection.request('/snapshots', method='POST', + data=data).object) def destroy_volume_snapshot(self, snapshot): resp = self.volumev2_connection.request('/snapshots/%s' % snapshot.id, - method='DELETE') + method='DELETE') return resp.status == httplib.ACCEPTED From a65f4a6cc4439355ae3d39dec6f606e0470a992e Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 16:32:58 +0200 Subject: [PATCH 06/23] Add cinder support LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 2 +- libcloud/test/common/test_openstack_identity.py | 2 +- libcloud/test/compute/fixtures/openstack/_v2_0__auth.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index d3b8c0f20d..3e3b8d61bd 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -91,7 +91,7 @@ class OpenStackNetworkConnection(OpenStackBaseConnection): class OpenStackVolumeV2Connection(OpenStackBaseConnection): service_type = 'volumev2' - service_name = 'cinder' + service_name = 'cinderv2' service_region = 'RegionOne' diff --git a/libcloud/test/common/test_openstack_identity.py b/libcloud/test/common/test_openstack_identity.py index 64b9eea48d..4d052fd815 100644 --- a/libcloud/test/common/test_openstack_identity.py +++ b/libcloud/test/common/test_openstack_identity.py @@ -614,7 +614,7 @@ def test_get_service_names(self): auth_version='2.0') service_names = catalog.get_service_names() - self.assertEqual(service_names, ['cinder', 'cloudFiles', + self.assertEqual(service_names, ['cinderv2', 'cloudFiles', 'cloudFilesCDN', 'cloudServers', 'cloudServersOpenStack', 'cloudServersPreprod', diff --git a/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json b/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json index d2bd3c3743..3727df8bb2 100644 --- a/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json +++ b/libcloud/test/compute/fixtures/openstack/_v2_0__auth.json @@ -139,7 +139,7 @@ "versionId": "2" } ], - "name": "cinder", + "name": "cinderv2", "type": "volumev2" }, { From b6103c7be4f22960b45cb5d24ee50a18fe16cc28 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 17:13:29 +0200 Subject: [PATCH 07/23] Move sec groups to neutron api LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 44 ++++++++++++++++++- .../openstack_v1.1/_v2_0__security_group.json | 20 +++++++++ .../_v2_0__security_groups.json | 29 ++++++++++++ libcloud/test/compute/test_openstack.py | 13 ++++-- 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 3e3b8d61bd..c9c43bf6ed 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -1812,7 +1812,8 @@ def _to_security_groups(self, obj): security_groups] def _to_security_group(self, obj): - rules = self._to_security_group_rules(obj.get('rules', [])) + rules = self._to_security_group_rules(obj.get('security_group_rules', + obj.get('rules', []))) return OpenStackSecurityGroup(id=obj['id'], tenant_id=obj['tenant_id'], name=obj['name'], @@ -3114,6 +3115,47 @@ def destroy_volume_snapshot(self, snapshot): return resp.status == httplib.ACCEPTED + def ex_list_security_groups(self): + """ + Get a list of Security Groups that are available. + + :rtype: ``list`` of :class:`OpenStackSecurityGroup` + """ + return self._to_security_groups( + self.network_connection.request('/v2.0/security-groups').object) + + def ex_create_security_group(self, name, description): + """ + Create a new Security Group + + :param name: Name of the new Security Group + :type name: ``str`` + + :param description: Description of the new Security Group + :type description: ``str`` + + :rtype: :class:`OpenStackSecurityGroup` + """ + return self._to_security_group(self.network_connection .request( + '/v2.0/security-groups', method='POST', + data={'security_group': {'name': name, 'description': description}} + ).object['security_group']) + + def ex_delete_security_group(self, security_group): + """ + Delete a Security Group. + + :param security_group: Security Group should be deleted + :type security_group: :class:`OpenStackSecurityGroup` + + :rtype: ``bool`` + """ + resp = self.network_connection.request('/v2.0/security-groups/%s' % + (security_group.id), + method='DELETE') + return resp.status==httplib.NO_CONTENT + + class OpenStack_1_1_FloatingIpPool(object): """ Floating IP Pool info. diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json new file mode 100644 index 0000000000..df20ef4cf9 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json @@ -0,0 +1,20 @@ +{ + "security_group": + { + "description": "FTP Client-Server - Open 20-21 ports", + "id": 4, + "name": "ftp", + "security_group_rules": [ + { + "security_group_id": "4", + "direction": "ingress", + "port_range_max": 21, + "port_range_min": 20, + "remote_ip_prefix": "0.0.0.0/0", + "id": 1, + "protocol": "tcp" + } + ], + "tenant_id": "68" + } +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json new file mode 100644 index 0000000000..5fd84f4c38 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json @@ -0,0 +1,29 @@ +{ + "security_groups": [ + { + "description": "default", + "id": 2, + "name": "default", + "security_group_rules": [], + "tenant_id": "68" + }, + { + "description": "FTP Client-Server - Open 20-21 ports", + "id": 4, + "name": "ftp", + "security_group_rules": [ + { + "security_group_id": "4", + "direction": "ingress", + "port_range_max": 21, + "port_range_min": 20, + "remote_ip_prefix": "0.0.0.0/0", + "id": 1, + "protocol": "tcp" + } + ], + "tenant_id": "68" + } + ] +} + diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index 6b7eeb8463..838bbaf0ce 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -2383,7 +2383,7 @@ def _v2_1337_volumes_detail(self, method, url, body, headers): def _v2_1337_volumes(self, method, url, body, headers): if method == 'POST': body = self.fixtures.load('_v2_0__volume.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_volumes_cd76a3a1_c4ce_40f6_9b9f_07a61508938d(self, method, url, body, headers): if method == 'GET': @@ -2400,7 +2400,7 @@ def _v2_1337_snapshots_detail(self, method, url, body, headers): def _v2_1337_snapshots(self, method, url, body, headers): if method == 'POST': body = self.fixtures.load('_v2_0__snapshot.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, body, headers): if method == 'GET': @@ -2409,7 +2409,14 @@ def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, b if method == 'DELETE': body = '' return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) - + + def _v2_1337_v2_0_security_groups(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__security_group.json') + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'GET': + body = self.fixtures.load('_v2_0__security_group.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) # This exists because the nova compute url in devstack has v2 in there but the v1.1 fixtures # work fine. From bd8318b53676f84e91ed775dc174545f00ac834b Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 24 Sep 2018 17:16:19 +0200 Subject: [PATCH 08/23] Add cinder support LIBCLOUD-874 --- libcloud/test/compute/test_openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index 6b7eeb8463..2ababff051 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -2383,7 +2383,7 @@ def _v2_1337_volumes_detail(self, method, url, body, headers): def _v2_1337_volumes(self, method, url, body, headers): if method == 'POST': body = self.fixtures.load('_v2_0__volume.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_volumes_cd76a3a1_c4ce_40f6_9b9f_07a61508938d(self, method, url, body, headers): if method == 'GET': @@ -2400,7 +2400,7 @@ def _v2_1337_snapshots_detail(self, method, url, body, headers): def _v2_1337_snapshots(self, method, url, body, headers): if method == 'POST': body = self.fixtures.load('_v2_0__snapshot.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, body, headers): if method == 'GET': From c5add2fc8436088e07f20e9da3a6c7a491257101 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 25 Sep 2018 08:57:05 +0200 Subject: [PATCH 09/23] Move sec groups to neutron api LIBCLOUD-874 --- libcloud/compute/drivers/openstack.py | 114 ++++++++++++++++-- .../openstack_v1.1/_v2_0__security_group.json | 18 +-- .../_v2_0__security_group_rule.json | 13 ++ .../_v2_0__security_groups.json | 2 +- libcloud/test/compute/test_openstack.py | 24 +++- 5 files changed, 146 insertions(+), 25 deletions(-) create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group_rule.json diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index c9c43bf6ed..76de1b8a52 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -1112,7 +1112,7 @@ class OpenStackSecurityGroupRule(object): def __init__(self, id, parent_group_id, ip_protocol, from_port, to_port, driver, ip_range=None, group=None, tenant_id=None, - extra=None): + direction=None, extra=None): """ Constructor. @@ -1140,6 +1140,9 @@ def __init__(self, id, parent_group_id, ip_protocol, from_port, to_port, :keyword tenant_id: Owner of the security group. :type tenant_id: ``str`` + :keyword direction: Security group Direction (ingress or egress). + :type direction: ``str`` + :keyword extra: Extra attributes associated with this rule. :type extra: ``dict`` """ @@ -1151,12 +1154,21 @@ def __init__(self, id, parent_group_id, ip_protocol, from_port, to_port, self.driver = driver self.ip_range = '' self.group = {} + self.direction = 'ingress' if group is None: self.ip_range = ip_range else: self.group = {'name': group, 'tenant_id': tenant_id} + # by default in old versions only ingress was used + if direction is not None: + if direction in ['ingress', 'egress']: + self.direction = direction + else: + raise OpenStackException("Security group direction incorrect " + "value: ingress or egress.") + self.tenant_id = tenant_id self.extra = extra or {} @@ -1678,7 +1690,7 @@ def ex_delete_network(self, network): resp = self.connection.request('%s/%s' % (self._networks_url_prefix, network.id), method='DELETE') - return resp.status == httplib.ACCEPTED + return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) def ex_get_console_output(self, node, length=None): """ @@ -1836,6 +1848,8 @@ def ex_get_node_security_groups(self, node): :rtype: ``list`` of :class:`OpenStackSecurityGroup` """ + print(vars(self.connection.request( + '/servers/%s/os-security-groups' % (node.id)))) return self._to_security_groups( self.connection.request('/servers/%s/os-security-groups' % (node.id)).object) @@ -3112,8 +3126,7 @@ def create_volume_snapshot(self, volume, name=None, ex_description=None, def destroy_volume_snapshot(self, snapshot): resp = self.volumev2_connection.request('/snapshots/%s' % snapshot.id, method='DELETE') - return resp.status == httplib.ACCEPTED - + return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) def ex_list_security_groups(self): """ @@ -3151,9 +3164,96 @@ def ex_delete_security_group(self, security_group): :rtype: ``bool`` """ resp = self.network_connection.request('/v2.0/security-groups/%s' % - (security_group.id), - method='DELETE') - return resp.status==httplib.NO_CONTENT + (security_group.id), + method='DELETE') + return resp.status == httplib.NO_CONTENT + + def _to_security_group_rule(self, obj): + ip_range = group = tenant_id = parent_id = None + protocol = from_port = to_port = direction = None + + if 'parent_group_id' in obj: + if obj['group'] == {}: + ip_range = obj['ip_range'].get('cidr', None) + else: + group = obj['group'].get('name', None) + tenant_id = obj['group'].get('tenant_id', None) + + parent_id = obj['parent_group_id'] + from_port = obj['from_port'] + to_port = obj['to_port'] + protocol = obj['ip_protocol'] + else: + ip_range = obj.get('remote_ip_prefix', None) + group = obj.get('remote_group_id', None) + tenant_id = obj.get('tenant_id', None) + + parent_id = obj['security_group_id'] + from_port = obj['port_range_min'] + to_port = obj['port_range_max'] + protocol = obj['protocol'] + + return OpenStackSecurityGroupRule( + id=obj['id'], parent_group_id=parent_id, + ip_protocol=protocol, from_port=from_port, + to_port=to_port, driver=self, ip_range=ip_range, + group=group, tenant_id=tenant_id, direction=direction) + + def ex_create_security_group_rule(self, security_group, ip_protocol, + from_port, to_port, cidr=None, + source_security_group=None): + """ + Create a new Rule in a Security Group + + :param security_group: Security Group in which to add the rule + :type security_group: :class:`OpenStackSecurityGroup` + + :param ip_protocol: Protocol to which this rule applies + Examples: tcp, udp, ... + :type ip_protocol: ``str`` + + :param from_port: First port of the port range + :type from_port: ``int`` + + :param to_port: Last port of the port range + :type to_port: ``int`` + + :param cidr: CIDR notation of the source IP range for this rule + :type cidr: ``str`` + + :param source_security_group: Existing Security Group to use as the + source (instead of CIDR) + :type source_security_group: L{OpenStackSecurityGroup + + :rtype: :class:`OpenStackSecurityGroupRule` + """ + source_security_group_id = None + if type(source_security_group) == OpenStackSecurityGroup: + source_security_group_id = source_security_group.id + + return self._to_security_group_rule(self.network_connection.request( + '/v2.0/security-group-rules', method='POST', + data={'security_group_rule': { + 'protocol': ip_protocol, + 'port_range_min': from_port, + 'port_range_max': to_port, + 'remote_ip_prefix': cidr, + 'remote_group_id': source_security_group_id, + 'security_group_id': security_group.id}} + ).object['security_group_rule']) + + def ex_delete_security_group_rule(self, rule): + """ + Delete a Rule from a Security Group. + + :param rule: Rule should be deleted + :type rule: :class:`OpenStackSecurityGroupRule` + + :rtype: ``bool`` + """ + resp = self.connection.request('/v2.0/security-group-rules/%s' % + (rule.id), method='DELETE') + return resp.status == httplib.NO_CONTENT class OpenStack_1_1_FloatingIpPool(object): diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json index df20ef4cf9..d01f206b50 100644 --- a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group.json @@ -1,20 +1,10 @@ { "security_group": { - "description": "FTP Client-Server - Open 20-21 ports", - "id": 4, - "name": "ftp", - "security_group_rules": [ - { - "security_group_id": "4", - "direction": "ingress", - "port_range_max": 21, - "port_range_min": 20, - "remote_ip_prefix": "0.0.0.0/0", - "id": 1, - "protocol": "tcp" - } - ], + "description": "Test Security Group", + "id": 6, + "name": "test", + "security_group_rules": [], "tenant_id": "68" } } \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group_rule.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group_rule.json new file mode 100644 index 0000000000..2004b5de8f --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_group_rule.json @@ -0,0 +1,13 @@ +{ + "security_group_rule": + { + "security_group_id": 6, + "direction": "ingress", + "port_range_max": 16, + "port_range_min": 14, + "remote_ip_prefix": "0.0.0.0/0", + "id": 2, + "protocol": "tcp" + } +} + diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json index 5fd84f4c38..57209014a0 100644 --- a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__security_groups.json @@ -13,7 +13,7 @@ "name": "ftp", "security_group_rules": [ { - "security_group_id": "4", + "security_group_id": 4, "direction": "ingress", "port_range_max": 21, "port_range_min": 20, diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index 838bbaf0ce..ac293c23af 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -2370,7 +2370,7 @@ def _v2_1337_v2_0_networks_d32019d3_bc6e_4319_9c1d_6722fc136a22(self, method, ur return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) if method == 'DELETE': body = '' - return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_v2_0_subnets(self, method, url, body, headers): body = self.fixtures.load('_v2_0__subnets.json') @@ -2391,7 +2391,7 @@ def _v2_1337_volumes_cd76a3a1_c4ce_40f6_9b9f_07a61508938d(self, method, url, bod return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) if method == 'DELETE': body = '' - return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_snapshots_detail(self, method, url, body, headers): body = self.fixtures.load('_v2_0__snapshots.json') @@ -2408,15 +2408,33 @@ def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, b return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) if method == 'DELETE': body = '' - return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_v2_0_security_groups(self, method, url, body, headers): if method == 'POST': body = self.fixtures.load('_v2_0__security_group.json') return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'GET': + body = self.fixtures.load('_v2_0__security_groups.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_v2_0_security_groups_6(self, method, url, body, headers): if method == 'GET': body = self.fixtures.load('_v2_0__security_group.json') return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_v2_0_security_group_rules(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__security_group_rule.json') + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_v2_0_security_group_rules_2(self, method, url, body, headers): + if method == 'DELETE': + body = '' + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) # This exists because the nova compute url in devstack has v2 in there but the v1.1 fixtures # work fine. From 9cc717f2643e3375d729f04816e8df65ed5a52db Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 25 Sep 2018 15:35:28 +0200 Subject: [PATCH 10/23] Move floatingips to neutron api LIBCLOUD-874 --- libcloud/common/openstack.py | 5 +- libcloud/compute/drivers/openstack.py | 122 +++++++++++++++++- .../openstack_v1.1/_v2_0__floatingip.json | 23 ++++ .../openstack_v1.1/_v2_0__floatingips.json | 52 ++++++++ .../_v2_0__networks_public.json | 64 +++++++++ libcloud/test/compute/test_openstack.py | 70 +++++++++- 6 files changed, 329 insertions(+), 7 deletions(-) create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingip.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingips.json create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__networks_public.json diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py index b64f6692f5..db9f43fdfb 100644 --- a/libcloud/common/openstack.py +++ b/libcloud/common/openstack.py @@ -390,8 +390,9 @@ def parse_error(self): context = self.connection.context driver = self.connection.driver key_pair_name = context.get('key_pair_name', None) - - if len(values) > 0 and values[0]['code'] == 404 and key_pair_name: + print(values) + if len(values) > 0 and 'code' in values[0] and \ + values[0]['code'] == 404 and key_pair_name: raise KeyPairDoesNotExistError(name=key_pair_name, driver=driver) elif len(values) > 0 and 'message' in values[0]: diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 76de1b8a52..cbeed73c06 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -59,6 +59,7 @@ 'OpenStack_1_1_Connection', 'OpenStack_1_1_NodeDriver', 'OpenStack_1_1_FloatingIpPool', + 'OpenStack_2_FloatingIpPool', 'OpenStack_1_1_FloatingIpAddress', 'OpenStack_2_PortInterfaceState', 'OpenStack_2_PortInterface', @@ -3251,10 +3252,29 @@ def ex_delete_security_group_rule(self, rule): :rtype: ``bool`` """ - resp = self.connection.request('/v2.0/security-group-rules/%s' % - (rule.id), method='DELETE') + resp = self.network_connection.request( + '/v2.0/security-group-rules/%s' % (rule.id), method='DELETE') return resp.status == httplib.NO_CONTENT + def _to_floating_ip_pool(self, obj): + return OpenStack_2_FloatingIpPool(obj['id'], obj['name'], + self.network_connection) + + def _to_floating_ip_pools(self, obj): + pool_elements = obj['networks'] + return [self._to_floating_ip_pool(pool) for pool in pool_elements] + + def ex_list_floating_ip_pools(self): + """ + List available floating IP pools + + :rtype: ``list`` of :class:`OpenStack_1_1_FloatingIpPool` + """ + return self._to_floating_ip_pools( + self.network_connection.request('/v2.0/networks?router:external' + '=True&fields=id&fields=' + 'name').object) + class OpenStack_1_1_FloatingIpPool(object): """ @@ -3361,6 +3381,104 @@ def __repr__(self): % (self.id, self.ip_address, self.pool, self.driver)) +class OpenStack_2_FloatingIpPool(OpenStack_1_1_FloatingIpPool): + """ + Floating IP Pool info. + """ + + def __init__(self, id, name, connection): + self.id = id + self.name = name + self.connection = connection + + def _to_floating_ips(self, obj): + ip_elements = obj['floatingips'] + return [self._to_floating_ip(ip) for ip in ip_elements] + + def _to_floating_ip(self, obj): + instance_id = None + + # In neutron version prior to 13.0.0 port_details does not exists + if 'port_details' not in obj and 'port_id' in obj: + port = self.connection.driver.ex_get_port(obj['port_id']) + if port: + obj['port_details'] = {"device_id": port.extra["device_id"], + "device_owner": + port.extra["device_owner"], + "mac_address": + port.extra["mac_address"]} + + if 'port_details' in obj and obj['port_details']: + if obj['port_details']['device_owner'] == 'compute:nova': + instance_id = obj['port_details']['device_id'] + + ip_address = obj['floating_ip_address'] + return OpenStack_1_1_FloatingIpAddress(id=obj['id'], + ip_address=ip_address, + pool=self, + node_id=instance_id, + driver=self.connection.driver) + + def list_floating_ips(self): + """ + List floating IPs in the pool + + :rtype: ``list`` of :class:`OpenStack_1_1_FloatingIpAddress` + """ + return self._to_floating_ips( + self.connection.request('/v2.0/floatingips').object) + + def get_floating_ip(self, ip): + """ + Get specified floating IP from the pool + + :param ip: floating IP to get + :type ip: ``str`` + + :rtype: :class:`OpenStack_1_1_FloatingIpAddress` + """ + floating_ips = self._to_floating_ips( + self.connection.request('/v2.0/floatingips?floating_ip_address' + '=%s' % ip).object) + return floating_ips[0] + + def create_floating_ip(self): + """ + Create new floating IP in the pool + + :rtype: :class:`OpenStack_1_1_FloatingIpAddress` + """ + resp = self.connection.request('/v2.0/floatingips', + method='POST', + data={'floatingip': + {'floating_network_id': self.id}} + ) + data = resp.object['floatingip'] + id = data['id'] + ip_address = data['floating_ip_address'] + return OpenStack_1_1_FloatingIpAddress(id=id, + ip_address=ip_address, + pool=self, + node_id=None, + driver=self.connection.driver) + + def delete_floating_ip(self, ip): + """ + Delete specified floating IP from the pool + + :param ip: floating IP to remove + :type ip: :class:`OpenStack_1_1_FloatingIpAddress` + + :rtype: ``bool`` + """ + resp = self.connection.request('/v2.0/floatingips/%s' % ip.id, + method='DELETE') + return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) + + def __repr__(self): + return ('' % self.name) + + class OpenStack_2_SubNet(object): """ A Virtual SubNet. diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingip.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingip.json new file mode 100644 index 0000000000..0ef2ef55b7 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingip.json @@ -0,0 +1,23 @@ +{ + "floatingip": + { + "router_id": null, + "description": "for test", + "dns_domain": "my-domain.org.", + "dns_name": "myfip2", + "created_at": "2016-12-21T11:55:50Z", + "updated_at": "2016-12-21T11:55:53Z", + "revision_number": 2, + "project_id": "4969c491a3c74ee4af974e6d800c62de", + "tenant_id": "4969c491a3c74ee4af974e6d800c62de", + "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57", + "fixed_ip_address": null, + "floating_ip_address": "10.3.1.42", + "port_id": null, + "id": "09ea1784-2f81-46dc-8c91-244b4df75bde", + "status": "DOWN", + "port_details": null, + "tags": ["tag1,tag2"], + "port_forwardings": [] + } +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingips.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingips.json new file mode 100644 index 0000000000..fa71752a78 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__floatingips.json @@ -0,0 +1,52 @@ +{ + "floatingips": [ + { + "router_id": null, + "description": "for test", + "dns_domain": "my-domain.org.", + "dns_name": "myfip2", + "created_at": "2016-12-21T11:55:50Z", + "updated_at": "2016-12-21T11:55:53Z", + "revision_number": 2, + "project_id": "4969c491a3c74ee4af974e6d800c62de", + "tenant_id": "4969c491a3c74ee4af974e6d800c62de", + "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57", + "fixed_ip_address": null, + "floating_ip_address": "10.3.1.42", + "port_id": null, + "id": "09ea1784-2f81-46dc-8c91-244b4df75bde", + "status": "DOWN", + "port_details": null, + "tags": ["tag1,tag2"], + "port_forwardings": [] + }, + { + "router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f", + "description": "for test", + "dns_domain": "my-domain.org.", + "dns_name": "myfip", + "created_at": "2016-12-21T10:55:50Z", + "updated_at": "2016-12-21T10:55:53Z", + "revision_number": 1, + "project_id": "4969c491a3c74ee4af974e6d800c62de", + "tenant_id": "4969c491a3c74ee4af974e6d800c62de", + "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57", + "fixed_ip_address": "10.0.0.3", + "floating_ip_address": "10.3.1.1", + "port_id": "ce705c24-c1ef-408a-bda3-7bbd946164ab", + "id": "04c5336a-0629-4694-ba30-04b0bdfa88a4", + "status": "ACTIVE", + "port_details": { + "status": "ACTIVE", + "name": "", + "admin_state_up": true, + "network_id": "02dd8479-ef26-4398-a102-d19d0a7b3a1f", + "device_owner": "compute:nova", + "mac_address": "fa:16:3e:b1:3b:30", + "device_id": "fcfc96da-19e2-40fd-8497-f29da1b21143" + }, + "tags": ["tag1,tag2"], + "port_forwardings": [] + } + ] +} \ No newline at end of file diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__networks_public.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__networks_public.json new file mode 100644 index 0000000000..4ac5688f76 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__networks_public.json @@ -0,0 +1,64 @@ +{ + "networks": [ + { + "admin_state_up": true, + "availability_zone_hints": [], + "availability_zones": [ + "nova" + ], + "created_at": "2016-03-08T20:19:41", + "dns_domain": "my-domain.org.", + "id": "d32019d3-bc6e-4319-9c1d-6722fc136a22", + "ipv4_address_scope": null, + "ipv6_address_scope": null, + "l2_adjacency": false, + "mtu": 1500, + "name": "public", + "port_security_enabled": true, + "project_id": "4fd44f30292945e481c7b8a0c8908869", + "qos_policy_id": "6a8454ade84346f59e8d40665f878b2e", + "revision_number": 1, + "router:external": true, + "shared": false, + "status": "ACTIVE", + "subnets": [ + "54d6f61d-db07-451c-9ab3-b9609b6b6f0b" + ], + "tenant_id": "4fd44f30292945e481c7b8a0c8908869", + "updated_at": "2016-03-08T20:19:41", + "vlan_transparent": true, + "description": "", + "is_default": false + }, + { + "admin_state_up": true, + "availability_zone_hints": [], + "availability_zones": [ + "nova" + ], + "created_at": "2016-03-08T20:19:41", + "dns_domain": "my-domain.org.", + "id": "d32019d3-bc6e-4319-9c1d-6722fc136a22", + "ipv4_address_scope": null, + "ipv6_address_scope": null, + "l2_adjacency": false, + "mtu": 1500, + "name": "foobar", + "port_security_enabled": true, + "project_id": "4fd44f30292945e481c7b8a0c8908869", + "qos_policy_id": "6a8454ade84346f59e8d40665f878b2e", + "revision_number": 1, + "router:external": true, + "shared": false, + "status": "ACTIVE", + "subnets": [ + "54d6f61d-db07-451c-9ab3-b9609b6b6f0b" + ], + "tenant_id": "4fd44f30292945e481c7b8a0c8908869", + "updated_at": "2016-03-08T20:19:41", + "vlan_transparent": true, + "description": "", + "is_default": false + } + ] +} diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index ac293c23af..cdb688ee32 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -45,7 +45,7 @@ OpenStack_1_1_NodeDriver, OpenStackSecurityGroup, OpenStackSecurityGroupRule, OpenStack_1_1_FloatingIpPool, OpenStack_1_1_FloatingIpAddress, OpenStackKeyPair, - OpenStack_1_0_Connection, + OpenStack_1_0_Connection, OpenStack_2_FloatingIpPool, OpenStackNodeDriver, OpenStack_2_NodeDriver, OpenStack_2_PortInterfaceState, OpenStackNetwork) from libcloud.compute.base import Node, NodeImage, NodeSize @@ -1399,6 +1399,53 @@ def test_OpenStack_1_1_FloatingIpAddress_delete(self): self.assertEqual(pool.delete_floating_ip.call_count, 1) + def test_OpenStack_2_FloatingIpPool_list_floating_ips(self): + pool = OpenStack_2_FloatingIpPool(1, 'foo', self.driver.connection) + ret = pool.list_floating_ips() + + self.assertEqual(ret[0].id, '09ea1784-2f81-46dc-8c91-244b4df75bde') + self.assertEqual(ret[0].pool, pool) + self.assertEqual(ret[0].ip_address, '10.3.1.42') + self.assertEqual(ret[0].node_id, None) + self.assertEqual(ret[1].id, '04c5336a-0629-4694-ba30-04b0bdfa88a4') + self.assertEqual(ret[1].pool, pool) + self.assertEqual(ret[1].ip_address, '10.3.1.1') + self.assertEqual( + ret[1].node_id, 'fcfc96da-19e2-40fd-8497-f29da1b21143') + + def test_OpenStack_2_FloatingIpPool_get_floating_ip(self): + pool = OpenStack_2_FloatingIpPool(1, 'foo', self.driver.connection) + ret = pool.get_floating_ip('10.3.1.42') + + self.assertEqual(ret.id, '09ea1784-2f81-46dc-8c91-244b4df75bde') + self.assertEqual(ret.pool, pool) + self.assertEqual(ret.ip_address, '10.3.1.42') + self.assertEqual(ret.node_id, None) + + def test_OpenStack_2_FloatingIpPool_create_floating_ip(self): + pool = OpenStack_2_FloatingIpPool(1, 'foo', self.driver.connection) + ret = pool.create_floating_ip() + + self.assertEqual(ret.id, '09ea1784-2f81-46dc-8c91-244b4df75bde') + self.assertEqual(ret.pool, pool) + self.assertEqual(ret.ip_address, '10.3.1.42') + self.assertEqual(ret.node_id, None) + + def test_OpenStack_2_FloatingIpPool_delete_floating_ip(self): + pool = OpenStack_2_FloatingIpPool(1, 'foo', self.driver.connection) + ip = OpenStack_1_1_FloatingIpAddress('foo-bar-id', '42.42.42.42', pool) + + self.assertTrue(pool.delete_floating_ip(ip)) + + def test_OpenStack_2_FloatingIpAddress_delete(self): + pool = OpenStack_2_FloatingIpPool(1, 'foo', self.driver.connection) + pool.delete_floating_ip = Mock() + ip = OpenStack_1_1_FloatingIpAddress('foo-bar-id', '42.42.42.42', pool) + + ip.pool.delete_floating_ip() + + self.assertEqual(pool.delete_floating_ip.call_count, 1) + def test_ex_get_metadata_for_node(self): image = NodeImage(id=11, name='Ubuntu 8.10 (intrepid)', driver=self.driver) size = NodeSize(1, '256 slice', None, None, None, None, driver=self.driver) @@ -2357,8 +2404,12 @@ def _v1_1_slug_os_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5_RACKSPACE(self, def _v2_1337_v2_0_networks(self, method, url, body, headers): if method == 'GET': - body = self.fixtures.load('_v2_0__networks.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if "router:external=True" in url: + body = self.fixtures.load('_v2_0__networks_public.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + else: + body = self.fixtures.load('_v2_0__networks.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) elif method == 'POST': body = self.fixtures.load('_v2_0__networks_POST.json') return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) @@ -2436,6 +2487,19 @@ def _v2_1337_v2_0_security_group_rules_2(self, method, url, body, headers): body = '' return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) + def _v2_1337_v2_0_floatingips(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__floatingip.json') + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'GET': + body = self.fixtures.load('_v2_0__floatingips.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_v2_0_floatingips_foo_bar_id(self, method, url, body, headers): + if method == 'DELETE': + body = '' + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) + # This exists because the nova compute url in devstack has v2 in there but the v1.1 fixtures # work fine. From ef30fe7f9ec8f6fbe09e384485f933a1ec7904c9 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 25 Sep 2018 16:24:14 +0200 Subject: [PATCH 11/23] Fix _to_port to make it work with old versions --- libcloud/compute/drivers/openstack.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index cbeed73c06..a3a5e18b0e 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2688,11 +2688,12 @@ def _to_port(self, element): mac_address=element['mac_address'], name=element['name'], network_id=element['network_id'], - project_id=element['project_id'], - port_security_enabled=element['port_security_enabled'], - revision_number=element['revision_number'], + project_id=element.get('project_id', None), + port_security_enabled=element.get('port_security_enabled', + None), + revision_number=element.get('revision_number', None), security_groups=element['security_groups'], - tags=element['tags'], + tags=element.get('tags', None), tenant_id=element['tenant_id'], updated=updated, ) @@ -3398,8 +3399,9 @@ def _to_floating_ips(self, obj): def _to_floating_ip(self, obj): instance_id = None + print(obj) # In neutron version prior to 13.0.0 port_details does not exists - if 'port_details' not in obj and 'port_id' in obj: + if 'port_details' not in obj and 'port_id' in obj and obj['port_id']: port = self.connection.driver.ex_get_port(obj['port_id']) if port: obj['port_details'] = {"device_id": port.extra["device_id"], From bcbd0ca33c4ce4649e12d8289091a850bdc38ffd Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 25 Sep 2018 16:38:45 +0200 Subject: [PATCH 12/23] Remove print --- libcloud/common/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py index db9f43fdfb..6e7d5b5718 100644 --- a/libcloud/common/openstack.py +++ b/libcloud/common/openstack.py @@ -390,7 +390,7 @@ def parse_error(self): context = self.connection.context driver = self.connection.driver key_pair_name = context.get('key_pair_name', None) - print(values) + if len(values) > 0 and 'code' in values[0] and \ values[0]['code'] == 404 and key_pair_name: raise KeyPairDoesNotExistError(name=key_pair_name, From fbe0e2613b1f8e6a40164d7513b357602b73de8e Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 25 Sep 2018 16:54:01 +0200 Subject: [PATCH 13/23] Update comment --- libcloud/compute/drivers/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index a3a5e18b0e..7893694e91 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -3269,7 +3269,7 @@ def ex_list_floating_ip_pools(self): """ List available floating IP pools - :rtype: ``list`` of :class:`OpenStack_1_1_FloatingIpPool` + :rtype: ``list`` of :class:`OpenStack_2_FloatingIpPool` """ return self._to_floating_ip_pools( self.network_connection.request('/v2.0/networks?router:external' From 7e1ccd5e6eeb8dac3060bc4fb10e45ed781738ed Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 28 Sep 2018 08:44:55 +0200 Subject: [PATCH 14/23] add direction in security_group_rule creation --- libcloud/compute/drivers/openstack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 7893694e91..b571e9f4f6 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -3236,6 +3236,7 @@ def ex_create_security_group_rule(self, security_group, ip_protocol, return self._to_security_group_rule(self.network_connection.request( '/v2.0/security-group-rules', method='POST', data={'security_group_rule': { + 'direction': 'ingress', 'protocol': ip_protocol, 'port_range_min': from_port, 'port_range_max': to_port, From 46e00c62be47cd1061f547d7f17a937b04567262 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 23 Oct 2018 12:19:50 +0200 Subject: [PATCH 15/23] Add create and delete subnets --- libcloud/compute/drivers/openstack.py | 41 ++++++++++++++++++- .../openstack_v1.1/_v2_0__subnet.json | 32 +++++++++++++++ libcloud/test/compute/test_openstack.py | 27 +++++++++++- 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__subnet.json diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 7893694e91..15c96b2c16 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -1681,7 +1681,7 @@ def ex_create_network(self, name, cidr): def ex_delete_network(self, network): """ - Get a list of NodeNetorks that are available. + Delete a Network :param network: Network which should be used :type network: :class:`OpenStackNetwork` @@ -2910,6 +2910,44 @@ def ex_list_subnets(self): self._subnets_url_prefix).object return self._to_subnets(response) + def ex_create_subnet(self, name, network, cidr, ip_version=4): + """ + Create a new Subnet + + :param name: Name of subnet which should be used + :type name: ``str`` + + :param network: Parent network of the subnet + :type network: ``OpenStackNetwork`` + + :param cidr: cidr of network which should be used + :type cidr: ``str`` + + :param ip_version: ip_version of subnet which should be used + :type ip_version: ``int`` + + :rtype: :class:`OpenStack_2_SubNet` + """ + data = {'subnet': {'cidr': cidr, 'network_id': network.id, + 'ip_version': ip_version, 'name': name}} + response = self.connection.request(self._subnets_url_prefix, + method='POST', data=data).object + return self._to_subnet(response['subnet']) + + def ex_delete_subnet(self, subnet): + """ + Delete a Subnet + + :param subnet: Subnet which should be deleted + :type subnet: :class:`OpenStack_2_SubNet` + + :rtype: ``bool`` + """ + resp = self.connection.request('%s/%s' % (self._subnets_url_prefix, + subnet.id), + method='DELETE') + return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) + def ex_list_ports(self): """ List all OpenStack_2_PortInterfaces @@ -3236,6 +3274,7 @@ def ex_create_security_group_rule(self, security_group, ip_protocol, return self._to_security_group_rule(self.network_connection.request( '/v2.0/security-group-rules', method='POST', data={'security_group_rule': { + 'direction': 'ingress', 'protocol': ip_protocol, 'port_range_min': from_port, 'port_range_max': to_port, diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__subnet.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__subnet.json new file mode 100644 index 0000000000..dba344a602 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__subnet.json @@ -0,0 +1,32 @@ +{ + "subnet": + { + "name": "name", + "enable_dhcp": true, + "network_id": "db193ab3-96e3-4cb3-8fc5-05f4296d0324", + "segment_id": null, + "project_id": "26a7980765d0414dbc1fc1f88cdb7e6e", + "tenant_id": "26a7980765d0414dbc1fc1f88cdb7e6e", + "dns_nameservers": [], + "allocation_pools": [ + { + "start": "10.0.0.2", + "end": "10.0.0.254" + } + ], + "host_routes": [], + "ip_version": 4, + "gateway_ip": "10.0.0.1", + "cidr": "10.0.0.0/24", + "id": "08eae331-0402-425a-923c-34f7cfe39c1b", + "created_at": "2016-10-10T14:35:34Z", + "description": "", + "ipv6_address_mode": null, + "ipv6_ra_mode": null, + "revision_number": 2, + "service_types": [], + "subnetpool_id": null, + "tags": ["tag1,tag2"], + "updated_at": "2016-10-10T14:35:34Z" + } +} \ No newline at end of file diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index cdb688ee32..56b0a3f7c3 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -1747,6 +1747,17 @@ def test_ex_list_subnets(self): self.assertEqual(subnet.name, 'private-subnet') self.assertEqual(subnet.cidr, '10.0.0.0/24') + def test_ex_create_subnet(self): + network = self.driver.ex_list_networks()[0] + subnet = self.driver.ex_create_subnet('name', network, '10.0.0.0/24') + + self.assertEqual(subnet.name, 'name') + self.assertEqual(subnet.cidr, '10.0.0.0/24') + + def test_ex_delete_subnet(self): + subnet = self.driver.ex_list_subnets()[0] + self.assertTrue(self.driver.ex_delete_subnet(subnet=subnet)) + def test_ex_list_network(self): networks = self.driver.ex_list_networks() network = networks[0] @@ -2423,9 +2434,21 @@ def _v2_1337_v2_0_networks_d32019d3_bc6e_4319_9c1d_6722fc136a22(self, method, ur body = '' return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) + def _v2_1337_v2_0_subnets_08eae331_0402_425a_923c_34f7cfe39c1b(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('_v2_0__subnet.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) + def _v2_1337_v2_0_subnets(self, method, url, body, headers): - body = self.fixtures.load('_v2_0__subnets.json') - return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'POST': + body = self.fixtures.load('_v2_0__subnet.json') + return (httplib.CREATED, body, self.json_content_headers, httplib.responses[httplib.OK]) + else: + body = self.fixtures.load('_v2_0__subnets.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_volumes_detail(self, method, url, body, headers): body = self.fixtures.load('_v2_0__volumes.json') From 57c674a1d45d9c9840c050cf02f807b3233037cd Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 23 Oct 2018 12:20:16 +0200 Subject: [PATCH 16/23] Remove prints --- libcloud/compute/drivers/openstack.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 15c96b2c16..d67b7538f8 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -1849,8 +1849,6 @@ def ex_get_node_security_groups(self, node): :rtype: ``list`` of :class:`OpenStackSecurityGroup` """ - print(vars(self.connection.request( - '/servers/%s/os-security-groups' % (node.id)))) return self._to_security_groups( self.connection.request('/servers/%s/os-security-groups' % (node.id)).object) @@ -3438,7 +3436,6 @@ def _to_floating_ips(self, obj): def _to_floating_ip(self, obj): instance_id = None - print(obj) # In neutron version prior to 13.0.0 port_details does not exists if 'port_details' not in obj and 'port_id' in obj and obj['port_id']: port = self.connection.driver.ex_get_port(obj['port_id']) From a3944fe9292ef84c87a6321afe29919f1a6c33d5 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 23 Oct 2018 13:05:07 +0200 Subject: [PATCH 17/23] Solve PR issues --- libcloud/compute/drivers/openstack.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index d67b7538f8..c081a8bc77 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2216,27 +2216,26 @@ def _to_volume(self, api_node): return StorageVolume( id=api_node['id'], - name=api_node.get('name', api_node.get('displayName', None)), + name=api_node.get('name', api_node.get('displayName')), size=api_node['size'], state=state, driver=self, extra={ 'description': api_node.get('description', - api_node.get('displayDescription', - None)), + api_node.get('displayDescription') + ), 'attachments': [att for att in api_node['attachments'] if att], # TODO: remove in 1.18.0 'state': api_node.get('status', None), 'snapshot_id': api_node.get('snapshot_id', - api_node.get('snapshotId', None)), + api_node.get('snapshotId')), 'location': api_node.get('availability_zone', - api_node.get('availabilityZone', - None)), + api_node.get('availabilityZone')), 'volume_type': api_node.get('volume_type', - api_node.get('volumeType', None)), + api_node.get('volumeType')), 'metadata': api_node.get('metadata', None), 'created_at': api_node.get('created_at', - api_node.get('createdAt', None)) + api_node.get('createdAt')) } ) @@ -3162,6 +3161,14 @@ def create_volume_snapshot(self, volume, name=None, ex_description=None, data=data).object) def destroy_volume_snapshot(self, snapshot): + """ + Delete a Volume Snapshot. + + :param snapshot: Snapshot to be deleted + :type snapshot: :class:`VolumeSnapshot` + + :rtype: ``bool`` + """ resp = self.volumev2_connection.request('/snapshots/%s' % snapshot.id, method='DELETE') return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) @@ -3419,7 +3426,7 @@ def __repr__(self): % (self.id, self.ip_address, self.pool, self.driver)) -class OpenStack_2_FloatingIpPool(OpenStack_1_1_FloatingIpPool): +class OpenStack_2_FloatingIpPool(object): """ Floating IP Pool info. """ From d0be0552614764cfc24597154077a3abc118b870 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 23 Oct 2018 16:34:43 +0200 Subject: [PATCH 18/23] Fix conn issues --- libcloud/compute/drivers/openstack.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index c081a8bc77..1bed0f98e6 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2927,8 +2927,8 @@ def ex_create_subnet(self, name, network, cidr, ip_version=4): """ data = {'subnet': {'cidr': cidr, 'network_id': network.id, 'ip_version': ip_version, 'name': name}} - response = self.connection.request(self._subnets_url_prefix, - method='POST', data=data).object + response = self.network_connection.request( + self._subnets_url_prefix, method='POST', data=data).object return self._to_subnet(response['subnet']) def ex_delete_subnet(self, subnet): @@ -2940,9 +2940,8 @@ def ex_delete_subnet(self, subnet): :rtype: ``bool`` """ - resp = self.connection.request('%s/%s' % (self._subnets_url_prefix, - subnet.id), - method='DELETE') + resp = self.network_connection.request('%s/%s' % ( + self._subnets_url_prefix, subnet.id), method='DELETE') return resp.status in (httplib.NO_CONTENT, httplib.ACCEPTED) def ex_list_ports(self): From 7ff6a15db791ea4064cd320f53d05003cb763561 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 28 Nov 2018 12:10:50 +0100 Subject: [PATCH 19/23] Change order in gets --- libcloud/compute/drivers/openstack.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 1bed0f98e6..5ac55cb246 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2216,14 +2216,13 @@ def _to_volume(self, api_node): return StorageVolume( id=api_node['id'], - name=api_node.get('name', api_node.get('displayName')), + name=api_node.get('displayName', api_node.get('name')), size=api_node['size'], state=state, driver=self, extra={ - 'description': api_node.get('description', - api_node.get('displayDescription') - ), + 'description': api_node.get('displayDescription', + api_node.get('description')), 'attachments': [att for att in api_node['attachments'] if att], # TODO: remove in 1.18.0 'state': api_node.get('status', None), From dc5eb531c01c8e6d73b3c31c4bf70722b38cd720 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 28 Nov 2018 16:03:39 +0100 Subject: [PATCH 20/23] Complete docstring --- libcloud/compute/drivers/openstack.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 5ac55cb246..32ed66c01b 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -3060,10 +3060,23 @@ def ex_get_port(self, port_interface_id): return self._to_port(response.object['port']) def list_volumes(self): + """ + Get a list of Volumes that are available. + + :rtype: ``list`` of :class:`StorageVolume` + """ return self._to_volumes( self.volumev2_connection.request('/volumes/detail').object) def ex_get_volume(self, volumeId): + """ + Retrieve the StorageVolume with the given ID + + :param volumeId: ID of the volume + :type volumeId: ``string`` + + :return: :class:`StorageVolume` + """ return self._to_volume( self.volumev2_connection.request('/volumes/%s' % volumeId).object) @@ -3118,10 +3131,23 @@ def create_volume(self, size, name, location=None, snapshot=None, return self._to_volume(resp.object) def destroy_volume(self, volume): + """ + Delete a Volume. + + :param volume: Volume to be deleted + :type volume: :class:`StorageVolume` + + :rtype: ``bool`` + """ return self.volumev2_connection.request('/volumes/%s' % volume.id, method='DELETE').success() def ex_list_snapshots(self): + """ + Get a list of Snapshot that are available. + + :rtype: ``list`` of :class:`VolumeSnapshot` + """ return self._to_snapshots( self.volumev2_connection.request('/snapshots/detail').object) From 94571b9b4eaec971f75b7d4f3ad64c4cfc328db4 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 28 Nov 2018 16:37:19 +0100 Subject: [PATCH 21/23] Improve comment --- libcloud/compute/drivers/openstack.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 32ed66c01b..94ae4320cc 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2596,10 +2596,9 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): network_connectionCls = OpenStack_2_NetworkConnection network_connection = None - # Similarly not all node-related operations are exposed through the - # compute API - # See https://developer.openstack.org/api-ref/compute/ - # For example, volume management are made in the cinder service + # Similarly all image operations are noe exposed through the block-storage + # API of the cinde service: + # https://developer.openstack.org/api-ref/block-storage/ volumev2_connectionCls = OpenStack_2_VolumeV2Connection volumev2_connection = None From 9f6041f3b8edc88b983a6b6f80959e8c9985c24c Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 29 Nov 2018 08:16:38 +0100 Subject: [PATCH 22/23] Fix OpenStackException issue --- libcloud/compute/drivers/openstack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 94ae4320cc..def9b6a4c5 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -1168,7 +1168,8 @@ def __init__(self, id, parent_group_id, ip_protocol, from_port, to_port, self.direction = direction else: raise OpenStackException("Security group direction incorrect " - "value: ingress or egress.") + "value: ingress or egress.", 500, + driver) self.tenant_id = tenant_id self.extra = extra or {} From c7b2b293b13c20a0a7eebc12a1f006f637c24259 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 29 Nov 2018 16:47:54 +0100 Subject: [PATCH 23/23] Fix error in ex_list_ports if created_at is not in result --- libcloud/compute/drivers/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index def9b6a4c5..4d39f6d888 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -2664,7 +2664,7 @@ def __init__(self, *args, **kwargs): super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) def _to_port(self, element): - created = element['created_at'] + created = element.get('created_at') updated = element.get('updated_at') return OpenStack_2_PortInterface( id=element['id'],