From 72cb6b4f12cc4540e1dce00aceb75e55ed971f64 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 15:45:53 -0400 Subject: [PATCH 1/9] GCE router and subnet implementation Also fixed some lint errors and unified all GCE API calls to have the following format (currently, there are 5-6 different formats): (self.provider .(gce_compute | gcp_storage) . .method(...) .execute()) --- cloudbridge/cloud/providers/gce/helpers.py | 2 +- cloudbridge/cloud/providers/gce/provider.py | 15 +- cloudbridge/cloud/providers/gce/resources.py | 485 ++++++++++++------- cloudbridge/cloud/providers/gce/services.py | 332 +++++++++---- 4 files changed, 559 insertions(+), 275 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/helpers.py b/cloudbridge/cloud/providers/gce/helpers.py index 1950c98d..6d325695 100644 --- a/cloudbridge/cloud/providers/gce/helpers.py +++ b/cloudbridge/cloud/providers/gce/helpers.py @@ -1,7 +1,7 @@ # based on http://stackoverflow.com/a/39126754 +from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization as crypt_serialization from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.backends import default_backend def generate_key_pair(): diff --git a/cloudbridge/cloud/providers/gce/provider.py b/cloudbridge/cloud/providers/gce/provider.py index af3b81c3..7dd5681a 100644 --- a/cloudbridge/cloud/providers/gce/provider.py +++ b/cloudbridge/cloud/providers/gce/provider.py @@ -2,18 +2,19 @@ Provider implementation based on google-api-python-client library for GCE. """ - - -from cloudbridge.cloud.base import BaseCloudProvider -import httplib2 import json import os import re -from string import Template import time +from string import Template + +from cloudbridge.cloud.base import BaseCloudProvider -from googleapiclient import discovery import googleapiclient.http +from googleapiclient import discovery + +import httplib2 + from oauth2client.client import GoogleCredentials from oauth2client.service_account import ServiceAccountCredentials @@ -230,7 +231,7 @@ def _postproc(*kwargs): def wait_for_global_operation(self, operation): while True: - result = self.gce_compute.globalOperations().get( + self.gce_compute.globalOperations().get( project=self.project_name, operation=operation['name']).execute() diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index fd291483..52f6b00a 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -1,6 +1,13 @@ """ DataTypes used by this provider """ +import hashlib +import inspect +import json +import re +import uuid + +import cloudbridge as cb from cloudbridge.cloud.base.resources import BaseAttachmentInfo from cloudbridge.cloud.base.resources import BaseFloatingIP from cloudbridge.cloud.base.resources import BaseInstance @@ -10,16 +17,19 @@ from cloudbridge.cloud.base.resources import BaseNetwork from cloudbridge.cloud.base.resources import BasePlacementZone from cloudbridge.cloud.base.resources import BaseRegion +from cloudbridge.cloud.base.resources import BaseRouter from cloudbridge.cloud.base.resources import BaseSecurityGroup from cloudbridge.cloud.base.resources import BaseSecurityGroupRule from cloudbridge.cloud.base.resources import BaseSnapshot +from cloudbridge.cloud.base.resources import BaseSubnet from cloudbridge.cloud.base.resources import BaseVolume from cloudbridge.cloud.interfaces.resources import InstanceState from cloudbridge.cloud.interfaces.resources import MachineImageState +from cloudbridge.cloud.interfaces.resources import RouterState from cloudbridge.cloud.interfaces.resources import SnapshotState from cloudbridge.cloud.interfaces.resources import VolumeState -import cloudbridge as cb +import googleapiclient # Older versions of Python do not have a built-in set data-structure. try: @@ -27,12 +37,6 @@ except NameError: from sets import Set as set -import hashlib -import inspect -import json -import re -import uuid - class GCEKeyPair(BaseKeyPair): @@ -187,8 +191,11 @@ def zones(self): """ Accesss information about placement zones within this region. """ - zones_response = self._provider.gce_compute.zones().list( - project=self._provider.project_name).execute() + zones_response = (self._provider + .gce_compute + .zones() + .list(project=self._provider.project_name) + .execute()) zones = [zone for zone in zones_response['items'] if zone['region'] == self._gce_region['selfLink']] return [GCEPlacementZone(self._provider, zone['name'], self.name) @@ -285,11 +292,12 @@ def add_firewall(self, tag, ip_protocol, port, source_range, source_tag, firewall['sourceTags'] = [source_tag] project_name = self._provider.project_name try: - response = (self._provider.gce_compute - .firewalls() - .insert(project=project_name, - body=firewall) - .execute()) + response = (self._provider + .gce_compute + .firewalls() + .insert(project=project_name, + body=firewall) + .execute()) self._provider.wait_for_operation(response) # TODO: process the response and handle errors. except: @@ -314,7 +322,8 @@ def find_firewall(self, tag, ip_protocol, port, source_range, source_tag, if not self._check_list_in_dict(firewall, 'sourceRanges', source_range): continue - if not self._check_list_in_dict(firewall, 'sourceTags', source_tag): + if not self._check_list_in_dict(firewall, 'sourceTags', + source_tag): continue return firewall['id'] return None @@ -328,7 +337,7 @@ def get_firewall_info(self, firewall_id): if firewall['id'] != firewall_id: continue if ('sourceRanges' in firewall and - len(firewall['sourceRanges']) == 1): + len(firewall['sourceRanges']) == 1): info['source_range'] = firewall['sourceRanges'][0] if 'sourceTags' in firewall and len(firewall['sourceTags']) == 1: info['source_tag'] = firewall['sourceTags'][0] @@ -337,7 +346,7 @@ def get_firewall_info(self, firewall_id): if 'IPProtocol' in firewall['allowed'][0]: info['ip_protocol'] = firewall['allowed'][0]['IPProtocol'] if ('ports' in firewall['allowed'][0] and - len(firewall['allowed'][0]['ports']) == 1): + len(firewall['allowed'][0]['ports']) == 1): info['port'] = firewall['allowed'][0]['ports'][0] info['network_name'] = self.network_name(firewall) return info @@ -362,7 +371,8 @@ def iter_firewalls(self, tag=None, network_name=None): if 'items' not in self._list_response: return for firewall in self._list_response['items']: - if 'targetTags' not in firewall or len(firewall['targetTags']) != 1: + if ('targetTags' not in firewall or + len(firewall['targetTags']) != 1): continue if 'allowed' not in firewall or len(firewall['allowed']) != 1: continue @@ -381,11 +391,12 @@ def _delete_firewall(self, firewall): """ project_name = self._provider.project_name try: - response = (self._provider.gce_compute - .firewalls() - .delete(project=project_name, - firewall=firewall['name']) - .execute()) + response = (self._provider + .gce_compute + .firewalls() + .delete(project=project_name, + firewall=firewall['name']) + .execute()) self._provider.wait_for_operation(response) except: return False @@ -396,11 +407,11 @@ def _update_list_response(self): """ Sync the local cache of all firewalls with the server. """ - self._list_response = ( - self._provider.gce_compute - .firewalls() - .list(project=self._provider.project_name) - .execute()) + self._list_response = (self._provider + .gce_compute + .firewalls() + .list(project=self._provider.project_name) + .execute()) def _check_list_in_dict(self, dictionary, field_name, value): """ @@ -408,9 +419,8 @@ def _check_list_in_dict(self, dictionary, field_name, value): """ if field_name not in dictionary: return value is None - if (value is None or - len(dictionary[field_name]) != 1 or - dictionary[field_name][0] != value): + if (value is None or len(dictionary[field_name]) != 1 or + dictionary[field_name][0] != value): return False return True @@ -532,7 +542,7 @@ def parent(self): return None if 'target_tag' not in info or info['network_name'] is None: return None - network = delegate.network.get_by_name(info['network_name']) + network = self._delegate.network.get_by_name(info['network_name']) if network is None: return None return GCESecurityGroup(self._delegate, info['target_tag'], network) @@ -666,9 +676,12 @@ def delete(self): """ Delete this image """ - request = self._provider.gce_compute.images().delete( - project=self._provider.project_name, image=self.name) - request.execute() + (self._provider + .gce_compute + .images() + .delete(project=self._provider.project_name, + image=self.name) + .execute()) @property def state(self): @@ -689,11 +702,11 @@ def refresh(self): cb.log.warning("Project name is not found.") return try: - response = self._provider.gce_compute \ - .images() \ - .get(project=project, - image=self.name) \ - .execute() + response = (self._provider + .gce_compute + .images() + .get(project=project, image=self.name) + .execute()) if response: # pylint:disable=protected-access self._gce_image = response @@ -767,10 +780,10 @@ def public_ips(self): access_configs = network_interfaces[0].get('accessConfigs') if access_configs is not None and len(access_configs) > 0: # https://cloud.google.com/compute/docs/reference/beta/instances - # An array of configurations for this interface. Currently, only - # one access config, ONE_TO_ONE_NAT, is supported. If there are - # no accessConfigs specified, then this instance will have no - # external internet access. + # An array of configurations for this interface. Currently, + # only one access config, ONE_TO_ONE_NAT, is supported. If + # there are no accessConfigs specified, then this instance will + # have no external internet access. access_config = access_configs[0] if 'natIP' in access_config: ips.append(access_config['natIP']) @@ -820,41 +833,45 @@ def reboot(self): Reboot this instance. """ if self.state == InstanceState.STOPPED: - self._provider.gce_compute \ - .instances() \ - .start(project=self._provider.project_name, - zone=self._provider.default_zone, - instance=self.name) \ - .execute() + (self._provider + .gce_compute + .instances() + .start(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=self.name) + .execute()) else: - self._provider.gce_compute \ - .instances() \ - .reset(project=self._provider.project_name, - zone=self._provider.default_zone, - instance=self.name) \ - .execute() + (self._provider + .gce_compute + .instances() + .reset(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=self.name) + .execute()) def terminate(self): """ Permanently terminate this instance. """ - self._provider.gce_compute \ - .instances() \ - .delete(project=self._provider.project_name, - zone=self._provider.default_zone, - instance=self.name) \ - .execute() + (self._provider + .gce_compute + .instances() + .delete(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=self.name) + .execute()) def stop(self): """ Stop this instance. """ - self._provider.gce_compute \ - .instances() \ - .stop(project=self._provider.project_name, - zone=self._provider.default_zone, - instance=self.name) \ - .execute() + (self._provider + .gce_compute + .instances() + .stop(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=self.name) + .execute()) @property def image_id(self): @@ -928,11 +945,11 @@ def _get_existing_target_instance(self): self_url = self._provider.parse_url(self._gce_instance['selfLink']) try: response = (self._provider - .gce_compute - .targetInstances() - .list(project=self_url.parameters['project'], - zone=self_url.parameters['zone']) - .execute()) + .gce_compute + .targetInstances() + .list(project=self_url.parameters['project'], + zone=self_url.parameters['zone']) + .execute()) if 'items' not in response: return None for target_instance in response['items']: @@ -959,16 +976,17 @@ def _get_target_instance(self): 'instance': self._gce_instance['selfLink']} try: response = (self._provider - .gce_compute - .targetInstances() - .insert(project=self_url.parameters['project'], - zone=self_url.parameters['zone'], - body=body) - .execute()) + .gce_compute + .targetInstances() + .insert(project=self_url.parameters['project'], + zone=self_url.parameters['zone'], + body=body) + .execute()) self._provider.wait_for_operation( response, zone=self_url.parameters['zone']) except Exception as e: - cb.log.warning('Exception while inserting a target instance: %s', e) + cb.log.warning('Exception while inserting a target instance: %s', + e) return None # The following method should find the target instance that we @@ -984,16 +1002,18 @@ def _redirect_existing_rule(self, ip, target_instance): new_name = target_instance['name'] new_url = target_instance['selfLink'] try: - response = (self._provider.gce_compute - .forwardingRules() - .list(project=self._provider.project_name, - region=ip.region) - .execute()) + response = (self._provider + .gce_compute + .forwardingRules() + .list(project=self._provider.project_name, + region=ip.region) + .execute()) if 'items' not in response: return False for rule in response['items']: if rule['IPAddress'] == ip.public_ip: - parsed_target_url = self._provider.parse_url(rule['target']) + parsed_target_url = self._provider.parse_url( + rule['target']) old_zone = parsed_target_url.parameters['zone'] old_name = parsed_target_url.parameters['targetInstance'] if old_zone == new_zone and old_name == new_name: @@ -1028,16 +1048,17 @@ def _forward(self, ip, target_instance): 'IPAddress': ip.public_ip, 'target': target_instance['selfLink']} try: - response = (self._provider.gce_compute - .forwardingRules() - .insert( - project=self._provider.project_name, - region=ip.region, - body=body) - .execute()) + response = (self._provider + .gce_compute + .forwardingRules() + .insert(project=self._provider.project_name, + region=ip.region, + body=body) + .execute()) self._provider.wait_for_operation(response, region=ip.region) except Exception as e: - cb.log.warning('Exception while inserting a forwarding rule: %s', e) + cb.log.warning('Exception while inserting a forwarding rule: %s', + e) return False return True @@ -1049,21 +1070,24 @@ def _delete_existing_rule(self, ip, target_instance): .parameters['zone']) name = target_instance['name'] try: - response = (self._provider.gce_compute - .forwardingRules() - .list(project=self._provider.project_name, - region=ip.region) - .execute()) + response = (self._provider + .gce_compute + .forwardingRules() + .list(project=self._provider.project_name, + region=ip.region) + .execute()) if 'items' not in response: return False for rule in response['items']: if rule['IPAddress'] == ip.public_ip: - parsed_target_url = self._provider.parse_url(rule['target']) + parsed_target_url = self._provider.parse_url( + rule['target']) temp_zone = parsed_target_url.parameters['zone'] temp_name = parsed_target_url.parameters['targetInstance'] if temp_zone != zone or temp_name != name: - cb.log.warning('"%s" is forwarded to "%s" in zone "%s"', - ip.public_ip, temp_name, temp_zone) + cb.log.warning( + '"%s" is forwarded to "%s" in zone "%s"', + ip.public_ip, temp_name, temp_zone) return False response = (self._provider .gce_compute @@ -1080,7 +1104,7 @@ def _delete_existing_rule(self, ip, target_instance): 'Exception while listing/deleting forwarding rules: %s', e) return False return True - + def add_floating_ip(self, ip_address): """ Add an elastic IP address to this instance. @@ -1095,8 +1119,9 @@ def add_floating_ip(self, ip_address): return target_instance = self._get_target_instance() if not target_instance: - cb.log.warning('Could not create a targetInstance for "%s"', - self.name) + cb.log.warning( + 'Could not create a targetInstance for "%s"', + self.name) return if not self._forward(ip, target_instance): cb.log.warning('Could not forward "%s" to "%s"', @@ -1113,7 +1138,7 @@ def remove_floating_ip(self, ip_address): if not ip.in_use() or ip.private_ip not in self.private_ips: cb.log.warning( 'Floating IP "%s" is not associated to "%s".', - ip_address, self.name) + ip_address, self.name) return target_instance = self._get_target_instance() if not target_instance: @@ -1142,6 +1167,7 @@ def refresh(self): self._gce_instance = self._provider.get_gce_resource_data( self._gce_instance.get('selfLink')) + class GCENetwork(BaseNetwork): def __init__(self, provider, network): @@ -1175,11 +1201,11 @@ def cidr_block(self): def delete(self): try: response = (self._provider - .gce_compute - .networks() - .delete(project=self._provider.project_name, - network=self.name) - .execute()) + .gce_compute + .networks() + .delete(project=self._provider.project_name, + network=self.name) + .execute()) if 'error' in response: return False self._provider.wait_for_operation(response) @@ -1188,14 +1214,15 @@ def delete(self): return True def subnets(self): - raise NotImplementedError("To be implemented") + return self._provider.network.subnets.list() def create_subnet(self, cidr_block, name=None): - raise NotImplementedError("To be implemented") + return self._provider.network.subnets.create(self, cidr_block, name) def refresh(self): return self.state + class GCEFloatingIP(BaseFloatingIP): _DEAD_INSTANCE = 'dead instance' @@ -1223,9 +1250,9 @@ def __init__(self, provider, floating_ip): if target['kind'] == 'compute#targetInstance': url = provider.parse_url(target['instance']) try: - self._target_instance = url.get() + self._target_instance = url.get() except: - self._target_instance = GCEFloatingIP._DEAD_INSTANCE + self._target_instance = GCEFloatingIP._DEAD_INSTANCE else: cb.log.warning('Address "%s" is forwarded to a %s', floating_ip['address'], target['kind']) @@ -1248,7 +1275,7 @@ def public_ip(self): @property def private_ip(self): if (not self._target_instance or - self._target_instance == GCEFloatingIP._DEAD_INSTANCE): + self._target_instance == GCEFloatingIP._DEAD_INSTANCE): return None return self._target_instance['networkInterfaces'][0]['networkIP'] @@ -1256,25 +1283,125 @@ def in_use(self): return True if self._target_instance else False def delete(self): - project_name = self._provider.project_name - # First, delete the forwarding rule, if there is any. - if self._rule: - response = (self._provider.gce_compute - .forwardingRules() - .delete(project=project_name, - region=self._region, - forwardingRule=self._rule['name']) - .execute()) - self._provider.wait_for_operation(response, region=self._region) - - # Release the address. - response = (self._provider.gce_compute - .addresses() - .delete(project=project_name, - region=self._region, - address=self._ip['name']) - .execute()) - self._provider.wait_for_operation(response, region=self._region) + project_name = self._provider.project_name + # First, delete the forwarding rule, if there is any. + if self._rule: + response = (self._provider + .gce_compute + .forwardingRules() + .delete(project=project_name, + region=self._region, + forwardingRule=self._rule['name']) + .execute()) + self._provider.wait_for_operation(response, region=self._region) + + # Release the address. + response = (self._provider + .gce_compute + .addresses() + .delete(project=project_name, + region=self._region, + address=self._ip['name']) + .execute()) + self._provider.wait_for_operation(response, region=self._region) + + +class GCERouter(BaseRouter): + + def __init__(self, provider, router): + super(GCERouter, self).__init__(provider) + self._router = router + + @property + def id(self): + return self._router['id'] + + @property + def name(self): + return self._router['name'] + + def refresh(self): + self._router = self._provider.parse_url(self._router['selfLink']).get() + + @property + def state(self): + # GCE routers are always attached to a network. + return RouterState.ATTACHED + + @property + def network_id(self): + network = self._provider.parse_url(self._router['network']).get() + return network['id'] + + def delete(self): + response = (self._provider + .gce_compute + .routers() + .delete(project=self._provider.project_name, + region=self._router['region'], + router=self._router['name']) + .execute()) + self._provider.wait_for_operation(response, + region=self._router['region']) + + def attach_network(self, network_id): + if network_id == self.network_id: + return + cb.log.warning('GCE routers should be attached at creation time') + + def detach_network(self, network_id): + cb.log.warning('GCE routers are always attached') + + def add_route(self, subnet_id): + cb.log.warning('Not implemented') + + def remove_route(self, subnet_id): + cb.log.warning('Not implemented') + + +class GCESubnet(BaseSubnet): + + def __init__(self, provider, subnet): + super(GCESubnet, self).__init__(provider) + self._subnet = subnet + + @property + def id(self): + return self._subnet['id'] + + @property + def name(self): + return self._subnet['name'] + + @name.setter + def name(self, value): + if value == self.name: + return + cb.log.warning('Cannot change the name of a GCE subnetwork') + + @property + def cidr_block(self): + return self._subnet['ipCidrRange'] + + @property + def network_url(self): + return self._subnet['network'] + + @property + def network_id(self): + return self._provider.parse_url(self.network_url).get()['id'] + + @property + def region(self): + return self._subnet['region'] + + @property + def zone(self): + raise NotImplementedError('To be implemented') + + @property + def delete(self): + return self._provider.network.subnets.delete(self) class GCEVolume(BaseVolume): @@ -1319,21 +1446,22 @@ def description(self): @description.setter def description(self, value): request_body = { - 'labels': {'description': value.replace(' ', '_').lower(),}, + 'labels': {'description': value.replace(' ', '_').lower()}, 'labelFingerprint': self._volume.get('labelFingerprint'), } try: - response = (self._provider.gce_compute - .disks() - .setLabels( - project=self._provider.project_name, + (self._provider + .gce_compute + .disks() + .setLabels(project=self._provider.project_name, zone=self._provider.default_zone, resource=self.name, - body=request_body).execute()) + body=request_body) + .execute()) except Exception as e: - cb.log.warning('Exception while setting volume description: %s.' - 'Check for invalid characters in description. Should' - 'confirm to RFC1035.', e) + cb.log.warning('Exception while setting volume description: %s. ' + 'Check for invalid characters in description. ' + 'Should confirm to RFC1035.', e) raise e self.refresh() @@ -1361,7 +1489,7 @@ def attachments(self): # the first user of a disk. if 'users' in self._volume and len(self._volume) > 0: if len(self._volume) > 1: - cb.log.warning("This volume is attached to multiple instances.") + cb.log.warning("This volume is attached to multiple instances") return BaseAttachmentInfo(self, self._volume.get('users')[0], None) @@ -1386,13 +1514,14 @@ def attach(self, instance, device): instance_name = instance.name if isinstance( instance, GCEInstance) else instance - response = (self._provider.gce_compute - .instances() - .attachDisk( - project=self._provider.project_name, - zone=self._provider.default_zone, - instance=instance_name, - body=attach_disk_body).execute()) + (self._provider + .gce_compute + .instances() + .attachDisk(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=instance_name, + body=attach_disk_body) + .execute()) def detach(self, force=False): """ @@ -1409,17 +1538,18 @@ def detach(self, force=False): device_name = None for disk in instance_data['disks']: if ('source' in disk and 'deviceName' in disk and - disk['source'] == self.id): + disk['source'] == self.id): device_name = disk['deviceName'] if not device_name: return - response = (self._provider.gce_compute - .instances() - .detachDisk( - project=self._provider.project_name, - zone=self._provider.default_zone, - instance=instance_data.get('name'), - deviceName=device_name).execute()) + (self._provider + .gce_compute + .instances() + .detachDisk(project=self._provider.project_name, + zone=self._provider.default_zone, + instance=instance_data.get('name'), + deviceName=device_name) + .execute()) def create_snapshot(self, name, description=None): """ @@ -1432,12 +1562,13 @@ def delete(self): """ Delete this volume. """ - response = (self._provider.gce_compute - .disks() - .delete( - project=self._provider.project_name, - zone=self._provider.default_zone, - disk=self.name).execute()) + (self._provider + .gce_compute + .disks() + .delete(project=self._provider.project_name, + zone=self._provider.default_zone, + disk=self.name) + .execute()) @property def state(self): @@ -1520,11 +1651,12 @@ def delete(self): """ Delete this snapshot. """ - response = (self._provider.gce_compute - .snapshots() - .delete( - project=self._provider.project_name, - snapshot=self.name).execute()) + (self._provider + .gce_compute + .snapshots() + .delete(project=self._provider.project_name, + snapshot=self.name) + .execute()) def create_volume(self, placement, size=None, volume_type=None, iops=None): """ @@ -1548,11 +1680,12 @@ def create_volume(self, placement, size=None, volume_type=None, iops=None): 'type': vol_type, 'sourceSnapshot': self.id } - operation = (self._provider.gce_compute + operation = (self._provider + .gce_compute .disks() - .insert( - project=self._provider.project_name, - zone=placement, - body=disk_body).execute()) + .insert(project=self._provider.project_name, + zone=placement, + body=disk_body) + .execute()) return self._provider.block_store.volumes.get( operation.get('targetLink')) diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index 3f5d5b5c..a685233b 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -1,3 +1,8 @@ +import hashlib +import uuid +from collections import namedtuple + +import cloudbridge as cb from cloudbridge.cloud.base.resources import ClientPagedResultList from cloudbridge.cloud.base.resources import ServerPagedResultList from cloudbridge.cloud.base.services import BaseBlockStoreService @@ -11,20 +16,15 @@ from cloudbridge.cloud.base.services import BaseSecurityGroupService from cloudbridge.cloud.base.services import BaseSecurityService from cloudbridge.cloud.base.services import BaseSnapshotService +from cloudbridge.cloud.base.services import BaseSubnetService from cloudbridge.cloud.base.services import BaseVolumeService from cloudbridge.cloud.interfaces.resources import PlacementZone from cloudbridge.cloud.interfaces.resources import SecurityGroup from cloudbridge.cloud.providers.gce import helpers -import cloudbridge as cb -from collections import namedtuple -import hashlib import googleapiclient from retrying import retry -import sys - -import uuid from .resources import GCEFirewallsDelegate from .resources import GCEFloatingIP @@ -33,10 +33,12 @@ from .resources import GCEKeyPair from .resources import GCEMachineImage from .resources import GCENetwork +from .resources import GCEPlacementZone from .resources import GCERegion +from .resources import GCERouter from .resources import GCESecurityGroup -from .resources import GCESecurityGroupRule from .resources import GCESnapshot +from .resources import GCESubnet from .resources import GCEVolume @@ -125,8 +127,8 @@ def _iter_gce_ssh_keys(self, metadata): # elems should be "ssh-rsa " elems = key.split(" ") if elems and elems[0]: # ignore blank lines - yield GCEKeyPairService.GCEKeyInfo(elems[0], elems[1].encode('ascii'), - elems[2]) + yield GCEKeyPairService.GCEKeyInfo( + elems[0], elems[1].encode('ascii'), elems[2]) def gce_metadata_save_op(self, callback): """ @@ -276,11 +278,12 @@ def __init__(self, provider): @property def instance_data(self): - response = self.provider.gce_compute \ - .machineTypes() \ - .list(project=self.provider.project_name, - zone=self.provider.default_zone) \ - .execute() + response = (self.provider + .gce_compute + .machineTypes() + .list(project=self.provider.project_name, + zone=self.provider.default_zone) + .execute()) return response['items'] def get(self, instance_type_id): @@ -318,11 +321,12 @@ def __init__(self, provider): def get(self, region_id): try: - region = self.provider.gce_compute \ - .regions() \ - .get(project=self.provider.project_name, - region=region_id) \ - .execute() + region = (self.provider + .gce_compute + .regions() + .get(project=self.provider.project_name, + region=region_id) + .execute()) # Handle the case when region_id is not valid except googleapiclient.errors.HttpError: return None @@ -332,8 +336,11 @@ def get(self, region_id): return None def list(self, limit=None, marker=None): - regions_response = self.provider.gce_compute.regions().list( - project=self.provider.project_name).execute() + regions_response = (self.provider + .gce_compute + .regions() + .list(project=self.provider.project_name) + .execute()) regions = [GCERegion(self.provider, region) for region in regions_response['items']] return ClientPagedResultList(self.provider, regions, @@ -351,7 +358,7 @@ def __init__(self, provider): self._public_images = None _PUBLIC_IMAGE_PROJECTS = ['centos-cloud', 'coreos-cloud', 'debian-cloud', - 'opensuse-cloud', 'ubuntu-os-cloud'] + 'opensuse-cloud', 'ubuntu-os-cloud'] def _retrieve_public_images(self): if self._public_images is not None: @@ -359,10 +366,11 @@ def _retrieve_public_images(self): self._public_images = [] for project in GCEImageService._PUBLIC_IMAGE_PROJECTS: try: - response = self.provider.gce_compute \ - .images() \ - .list(project=project) \ - .execute() + response = (self.provider + .gce_compute + .images() + .list(project=project) + .execute()) except googleapiclient.errors.HttpError as http_error: cb.log.warning("googleapiclient.errors.HttpError: {0}".format( http_error)) @@ -376,11 +384,12 @@ def get(self, image_id): Returns an Image given its id """ try: - image = self.provider.gce_compute \ - .images() \ - .get(project=self.provider.project_name, - image=image_id) \ - .execute() + image = (self.provider + .gce_compute + .images() + .get(project=self.provider.project_name, + image=image_id) + .execute()) if image: return GCEMachineImage(self.provider, image) except TypeError as type_error: @@ -415,13 +424,13 @@ def list(self, limit=None, marker=None): self._retrieve_public_images() images = [] if (self.provider.project_name not in - GCEImageService._PUBLIC_IMAGE_PROJECTS): + GCEImageService._PUBLIC_IMAGE_PROJECTS): try: - response = self.provider \ - .gce_compute \ - .images() \ - .list(project=self.provider.project_name) \ - .execute() + response = (self.provider + .gce_compute + .images() + .list(project=self.provider.project_name) + .execute()) if 'items' in response: images = [GCEMachineImage(self.provider, image) for image in response['items']] @@ -481,11 +490,11 @@ def create(self, name, image, instance_type, subnet, zone=None, config['tags']['items'] = sg_names else: config = launch_config - operation = (self.provider.gce_compute.instances() - .insert( - project=self.provider.project_name, - zone=self.provider.default_zone, - body=config) + operation = (self.provider + .gce_compute.instances() + .insert(project=self.provider.project_name, + zone=self.provider.default_zone, + body=config) .execute()) if 'zone' not in operation: return None @@ -531,17 +540,21 @@ def list(self, limit=None, marker=None): # For GCE API, Acceptable values are 0 to 500, inclusive. # (Default: 500). max_result = limit if limit is not None and limit < 500 else 500 - response = self.provider.gce_compute.instances().list( - project=self.provider.project_name, - zone=self.provider.default_zone, - maxResults=max_result, - pageToken=marker).execute() + response = (self.provider + .gce_compute + .instances() + .list(project=self.provider.project_name, + zone=self.provider.default_zone, + maxResults=max_result, + pageToken=marker) + .execute()) instances = [GCEInstance(self.provider, inst) for inst in response['items']] return ServerPagedResultList(len(instances) > max_result, response.get('nextPageToken'), False, data=instances) + class GCEComputeService(BaseComputeService): # TODO: implement GCEComputeService def __init__(self, provider): @@ -572,14 +585,14 @@ class GCENetworkService(BaseNetworkService): def __init__(self, provider): super(GCENetworkService, self).__init__(provider) + self._subnet_svc = GCESubnetService(self.provider) def get(self, network_id): if network_id is None: return None - # networks = self.list(filter='id eq %s' % network_id) would be better. - # But, there is a GCE API bug that causes an error if the network_id - # has more than 19 digits. So, we list all networks and filter - # ourselves. + # Note: networks = self.list(filter='id eq %s' % network_id) does not + # work due to a GCE API bug that causes an error if the network_id has + # has more than 19 digits. networks = self.list() for network in networks: if network.id == network_id: @@ -594,11 +607,12 @@ def get_by_name(self, network_name): def list(self, limit=None, marker=None, filter=None): try: - response = (self.provider.gce_compute - .networks() - .list(project=self.provider.project_name, - filter=filter) - .execute()) + response = (self.provider + .gce_compute + .networks() + .list(project=self.provider.project_name, + filter=filter) + .execute()) networks = [] if 'items' in response: for network in response['items']: @@ -608,16 +622,28 @@ def list(self, limit=None, marker=None, filter=None): return [] def create(self, name): + """ + Creates a custom mode VPC network. + """ try: networks = self.list(filter='name eq %s' % name) if len(networks) > 0: return networks[0] - response = (self.provider.gce_compute - .networks() - .insert(project=self.provider.project_name, - body={'name': name}) - .execute()) + # Possible values for 'autoCreateSubnetworks' are: + # + # None: For creating a legacy (non-subnetted) network. + # True: For creating an auto mode VPC network. This also creates a + # subnetwork in every region. + # False: For creating a custom mode VPC network. Subnetworks should + # be created manually. + response = (self.provider + .gce_compute + .networks() + .insert(project=self.provider.project_name, + body={'name': name, + 'autoCreateSubnetworks': False}) + .execute()) if 'error' in response: return None self.provider.wait_for_operation(response) @@ -628,17 +654,18 @@ def create(self, name): @property def subnets(self): - raise NotImplementedError('To be implemented') + return self._subnet_svc def floating_ips(self, network_id=None, region=None): if not region: region = self.provider.region_name try: - response = (self.provider.gce_compute - .addresses() - .list(project=self.provider.project_name, - region=region) - .execute()) + response = (self.provider + .gce_compute + .addresses() + .list(project=self.provider.project_name, + region=region) + .execute()) ips = [] if 'items' in response: for ip in response['items']: @@ -654,28 +681,138 @@ def create_floating_ip(self, region=None): region = self.provider.region_name ip_name = 'ip-{0}'.format(uuid.uuid4()) try: - response = (self.provider.gce_compute - .addresses() - .insert(project=self.provider.project_name, - region=region, - body={'name': ip_name}) - .execute()) + response = (self.provider + .gce_compute + .addresses() + .insert(project=self.provider.project_name, + region=region, + body={'name': ip_name}) + .execute()) if 'error' in response: return None self.provider.wait_for_operation(response, region=region) ips = self.floating_ips() for ip in ips: - if ip.id == response["targetId"]: + if ip.id == response['targetId']: return ip except: return None - def routers(self): - raise NotImplementedError('To be implemented') + def routers(self, region=None): + if not region: + region = self.provider.region_name + try: + response = (self.provider + .gce_compute + .routers() + .list(project=self.provider.project_name, + region=region) + .execute()) + routers = [] + if 'items' in response: + for router in response['items']: + routers.append(GCERouter(self.provider, router)) + return routers + except: + return [] - def create_router(self, name=None): + def create_router(self, name=None, network=None, region=None): + network_url = 'global/networks/default' + if isinstance(network, GCENetwork): + network_url = network.resource_url + if not region: + region = self.provider.region_name + try: + response = (self.provider + .gce_compute + .routers() + .insert(project=self.provider.project_name, + region=region, + body={'name': name, + 'network': network_url}) + .execute()) + if 'error' in response: + return None + self.provider.wait_for_opeartion(response, region=region) + routers = self.routers() + for router in routers: + if router.id == response['targetId']: + return router + except: + return None + + +class GCESubnetService(BaseSubnetService): + + def __init__(self, provider): + super(GCESubnetService, self).__init__(provider) + + def get(self, subnet_id): + for subnet in self.list(): + if subnet.id == subnet_id: + return subnet + return None + + def list(self, network=None, region=None): + if not region: + region = self.provider.region_name + try: + response = (self.provider + .gce_compute + .subnetworks() + .list(project=self.provider.project_name, + region=region) + .execute()) + subnets = [] + if 'items' in response: + for subnet in response['items']: + subnets.append(GCESubnet(self.provider, subnet)) + return subnets + except: + return [] + + def create(self, network, cidr_block, name=None, zone=None): + if not name: + name = 'subnet-{0}'.format(uuid.uuid4()) + region = self.provider.region_name + if isinstance(zone, GCEPlacementZone): + region = zone.region_name + body = {'ipCidrRange': cidr_block, + 'name': name, + 'network': network.resource_url, + 'region': region} + try: + response = (self.provider + .gce_compute + .subnetworks() + .insert(project=self.provider.project_name, + region=region, + body=body) + .execute()) + self.provider.wait_for_operation(response, region=region) + if 'error' in response: + return None + subnets = self.list(network, region) + for subnet in subnets: + cb.log.warning('subnet ID: %s', subnet.id) + if subnet.id == response['targetId']: + return subnet + except: + return None + + def get_or_create_default(self, zone=None): raise NotImplementedError('To be implemented') + def delete(self, subnet): + response = (self.provider + .gce_compute + .subnetworks() + .delete(project=self.provider.project_name, + region=subnet.region, + router=subnet.name) + .execute()) + self._provider.wait_for_operation(response, region=subnet.region) + class GCEBlockStoreService(BaseBlockStoreService): @@ -722,12 +859,14 @@ def find(self, name, limit=None, marker=None): filtr = 'name eq ' + name max_result = limit if limit is not None and limit < 500 else 500 response = (self.provider - .gce_compute.disks() + .gce_compute + .disks() .list(project=self.provider.project_name, zone=self.provider.default_zone, filter=filtr, maxResults=max_result, - pageToken=marker).execute()) + pageToken=marker) + .execute()) if 'items' not in response: return [] gce_vols = [GCEVolume(self.provider, vol) @@ -748,11 +887,13 @@ def list(self, limit=None, marker=None): # (Default: 500). max_result = limit if limit is not None and limit < 500 else 500 response = (self.provider - .gce_compute.disks() + .gce_compute + .disks() .list(project=self.provider.project_name, zone=self.provider.default_zone, maxResults=max_result, - pageToken=marker).execute()) + pageToken=marker) + .execute()) if 'items' not in response: return [] gce_vols = [GCEVolume(self.provider, vol) @@ -782,11 +923,14 @@ def create(self, name, size, zone, snapshot=None, description=None): 'sourceSnapshot': snapshot_id, 'description': description, } - operation = (self.provider.gce_compute.disks() + operation = (self.provider + .gce_compute + .disks() .insert( project=self._provider.project_name, zone=zone_name, - body=disk_body).execute()) + body=disk_body) + .execute()) return self.get(operation.get('targetLink')) @@ -817,15 +961,17 @@ def find(self, name, limit=None, marker=None): filtr = 'name eq ' + name max_result = limit if limit is not None and limit < 500 else 500 response = (self.provider - .gce_compute.snapshots() + .gce_compute + .snapshots() .list(project=self.provider.project_name, filter=filtr, maxResults=max_result, - pageToken=marker).execute()) + pageToken=marker) + .execute()) if 'items' not in response: return [] snapshots = [GCESnapshot(self.provider, snapshot) - for snapshot in response['items']] + for snapshot in response['items']] return ServerPagedResultList(len(snapshots) > max_result, response.get('nextPageToken'), False, data=snapshots) @@ -836,14 +982,16 @@ def list(self, limit=None, marker=None): """ max_result = limit if limit is not None and limit < 500 else 500 response = (self.provider - .gce_compute.snapshots() + .gce_compute + .snapshots() .list(project=self.provider.project_name, maxResults=max_result, - pageToken=marker).execute()) + pageToken=marker) + .execute()) if 'items' not in response: return [] snapshots = [GCESnapshot(self.provider, snapshot) - for snapshot in response['items']] + for snapshot in response['items']] return ServerPagedResultList(len(snapshots) > max_result, response.get('nextPageToken'), False, data=snapshots) @@ -858,11 +1006,13 @@ def create(self, name, volume, description=None): "description": description } operation = (self.provider - .gce_compute.disks() + .gce_compute + .disks() .createSnapshot( project=self.provider.project_name, zone=self.provider.default_zone, - disk=volume_name, body=snapshot_body).execute()) + disk=volume_name, body=snapshot_body) + .execute()) if 'zone' not in operation: return None gce_zone = self.provider.get_gce_resource_data(operation['zone']) From 0e5e74ddb55c8128e7b474e5490a8051a8fa6091 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 15:49:55 -0400 Subject: [PATCH 2/9] The GCS object store implementation --- cloudbridge/cloud/providers/gce/provider.py | 5 +- cloudbridge/cloud/providers/gce/services.py | 71 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/provider.py b/cloudbridge/cloud/providers/gce/provider.py index 7dd5681a..a5071b9f 100644 --- a/cloudbridge/cloud/providers/gce/provider.py +++ b/cloudbridge/cloud/providers/gce/provider.py @@ -21,6 +21,7 @@ from .services import GCEBlockStoreService from .services import GCEComputeService from .services import GCENetworkService +from .services import GCSObjectStoreService from .services import GCESecurityService @@ -165,6 +166,7 @@ def __init__(self, config): self._security = GCESecurityService(self) self._network = GCENetworkService(self) self._block_store = GCEBlockStoreService(self) + self._object_store = GCSObjectStoreService(self) self._compute_resources = GCPResources(self.gce_compute) self._storage_resources = GCPResources(self.gcp_storage) @@ -187,8 +189,7 @@ def block_store(self): @property def object_store(self): - raise NotImplementedError( - "GCECloudProvider does not implement this service") + return self._object_store @property def gce_compute(self): diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index a685233b..7e634588 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -1022,3 +1022,74 @@ def create(self, name, volume, description=None): return snapshots[0] else: return None + + +class GCSObjectStoreService(BaseObjectStoreService): + + def __init__(self, provider): + super(GCSObjectStoreService, self).__init__(provider) + + def get(self, bucket_id): + """ + Returns a bucket given its ID. Returns ``None`` if the bucket + does not exist or if the user does not have permission to access the + bucket. + """ + try: + response = (self.provider + .gcp_storage + .buckets() + .get(bucket=bucket_id) + .execute()) + if 'error' in response: + # response['error']['code'] is 404 if the bucket does not exist + # and 403 if the user does not have permission to access it. + return None + return GCSBucket(self.provider, response) + except: + return None + + def find(self, name, limit=None, marker=None): + """ + Searches in bucket names for a substring. + """ + buckets = [bucket for bucket in self.list() if name in bucket.name] + return ClientPagedResultList(self.provider, buckets, limit=limit, + marker=marker) + + def list(self, limit=None, marker=None): + """ + List all containers. + """ + try: + response = (self.provider + .gcp_storage + .buckets() + .list(project=self.provider.project_name) + .execute()) + if 'error' in response or 'items' not in response: + return [] + buckets = [GCSBucket(self.provider, bucket) + for bucket in response['items']] + return ClientPagedResultList(self.provider, buckets, + limit=limit, marker=marker) + except: + return [] + + def create(self, name, location=None): + """ + Create a new bucket and returns it. Returns None if creation fails. + """ + try: + response = (self.provider + .gcp_storage + .buckets() + .insert(project=self.provider.project_name, + name=name, + location=location if location else '') + .execute()) + if 'error' in response: + return None + return GCSBucket(self.provider, response) + except: + return None From 4f5972c18fade1596b9b731ae8de318fffd18cae Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 15:50:58 -0400 Subject: [PATCH 3/9] The GCS bucket implementation --- cloudbridge/cloud/providers/gce/resources.py | 82 ++++++++++++++++++++ cloudbridge/cloud/providers/gce/services.py | 6 +- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index 52f6b00a..f2cfa8e9 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -9,6 +9,8 @@ import cloudbridge as cb from cloudbridge.cloud.base.resources import BaseAttachmentInfo +from cloudbridge.cloud.base.resources import BaseBucket +from cloudbridge.cloud.base.resources import BaseBucketObject from cloudbridge.cloud.base.resources import BaseFloatingIP from cloudbridge.cloud.base.resources import BaseInstance from cloudbridge.cloud.base.resources import BaseInstanceType @@ -1689,3 +1691,83 @@ def create_volume(self, placement, size=None, volume_type=None, iops=None): .execute()) return self._provider.block_store.volumes.get( operation.get('targetLink')) + + +class GCSBucket(BaseBucket): + + def __init__(self, provider, bucket): + super(GCSBucket, self).__init__(provider) + self._bucket = bucket + + @property + def id(self): + return self._bucket['id'] + + @property + def name(self): + """ + Get this bucket's name. + """ + return self._bucket['name'] + + def get(self, name): + """ + Retrieve a given object from this bucket. + """ + try: + response = (self._provider + .gcp_storage + .objects() + .get(bucket=self.name, object=name) + .execute()) + if 'error' in response: + return None + return GCSObject(self._provider, response) + except: + return None + + def list(self, limit=None, marker=None, prefix=None): + """ + List all objects within this bucket. + """ + body = {} + if prefix: + body['prefix'] = prefix + try: + response = (self._provider + .gcp_storage + .objects() + .list(bucket=self.name, body=body) + .execute()) + if 'error' in response or 'items' not in response: + return None + objects = [GCSObject(self._provider, obj) + for obj in response['items']] + return ClientPagedResultList(self._provider, objects, limit=limit, + marker=marker) + except: + return [] + + def delete(self, delete_contents=False): + """ + Delete this bucket. + """ + (self._provider + .gcp_storage + .buckets() + .delete(bucket=self.name) + .execute()) + + def create_object(self, name): + try: + response = (self._provider + .gcp_storage + .objects() + .insert(bucket=self.name, body={'name': name}) + .execute()) + if 'error' in response: + return None + return GCSObject(self._provider, response) + except: + return None + diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index 7e634588..01f46292 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -40,6 +40,7 @@ from .resources import GCESnapshot from .resources import GCESubnet from .resources import GCEVolume +from .resources import GCSBucket class GCESecurityService(BaseSecurityService): @@ -1080,13 +1081,16 @@ def create(self, name, location=None): """ Create a new bucket and returns it. Returns None if creation fails. """ + body = {} + if location: + body['location'] = location try: response = (self.provider .gcp_storage .buckets() .insert(project=self.provider.project_name, name=name, - location=location if location else '') + body=body) .execute()) if 'error' in response: return None From c0b71ad3b645f94225c4d8ca68d284df623c9019 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 15:51:21 -0400 Subject: [PATCH 4/9] The GCS object implementation --- cloudbridge/cloud/providers/gce/resources.py | 44 ++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index f2cfa8e9..e4bd5d24 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -1693,6 +1693,50 @@ def create_volume(self, placement, size=None, volume_type=None, iops=None): operation.get('targetLink')) +class GCSObject(BaseBucketObject): + + def __init__(self, provider, obj): + super(GCSObject, self).__init__(provider) + self._obj = obj + + @property + def id(self): + return self._obj['id'] + + @property + def name(self): + return self._obj['name'] + + @property + def size(self): + return self._obj['size'] + + @property + def last_modified(self): + return self._obj['updated'] + + def iter_content(self): + # TODO: It's not clear what does this method do. Does it return an + # iterator for metadata fields? + raise NotImplementedError('Not Implemented') + + def upload(self, data): + raise NotImplementedError('Not Implemented') + + def upload_from_file(self, path): + raise NotImplementedError('Not Implemented') + + def delete(self): + (self._provider + .gcp_storage + .objects() + .delete(bucket=self._obj['bucket'], object=self.name) + .execute()) + + def generate_url(self, expires_in=0): + return self._obj['selfLink'] + + class GCSBucket(BaseBucket): def __init__(self, provider, bucket): From 4f66131a6621c91d039864f05c756f9c534d65d6 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 15:51:39 -0400 Subject: [PATCH 5/9] Fix small lint errors --- cloudbridge/cloud/providers/gce/provider.py | 2 +- cloudbridge/cloud/providers/gce/resources.py | 2 +- cloudbridge/cloud/providers/gce/services.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/provider.py b/cloudbridge/cloud/providers/gce/provider.py index a5071b9f..b009ffe9 100644 --- a/cloudbridge/cloud/providers/gce/provider.py +++ b/cloudbridge/cloud/providers/gce/provider.py @@ -21,8 +21,8 @@ from .services import GCEBlockStoreService from .services import GCEComputeService from .services import GCENetworkService -from .services import GCSObjectStoreService from .services import GCESecurityService +from .services import GCSObjectStoreService class GCPResourceUrl(object): diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index e4bd5d24..381dc0b2 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -25,6 +25,7 @@ from cloudbridge.cloud.base.resources import BaseSnapshot from cloudbridge.cloud.base.resources import BaseSubnet from cloudbridge.cloud.base.resources import BaseVolume +from cloudbridge.cloud.base.resources import ClientPagedResultList from cloudbridge.cloud.interfaces.resources import InstanceState from cloudbridge.cloud.interfaces.resources import MachineImageState from cloudbridge.cloud.interfaces.resources import RouterState @@ -1814,4 +1815,3 @@ def create_object(self, name): return GCSObject(self._provider, response) except: return None - diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index 01f46292..bb0f8338 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -12,6 +12,7 @@ from cloudbridge.cloud.base.services import BaseInstanceTypesService from cloudbridge.cloud.base.services import BaseKeyPairService from cloudbridge.cloud.base.services import BaseNetworkService +from cloudbridge.cloud.base.services import BaseObjectStoreService from cloudbridge.cloud.base.services import BaseRegionService from cloudbridge.cloud.base.services import BaseSecurityGroupService from cloudbridge.cloud.base.services import BaseSecurityService From d9cbb61bd806d94dce9b8bca38102c1643798ca5 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 17:55:11 -0400 Subject: [PATCH 6/9] Fix GCS object creation --- cloudbridge/cloud/providers/gce/resources.py | 25 +++++++++++++------- cloudbridge/cloud/providers/gce/services.py | 3 +-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index 381dc0b2..a71bfcee 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -3,6 +3,7 @@ """ import hashlib import inspect +import io import json import re import uuid @@ -1775,22 +1776,22 @@ def list(self, limit=None, marker=None, prefix=None): """ List all objects within this bucket. """ - body = {} - if prefix: - body['prefix'] = prefix try: response = (self._provider .gcp_storage .objects() - .list(bucket=self.name, body=body) + .list(bucket=self.name, + prefix=prefix if prefix else '') .execute()) if 'error' in response or 'items' not in response: - return None + cb.log.warning('response: %s', response) + return [] objects = [GCSObject(self._provider, obj) for obj in response['items']] return ClientPagedResultList(self._provider, objects, limit=limit, marker=marker) - except: + except Exception as e: + cb.log.warning('error: %s', e) return [] def delete(self, delete_contents=False): @@ -1804,14 +1805,22 @@ def delete(self, delete_contents=False): .execute()) def create_object(self, name): + """ + Create an empty plain text object. + """ try: + media_body = googleapiclient.http.MediaIoBaseUpload( + io.BytesIO(''), mimetype='plain/text') response = (self._provider .gcp_storage .objects() - .insert(bucket=self.name, body={'name': name}) + .insert(bucket=self.name, + body={'name': name}, + media_body=media_body) .execute()) if 'error' in response: return None return GCSObject(self._provider, response) - except: + except Exception as e: + cb.log.warning('error: %s', e) return None diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index bb0f8338..1fa99b0a 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -1082,7 +1082,7 @@ def create(self, name, location=None): """ Create a new bucket and returns it. Returns None if creation fails. """ - body = {} + body = {'name': name} if location: body['location'] = location try: @@ -1090,7 +1090,6 @@ def create(self, name, location=None): .gcp_storage .buckets() .insert(project=self.provider.project_name, - name=name, body=body) .execute()) if 'error' in response: From 2eb05ff04d7724768565200a89d93de21d5355ca Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 18:35:37 -0400 Subject: [PATCH 7/9] Implement uploading GCS objects --- cloudbridge/cloud/providers/gce/resources.py | 46 ++++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index a71bfcee..f2bdfcc9 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -1697,8 +1697,9 @@ def create_volume(self, placement, size=None, volume_type=None, iops=None): class GCSObject(BaseBucketObject): - def __init__(self, provider, obj): + def __init__(self, provider, bucket, obj): super(GCSObject, self).__init__(provider) + self._bucket = bucket self._obj = obj @property @@ -1723,10 +1724,27 @@ def iter_content(self): raise NotImplementedError('Not Implemented') def upload(self, data): - raise NotImplementedError('Not Implemented') + """ + Set the contents of this object to the given text. + """ + media_body = googleapiclient.http.MediaIoBaseUpload( + io.BytesIO(data), mimetype='application/octet-stream') + response = self._bucket.create_object_with_media_body(self.name, + media_body) + if response: + self._obj = response def upload_from_file(self, path): - raise NotImplementedError('Not Implemented') + """ + Upload a binary file. + """ + with open(path, 'rb') as f: + media_body = googleapiclient.http.MediaIoBaseUpload( + f, 'application/octet-stream') + response = self._bucket.create_object_with_media_body(self.name, + media_body) + if response: + self._obj = response def delete(self): (self._provider @@ -1768,7 +1786,7 @@ def get(self, name): .execute()) if 'error' in response: return None - return GCSObject(self._provider, response) + return GCSObject(self._provider, self, response) except: return None @@ -1784,14 +1802,12 @@ def list(self, limit=None, marker=None, prefix=None): prefix=prefix if prefix else '') .execute()) if 'error' in response or 'items' not in response: - cb.log.warning('response: %s', response) return [] - objects = [GCSObject(self._provider, obj) + objects = [GCSObject(self._provider, self, obj) for obj in response['items']] return ClientPagedResultList(self._provider, objects, limit=limit, marker=marker) - except Exception as e: - cb.log.warning('error: %s', e) + except: return [] def delete(self, delete_contents=False): @@ -1808,9 +1824,14 @@ def create_object(self, name): """ Create an empty plain text object. """ + response = self.create_object_with_media_body( + name, + googleapiclient.http.MediaIoBaseUpload( + io.BytesIO(''), mimetype='application/octet-stream')) + return GCSObject(self._provider, self, response) if response else None + + def create_object_with_media_body(self, name, media_body): try: - media_body = googleapiclient.http.MediaIoBaseUpload( - io.BytesIO(''), mimetype='plain/text') response = (self._provider .gcp_storage .objects() @@ -1820,7 +1841,6 @@ def create_object(self, name): .execute()) if 'error' in response: return None - return GCSObject(self._provider, response) - except Exception as e: - cb.log.warning('error: %s', e) + return response + except: return None From ba70df6a5f238f927ee7c59ad383771a539be823 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 18:55:50 -0400 Subject: [PATCH 8/9] Better MIME types --- cloudbridge/cloud/providers/gce/resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/resources.py b/cloudbridge/cloud/providers/gce/resources.py index f2bdfcc9..7b6441da 100644 --- a/cloudbridge/cloud/providers/gce/resources.py +++ b/cloudbridge/cloud/providers/gce/resources.py @@ -1728,7 +1728,7 @@ def upload(self, data): Set the contents of this object to the given text. """ media_body = googleapiclient.http.MediaIoBaseUpload( - io.BytesIO(data), mimetype='application/octet-stream') + io.BytesIO(data), mimetype='plain/text') response = self._bucket.create_object_with_media_body(self.name, media_body) if response: @@ -1827,7 +1827,7 @@ def create_object(self, name): response = self.create_object_with_media_body( name, googleapiclient.http.MediaIoBaseUpload( - io.BytesIO(''), mimetype='application/octet-stream')) + io.BytesIO(''), mimetype='plain/text')) return GCSObject(self._provider, self, response) if response else None def create_object_with_media_body(self, name, media_body): From dad1f84f0db9cba1c79ddf8e732e93718dd00e44 Mon Sep 17 00:00:00 2001 From: Ehsan Chiniforooshan Date: Mon, 26 Jun 2017 22:37:01 -0400 Subject: [PATCH 9/9] Review comments Also removed the wait_for_global_operation method, since the more general wait_for_operation method is used everywhere. --- cloudbridge/cloud/providers/gce/provider.py | 6 ------ cloudbridge/cloud/providers/gce/services.py | 6 +++++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cloudbridge/cloud/providers/gce/provider.py b/cloudbridge/cloud/providers/gce/provider.py index b009ffe9..0b27480c 100644 --- a/cloudbridge/cloud/providers/gce/provider.py +++ b/cloudbridge/cloud/providers/gce/provider.py @@ -230,12 +230,6 @@ def _postproc(*kwargs): response = request.execute() return response - def wait_for_global_operation(self, operation): - while True: - self.gce_compute.globalOperations().get( - project=self.project_name, - operation=operation['name']).execute() - def _connect_gcp_storage(self): return discovery.build('storage', 'v1', credentials=self._credentials) diff --git a/cloudbridge/cloud/providers/gce/services.py b/cloudbridge/cloud/providers/gce/services.py index 1fa99b0a..39917771 100644 --- a/cloudbridge/cloud/providers/gce/services.py +++ b/cloudbridge/cloud/providers/gce/services.py @@ -735,7 +735,7 @@ def create_router(self, name=None, network=None, region=None): .execute()) if 'error' in response: return None - self.provider.wait_for_opeartion(response, region=region) + self.provider.wait_for_operation(response, region=region) routers = self.routers() for router in routers: if router.id == response['targetId']: @@ -1046,6 +1046,10 @@ def get(self, bucket_id): if 'error' in response: # response['error']['code'] is 404 if the bucket does not exist # and 403 if the user does not have permission to access it. + if response['error']['code'] not in (403, 404): + cb.log.warning('Unexpected error code (%d) when accessing ' + 'bucket %s', response['error']['code'], + bucket_id) return None return GCSBucket(self.provider, response) except: