Browse files

initial EC2 support with very basic documentation

  • Loading branch information...
1 parent c32ce70 commit 56b155fc308adb2558dbd3b30bc5c8256605e6b2 @rongoro rongoro committed with JeremyGrosser Jul 26, 2010
View
38 doc/using.rst
@@ -140,6 +140,44 @@ If you already have IPManager and SimpleEntityNameManager instances setup, the c
Created v1000
$ clusto vm start v1000
+EC2 virtual machines
+~~~~~~~~~~~~~~~~~~~~
+ALPHA!
+
+EC2 support is in it's very early stages but works like this::
+
+ from clusto.drivers import EC2VirtualServer, EC2VMManager
+
+ ec2manager = EC2VMManager("myec2", aws_access_key_id="...", aws_secret_access_key="...")
+
+ vserver = EC2VirtualServer("v01")
+ vserver.add_attr(key='aws', subkey='ec2_instance_type',value='m1.small')
+ vserver.add_attr(key='aws', subkey='ec2_region',value='eu-west-1')
+ vserver.add_attr(key='aws', subkey='ec2_ami',value='ami-cf4d67bb')
+
+
+ ec2manager.allocate(vserver)
+
+ vserver.get_state() # checks the instance state with EC2
+
+ vserver.get_ips() # gets the public and private IPs from EC2
+
+ vserver.set_attr(key='aws', subkey='ec2_allow_termination', value=False)
+
+ ec2manager.deallocate(vserver)
+ >>> EC2VMManagerException: Not Allowed to terminate v01.
+
+ vserver.set_attr(key='aws', subkey='ec2_allow_termination', value=True)
+
+ ec2manager.deallocate(vserver) # terminates the instance at EC2
+
+The EC2VMManager has stubs for budgeting which aren't fleshed out yet (to restrict $/hr spent). There is also a helper function::
+
+ ec2manager.get_all_ec2_instance_resources()
+
+That returns all the instance information for all regions as seen by EC2. This might not be the greatest idea when you start to get a lot of instances.
+
+
Pool types
~~~~~~~~~~
Clusto currently supports three different types of "pools." Pools are used to group entities together and apply attributes to the groupings. The supported pool types are:
View
1 src/clusto/drivers/devices/servers/__init__.py
@@ -1,4 +1,5 @@
from clusto.drivers.devices.servers.basicserver import *
from clusto.drivers.devices.servers.penguincomputing import *
from clusto.drivers.devices.servers.kvmvirtualserver import *
+from clusto.drivers.devices.servers.ec2server import *
View
20 src/clusto/drivers/devices/servers/ec2server.py
@@ -1,9 +1,27 @@
-from clusto.drivers.devices import BasicVirtualServer
+from clusto.drivers.devices.servers import BasicVirtualServer
+from clusto.drivers.resourcemanagers.ec2vmmanager import EC2VMManager
class EC2VirtualServer(BasicVirtualServer):
_driver_name = "ec2virtualserver"
_port_meta = {}
+
+ @property
+ def _instance(self):
+ res = EC2VMManager.resources(self)[0]
+ manager = EC2VMManager.get_resource_manager(res)
+
+ instance = manager._get_instance_from_resource(res.value)
+ return instance
+ def get_ips(self):
+ """Get the IPs for this EC2 server."""
+
+ return [self._instance.private_ip_address, self._instance.ip_address]
+
+ def get_state(self):
+ """Get the instance state."""
+
+ return self._instance.state
View
1 src/clusto/drivers/resourcemanagers/__init__.py
@@ -2,3 +2,4 @@
from clusto.drivers.resourcemanagers.simplenamemanager import *
from clusto.drivers.resourcemanagers.ipmanager import *
from clusto.drivers.resourcemanagers.vmmanager import *
+from clusto.drivers.resourcemanagers.ec2vmmanager import *
View
183 src/clusto/drivers/resourcemanagers/ec2vmmanager.py
@@ -0,0 +1,183 @@
+from clusto.drivers.base import ResourceManager
+from clusto.exceptions import ResourceException
+
+
+import boto
+
+class EC2VMManagerException(ResourceException):
+ pass
+
+class EC2VMManager(ResourceManager):
+
+ _driver_name = "ec2vmmanager"
+ _attr_name = "ec2vmmanager"
+
+ _properties = {'budget':None, # hourly budget
+ 'aws_access_key_id':None,
+ 'aws_secret_access_key':None}
+
+
+ _type_costs = {'eu-west-1':{'unix':{'m1.small':0.095,
+ 'm1.large':0.38,
+ 'm1.xlarge':0.76,
+ 'm2.xlarge':0.57,
+ 'm2.2xlarge':1.34,
+ 'm2.4xlarge':2.68,
+ 'c1.medium':0.19,
+ 'c1.xlarge':0.76}
+ },
+
+ 'us-east-1':{'unix':{'m1.small':0.085,
+ 'm1.large':0.34,
+ 'm1.xlarge':0.68,
+ 'm2.xlarge':0.50,
+ 'm2.2xlarge':1.20,
+ 'm2.4xlarge':2.40,
+ 'c1.medium':0.17,
+ 'c1.xlarge':0.68,
+ 'cc1.4xlarge':1.60}
+ },
+ 'us-west-1':{'unix':{'m1.small':0.095,
+ 'm1.large':0.38,
+ 'm1.xlarge':0.76,
+ 'm2.xlarge':0.57,
+ 'm2.2xlarge':1.34,
+ 'm2.4xlarge':2.68,
+ 'c1.medium':0.19,
+ 'c1.xlarge':0.76}
+ },
+ 'ap-southeast-1':{'unix':{'m1.small':0.095,
+ 'm1.large':0.38,
+ 'm1.xlarge':0.76,
+ 'm2.xlarge':0.57,
+ 'm2.2xlarge':1.34,
+ 'm2.4xlarge':2.68,
+ 'c1.medium':0.19,
+ 'c1.xlarge':0.76}
+ },
+ }
+
+ def _ec2_connection(self, region=None):
+ c = boto.connect_ec2(aws_access_key_id=str(self.aws_access_key_id),
+ aws_secret_access_key=str(self.aws_secret_access_key))
+
+ if not region:
+ return c
+ else:
+ return boto.ec2.connect_to_region(region)
+
+
+ def _instance_to_dict(self, instance):
+
+ return {'placement':instance.placement,
+ 'instance_id':instance.id}
+
+ def _get_instance_from_resource(self, resource):
+
+ conn = self._ec2_connection(resource['placement'][:-1])
+
+ il = conn.get_all_instances(instance_ids=[resource['instance_id']])
+
+ return il[0].instances[0]
+
+ def _stop_instance(self, resource):
+
+ conn = self._ec2_connection(resource['placement'][:-1])
+
+ reservations = conn.get_all_instances([resource['instance_id']])
+
+ for reservation in reservations:
+ for instance in reservation.instances:
+ if instance.id == resource['instance_id']:
+ instance.stop()
+ return
+
+
+ def get_all_ec2_instance_resources(self):
+ """Query AWS and return all active ec2 instances and their state"""
+
+ instance_resources = []
+
+ regions = self._ec2_connection().get_all_regions()
+
+ for region in regions:
+
+ conn = self._ec2_connection(region.name)
+
+ for reservation in conn.get_all_instances():
+ for instance in reservation.instances:
+ instance_resources.append({'resource':self._instance_to_dict(instance),
+ 'state': instance.state
+ })
+
+ return instance_resources
+
+
+ def allocator(self, thing):
+ """Allocate VMs on ec2 while keeping track of current costs and staying within the budget
+
+ """
+
+ for res in self.resources(thing):
+ raise ResourceException("%s is already assigned to %s"
+ % (thing.name, res.value))
+
+
+ region = thing.attr_value(key='aws', subkey='ec2_region',
+ merge_container_attrs=True) or 'us-east-1'
+
+ instance_type = thing.attr_value(key='aws', subkey='ec2_instance_type',
+ merge_container_attrs=True)
+
+ if not instance_type:
+ raise ResourceException("No instance type specified for %s"
+ % thing.name)
+
+ image_id = thing.attr_value(key='aws', subkey='ec2_ami',
+ merge_container_attrs=True)
+
+ if not image_id:
+ raise ResourceException("No AMI specified for %s" % thing.name)
+
+ placement = thing.attr_value(key='aws', subkey='ec2_placement',
+ merge_container_attrs=True)
+ user_data = thing.attr_value(key='aws', subkey='ec2_user_data',
+ merge_container_attrs=True)
+ key_name = thing.attr_value(key='aws', subkey='ec2_key_name',
+ merge_container_attrs=True)
+
+
+ res = self.resources(thing)
+ if len(res) > 1:
+ raise ResourceException("%s is somehow already assigned more than one instance")
+ elif len(res) == 1:
+ raise ResourceException("%s is already running as %s"
+ % res[0].value)
+ else:
+
+ c = self._ec2_connection(region)
+ image = c.get_image(image_id)
+
+ reservation = image.run(instance_type=instance_type,
+ placement=placement,
+ key_name=key_name,
+ user_data=user_data)
+
+ i = reservation.instances[0]
+
+ return (self._instance_to_dict(i), True)
+
+
+ def deallocate(self, thing, resource=(), number=True):
+ """deallocates a resource from the given thing."""
+
+ if thing.attr_value(key='aws', subkey='ec2_allow_termination') == False:
+ raise EC2VMManagerException("Not Allowed to terminate %s." % thing.name)
+
+ if not resource:
+ for resource in self.resources(thing):
+ self._stop_instance(resource.value)
+ super(EC2VMManager, self).deallocate(thing, resource.value,
+ number)
+ else:
+ super(EC2VMManager, self).deallocate(thing, resource.value, number)
View
12 src/clusto/drivers/resourcemanagers/vmmanager.py
@@ -158,18 +158,6 @@ def allocate(self, thing, resource=(), number=True, **kwargs):
return attr
-class EC2VMManager(VMManager):
-
- _driver_name = "ec2vmmanager"
-
- _properties = {'budget':None,
- 'current_cost':None,
- 'accountstuff':None} # i'd have to lookup again what ec2 actually needs
-
- def allocator(self):
- """allocate VMs on ec2 while keeping track of current costs and staying within the budget"""
- pass
-
class XenVMManager(VMManager):
"""Manage Xen Instances

0 comments on commit 56b155f

Please sign in to comment.