Merge pull request #49 from onefinestay/ebs_functionality

ability to create + attach volume to new instance

CHANGES

@@ -1,3 +1,9 @@
+Version 0.4.1
+-------------
+
+- Create and attach volumes at instance launch
+
+
 Version 0.4.0
 --------------
 

gonzo/clouds/compute.py

@@ -1,4 +1,5 @@
 import logging
+import time
 
 from datetime import datetime
 
@@ -97,7 +98,8 @@ def get_next_az(self, environment, server_type):
                         return available_azs[index + 1]
 
     def create_instance(self, image_name, name, owner, user_data=None,
-                        security_groups=None, size=None, key_name=None):
+                        security_groups=None, size=None, key_name=None,
+                        volume_size=None):
         instance_name = name.split('-')
         environment = instance_name[0]
         server_type = '-'.join(instance_name[1:-1])
@@ -146,8 +148,52 @@ def create_instance(self, image_name, name, owner, user_data=None,
         )
         self.compute_session.wait_until_running([instance])
         new_instance = self.get_instance_by_uuid(instance.uuid)
+
+        if volume_size is not None:
+            self.create_and_attach_volume(new_instance, volume_size)
+
         return new_instance
 
+    def create_and_attach_volume(self, instance, vol_size, vol_type=None,
+                                 vol_name=None):
+        created_volume = self.create_volume(
+            instance, instance.name,
+            vol_size, vol_type)
+
+        self.wait_until_volume_available(created_volume)
+
+        self.compute_session.attach_volume(
+            node=instance,
+            volume=created_volume,
+            device='/dev/xvdf'
+        )
+
+    def wait_until_volume_available(self, volume):
+
+        while not self.volume_is_created(volume):
+            time.sleep(1)
+        return True
+
+    def volume_is_created(self, volume):
+        for vol in self.compute_session.list_volumes():
+            if vol.id == volume.id:
+                if vol.extra['state'] == 'available':
+                    return True
+                elif vol.extra['state'] == 'creating':
+                    return False
+                else:
+                    raise LookupError("Unknown volume state `{}`".format(
+                        vol.extra['state']))
+
+        raise LookupError("Unknown volume `{}`".format(volume))
+
+    def get_az_of_instance(self, instance):
+        instance_az = instance.extra['gonzo_az']
+
+        for az in self.list_availability_zones():
+            if az.name == instance_az:
+                return az
+
     def get_instance_size_by_name(self, size_name):
         for size in self.compute_session.list_sizes():
             if size_name == getattr(size, self.INSTANCE_SIZE_ATTRIBUTE):
@@ -222,6 +268,16 @@ def _monkeypatch_instance(self, instance):
     def security_groups_for_launch(self, security_group_names):
         return security_group_names
 
+    def create_volume(self, instance, vol_name,
+                      vol_size, vol_type='gp2'):
+        instance_az = self.get_az_of_instance(instance)
+        return self.compute_session.create_volume(
+            name=vol_name,
+            size=vol_size,
+            location=instance_az,
+            ex_volume_type=vol_type,
+        )
+
 
 @backend_for(ComputeProvider.OPENSTACK)
 class Openstack(Cloud):
@@ -271,3 +327,9 @@ def security_groups_for_launch(self, security_group_names):
             self.get_security_group(name)
             for name in security_group_names
         ]
+
+    def create_volume(self, instance, vol_name, vol_size, vol_type='gp2'):
+        return self.compute_session.create_volume(
+            name=vol_name,
+            size=vol_size,
+        )

gonzo/scripts/launch.py

@@ -105,7 +105,9 @@ def launch(args):
         security_groups=security_groups,
         owner=username,
         key_name=cloud_config.get('PUBLIC_KEY_NAME'),
+        volume_size=args.volume_size
     )
+
     print "Instance created: {}.{}".format(
         instance.name,
         cloud_config['DNS_ZONE']
@@ -150,6 +152,8 @@ def main(args):
 
 
 def init_parser(parser):
+    parser.add_argument(
+        '--volume-size', dest="volume_size")
     parser.add_argument(
         'env_type', metavar='environment-server_type', help=env_type_pair_help)
     parser.add_argument(

setup.py

@@ -36,7 +36,7 @@ def parse_requirments(fn, dependency_links):
 setup(
     name='gonzo',
     packages=find_packages(exclude=['tests', 'tests.*']),
-    version='0.4.0',
+    version='0.4.1',
     author='onefinestay',
     author_email='engineering@onefinestay.com',
     url='https://github.com/onefinestay/gonzo',

tests/integration/test_end_to_end.py

@@ -53,16 +53,20 @@ def openstack_session(fake_get_config):
     openstack = Openstack(fake_get_config().CLOUDS['cloudname'], region=None)
     instance_list = openstack.compute_session.list_nodes()
 
-    print "deleting instances"
     for instance in instance_list:
         openstack.compute_session.destroy_node(instance)
 
     time.sleep(10)  # wait for instance to be deleted
 
-    print "deleting security groups"
     sec_groups = openstack.compute_session.ex_list_security_groups()
     for sec_group in sec_groups[1:]:  # Skip default group
         openstack.compute_session.ex_delete_security_group(sec_group)
+
+    volume_list = openstack.compute_session.list_volumes()
+    if volume_list:
+        for volume in volume_list:
+            openstack.compute_session.destroy_volume(volume)
+
     yield openstack
 
 
@@ -80,13 +84,14 @@ def test_end_to_end(capsys, fake_dns, fake_get_config, openstack_session):
 
     image_list = openstack_session.compute_session.list_images()
     image_map = {image.name: image.id for image in image_list}
-    image_id = image_map['cirros-0.3.2-x86_64-uec']
+    image_id = image_map['cirros-0.3.4-x86_64-uec']
 
     # launch an instance
     instance_name = ["test", "launch", "instance"]
     args = parser.parse_args(["launch", "-".join(
-        instance_name), "--image-id={}".format(image_id)]
+        instance_name), "--image-id={}".format(image_id), '--volume-size=3']
     )
+
     launch.launch(args)
     full_instance_name = "{}-001".format("-".join(instance_name))
     instance = openstack_session.get_instance_by_name(full_instance_name)
@@ -99,6 +104,16 @@ def test_end_to_end(capsys, fake_dns, fake_get_config, openstack_session):
         instance_name[-2:]
     )
 
+    # check ebs created
+    time.sleep(10)
+    volumes = openstack_session.compute_session.list_volumes()
+    assert len(volumes) == 1
+
+    # check ebs attached to instance
+    volume = volumes[0]
+    assert volume.extra['state'] == 'in-use'
+    assert volume.extra['attachments'][0]['serverId'] == instance.id
+
     # check security groups created
     assert openstack_session.get_security_group("launch-instance")
     assert openstack_session.get_security_group("gonzo")