server.py

# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
# Copyright (c) 2010 Chris Moyer http://coredumped.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

"""
High-level abstraction of an EC2 server
"""
from __future__ import with_statement
import boto.ec2
from boto.mashups.iobject import IObject
from boto.pyami.config import BotoConfigPath, Config
from boto.sdb.db.model import Model
from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty, CalculatedProperty
from boto.manage import propget
from boto.ec2.zone import Zone
from boto.ec2.keypair import KeyPair
import os, time, StringIO
from contextlib import closing
from boto.exception import EC2ResponseError

InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge',
                 'c1.medium', 'c1.xlarge',
                 'm2.2xlarge', 'm2.4xlarge']

class Bundler(object):

    def __init__(self, server, uname='root'):
        from boto.manage.cmdshell import SSHClient
        self.server = server
        self.uname = uname
        self.ssh_client = SSHClient(server, uname=uname)

    def copy_x509(self, key_file, cert_file):
        print '\tcopying cert and pk over to /mnt directory on server'
        self.ssh_client.open_sftp()
        path, name = os.path.split(key_file)
        self.remote_key_file = '/mnt/%s' % name
        self.ssh_client.put_file(key_file, self.remote_key_file)
        path, name = os.path.split(cert_file)
        self.remote_cert_file = '/mnt/%s' % name
        self.ssh_client.put_file(cert_file, self.remote_cert_file)
        print '...complete!'

    def bundle_image(self, prefix, size, ssh_key):
        command = ""
        if self.uname != 'root':
            command = "sudo "
        command += 'ec2-bundle-vol '
        command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file)
        command += '-u %s ' % self.server._reservation.owner_id
        command += '-p %s ' % prefix
        command += '-s %d ' % size
        command += '-d /mnt '
        if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium':
            command += '-r i386'
        else:
            command += '-r x86_64'
        return command

    def upload_bundle(self, bucket, prefix, ssh_key):
        command = ""
        if self.uname != 'root':
            command = "sudo "
        command += 'ec2-upload-bundle '
        command += '-m /mnt/%s.manifest.xml ' % prefix
        command += '-b %s ' % bucket
        command += '-a %s ' % self.server.ec2.aws_access_key_id
        command += '-s %s ' % self.server.ec2.aws_secret_access_key
        return command

    def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None,
               size=None, ssh_key=None, fp=None, clear_history=True):
        iobject = IObject()
        if not bucket:
            bucket = iobject.get_string('Name of S3 bucket')
        if not prefix:
            prefix = iobject.get_string('Prefix for AMI file')
        if not key_file:
            key_file = iobject.get_filename('Path to RSA private key file')
        if not cert_file:
            cert_file = iobject.get_filename('Path to RSA public cert file')
        if not size:
            size = iobject.get_int('Size (in MB) of bundled image')
        if not ssh_key:
            ssh_key = self.server.get_ssh_key_file()
        self.copy_x509(key_file, cert_file)
        if not fp:
            fp = StringIO.StringIO()
        fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
        fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
        if clear_history:
            fp.write('history -c; ')
        fp.write(self.bundle_image(prefix, size, ssh_key))
        fp.write('; ')
        fp.write(self.upload_bundle(bucket, prefix, ssh_key))
        fp.write('; ')
        fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
        fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
        command = fp.getvalue()
        print 'running the following command on the remote server:'
        print command
        t = self.ssh_client.run(command)
        print '\t%s' % t[0]
        print '\t%s' % t[1]
        print '...complete!'
        print 'registering image...'
        self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix))
        return self.image_id

class CommandLineGetter(object):

    def get_ami_list(self):
        my_amis = []
        for ami in self.ec2.get_all_images():
            # hack alert, need a better way to do this!
            if ami.location.find('pyami') >= 0:
                my_amis.append((ami.location, ami))
        return my_amis
    
    def get_region(self, params):
        region = params.get('region', None)
        if isinstance(region, str) or isinstance(region, unicode):
            region = boto.ec2.get_region(region)
            params['region'] = region
        if not region:
            prop = self.cls.find_property('region_name')
            params['region'] = propget.get(prop, choices=boto.ec2.regions)

    def get_name(self, params):
        if not params.get('name', None):
            prop = self.cls.find_property('name')
            params['name'] = propget.get(prop)

    def get_description(self, params):
        if not params.get('description', None):
            prop = self.cls.find_property('description')
            params['description'] = propget.get(prop)

    def get_instance_type(self, params):
        if not params.get('instance_type', None):
            prop = StringProperty(name='instance_type', verbose_name='Instance Type',
                                  choices=InstanceTypes)
            params['instance_type'] = propget.get(prop)

    def get_quantity(self, params):
        if not params.get('quantity', None):
            prop = IntegerProperty(name='quantity', verbose_name='Number of Instances')
            params['quantity'] = propget.get(prop)

    def get_zone(self, params):
        if not params.get('zone', None):
            prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone',
                                  choices=self.ec2.get_all_zones)
            params['zone'] = propget.get(prop)
            
    def get_ami_id(self, params):
        ami = params.get('ami', None)
        if isinstance(ami, str) or isinstance(ami, unicode):
            for a in self.ec2.get_all_images():
                if a.id == ami:
                    params['ami'] = a
        if not params.get('ami', None):
            prop = StringProperty(name='ami', verbose_name='AMI',
                                  choices=self.get_ami_list)
            params['ami'] = propget.get(prop)

    def get_group(self, params):
        group = params.get('group', None)
        if isinstance(group, str) or isinstance(group, unicode):
            group_list = self.ec2.get_all_security_groups()
            for g in group_list:
                if g.name == group:
                    group = g
                    params['group'] = g
        if not group:
            prop = StringProperty(name='group', verbose_name='EC2 Security Group',
                                  choices=self.ec2.get_all_security_groups)
            params['group'] = propget.get(prop)

    def get_key(self, params):
        keypair = params.get('keypair', None)
        if isinstance(keypair, str) or isinstance(keypair, unicode):
            key_list = self.ec2.get_all_key_pairs()
            for k in key_list:
                if k.name == keypair:
                    keypair = k.name
                    params['keypair'] = k.name
        if not keypair:
            prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair',
                                  choices=self.ec2.get_all_key_pairs)
            params['keypair'] = propget.get(prop).name

    def get(self, cls, params):
        self.cls = cls
        self.get_region(params)
        self.ec2 = params['region'].connect()
        self.get_name(params)
        self.get_description(params)
        self.get_instance_type(params)
        self.get_zone(params)
        self.get_quantity(params)
        self.get_ami_id(params)
        self.get_group(params)
        self.get_key(params)

class Server(Model):

    #
    # The properties of this object consists of real properties for data that
    # is not already stored in EC2 somewhere (e.g. name, description) plus
    # calculated properties for all of the properties that are already in
    # EC2 (e.g. hostname, security groups, etc.)
    #
    name = StringProperty(unique=True, verbose_name="Name")
    description = StringProperty(verbose_name="Description")
    region_name = StringProperty(verbose_name="EC2 Region Name")
    instance_id = StringProperty(verbose_name="EC2 Instance ID")
    elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address")
    production = BooleanProperty(verbose_name="Is This Server Production", default=False)
    ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True)
    zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True)
    hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True)
    private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True)
    groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True)
    security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True)
    key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True)
    instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True)
    status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True)
    launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True)
    console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True)

    packages = []
    plugins = []

    @classmethod
    def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key):
        if not cfg.has_section('Credentials'):
            cfg.add_section('Credentials')
        cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id)
        cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key)
        if not cfg.has_section('DB_Server'):
            cfg.add_section('DB_Server')
        cfg.set('DB_Server', 'db_type', 'SimpleDB')
        cfg.set('DB_Server', 'db_name', cls._manager.domain.name)

    '''
    Create a new instance based on the specified configuration file or the specified
    configuration and the passed in parameters.
    
    If the config_file argument is not None, the configuration is read from there. 
    Otherwise, the cfg argument is used.

    The config file may include other config files with a #import reference. The included
    config files must reside in the same directory as the specified file. 
    
    The logical_volume argument, if supplied, will be used to get the current physical 
    volume ID and use that as an override of the value specified in the config file. This 
    may be useful for debugging purposes when you want to debug with a production config 
    file but a test Volume. 
    
    The dictionary argument may be used to override any EC2 configuration values in the 
    config file. 
    '''
    @classmethod
    def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
        if config_file:
            cfg = Config(path=config_file)
        if cfg.has_section('EC2'):
            # include any EC2 configuration values that aren't specified in params:
            for option in cfg.options('EC2'):
                if option not in params:
                    params[option] = cfg.get('EC2', option)
        getter = CommandLineGetter()
        getter.get(cls, params)
        region = params.get('region')
        ec2 = region.connect()
        cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
        ami = params.get('ami')
        kp = params.get('keypair')
        group = params.get('group')
        zone = params.get('zone')
        # deal with possibly passed in logical volume:
        if logical_volume != None:
           cfg.set('EBS', 'logical_volume_name', logical_volume.name) 
        cfg_fp = StringIO.StringIO()
        cfg.write(cfg_fp)
        # deal with the possibility that zone and/or keypair are strings read from the config file:
        if isinstance(zone, Zone):
            zone = zone.name
        if isinstance(kp, KeyPair):
            kp = kp.name
        reservation = ami.run(min_count=1,
                              max_count=params.get('quantity', 1),
                              key_name=kp,
                              security_groups=[group],
                              instance_type=params.get('instance_type'),
                              placement = zone,
                              user_data = cfg_fp.getvalue())
        l = []
        i = 0
        elastic_ip = params.get('elastic_ip')
        instances = reservation.instances
        if elastic_ip != None and instances.__len__() > 0:
            instance = instances[0]
            print 'Waiting for instance to start so we can set its elastic IP address...'
            # Sometimes we get a message from ec2 that says that the instance does not exist.
            # Hopefully the following delay will giv eec2 enough time to get to a stable state:
            time.sleep(5) 
            while instance.update() != 'running':
                time.sleep(1)
            instance.use_ip(elastic_ip)
            print 'set the elastic IP of the first instance to %s' % elastic_ip
        for instance in instances:
            s = cls()
            s.ec2 = ec2
            s.name = params.get('name') + '' if i==0 else str(i)
            s.description = params.get('description')
            s.region_name = region.name
            s.instance_id = instance.id
            if elastic_ip and i == 0:
                s.elastic_ip = elastic_ip
            s.put()
            l.append(s)
            i += 1
        return l
    
    @classmethod
    def create_from_instance_id(cls, instance_id, name, description=''):
        regions = boto.ec2.regions()
        for region in regions:
            ec2 = region.connect()
            try:
                rs = ec2.get_all_instances([instance_id])
            except:
                rs = []
            if len(rs) == 1:
                s = cls()
                s.ec2 = ec2
                s.name = name
                s.description = description
                s.region_name = region.name
                s.instance_id = instance_id
                s._reservation = rs[0]
                for instance in s._reservation.instances:
                    if instance.id == instance_id:
                        s._instance = instance
                s.put()
                return s
        return None

    @classmethod
    def create_from_current_instances(cls):
        servers = []
        regions = boto.ec2.regions()
        for region in regions:
            ec2 = region.connect()
            rs = ec2.get_all_instances()
            for reservation in rs:
                for instance in reservation.instances:
                    try:
                        Server.find(instance_id=instance.id).next()
                        boto.log.info('Server for %s already exists' % instance.id)
                    except StopIteration:
                        s = cls()
                        s.ec2 = ec2
                        s.name = instance.id
                        s.region_name = region.name
                        s.instance_id = instance.id
                        s._reservation = reservation
                        s.put()
                        servers.append(s)
        return servers
    
    def __init__(self, id=None, **kw):
        Model.__init__(self, id, **kw)
        self.ssh_key_file = None
        self.ec2 = None
        self._cmdshell = None
        self._reservation = None
        self._instance = None
        self._setup_ec2()

    def _setup_ec2(self):
        if self.ec2 and self._instance and self._reservation:
            return
        if self.id:
            if self.region_name:
                for region in boto.ec2.regions():
                    if region.name == self.region_name:
                        self.ec2 = region.connect()
                        if self.instance_id and not self._instance:
                            try:
                                rs = self.ec2.get_all_instances([self.instance_id])
                                if len(rs) >= 1:
                                    for instance in rs[0].instances:
                                        if instance.id == self.instance_id:
                                            self._reservation = rs[0]
                                            self._instance = instance
                            except EC2ResponseError:
                                pass
                            
    def _status(self):
        status = ''
        if self._instance:
            self._instance.update()
            status = self._instance.state
        return status

    def _hostname(self):
        hostname = ''
        if self._instance:
            hostname = self._instance.public_dns_name
        return hostname

    def _private_hostname(self):
        hostname = ''
        if self._instance:
            hostname = self._instance.private_dns_name
        return hostname

    def _instance_type(self):
        it = ''
        if self._instance:
            it = self._instance.instance_type
        return it

    def _launch_time(self):
        lt = ''
        if self._instance:
            lt = self._instance.launch_time
        return lt

    def _console_output(self):
        co = ''
        if self._instance:
            co = self._instance.get_console_output()
        return co

    def _groups(self):
        gn = []
        if self._reservation:
            gn = self._reservation.groups
        return gn

    def _security_group(self):
        groups = self._groups()
        if len(groups) >= 1:
            return groups[0].id
        return ""

    def _zone(self):
        zone = None
        if self._instance:
            zone = self._instance.placement
        return zone

    def _key_name(self):
        kn = None
        if self._instance:
            kn = self._instance.key_name
        return kn

    def put(self):
        Model.put(self)
        self._setup_ec2()

    def delete(self):
        if self.production:
            raise ValueError, "Can't delete a production server"
        #self.stop()
        Model.delete(self)

    def stop(self):
        if self.production:
            raise ValueError, "Can't delete a production server"
        if self._instance:
            self._instance.stop()

    def terminate(self):
        if self.production:
            raise ValueError, "Can't delete a production server"
        if self._instance:
            self._instance.terminate()

    def reboot(self):
        if self._instance:
            self._instance.reboot()

    def wait(self):
        while self.status != 'running':
            time.sleep(5)

    def get_ssh_key_file(self):
        if not self.ssh_key_file:
            ssh_dir = os.path.expanduser('~/.ssh')
            if os.path.isdir(ssh_dir):
                ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name)
                if os.path.isfile(ssh_file):
                    self.ssh_key_file = ssh_file
            if not self.ssh_key_file:
                iobject = IObject()
                self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file')
        return self.ssh_key_file

    def get_cmdshell(self):
        if not self._cmdshell:
            import cmdshell
            self.get_ssh_key_file()
            self._cmdshell = cmdshell.start(self)
        return self._cmdshell

    def reset_cmdshell(self):
        self._cmdshell = None

    def run(self, command):
        with closing(self.get_cmdshell()) as cmd:
            status = cmd.run(command)
        return status

    def get_bundler(self, uname='root'):
        self.get_ssh_key_file()
        return Bundler(self, uname)

    def get_ssh_client(self, uname='root'):
        from boto.manage.cmdshell import SSHClient
        self.get_ssh_key_file()
        return SSHClient(self, uname=uname)

    def install(self, pkg):
        return self.run('apt-get -y install %s' % pkg)