Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #8 from ooda/aws-fabric-tasks

Aws fabric tasks
  • Loading branch information...
commit 3c171c2d7c13f1e52b6158baea18f59ecd1563ee 2 parents 383bf68 + 8eb75ac
Hugues Demers hdemers authored
33 CONTRIBUTING.md
View
@@ -0,0 +1,33 @@
+Guidelines for contributing
+===========================
+
+Here are some guidelines for contributing new features, fixing bugs and overall
+adding new code to this project.
+
+
+Pull request checklist
+----------------------
+For the impatient:
+
+ 1. Make sure it's PEP8 compliant.
+ 3. Rebase onto master.
+ 4. Open a pull request on GitHub.
+
+
+Coding style guides
+-------------------
+
+### Python
+You should closely follow the PEP8 style guide described
+[here](http://www.python.org/dev/peps/pep-0008/). However, the best way to
+ensure your code is compliant is to use a tool that automatically checks your
+code, like *pep8.py*. See [this](http://stackoverflow.com/q/399956) for hints
+on how to do that.
+
+
+Versioning
+----------
+
+We use the [semantic versioning](http://semver.org) scheme. The master branch
+should be tagged appropriately.
+
44 README.md
View
@@ -0,0 +1,44 @@
+Cloudly
+=======
+
+We use a lot of cloud stuff here at OODA Technologies. This library
+contains all sort of small helper modules from quickly launching an EC2 instance
+and configuring it according to our taste to wrappers around pub/sub providers,
+to easy connection to a CouchDB.
+
+Currently, this includes wrappers and tools for:
+
+ - AWS
+ - Redis
+ - Redis queues using [RQ](http://python-rq.org/)
+ - CouchDB
+ - memoization
+ - metrics using [Cube](http://square.github.com/cube/)
+ - logging
+ - notification using SNS
+ - publish/subscribe using [PubNub](http://www.pubnub.com/) or
+ [Pusher](http://pusher.com/)
+
+Installation instructions are below. Contribution instructions are in
+[CONTRIBUTING.md](CONTRIBUTING.md).
+
+
+Installation
+------------
+
+You'll need:
+
+ - a redis server
+ - AWS EC2 API tools
+
+On Ubuntu do:
+
+ sudo apt-get install redis-server ec2-api-tools
+
+Then clone this repository:
+
+ git clone git@github.com:ooda/cloudly.git
+
+And install all requirements:
+
+ pip install -r requirements.txt
257 cloudly/aws/fabfile.py
View
@@ -0,0 +1,257 @@
+"""
+Fabric file to handle status and deploy.
+
+Usage:
+
+Either use -H to specify a host, or -R for a role. Otherwise will prompt
+you with list of hosts.
+"""
+
+import os
+import getpass
+import functools
+from time import sleep
+
+from fabric.api import (
+ run, env, cd, settings, puts, runs_once, task
+)
+from fabric.operations import prompt
+from cuisine import (package_ensure, package_update, file_write,
+ python_package_ensure, mode_sudo)
+
+from cloudly.aws import ec2
+import boto
+
+DEFAULT_AMI = "ami-82fa58eb"
+AWS_USERNAME = os.environ.get("AWS_USERNAME", "ubuntu")
+GIT_WORK_TREE = os.environ.get("AWS_DEPLOY_DIR", "service")
+DEPLOY_LOCATION = os.path.join(
+ "/home", AWS_USERNAME, GIT_WORK_TREE
+)
+
+ROLES = ["development", "production"]
+
+INSTANCES_HEADER = "{0:<4} {1:<20} {2:<16} {3:<12} {4:<12} {5:<13}".format(
+ "", "Name", "IP address", "Launch time", "Instance ID", "Image ID"
+)
+
+
+def get_role_def(rolename):
+ """For the specific rolename, yields a list of hosts for that role."""
+ for instance in ec2.all():
+ if instance.public_dns_name:
+ if instance.role == rolename:
+ yield instance.public_dns_name
+
+env.roledefs = dict([
+ # Use a partial to create a callable that will only get called when a role
+ # is requested.
+ (n, functools.partial(get_role_def, n))
+ for n in ROLES
+])
+
+
+def set_environment(fn):
+ """Dynamically set environment variables for tasks."""
+ @functools.wraps(fn)
+ def wrapped_fn(*args, **kwargs):
+ key_filename = "{}/key_pairs/{}.pem".format(
+ os.environ["AWS_DIR"],
+ getpass.getuser()
+ )
+ with settings(user=AWS_USERNAME, key_filename=key_filename,
+ key_name=getpass.getuser()):
+ return fn(*args, **kwargs)
+
+ return wrapped_fn
+
+
+def print_instances(instances, show_terminated=False):
+ """Print out list of hosts. Set only_running to false
+ to also print out turned off machines."""
+ for index, instance in enumerate(instances):
+ if instance.state == "running" or show_terminated:
+ format_str = ("{index:>4}: {name:<20} {instance.ip_address:<16} "
+ "{launch_time:<12} {instance.id:<12} "
+ "{instance.image_id:<13}")
+ puts(format_str.format(
+ index=index,
+ instance=instance,
+ launch_time=instance.launch_time[:10],
+ name=instance.tags.get("Name", "no name")
+ ))
+
+
+def choose_ec2_host(node_type=None):
+ """Ask the user which running EC2 instance he wants to act upon."""
+ # Only do this if there is no host already set.
+ if env.hosts:
+ return
+
+ # Grab instances of a certain type or all of them.
+ if node_type:
+ instances = ec2.get(node_type)
+ else:
+ instances = ec2.all()
+
+ if len(instances) == 0:
+ answer = prompt("\nType in hostname: ")
+ else:
+ # Print out list of hosts
+ print_instances(instances)
+ # Let user pick host
+ answer = prompt(
+ "\nChoose AMI instance [0-%d, or type in your own]: "
+ % (len(instances) - 1,))
+
+ if answer.isdigit():
+ host = instances[int(answer)].public_dns_name
+ instance_id = instances[int(answer)].id
+ else:
+ host = answer
+
+ env.hosts = [host]
+ env.instance_id = instance_id
+
+
+def sv(command, services):
+ """Executes the supervisor command."""
+ with cd(DEPLOY_LOCATION):
+ for service in services:
+ run("%s service/%s" % (command, service))
+
+
+@task
+@set_environment
+def up(*services):
+ """Start remote services
+
+ :param services: A list of services to be started.
+ """
+ sv("svc -u", services)
+
+
+@task
+@set_environment
+def down(*services):
+ """Shutdown remote services.
+
+ :param services: A list of services to be started.
+ """
+ sv("svc -d", services)
+
+
+@task
+@set_environment
+def restart(*services):
+ """Restart remote services.
+
+ :param services: A list of services to be started.
+ """
+ down(*services)
+ up(*services)
+
+
+@task
+@set_environment
+def status(*services):
+ """Output the status of remote services.
+
+ :param services: A list of services to be started.
+ """
+ # TODO: The output of this is not very readable. Should change.
+ sv("svstat", services)
+
+
+@task
+@set_environment
+@runs_once
+def list():
+ """Lists all the available hosts by role."""
+ puts(INSTANCES_HEADER)
+ print_instances(
+ ec2.all(),
+ False
+ )
+
+
+PACKAGES = [
+ "python-distribute",
+ "python-dev",
+ "gcc",
+ "redis-server",
+ "couchdb",
+ "daemontools",
+ "git"
+]
+
+BASH_LOGIN = """
+export WORKON_HOME=$HOME/.virtualenvs
+mkdir -p $WORKON_HOME
+
+VIRTUALENVWRAPPER=/usr/local/bin/virtualenvwrapper.sh
+if [ -f $VIRTUALENVWRAPPER ]; then
+ source $VIRTUALENVWRAPPER
+fi
+"""
+
+
+def standard_setup():
+ """Set up an AWS EC2 instance:
+
+ 1. Update all packages
+ 2. Install packages we need
+ 3. Install pip
+ 4. Install virtualenvwrapper
+ """
+ package_update()
+ package_ensure(PACKAGES)
+ # Install pip
+ run("curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py | "
+ "sudo python")
+ # Install virtualenv and virtualenvwrapper
+ with mode_sudo():
+ python_package_ensure("virtualenvwrapper")
+ # Add virtualenvwrapper initialization to bashrc
+ file_write("$HOME/.bash_login", BASH_LOGIN)
+
+
+@task
+@set_environment
+def launch(ami_image_id=DEFAULT_AMI, count=1):
+ """Launch a new AWS EC2 instance.
+
+ The exact image is given by the global variable DEFAULT_AMI.
+ """
+ connection = boto.connect_ec2()
+ reservation = connection.run_instances(ami_image_id,
+ min_count=count,
+ key_name=env.key_name,
+ instance_type="t1.micro",
+ security_groups=['basic'])
+ ready = False
+ instances = reservation.instances
+ puts("Launching {!r} instance(s)".format(len(instances)))
+ while not ready:
+ puts(".", end='', flush=True)
+ sleep(2)
+ [instance.update() for instance in instances]
+ ready = all([instance.state == 'running' for instance in instances])
+
+ puts(" done")
+ puts("Your new instance(s):")
+ puts(INSTANCES_HEADER)
+ print_instances(instances)
+ puts("Now configuring.")
+ with settings(hosts=[instance.public_dns_name for instance in instances]):
+ puts(env.hosts)
+ puts(env.user)
+ standard_setup()
+
+
+@task
+@set_environment
+def terminate():
+ """Terminate the selected instance."""
+ connection = boto.connect_ec2()
+ connection.terminate_instances([env.instance_id])
1  requirements.txt
View
@@ -4,3 +4,4 @@ rq==0.3.4
couchdb==0.8
python-memcached==1.48
isodate==0.4.9
+cuisine==0.5.1
2  setup.py
View
@@ -16,5 +16,5 @@
packages=find_packages(exclude=('tests', 'docs')),
scripts=['bin/sshaws', 'bin/launch'],
install_requires=['distribute', 'boto', 'redis', 'rq', 'couchdb',
- 'python-memcached', 'isodate']
+ 'python-memcached', 'isodate', 'cuisine']
)
Please sign in to comment.
Something went wrong with that request. Please try again.