Initial commit

.gitignore

@@ -0,0 +1,14 @@
+*.swp
+*.pyc
+*.swo
+*.swn
+*.*~
+*.sqlite3
+.project
+.pydevproject
+.settings
+.vagrant
+.DS_Store
+.coverage
+htmlcov/*
+*.~lock*

README.rst

@@ -0,0 +1,22 @@
+=================================
+{{ client }} / {{ project_code }}
+=================================
+
+.. warning::
+
+    This is a boilerplate README.rst - please complete the following sections.
+
+Communication
+-------------
+
+[List mailing lists for projects]
+
+Installation for developers
+---------------------------
+
+[Explain how to set-up the project and run the unit tests]
+
+Environments
+------------
+
+[List IP addresses and auth details for the various environments]

TODO

@@ -0,0 +1 @@
+Vagrant integration based on non-Tangent box

deploy-to-prod.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+fab prod prepare deploy

deploy-to-stage.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+fab stage prepare deploy

deploy-to-test.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+fab test prepare deploy

fabconfig.py

@@ -0,0 +1,45 @@
+"""
+Project-specific environment information.  
+
+This module provides configuration for the fabfile to run with.  The idea is
+that the fabfile is project-agnostic and all configuration takes place within
+this file.
+
+In reality, this won't be entirely true as each project will evolve specific
+deployment needs.  Nevertheless, this still provides a good starting point.
+"""
+from fabric.api import env
+
+# Many things are configured using the client and project code
+env.client = '{{ client }}'
+env.project_code = '{{ project_code }}'
+
+# This is the name of the folder within the repo which houses all code
+# to be deployed.
+env.web_dir = 'www'
+
+# Environment-agnostic folders
+env.project_dir = '/var/www/%(client)s/%(project_code)s' % env
+env.builds_dir = '%(project_dir)s/builds' % env
+
+def _configure(build_name):
+    env.build = build_name
+    env.virtualenv = '%(project_dir)s/virtualenvs/%(build)s/' % env
+    env.code_dir = '%(project_dir)s/builds/%(build)s/' % env
+    env.data_dir = '%(project_dir)s/data/%(build)s/' % env
+    env.apache_conf = 'deploy/apache2/%(client)s-%(project_code)s-%(build)s.conf' % env
+    env.nginx_conf = 'deploy/nginx/%(client)s-%(project_code)s-%(build)s.conf' % env
+    env.wsgi = 'deploy/wsgi/%(build)s.wsgi' % env
+
+def test():
+    _configure('test')
+    env.hosts = ['test.%(project_code)s.%(client)s.tangentlabs.co.uk']
+
+def stage():
+    _configure('stage')
+    env.hosts = ['stage.%(project_code)s.%(client)s.tangentlabs.co.uk']
+
+def prod():
+    _configure('prod')
+    # Production hosts needs filling in
+    env.hosts = []

fabfile.py

@@ -0,0 +1,262 @@
+import datetime
+import os
+from os.path import normpath
+
+from fabric.decorators import runs_once, roles, task
+from fabric.operations import put, prompt
+from fabric.colors import green, red
+from fabric.api import local, cd, sudo
+
+# Import project settings 
+try:
+    from fabconfig import *
+except ImportError:
+    import sys
+    print "You need to define a fabconfig.py file with your project settings"
+    sys.exit()
+
+def _get_commit_id():
+    """
+    Return the commit ID for the branch about to be deployed
+    """
+    return local('git rev-parse HEAD', capture=True)[:20]
+
+def notify(msg):
+    bar = '+' + '-' * (len(msg) + 2) + '+'
+    print green('')
+    print green(bar)
+    print green("| %s |" % msg)
+    print green(bar)
+    print green('')
+
+# Deployment tasks
+
+@runs_once
+def update_codebase(branch, repo):
+    """
+    Update codebase from the Git repo
+    """
+    notify('Updating codebase from remote "%s", branch "%s"' % (repo, branch))
+    local('git pull %s %s' % (repo, branch))
+    notify('Push any local changes to remote %s' % branch)
+    local('git push %s %s' % (repo, branch))
+
+@runs_once
+def set_reference_to_deploy_from(branch):
+    """
+    Determine the refspec (tag or commit ID) to build from
+
+    The role of this task is simply to set the env.version variable
+    which is used later on.
+    """
+    notify("Determine the git reference to deploy from")
+    # Versioning - we either deploy from a tag or we create a new one
+    local('git fetch --tags')
+
+    if env.build == 'test':
+        # Allow a new tag to be set, or generate on automatically
+        print ''
+        create_tag = prompt(red('Tag this release? [y/N] '))
+        if create_tag.lower() == 'y':
+            notify("Showing latest tags for reference")
+            local('git tag | sort -V | tail -5')
+            env.version = prompt(red('Tag name [in format x.x.x]? '))
+            notify("Tagging version %s" % env.version)
+            local('git tag %s -m "Tagging version %s in fabfile"' % (env.version, env.version))
+            local('git push --tags')
+        else:
+            deploy_version = prompt(red('Build from a specific commit (useful for debugging)? [y/N] '))
+            print ''
+            if deploy_version.lower() == 'y':
+                env.version = prompt(red('Choose commit to build from: '))
+            else:
+                env.version = local('git describe %s' % branch, capture=True).strip()
+    else:
+        # An existing tag must be specified to deploy to QE or PE
+        local('git tag | sort -V | tail -5')
+        env.version = prompt(red('Choose tag to build from: '))
+        # Check this is valid
+        notify("Checking chosen tag exists")
+        local('git tag | grep "%s"' % env.version)
+
+    if env.build == 'prod':
+        # If a production build, then we ensure that the master branch
+        # gets updated to include all work up to this tag
+        notify("Merging tag into master")
+        local('git checkout master')
+        local('git merge %s' % env.version)
+        local('git push origin master')
+        local('git checkout develop')
+
+def set_ssh_user():
+    if 'TANGENT_USER' in os.environ:
+        env.user = os.environ['TANGENT_USER']
+    else:
+        env.user = prompt(red('Username for remote host? [default is current user] '))
+    if not env.user:
+        env.user = os.environ['USER']
+
+def deploy_codebase(archive_file, commit_id):
+    """
+    Push a tarball of the codebase up
+    """
+    upload(archive_file)
+    unpack(archive_file)
+
+def prepare(repo='origin'):
+    notify('BUILDING TO %s' % env.build.upper())
+
+    # Ensure we have latest code locally
+    branch = local('git branch | grep "^*" | cut -d" " -f2', capture=True)
+    update_codebase(branch, repo)
+    set_reference_to_deploy_from(branch)
+
+    # Create a build file ready to be pushed to the servers
+    notify("Building from refspec %s" % env.version)
+    env.build_file = '/tmp/build-%s.tar.gz' % str(env.version)
+    local('git archive --format tar %s %s | gzip > %s' % (env.version, env.web_dir, env.build_file))
+
+    # Set timestamp now so it is the same on all servers after deployment
+    now = datetime.datetime.now()
+    env.build_dir = '%s-%s' % (env.build, now.strftime('%Y-%m-%d-%H-%M'))
+    env.code_dir = '%s/%s' % (env.builds_dir, env.build_dir)
+
+def deploy():
+    """
+    Deploys the codebase
+    """
+    # Set SSH user and upload codebase to all servers, both
+    # app and proc.
+    set_ssh_user()
+    deploy_codebase(env.build_file, env.version)
+
+    update_virtualenv()
+    migrate()
+    collect_static_files()
+    deploy_apache_config()
+    deploy_nginx_config()
+    deploy_cronjobs()
+
+    switch_symlink()
+    reload_python_code()
+    reload_apache()
+    reload_nginx()
+    delete_old_builds()
+
+def switch_symlink():
+    notify("Switching symlinks")
+    with cd(env.builds_dir):
+        # Create new symlink for build folder
+        sudo('if [ -h %(build)s ]; then unlink %(build)s; fi' % env)
+        sudo('ln -s %(build_dir)s %(build)s' % env)
+
+def reload_python_code():
+    notify('Touching WSGI file to reload python code')
+    with cd(env.builds_dir):
+        sudo('touch %(build)s/%(wsgi)s' % env)
+
+def reload_apache():
+    notify('Reloading Apache2 configuration')
+    sudo('/etc/init.d/apache2 reload')
+
+def reload_nginx():
+    notify('Reloading nginx configuration')
+    sudo('/etc/init.d/nginx force-reload')
+
+def reload_tomcat():
+    sudo('/etc/init.d/tomcat6 force-reload')
+
+def upload(local_path, remote_path=None):
+    """
+    Uploads a file
+    """
+    if not remote_path:
+        remote_path = local_path
+    notify("Uploading %s to %s" % (local_path, remote_path))
+    put(local_path, remote_path)
+
+def unpack(archive_path):
+    """
+    Unpacks the tarball into the correct place but doesn't switch
+    the symlink
+    """
+    notify("Creating remote build folder")
+    with cd(env.builds_dir):
+        sudo('tar xzf %s' % archive_path)
+
+        # Create new build folder
+        sudo('if [ -d "%(build_dir)s" ]; then rm -rf "%(build_dir)s"; fi'% env)
+        sudo('mv %(web_dir)s %(build_dir)s' % env)
+
+        # Symlink in uploads folder
+        sudo('ln -s ../../../media/%(build)s %(build_dir)s/public/media' % env)
+
+        # Append release info to settings.py
+        sudo("sed -i 's/UNVERSIONED/%(version)s/' %(build_dir)s/settings.py" % env)
+
+        # Add file indicating Git commit
+        sudo('echo -e "refspec: %s\nuser: %s" > %s/build-info' % (env.version, env.user, env.build_dir))
+
+        # Remove archive
+        sudo('rm %s' % archive_path)
+
+def set_robots_and_sitemaps():
+    notify("Setting robots.txt and sitemaps")
+    with cd(env.builds_dir):
+        # create the proper symlinks to sitemaps and robots.txt
+        sudo("mkdir -p /mnt/static/landmark/%(build)s/sitemaps" % env)
+        sudo("""if [ -d /mnt/static/landmark/%(build)s/robots.txt ]; then
+                    cp %(build_dir)s/static/robots.txt /mnt/static/landmark/%(build)s/;
+                fi;""" % env)
+        with cd("%(build_dir)s/public/static/" % env):
+            sudo("rm -rf sitemaps && ln -s /mnt/static/landmark/%(build)s/sitemaps sitemaps" % env)
+            sudo("rm robots.txt && ln -s /mnt/static/landmark/%(build)s/robots.txt robots.txt" % env)
+
+def update_virtualenv():
+    """
+    Install the dependencies in the requirements file
+    """
+    with cd(env.code_dir):
+        sudo('source %s/bin/activate && pip install -r deploy/requirements.txt' % env.virtualenv)
+
+def collect_static_files():
+    notify("Collecting static files")
+    with cd(env.code_dir):
+        sudo('source %s/bin/activate && ./manage.py collectstatic --noinput > /dev/null' % env.virtualenv)
+        sudo('chmod -R g+w public' % env)
+
+def migrate():
+    """
+    Apply any schema alterations
+    """
+    notify("Applying database migrations")
+    with cd(env.code_dir):
+        sudo('source %s/bin/activate && ./manage.py syncdb --noinput > /dev/null' % env.virtualenv)
+        sudo('source %s/bin/activate && ./manage.py migrate --ignore-ghost-migrations' % env.virtualenv)
+
+def deploy_apache_config():
+    notify('Moving apache config into place')
+    with cd(env.code_dir):
+        sudo('mv %(apache_conf)s /etc/apache2/sites-enabled/' % env)
+
+def deploy_nginx_config():
+    notify('Moving nginx config into place')
+    with cd(env.code_dir):
+        sudo('mv %(nginx_conf)s /etc/nginx/sites-enabled/' % env)
+
+def deploy_cronjobs():
+    """
+    Deploy the app server cronjobs
+    """
+    notify('Deploying cronjobs')
+    with cd(env.code_dir):
+        # Replace variables in cron files
+        sudo("rename 's#BUILD#%(build)s#' deploy/cron.d/*" % env)
+        sudo("sed -i 's#VIRTUALENV_ROOT#%(virtualenv)s#g' deploy/cron.d/*" % env)
+        sudo("sed -i 's#BUILD_ROOT#%(code_dir)s#g' deploy/cron.d/*" % env)
+        sudo("mv deploy/cron.d/* /etc/cron.d" % env)
+
+def delete_old_builds():
+    notify('Deleting old builds')
+    with cd(env.builds_dir):
+        sudo('find . -maxdepth 1 -type d -name "%(build)s*" | sort -r | sed "1,9d" | xargs rm -rf' % env)