Skip to content
Find file
Fetching contributors…
Cannot retrieve contributors at this time
608 lines (518 sloc) 23.3 KB
"""
Server layout:
~/services/
This contains two subfolders
/apache/
/supervisor/
which hold the configurations for these applications
for each environment (staging, demo, etc) running on the server.
Theses folders are included in the global /etc/apache2 and
/etc/supervisor configurations.
~/www/
This folder contains the code, python environment, and logs
for each environment (staging, demo, etc) running on the server.
Each environment has its own subfolder named for its evironment
(i.e. ~/www/staging/logs and ~/www/demo/logs).
"""
import pdb
import uuid
from fabric.context_managers import settings, cd
from fabric.decorators import hosts
from fabric.operations import require, local, prompt
import os, sys
from fabric.api import run, roles, execute, task, sudo, env, serial, parallel
from fabric.contrib import files, console
from fabric import utils
import posixpath
from collections import defaultdict
PROJECT_ROOT = os.path.dirname(__file__)
RSYNC_EXCLUDE = (
'.DS_Store',
'.git',
'*.pyc',
'*.example',
'*.db',
)
env.project = 'commcare-hq'
env.code_repo = 'git://github.com/dimagi/commcare-hq.git'
env.home = "/home/cchq"
env.selenium_url = 'http://jenkins.dimagi.com/job/commcare-hq-post-deploy/buildWithParameters?token=%(token)s&TARGET=%(environment)s'
env.roledefs = {
'django_celery': [],
'django_app': [],
'django_public': [],
'formsplayer': [],
'staticfiles': [],
'django_monolith': [], # all of the above config - use this ONLY for single server config, lest deploy() will run multiple times in parallel causing bad contentions
#package level configs that are not quite config'ed yet in this fabfile
'couch': [],
'pg': [],
'rabbitmq': [],
'lb': [],
'deploy': [], #a placeholder to ensure deploy only runs once on a bogus, non functioning task, to split out the real tasks in the execute() block
}
@task
def _setup_path():
# using posixpath to ensure unix style slashes. See bug-ticket: http://code.fabfile.org/attachments/61/posixpath.patch
env.root = posixpath.join(env.home, 'www', env.environment)
env.log_dir = posixpath.join(env.home, 'www', env.environment, 'log')
env.code_root = posixpath.join(env.root, 'code_root')
env.project_root = posixpath.join(env.code_root, env.project)
env.project_media = posixpath.join(env.code_root, 'media')
env.virtualenv_root = posixpath.join(env.root, 'python_env')
env.services = posixpath.join(env.home, 'services')
@task
def _set_apache_user():
if what_os() == 'ubuntu':
env.apache_user = 'www-data'
elif what_os() == 'redhat':
env.apache_user = 'apache'
@roles('lb')
def setup_apache_dirs():
sudo('mkdir -p %(services)s/apache' % env, user=env.sudo_user)
@roles('django_celery', 'django_app', 'staticfiles') #'django_public','formsplayer','staticfiles'
def setup_dirs():
""" create (if necessary) and make writable uploaded media, log, etc. directories """
sudo('mkdir -p %(log_dir)s' % env, user=env.sudo_user)
sudo('chmod a+w %(log_dir)s' % env, user=env.sudo_user)
sudo('mkdir -p %(services)s/supervisor' % env, user=env.sudo_user)
#execute(setup_apache_dirs)
@task
def staging():
""" use staging environment on remote host"""
env.code_branch = 'develop'
env.sudo_user = 'commcare-hq'
env.environment = 'staging'
env.server_port = '9002'
env.server_name = 'noneset'
env.hosts = ['192.168.56.1']
env.settings = '%(project)s.localsettings' % env
env.host_os_map = None
env.db = '%s_%s' % (env.project, env.environment)
_setup_path()
env.user = prompt("Username: ", default='dimagivm')
env.make_bootstrap_command = 'python manage.py make_bootstrap direct-lessc'
@task
def india():
"""Our production server in India."""
env.home = '/home/commcarehq/'
env.root = root = '/home/commcarehq'
env.environment = 'india'
env.log_root = posixpath.join(root, 'log')
env.code_branch = 'master'
env.sudo_user = 'commcarehq'
env.hosts = ['220.226.209.82']
env.user = prompt("Username: ", default=env.user)
env.service_manager = "supervisor"
env.make_bootstrap_command = 'python manage.py make_bootstrap'
env.server_port = '8001'
_setup_path()
env.virtualenv_root = posixpath.join(root, '.virtualenvs/commcarehq')
env.roledefs = {
'couch': [],
'pg': [],
'rabbitmq': [],
'sofabed': [],
'django_celery': [],
'django_app': [],
'django_public': [],
'formsplayer': [],
'lb': [],
'staticfiles': [],
'deploy': [],
'django_monolith': ['220.226.209.82'],
}
env.jython_home = '/usr/local/lib/jython'
env.roles = ['django_monolith']
@task
def production():
""" use production environment on remote host"""
env.code_branch = 'master'
env.sudo_user = 'cchq'
env.environment = 'production'
env.server_port = '9010'
#env.hosts = None
env.roledefs = {
'couch': ['hqdb.internal.commcarehq.org'],
'pg': ['hqdb.internal.commcarehq.org'],
'rabbitmq': ['hqdb.internal.commcarehq.org'],
'sofabed': ['hqdb.internal.commcarehq.org'], #todo, right now group it with celery
'django_celery': ['hqdb.internal.commcarehq.org'],
'django_app': ['hqdjango0.internal.commcarehq.org', 'hqdjango2.internal.commcarehq.org'],
'django_public': ['hqdjango1.internal.commcarehq.org',],
'formsplayer': ['hqdjango0.internal.commcarehq.org'],
'lb': [], #todo on apache level config
'staticfiles': ['hqproxy0.internal.commcarehq.org'],
'deploy': ['hqdb.internal.commcarehq.org'], #this is a stub becuaue we don't want to be prompted for a host or run deploy too many times
'django_monolith': [] # fab complains if this doesn't exist
}
env.server_name = 'commcare-hq-production'
env.settings = '%(project)s.localsettings' % env
env.host_os_map = None # e.g. 'ubuntu' or 'redhat'. Gets autopopulated by what_os() if you don't know what it is or don't want to specify.
env.db = '%s_%s' % (env.project, env.environment)
env.roles = ['deploy', ]
env.jython_home = '/usr/local/lib/jython'
_setup_path()
@task
def install_packages():
"""Install packages, given a list of package names"""
require('environment', provided_by=('staging', 'production'))
packages_list = ''
installer_command = ''
if what_os() == 'ubuntu':
packages_list = 'apt-packages.txt'
installer_command = 'apt-get install -y'
elif what_os() == 'redhat':
packages_list = 'yum-packages.txt'
installer_command = 'yum install -y'
return
packages_file = posixpath.join(PROJECT_ROOT, 'requirements', packages_list)
with open(packages_file) as f:
packages = f.readlines()
sudo("%s %s" % (installer_command, " ".join(map(lambda x: x.strip('\n\r'), packages))))
@task
@roles('django_app','django_celery','staticfiles')
@parallel
def upgrade_packages():
"""
Bring all the installed packages up to date.
This is a bad idea in RedHat as it can lead to an
OS Upgrade (e.g RHEL 5.1 to RHEL 6).
Should be avoided. Run install packages instead.
"""
require('environment', provided_by=('staging', 'production'))
if what_os() == 'ubuntu':
sudo("apt-get update", shell=False)
sudo("apt-get upgrade -y", shell=False)
else:
return #disabled for RedHat (see docstring)
@task
def what_os():
with settings(warn_only=True):
require('environment', provided_by=('staging','production'))
if env.host_os_map is None:
#prior use case of setting a env.remote_os did not work when doing multiple hosts with different os! Need to keep state per host!
env.host_os_map = defaultdict(lambda: '')
if env.host_os_map[env.host_string] == '':
print 'Testing operating system type...'
if(files.exists('/etc/lsb-release',verbose=True) and files.contains(text='DISTRIB_ID=Ubuntu', filename='/etc/lsb-release')):
remote_os = 'ubuntu'
print 'Found lsb-release and contains "DISTRIB_ID=Ubuntu", this is an Ubuntu System.'
elif(files.exists('/etc/redhat-release',verbose=True)):
remote_os = 'redhat'
print 'Found /etc/redhat-release, this is a RedHat system.'
else:
print 'System OS not recognized! Aborting.'
exit()
env.host_os_map[env.host_string] = remote_os
return env.host_os_map[env.host_string]
#@parallel
@roles('pg','django_celery','django_app','staticfiles', 'django_monolith')
@task
def setup_server():
"""Set up a server for the first time in preparation for deployments."""
require('environment', provided_by=('staging', 'production', 'india'))
# Install required system packages for deployment, plus some extras
# Install pip, and use it to install virtualenv
install_packages()
sudo("easy_install -U pip", user=env.sudo_user)
sudo("pip install -U virtualenv", user=env.sudo_user)
upgrade_packages()
execute(create_db_user)
execute(create_db)
@roles('pg')
def create_db_user():
"""Create the Postgres user."""
require('environment', provided_by=('staging', 'production'))
sudo('createuser -D -A -R %(sudo_user)s' % env, user='postgres')
@roles('pg')
def create_db():
"""Create the Postgres database."""
require('environment', provided_by=('staging', 'production'))
sudo('createdb -O %(sudo_user)s %(db)s' % env, user='postgres')
@task
def bootstrap():
"""Initialize remote host environment (virtualenv, deploy, update) """
require('root', provided_by=('staging', 'production'))
sudo('mkdir -p %(root)s' % env, shell=False, user=env.sudo_user)
execute(clone_repo)
update_code()
execute(create_virtualenv)
execute(update_env)
execute(setup_dirs)
execute(generate_supervisorconf_file)
execute(fix_locale_perms)
#@parallel
@roles('django_celery', 'django_app', 'staticfiles', 'django_monolith') #'django_public','formsplayer'
def create_virtualenv():
""" setup virtualenv on remote host """
require('virtualenv_root', provided_by=('staging', 'production', 'india'))
with settings(warn_only=True):
sudo('rm -rf %(virtualenv_root)s' % env, user=env.sudo_user)
args = '--clear --distribute --no-site-packages'
sudo('virtualenv %s %s' % (args, env.virtualenv_root), user=env.sudo_user)
#@parallel
@roles('django_celery', 'django_app', 'staticfiles', 'django_monolith') #'django_public', 'formsplayer'
def clone_repo():
""" clone a new copy of the git repository """
with settings(warn_only=True):
with cd(env.root):
if not files.exists(env.code_root):
sudo('git clone %(code_repo)s %(code_root)s' % env, user=env.sudo_user)
with cd(env.code_root):
sudo('git submodule init', user=env.sudo_user)
@task
@roles('pg', 'django_monolith')
def preindex_views():
with cd(env.code_root):
update_code()
update_env()
sudo('echo "%(virtualenv_root)s/bin/python %(code_root)s/manage.py \
sync_prepare_couchdb_multi 8 %(user)s" | at -t `date -d "5 seconds" \
+%%m%%d%%H%%M.%%S`' % env, user=env.sudo_user)
@roles('django_app','django_celery', 'staticfiles', 'django_public', 'django_monolith')#,'formsplayer')
@parallel
def update_code():
with cd(env.code_root):
sudo('git checkout %(code_branch)s' % env, user=env.sudo_user)
sudo('git pull', user=env.sudo_user)
sudo('git submodule sync', user=env.sudo_user)
sudo('git submodule update --init --recursive', user=env.sudo_user)
@task
def deploy():
""" deploy code to remote host by checking out the latest via git """
if not console.confirm('Are you sure you want to deploy {env.environment}?'.format(env=env), default=False) or \
not console.confirm('Did you run "fab {env.environment} preindex_views"? '.format(env=env), default=False):
utils.abort('Deployment aborted.')
require('root', provided_by=('staging', 'production', 'india'))
run('echo ping!') #hack/workaround for delayed console response
try:
execute(update_code)
execute(update_env)
execute(clear_services_dir)
upload_and_set_supervisor_config()
execute(migrate)
execute(_do_collectstatic)
execute(version_static)
finally:
# hopefully bring the server back to life if anything goes wrong
execute(services_restart)
@task
@roles('django_app','django_celery','staticfiles', 'django_public', 'django_monolith')#,'formsplayer')
@parallel
def update_env():
""" update external dependencies on remote host assumes you've done a code update"""
require('code_root', provided_by=('staging', 'production', 'india'))
requirements = posixpath.join(env.code_root, 'requirements')
with cd(env.code_root):
cmd = ['%(virtualenv_root)s/bin/pip install' % env]
cmd += ['--requirement %s' % posixpath.join(requirements, 'prod-requirements.txt')]
cmd += ['--requirement %s' % posixpath.join(requirements, 'requirements.txt')]
sudo(' '.join(cmd), user=env.sudo_user)
@roles('lb')
def touch_apache():
"""Touch apache and supervisor conf files to trigger reload. Also calls supervisorctl update
to load latest supervisor.conf """
require('code_root', provided_by=('staging', 'production'))
apache_path = posixpath.join(posixpath.join(env.services, 'apache'), 'apache.conf')
sudo('touch %s' % apache_path, user=env.sudo_user)
@roles('django_celery', 'django_app', 'django_monolith')
def touch_supervisor():
""" touch apache and supervisor conf files to trigger reload. Also calls supervisorctl update
to load latest supervisor.conf """
require('code_root', provided_by=('staging', 'production'))
supervisor_path = posixpath.join(posixpath.join(env.services, 'supervisor'), 'supervisor.conf')
sudo('touch %s' % supervisor_path, user=env.sudo_user)
_supervisor_command('update')
@roles('django_app', 'django_celery', 'django_public', 'django_monolith')# 'formsplayer')
@parallel
def clear_services_dir():
#remove old confs from directory first
services_dir = posixpath.join(env.services, u'supervisor', 'supervisor_*.conf')
sudo('rm -f %s' % services_dir, user=env.sudo_user)
@roles('lb')
def configtest():
""" test Apache configuration """
require('root', provided_by=('staging', 'production'))
sudo('apache2ctl configtest')
@roles('lb')
def apache_reload():
""" reload Apache on remote host """
require('root', provided_by=('staging', 'production'))
if what_os() == 'redhat':
sudo('/etc/init.d/httpd reload')
elif what_os() == 'ubuntu':
sudo('/etc/init.d/apache2 reload')
@roles('lb')
def apache_restart():
""" restart Apache on remote host """
require('root', provided_by=('staging', 'production'))
sudo('/etc/init.d/apache2 restart')
@task
def netstat_plnt():
""" run netstat -plnt on a remote host """
require('hosts', provided_by=('production', 'staging'))
sudo('netstat -plnt')
############################################################3
#Start service functions
@roles('django_app', 'django_celery','django_public','django_monolith')# 'formsplayer'
def services_start():
''' Start the gunicorn servers '''
require('environment', provided_by=('staging', 'demo', 'production'))
_supervisor_command('update')
_supervisor_command('reload')
_supervisor_command('start all')
######################################################
########################################################
#Stop service Functions
@roles('django_app', 'django_celery','django_public', 'django_monolith')#, 'formsplayer')
def services_stop():
''' Stop the gunicorn servers '''
require('environment', provided_by=('staging', 'demo', 'production'))
_supervisor_command('stop all')
###########################################################
@roles('django_app', 'django_celery','django_public', 'django_monolith')#, 'formsplayer')
def services_restart():
''' Stop and restart all supervisord services'''
require('environment', provided_by=('staging', 'demo', 'production', 'india'))
_supervisor_command('stop all')
_supervisor_command('update')
_supervisor_command('reload')
_supervisor_command('start all')
#
@roles('django_celery','django_monolith')
def migrate():
""" run south migration on remote environment """
require('code_root', provided_by=('production', 'demo', 'staging', "india"))
with cd(env.code_root):
sudo('%(virtualenv_root)s/bin/python manage.py sync_finish_couchdb' % env, user=env.sudo_user)
sudo('%(virtualenv_root)s/bin/python manage.py syncdb --noinput' % env, user=env.sudo_user)
sudo('%(virtualenv_root)s/bin/python manage.py migrate --noinput' % env, user=env.sudo_user)
@roles('staticfiles', 'django_monolith')
def _do_collectstatic():
"""
Collect static after a code update
"""
with cd(env.code_root):
sudo('%(virtualenv_root)s/bin/python manage.py make_bootstrap' % env, user=env.sudo_user)
sudo('%(virtualenv_root)s/bin/python manage.py collectstatic --noinput' % env, user=env.sudo_user)
@roles('django_app', 'django_monolith')
@parallel
def version_static():
"""
Put refs on all static references to prevent stale browser cache hits when things change.
This needs to be run on the WEB WORKER since the web worker governs the actual static
reference.
"""
with cd(env.code_root):
sudo('rm -f tmp.sh resource_versions.py; %(virtualenv_root)s/bin/python manage.py \
printstatic > tmp.sh; bash tmp.sh > resource_versions.py' % env, user=env.sudo_user)
@task
@roles('staticfiles',)
def collectstatic():
""" run collectstatic on remote environment """
require('code_root', provided_by=('production', 'demo', 'staging'))
update_code()
_do_collectstatic()
@task
def reset_local_db():
""" Reset local database from remote host """
require('code_root', provided_by=('production', 'staging'))
if env.environment == 'production':
utils.abort('Local DB reset is for staging environment only')
question = 'Are you sure you want to reset your local '\
'database with the %(environment)s database?' % env
sys.path.append('.')
if not console.confirm(question, default=False):
utils.abort('Local database reset aborted.')
local_db = loc['default']['NAME']
remote_db = remote['default']['NAME']
with settings(warn_only=True):
local('dropdb %s' % local_db)
local('createdb %s' % local_db)
host = '%s@%s' % (env.user, env.hosts[0])
local('ssh -C %s sudo -u commcare-hq pg_dump -Ox %s | psql %s' % (host, remote_db, local_db))
@task
def fix_locale_perms():
""" Fix the permissions on the locale directory """
require('root', provided_by=('staging', 'production'))
_set_apache_user()
locale_dir = '%s/commcare-hq/locale/' % env.code_root
sudo('chown -R %s %s' % (env.sudo_user, locale_dir), user=env.sudo_user)
sudo('chgrp -R %s %s' % (env.apache_user, locale_dir), user=env.sudo_user)
sudo('chmod -R g+w %s' % (locale_dir), user=env.sudo_user)
@task
def commit_locale_changes():
""" Commit locale changes on the remote server and pull them in locally """
fix_locale_perms()
with cd(env.code_root):
sudo('-H -u %s git add commcare-hq/locale' % env.sudo_user, user=env.sudo_user)
sudo('-H -u %s git commit -m "updating translation"' % env.sudo_user, user=env.sudo_user)
local('git pull ssh://%s%s' % (env.host, env.code_root))
def _upload_supervisor_conf_file(filename):
upload_dict = {}
upload_dict["template"] = posixpath.join(os.path.dirname(__file__), 'services', 'templates', filename)
upload_dict["destination"] = '/tmp/%s.blah' % filename
upload_dict["enabled"] = posixpath.join(env.services, u'supervisor/%s' % filename)
upload_dict["main_supervisor_conf_dir"] = '/etc'
files.upload_template(upload_dict["template"], upload_dict["destination"], context=env, use_sudo=False)
sudo('chown -R %s %s' % (env.sudo_user, upload_dict["destination"]), shell=False)
#sudo('chgrp -R %s %s' % (env.apache_user, upload_dict["destination"]))
sudo('chmod -R g+w %(destination)s' % upload_dict, shell=False)
sudo('mv -f %(destination)s %(enabled)s' % upload_dict, shell=False)
@roles('django_celery', 'django_monolith')
def upload_celery_supervisorconf():
_upload_supervisor_conf_file('supervisor_celery.conf')
_upload_supervisor_conf_file('supervisor_celerybeat.conf')
_upload_supervisor_conf_file('supervisor_celerymon.conf')
_upload_supervisor_conf_file('supervisor_couchdb_lucene.conf')
@roles('django_celery', 'django_monolith')
def upload_sofabed_supervisorconf():
_upload_supervisor_conf_file('supervisor_sofabed.conf')
@roles('django_app', 'django_monolith')
def upload_djangoapp_supervisorconf():
_upload_supervisor_conf_file('supervisor_django.conf')
@roles('django_public')
def upload_django_public_supervisorconf():
_upload_supervisor_conf_file('supervisor_django_public.conf')
_upload_supervisor_conf_file('supervisor_sync_domains.conf')
@roles('formsplayer', 'django_monolith')
def upload_formsplayer_supervisorconf():
_upload_supervisor_conf_file('supervisor_formsplayer.conf')
def upload_and_set_supervisor_config():
"""Upload and link Supervisor configuration from the template."""
require('environment', provided_by=('staging', 'demo', 'production', 'india'))
execute(upload_celery_supervisorconf)
execute(upload_sofabed_supervisorconf)
execute(upload_djangoapp_supervisorconf)
execute(upload_django_public_supervisorconf)
execute(upload_formsplayer_supervisorconf)
#generate_supervisorconf_file()
def generate_supervisorconf_file():
#regenerate a brand new supervisor conf file from scratch.
#Set the line in the supervisord config file that points to our project's supervisor directory with conf files
replace_dict = {}
replace_dict["main_supervisor_conf_dir"] = '/etc'
replace_dict["tmp"] = posixpath.join('/','tmp', "supervisord_%s.tmp" % uuid.uuid4().hex)
#create brand new one
sudo("echo_supervisord_conf > %(tmp)s" % replace_dict)
files.uncomment(replace_dict['tmp'], "^;\[include\]", use_sudo=True, char=';')
files.sed(replace_dict["tmp"], ";files = relative/directory/\*\.ini", "files = %s/supervisor/*.conf" % env.services, use_sudo=True)
#sudo('mv -f %(tmp)s %(main_supervisor_conf_dir)s/supervisord.conf' % replace_dict)
sudo('cp -f %(tmp)s %(main_supervisor_conf_dir)s/supervisord.conf' % replace_dict)
def _supervisor_command(command):
require('hosts', provided_by=('staging', 'production'))
#if what_os() == 'redhat':
#cmd_exec = "/usr/bin/supervisorctl"
#elif what_os() == 'ubuntu':
#cmd_exec = "/usr/local/bin/supervisorctl"
sudo('supervisorctl %s' % (command), shell=False)
# tests
@task
def selenium_test():
require('environment', provided_by=('staging', 'demo', 'production', 'india'))
prompt("Jenkins username:", key="jenkins_user", default="selenium")
prompt("Jenkins password:", key="jenkins_password")
url = env.selenium_url % {"token": "foobar", "environment": env.environment}
local("curl --user %(user)s:%(pass)s '%(url)s'" % \
{'user': env.jenkins_user, 'pass': env.jenkins_password, 'url': url})
Something went wrong with that request. Please try again.