Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

initial commit

  • Loading branch information...
commit 00cbe1ce332e62fad7e0874767c5cd7b4d695fca 0 parents
@dexterbt1 authored
9 .gitignore
@@ -0,0 +1,9 @@
+py/bin
+py/lib
+py/include
+py/build
+*.egg
+.Python
+*.sw[op]
+*.pyc
+distribute*tar*
1  src/cuisine_sweet/__init__.py
@@ -0,0 +1 @@
+import git
7 src/cuisine_sweet/ensure/__init__.py
@@ -0,0 +1,7 @@
+import local_git
+import yum
+import supervisord
+import user_crontab
+import fs
+import cpan_module
+import git
75 src/cuisine_sweet/ensure/cpan_module.py
@@ -0,0 +1,75 @@
+"""
+Fabric checks and assertions for CPAN modules
+"""
+
+import os
+import cuisine
+
+from fabric.api import run, puts, env
+from fabric.colors import green, blue
+from cuisine_sweet.utils import completed_ok
+
+
+def perl_config_archname():
+ return run("perl -MConfig -e 'print $Config{archname}'")
+
+
+def cpanm_bin_installed(home='/tmp'):
+ cuisine.select_package(option='yum')
+ cuisine.package_ensure('perl-devel')
+ binpath = '%s/.deploy/bin' % home
+ cpanm = '%s/cpanm' % binpath
+ if not cuisine.file_exists(cpanm):
+ cuisine.dir_ensure(binpath, recursive=True, mode=755)
+ cuisine.package_ensure('curl')
+ run('curl -L http://cpanmin.us > %s' % cpanm)
+ run('chmod 755 %s' % cpanm)
+ cuisine.file_exists(cpanm)
+ return cpanm
+
+
+def _exists(module_name, home='/tmp', perlarch=None, locallib='perl5'):
+ path1 = os.path.join(home, locallib, 'lib', 'perl5')
+ path2 = os.path.join(home, locallib, 'lib', 'perl5', perlarch)
+ result = run("perl -I %s -I %s -e 'use %s' >& /dev/null && echo OK; true" % (path1, path2, module_name))
+ return result.endswith('OK')
+
+
+def _do_install(module_name, home='/tmp', cpanm='/tmp/.deploy/bin/cpanm', source=None, locallib='perl5'):
+ mod = module_name
+ if source is not None:
+ mod = source
+ locallib_base = os.path.join(home, locallib)
+ opts = '-l %s' % locallib_base
+ run('%s %s %s' % (cpanm, opts, mod))
+
+
+def _prepare_environment():
+ # --- perl arch name
+ arch = perl_config_archname()
+ if 'perl_arch' not in env:
+ env.perl_arch = { }
+ env.perl_arch[env.host] = arch
+ # --- cpanm
+ cpanm = cpanm_bin_installed(home='.')
+ if 'cpanm_bin' not in env:
+ env.cpanm_bin = { }
+ env.cpanm_bin[env.host] = cpanm
+
+
+@completed_ok(arg_output=[0])
+def installed(module, source=None, locallib='perl5', home='.'):
+ """
+ Ensure that the cpan module installed (in a localized locallib path)
+ """
+ try:
+ perlarch = env.perl_arch[env.host]
+ cpanm = env.cpanm_bin[env.host]
+ except:
+ _prepare_environment()
+ perlarch = env.perl_arch[env.host]
+ cpanm = env.cpanm_bin[env.host]
+ if not _exists(module, home=home, perlarch=perlarch, locallib=locallib):
+ _do_install(module, home=home, cpanm=cpanm, source=source, locallib=locallib)
+
+
26 src/cuisine_sweet/ensure/fs.py
@@ -0,0 +1,26 @@
+"""
+Fabric checks and assertions for Supervisord instances
+"""
+
+import cuisine
+from fabric.api import run, sudo
+from cuisine_sweet.utils import completed_ok
+
+
+@completed_ok(arg_output=[0])
+def dir_created(name, **kwargs):
+ """
+ Ensure that the directory is created and with proper permissions.
+
+ Wraps cuisine.dir_ensure function
+ """
+ cuisine.dir_ensure(name, **kwargs)
+
+
+@completed_ok(arg_output=[0,1])
+def dir_symlink_created(src, dest):
+ """
+ Ensure that a symlink at `dest` pointing to `src` is created.
+ """
+ run('ln -s %s %s >& /dev/null; true' % (src, dest))
+ run('test -L %s && test -d %s' % (dest, dest))
86 src/cuisine_sweet/ensure/git.py
@@ -0,0 +1,86 @@
+import os
+import sys
+import cuisine
+
+from fabric.api import env, cd, lcd, local, run, put, abort
+from fabric.utils import error
+from fabric.auth import get_password
+
+from cuisine_sweet import git
+from cuisine_sweet.utils import completed_ok, local_run_expect
+
+
+@completed_ok(arg_output=[0,1])
+def rsync(repo_url, repo_dir, refspec='master', home='.', base_dir='git', local_tmpdir='/tmp', save_history=False):
+ """
+ Does a git clone locally then rsync to remote
+ """
+ # ensure git + rsync is available locally
+ local('which git')
+ local('which rsync')
+
+ # ensure the temp paths are ready
+ local_user = local('whoami', capture=True)
+ clone_basepath_local = os.path.join(local_tmpdir, local_user, 'deploy', env.host, env.user, 'git')
+ local('mkdir -p %s' % clone_basepath_local)
+
+ # prepare remote path strings
+ clone_basepath_remote = os.path.join(home, base_dir)
+ cuisine.dir_ensure(clone_basepath_remote)
+
+ cuisine.command_ensure('git', package='git')
+
+ # prepare history (for recovery)
+ hist = git.GitHistory()
+ if save_history:
+ remote_hist_path = git.get_remote_git_history_path(home)
+ try:
+ tmphist = git.load_remote_git_history(remote_hist_path, local_user=local_user)
+ if tmphist is not None:
+ hist = tmphist
+ except Exception, e:
+ error("Warning: Unable to load history file %s: %s" % (remote_hist_path, e ))
+
+ # needed later for us to delete the unneeded repo clones
+ local_clones_dirs = local('cd %s && ls' % clone_basepath_local, capture=True).split("\n")
+ local_clones = { }
+ for lc in local_clones_dirs:
+ local_clones[lc] = True
+
+ # clone, reset, log to history
+ clone_path_remote = os.path.join(clone_basepath_remote, repo_dir)
+ clone_path_local = os.path.join(clone_basepath_local, repo_dir)
+
+ cloned_locally = local('test -d "%s" && echo OK; true' % clone_path_local, capture=True).endswith('OK')
+ if not cloned_locally:
+ with lcd(clone_basepath_local):
+ local('git clone %s %s' % (repo_url, repo_dir))
+
+ with lcd(clone_path_local):
+ local('git fetch')
+ local('git submodule update --init')
+ local('git reset --hard "origin/%s"' % refspec)
+ fuller_log = local('git log -n1 --pretty=fuller', capture=True)
+ hist.log_latest(repo_url, refspec, repo_dir, fuller_log)
+
+ if repo_dir in local_clones:
+ del local_clones[repo_dir]
+
+ # delete all undocumented local clones
+ for lc in local_clones.keys():
+ with lcd(clone_basepath_local):
+ local('rm -rf %s' % lc)
+
+ if save_history:
+ cuisine.file_write(remote_hist_path, hist.dump())
+
+ prompts = [ 'Are you sure you want to continue connecting', ".* password:" ]
+ answers = [ 'yes', get_password() ]
+ rsync_cmd = '''/bin/bash -l -c "rsync --delete --exclude \".git/" -lpthrvz --rsh='ssh -p 22' %s %s:%s"''' % (clone_basepath_local + "/", env.host_string, clone_basepath_remote)
+ local_run_expect(rsync_cmd, prompts, answers, logfile=sys.stdout)
+
+
+
+
+
+
38 src/cuisine_sweet/ensure/local_git.py
@@ -0,0 +1,38 @@
+"""
+Fabric checks and assertions for a local git checkout
+"""
+
+import cuisine
+from fabric.api import local, abort, puts, lcd
+from fabric.colors import green, blue, red
+from cuisine_sweet.utils import this_func
+
+
+def up_to_date(against=None, path='.'):
+ """
+ Check if the git checkout in `path` is up-to-date `against` another refspec.
+
+ The param `against` is usually set to 'origin/master'; then this function
+ will only succeed if the checkout has the latest commits from 'origin/master'.
+ """
+ if not against:
+ abort(red("Missing arg: against"))
+ with lcd(path):
+ num_changes_str = local('git fetch && git rev-list HEAD..%s | wc -l' % against, capture=True)
+ num_changes = int(num_changes_str)
+ if num_changes > 0:
+ abort(red("%s: Your git checkout at path=%s is not up-to-date against=%s (forgot to pull?)." % (this_func(), path, against)))
+ puts(green("%s(against=%s): OK" % (this_func(), against)))
+
+
+def clean(path='.'):
+ """
+ Check if the git checkout in `path` is free from dirty/uncommitted changes.
+ """
+ num_changes_str = local('git status --porcelain | wc -l', capture=True)
+ num_changes = int(num_changes_str)
+ if num_changes > 0:
+ abort(red("%s: Your current checkout has uncommitted changes." % this_func()))
+ puts(green("%s: OK" % this_func()))
+
+
82 src/cuisine_sweet/ensure/supervisord.py
@@ -0,0 +1,82 @@
+"""
+Fabric checks and assertions for Supervisord instances
+"""
+
+import re
+import cuisine
+from fabric.api import run, sudo
+from cuisine_sweet.utils import completed_ok
+
+
+@completed_ok()
+def installed():
+ """
+ Ensure that the supervisord is installed.
+ """
+ cuisine.select_package(option='yum')
+ cuisine.command_ensure('easy_install', package='python-setuptools')
+ if not cuisine.command_check('supervisord'):
+ sudo('easy_install supervisor')
+
+
+@completed_ok(arg_output=[0])
+def running(configfile, pidfile, basedir=None, envsource=None, sudo=False):
+ """
+ Ensure that the supervisord instance is running with the correct config.
+
+ Instance checking is based whether the process pid read from pidfile
+ is running.
+
+ If basedir is specified, it overrides the '-d' parameter passed when
+ running the supervisord daemon. If not specified, then the current
+ working directory (via `pwd`) is used.
+
+ If envsource is specified, prior to starting the supervisord daemon,
+ this path-to-shell-script-environment gets loaded (via `source`)
+
+ If sudo is true, then the supervisord daemon is started via sudo(),
+ otherwise, uses the default run() user.
+ """
+ if not basedir:
+ basedir = run('pwd') # current working directory
+ pid = run('cat %s; true' % pidfile)
+
+ running = False
+ if re.match(r'^\d+$', pid):
+ # check
+ pid_running = run('ps -p %s && echo OK; true' % pid).endswith('OK')
+ if pid_running:
+ # ensure that we are using the correct config, otherwise kill first
+ running_cmd = run('ps -p %s ho cmd' % pid)
+ running_cfg_match = re.match(r'.*-c\s+(\S+).*', running_cmd)
+ do_kill = True
+ if running_cfg_match:
+ running_cfg = running_cfg_match.group(1)
+ if running_cfg == configfile:
+ do_kill = False
+ running = True
+
+ if do_kill:
+ # kill impostor or old process
+ run('kill %s && sleep 1; true' % pid)
+
+ if not running:
+ xcmd = 'supervisord -d %s -c %s -j %s' % (basedir, configfile, pidfile)
+ if envsource:
+ xcmd = 'source %s && %s' % (envsource, xcmd)
+ if sudo:
+ sudo(xcmd)
+ else:
+ run(xcmd)
+
+
+
+@completed_ok(arg_output=[0])
+def updated_with_latest_config(configfile):
+ """
+ Ensure that the latest config of the supervisord is loaded and reflected.
+ """
+ run('which supervisorctl')
+ run('supervisorctl -c %s reread' % configfile)
+ run('supervisorctl -c %s update' % configfile)
+
24 src/cuisine_sweet/ensure/user_crontab.py
@@ -0,0 +1,24 @@
+"""
+Fabric checks and assertions for user's crontab
+"""
+
+from fabric.api import run, sudo
+from cuisine_sweet.utils import completed_ok
+
+
+@completed_ok(arg_output=[0])
+def loaded(crontab_file):
+ """
+ Ensure that the user's crontab file is loaded
+ """
+ run('which md5sum')
+ loaded = run('crontab -l >& /dev/null && echo OK; true').endswith('OK')
+ if loaded:
+ loaded_checksum = run('crontab -l | md5sum')
+ expected_checksum = run('cat %s | md5sum' % crontab_file)
+ if loaded_checksum != expected_checksum:
+ # reload
+ run('crontab %s' % crontab_file)
+ else:
+ run('crontab %s' % crontab_file)
+
32 src/cuisine_sweet/ensure/yum.py
@@ -0,0 +1,32 @@
+"""
+Fabric checks and assertions for RHEL-flavored yum installations
+"""
+
+import cuisine
+from fabric.api import run, sudo
+from cuisine_sweet.utils import completed_ok
+
+
+@completed_ok(arg_output=[0])
+def grouppackage_installed(groupname):
+ """
+ Ensure that the group package named `groupname` is installed.
+
+ If the groupname is not installed, then it will be installed
+ using sudo('yum -y groupinstall ...')
+ """
+ grp_installed = run('yum grouplist installed "%s" | grep -i installed && echo OK; true' % groupname).endswith('OK')
+ if not grp_installed:
+ sudo('yum -y groupinstall "%s"' % groupname)
+
+
+@completed_ok(arg_output=[0])
+def package_installed(package):
+ """
+ Ensure that a package named `package` is installed.
+
+ Wraps cuisine.package_ensure() + select_package(option='yum')
+ """
+ cuisine.select_package(option='yum')
+ cuisine.package_ensure(package)
+
116 src/cuisine_sweet/git.py
@@ -0,0 +1,116 @@
+import os
+import re
+import email.utils
+import datetime
+import time
+import yaml
+import cuisine
+from fabric.api import env, local, run, get, abort
+
+
+class GitHistory(object):
+ """
+ Encapsulates the history of git deploys
+ """
+
+ def __init__(self, initial_dump={ }):
+ self.commits = initial_dump
+
+ def log_latest(self, repo_url, repo_refspec, repo_dir, log):
+ c = self.parse_fuller_log(log)
+ if not (repo_dir in self.commits):
+ self.commits[repo_dir] = [ ]
+ do_append = True
+ if len(self.commits[repo_dir]) > 0:
+ # try to check first if the last commit is identical to this commit
+ last_c = self.commits[repo_dir][0][2]['commit_hash']
+ if last_c == c['commit_hash']:
+ # ignore dups
+ do_append = False
+ # but update last deploy timestamp
+ self.commits[repo_dir][0][0] = time.time()
+
+ if do_append:
+ self.commits[repo_dir].insert(0, [ time.time(), [ repo_url, repo_refspec, repo_dir ], c ])
+
+ def dump(self):
+ return yaml.dump(self.commits)
+
+
+ def repo_history(self, repo_dir):
+ if repo_dir in self.commits:
+ return self.commits[repo_dir]
+ return None
+
+
+ def parse_fuller_log(self, log):
+ lines = log.split("\n")
+ chash = lines.pop(0).split(' ')[1]
+ if re.match(r'Merge', lines[0]):
+ lines.pop(0)
+ author = re.match(r'Author:\s+(.*)$', lines.pop(0)).group(1)
+ date_raw = re.match(r'AuthorDate:\s+(.*)$', lines.pop(0)).group(1)
+ date_p = email.utils.parsedate_tz(date_raw)
+ date_args = date_p[0:6] + tuple([date_p[9]])
+ date = datetime.datetime(*date_args)
+ lines.pop(0)
+ lines.pop(0)
+ lines.pop(0)
+ title = lines.pop(0)
+ message_body = [ ]
+ if len(lines) > 0:
+ lines.pop(0)
+ message_body = lines
+ return { 'commit_hash': chash,
+ 'author': author,
+ 'date': date,
+ 'title': title,
+ 'message_body': message_body,
+ }
+
+
+
+# ======================
+
+def get_remote_git_history_path(home, filename='history.yml'):
+ """
+ returns the string path to the remote git history file
+ """
+ remote_hist_dir = os.path.join(home, '.deploy', 'git')
+ run('mkdir -p %s' % remote_hist_dir)
+ remote_hist_path = os.path.join(remote_hist_dir, 'history.yml')
+ return remote_hist_path
+
+
+def load_remote_git_history(remote_hist_path, local_user=None, tmpdir='/tmp'):
+ if not local_user:
+ local_user = local('whoami', capture=True)
+ hist = None
+ if cuisine.file_exists(remote_hist_path):
+ tmp_history_path = os.path.join(tmpdir, local_user, 'deploy', env.host, env.user, 'git-history')
+ local('mkdir -p %s' % tmp_history_path)
+ # download
+ downloaded = get(remote_hist_path, local_path=tmp_history_path)
+ hist_dump_yml = open(downloaded[0]).read()
+ local('rm %s' % downloaded[0])
+ # try loading the history dump
+ hist_dump = yaml.load(hist_dump_yml)
+ hist = GitHistory(initial_dump=hist_dump)
+ return hist
+
+
+
+def history_show_commits(repo_dir, limit=10, home='.'):
+ """
+ params(repo_dir, limit=10)
+ """
+ remote_hist_path = get_remote_git_history_path(home)
+ hist = load_remote_git_history(remote_hist_path)
+ if not hist:
+ abort("No history found")
+
+ repo_history = hist.repo_history(repo_dir)
+ print yaml.dump(repo_history[0:limit])
+
+
+
38 src/cuisine_sweet/utils.py
@@ -0,0 +1,38 @@
+import sys
+import inspect
+import pexpect
+from fabric.api import puts
+from fabric.colors import green
+
+def this_func(level=1):
+ frm = inspect.stack()[level]
+ mod = inspect.getmodule(frm[0])
+ return '%s.%s' % (mod.__name__, frm[3])
+
+
+def completed_ok(arg_output=[]):
+ def wrap(func):
+ def wrapped_f(*args, **kwargs):
+ r = func(*args, **kwargs)
+ out = []
+ for i in arg_output:
+ out.append(args[i])
+ puts(green('%s.%s(%s): OK' % (func.__module__, func.__name__, ', '.join(out))))
+ return wrapped_f
+ return wrap
+
+
+def local_run_expect(cmd, prompts, answers, logfile=sys.stdout):
+ child = pexpect.spawn(
+ cmd,
+ timeout=1800,
+ )
+ child.logfile = logfile
+ child.delaybeforesend = 0
+ while True:
+ try:
+ i = child.expect(prompts)
+ if i <= len(answers)-1:
+ child.sendline(answers[i])
+ except pexpect.EOF:
+ break
Please sign in to comment.
Something went wrong with that request. Please try again.