Permalink
Fetching contributors…
Cannot retrieve contributors at this time
executable file 695 lines (583 sloc) 18.4 KB
#!/usr/bin/env python
# *****************************************************************************
# build.sh by the Wunderful People at Wunderkraut
#
# https://github.com/wunderkraut/build.sh
# *****************************************************************************
import getopt
import sys
import yaml
import os
import subprocess
import shutil
import hashlib
import datetime
import shlex
import stat
import re
import gzip
import tarfile
# Build scripts version string.
build_sh_version_string = "build.sh 1.0"
build_sh_skip_backup = False
build_sh_disable_cache = False
# Site.make item (either a project/library from the site.make)
class MakeItem:
def __init__(self, type, name):
self.type = type
self.name = name
self.version = 'UNDEFINED'
if self.type == 'libraries':
self.project_type = 'library'
else:
self.project_type = 'module'
self.download_args = {}
# Parse a line from site.make for this project/lib
def parse(self, line):
# Download related items
type = re.compile("^[^\[]*\[[^\]]*\]\[download\]\[([^\]]*)\]\s*=\s*(.*)$")
t = type.match(line)
if t:
self.download_args[t.group(1)] = t.group(2)
# Version number
version = re.compile("^.*\[version\]\s*=\s*(.*)$")
v = version.match(line)
if v:
self.version = v.group(1)
# Project type
type = re.compile("^[^\[]*\[[^\]]*\]\[type\]\s*=\s*(.*)$")
t = type.match(line)
if t:
self.project_type = t.group(1)
# Validate site.make item, returns a string describing the issue or False if no issues
def validate(self):
if 'type' in self.download_args:
version = re.compile(".*[0-9]+\.[0-9]+.*")
if self.download_args['type'] == 'git':
if 'tag' not in self.download_args and 'revision' not in self.download_args:
return "No revision or tag defined for a git download"
elif self.download_args['type'] == 'file' and 'url' in self.download_args and not version.match(self.download_args['url']):
return "URL does not seem to have a version number in it (" + self.download_args['url'] + ")"
elif 'dev' in self.version:
return "Development version in use (" + self.version + ")"
return False
# BuildError exception class.
class BuildError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
# Maker class.
class Maker:
def __init__(self, settings):
self.composer = settings.get('composer', 'composer')
self.drush = settings.get('drush', 'drush')
self.type = settings.get('type', 'drush make')
self.drupal_subpath = settings.get('drupal_subpath', '')
self.temp_build_dir_name = settings['temporary']
self.temp_build_dir = os.path.abspath(self.temp_build_dir_name)
self.final_build_dir_name = settings['final']
self.final_build_dir = os.path.abspath(self.final_build_dir_name)
self.final_build_dir_bak = self.final_build_dir + "_bak"
self.old_build_dir = os.path.abspath(settings.get('previous', 'previous'))
self.profile_name = settings.get('profile', 'standard')
self.site_name = settings.get('site', 'A drupal site')
self.make_cache_dir = settings.get('make_cache', '.make_cache')
self.settings = settings
self.store_old_buids = True
self.linked = False
if self.type == 'drush make':
self.makefile = os.path.abspath(settings.get('makefile', 'conf/site.make'))
self.makefile_hash = hashlib.md5(open(self.makefile, 'rb').read()).hexdigest()
# See if drush is installed
if not self._which('drush'):
raise BuildError('Drush missing!?')
def test(self):
self._validate_makefile()
# Check if given program exists
# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
def _which(self, program):
import os
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
# Quickly validate the drush make file
def _validate_makefile(self):
f = open(self.makefile)
if f:
content = f.readlines()
projects = {}
prog = re.compile("^([^\[]*)\[([^\]]*)\]\[([^\]]*)\].*$")
for line in content:
m = prog.match(line)
if m:
name = m.group(2)
if name not in projects:
projects[name] = MakeItem(m.group(1), name)
projects[name].parse(line)
errors = False
for item in projects:
error = projects[item].validate()
if error:
errors = True
self.warning(projects[item].name + ': ' + error)
if errors:
raise BuildError("The make file is volatile - it is not ready for production use")
else:
self.notice("Everything looks good!")
# Run make
def make(self):
if self.type == 'drush make':
self._drush_make()
elif self.type == 'composer':
self._composer_make()
def _composer_make(self):
self._precheck()
self.link()
self._composer([
'-d=' + self.temp_build_dir,
'install'
]);
def _drush_make(self):
global build_sh_disable_cache
self._precheck()
self.notice("Building")
packaged_build = self.make_cache_dir + '/' + self.makefile_hash + '.tgz';
if not build_sh_disable_cache and os.path.exists(packaged_build):
# Existing build
self.notice("Make file unchanged - unpacking previous make")
tar = tarfile.open(packaged_build)
tar.extractall()
tar.close()
else:
if not self._drush(self._collect_make_args()):
raise BuildError("Make failed - check your makefile")
os.remove(self.temp_build_dir + "/sites/default/default.settings.php")
if not os.path.isdir(self.make_cache_dir):
os.makedirs(self.make_cache_dir)
tar = tarfile.open(packaged_build, "w:gz")
tar.add(self.temp_build_dir, arcname=self.temp_build_dir_name)
tar.close()
# f = open(self.temp_build_dir + "/buildhash", "w")
# f.write(self.makefile_hash)
# f.close()
# Remove default.settings.php
# Existing final build?
def hasExistingBuild(self):
return os.path.isdir(self.final_build_dir)
# Backup current final build
def backup(self, params):
global build_sh_skip_backup
if build_sh_skip_backup:
self.notice("Skipping backup!")
return
if not params:
params = {}
self.notice("Backing up current build")
if self.hasExistingBuild():
self._backup(params)
def cleanup(self):
import time
compare = time.time() - (60*60*24)
for f in os.listdir(self.old_build_dir):
fullpath = os.path.join(self.old_build_dir, f)
if os.stat(fullpath).st_mtime < compare:
if os.path.isdir(fullpath):
self.notice("Removing old build " + f)
shutil.rmtree(fullpath)
elif os.path.isfile(fullpath):
self.notice("Removing old build archive " + f)
os.remove(fullpath)
# Purge current final build
def purge(self):
self.notice("Purging current build")
if self.hasExistingBuild():
self._wipe()
# Link
def link(self):
# Link and copy required files
self._link()
self._copy()
self.linked = True
# Finalize new build to be the final build
def finalize(self):
self.notice("Finalizing new build")
if os.path.isdir(self.final_build_dir):
self._unlink()
self._ensure_writable(self.final_build_dir)
os.rename(self.final_build_dir, self.final_build_dir_bak)
# Make sure linking has happened
if not self.linked:
self.link()
os.rename(self.temp_build_dir, self.final_build_dir)
if os.path.isdir(self.final_build_dir_bak):
shutil.rmtree(self.final_build_dir_bak)
# Print notice
def notice(self, *args):
print "\033[92m** BUILD NOTICE: \033[0m" + ' '.join(str(a) for a in args)
# Print errror
def error(self, *args):
print "\033[91m** BUILD ERROR: \033[0m" + ' '.join(str(a) for a in args)
# Print warning
def warning(self, *args):
print "\033[93m** BUILD WARNING: \033[0m" + ' '.join(str(a) for a in args)
# Run install
def install(self):
if not self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
"site-install",
self.profile_name,
"install_configure_form.update_status_module='array(FALSE,FALSE)'"
"--account-name=admin",
"--account-pass=admin",
"--site-name=" + self.site_name,
"-y"
]):
raise BuildError("Install failed.");
# Update existing final build
def update(self):
if self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
'updatedb',
'--y'
]):
self.notice("Update process completed")
else:
self.warning("Unable to update")
# Ask user for verification
def verify(self, text):
if text:
response = raw_input(text)
else:
response = raw_input("Type yes to verify that you know what you are doing: ")
if response.lower() != "yes":
raise BuildError("Cancelled by user")
# Execute a shell command
def shell(self, command):
if isinstance(command, list):
for step in command:
value = os.system(command) == 0
if not value:
return False
return True
else:
return os.system(command) == 0
def append(self, command):
files = command.split(">")
if len(files) > 1:
with open(files[1].strip(), "a") as target:
target.write(open(files[0].strip(), "rb").read())
else:
raise BuildError("Append commands syntax is: source > target")
# Execute given step
def execute(self, step):
command = False
if isinstance(step, dict):
step, command = step.popitem()
if step == 'make':
self.make()
elif step == 'backup':
self.backup(command)
elif step == 'purge':
self.purge()
elif step == 'finalize':
self.finalize()
elif step == 'install':
self.install()
elif step == 'update':
self.update()
elif step == 'cleanup':
self.cleanup()
elif step == 'append':
self.append(command)
elif step == 'verify':
self.verify(command)
elif step == 'shell':
self.shell(command)
elif step == 'link':
self.link()
elif step == 'test':
self.test()
else:
print "Unknown step " + step
# Collect make args
def _collect_make_args(self):
return [
"--strict=0",
"--concurrency=20",
"-y",
"make",
self.makefile,
self.temp_build_dir
]
# Handle link
def _link(self):
if not "link" in self.settings:
return
for tuple in self.settings['link']:
source, target = tuple.items()[0]
target = self.temp_build_dir + "/" + target
if source.endswith('*'):
path = source[:-1]
paths = [path + name for name in os.listdir(path) if os.path.isdir(path + name)]
for source in paths:
self._link_files(source, target + "/" + os.path.basename(source))
else:
self._link_files(source, target)
# Handle unlink
def _unlink(self):
if not "link" in self.settings:
return
for tuple in self.settings['link']:
source, target = tuple.items()[0]
target = self.final_build_dir + "/" + target
if source.endswith('*'):
path = source[:-1]
paths = [path + name for name in os.listdir(path) if os.path.isdir(path + name)]
for source in paths:
self._unlink_files(target + "/" + os.path.basename(source))
else:
self._unlink_files(target)
# Handle copy
def _copy(self):
if not "copy" in self.settings:
return
for tuple in self.settings['copy']:
source, target = tuple.popitem()
target = self.temp_build_dir + "/" + target
self._copy_files(source, target)
# Execute a composer command
def _composer(self, args, quiet = False):
if quiet:
FNULL = open(os.devnull, 'w')
return subprocess.call([self.composer] + args, stdout=FNULL, stderr=FNULL) == 0
return subprocess.call([self.composer] + args) == 0
# Execute a drush command
def _drush(self, args, quiet = False):
if quiet:
FNULL = open(os.devnull, 'w')
return subprocess.call([self.drush] + args, stdout=FNULL, stderr=FNULL) == 0
return subprocess.call([self.drush] + args) == 0
# Ensure directories exist
def _precheck(self):
# Remove old build it if exists
if os.path.isdir(self.temp_build_dir):
shutil.rmtree(self.temp_build_dir)
if not os.path.isdir(self.old_build_dir):
os.mkdir(self.old_build_dir)
# TarFile exclude callback for _backup function
def _backup_exlude(self, file):
for exclude in self._build_exclude_files:
if file.endswith(exclude):
return True
return False
# Backup existing final build
def _backup(self, params):
if 'skip-database' in params:
self.notice("Database dump skipped as requested")
else:
dump_file = self.final_build_dir + '/db.sql'
gzdump_file = self.final_build_dir + '/db.sql.gz'
if self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
'sql-dump',
'--result-file=' + dump_file
], True):
self.notice("Database dump taken")
else:
self.warning("No database dump taken")
name = datetime.datetime.now()
name = name.isoformat()
backup_file = self.old_build_dir + "/" + name + ".tgz"
if 'ignore' in params:
self._build_exclude_files = params['ignore']
else:
self._build_exclude_files = {}
tar = tarfile.open(backup_file, "w:gz", dereference=True)
tar.add(self.final_build_dir, arcname=self.final_build_dir_name, exclude=self._backup_exlude)
tar.close()
# Wipe existing final build
def _wipe(self):
if self._drush([
'--root=' + format(self.final_build_dir + self.drupal_subpath),
'sql-drop',
'--y'
], True):
self.notice("Tables dropped")
else:
self.notice("No tables dropped")
self._ensure_writable(self.final_build_dir)
shutil.rmtree(self.final_build_dir)
# Ensure we have write access to the given dir
def _ensure_writable(self, path):
for root, dirs, files in os.walk(path):
for momo in dirs:
file = os.path.join(root, momo)
mode = os.stat(file).st_mode
os.chmod(file, mode|stat.S_IWRITE)
for momo in files:
file = os.path.join(root, momo)
mode = os.stat(file).st_mode
os.chmod(file, mode|stat.S_IWRITE)
def _ensure_container(self, filepath):
# Ensure target directory exists
target_container = os.path.dirname(filepath)
if not os.path.exists(target_container):
self.notice("Created directory " + target_container)
os.makedirs(target_container)
# Symlink file from source to target
def _link_files(self, source, target):
self._ensure_container(target)
if os.path.exists(source) and not os.path.exists(target):
source = os.path.relpath(source, os.path.dirname(target))
os.symlink(source, target)
else:
raise BuildError("Can't link " + source + " to " + target + ". Make sure that the source exists.")
# Unlink file from target
def _unlink_files(self, target):
self._ensure_container(target)
if os.path.exists(target):
os.unlink(target)
# Copy file from source to target
def _copy_files(self, source, target):
self._ensure_container(target)
if os.path.exists(source) and not os.path.exists(target):
if os.path.isdir(source):
shutil.copytree(source, target)
else:
shutil.copyfile(source, target)
else:
raise BuildError("Can't copy " + source + " to " + target + ". Make sure that the source exists.")
# Print help function
def help():
print 'build.sh [options] [command] [site]'
print '[command] is one of the commands defined in the configuration file'
print '[site] defines the site to build, defaults to default'
print 'Options:'
print ' -h --help'
print ' Print this help'
print ' -c --config'
print ' Configuration file to use, defaults to conf/site.yml'
print ' -o --commands'
print ' Configuration file to use, defaults to conf/commands.yml'
print ' -s --skip-backup'
print ' Do not take backups, ever'
print ' -d --disable-cache'
print ' Do not use caches'
print ' -v --version'
print ' Print version information'
# Print version function.
def version():
print build_sh_version_string
# Program main:
def main(argv):
# Default configuration file to use:
config_file = 'conf/site.yml'
commands_file = 'conf/commands.yml'
do_build = True
# Parse options:
try:
opts, args = getopt.getopt(argv, "hdc:o:vst", ["help", "config=", "commands=", "version", "test", "skip-backup", "disable-cache"])
except getopt.GetoptError:
help()
return
for opt, arg in opts:
if opt in ('-h', "--help"):
help()
return
elif opt in ("-c", "--config"):
config_file = arg
elif opt in ("-o", "--commands"):
commands_file = arg
elif opt in ("-s", "--skip-backup"):
global build_sh_skip_backup
build_sh_skip_backup = True
elif opt in ("-d", "--disable-cache"):
global build_sh_disable_cache
build_sh_disable_cache = True
elif opt in ("-v", "--version"):
version()
return
try:
# Get the settings file YAML contents.
f = open(config_file)
settings = yaml.safe_load(f)
f.close()
try:
command = args[0]
except IndexError:
help()
return
# Default site is "default"
site = 'default'
try:
site = args[1]
except IndexError:
if 'WKV_SITE_ENV' in os.environ:
site = os.environ['WKV_SITE_ENV']
else:
site = 'default'
sites = []
sites.append(site)
for site in sites:
# Copy defaults.
site_settings = settings["default"].copy()
if not site in settings:
new_site = False
for site_name in settings:
if 'aliases' in settings[site_name]:
if isinstance(settings[site_name]['aliases'], basestring):
site_aliases = [ settings[site_name]['aliases'] ]
else:
site_aliases = settings[site_name]['aliases']
if site in site_aliases:
new_site = site_name
break
if not new_site:
raise BuildError("The site " + site + " is not defined")
site = new_site
# If not the default site, update it with defaults.
if site != "default":
site_settings.update(settings[site])
# Create the site maker based on the settings
maker = Maker(site_settings)
maker.notice("Using configuration " + site)
commands = {}
if 'commands' in settings:
commands = settings['commands']
maker.warning("There are commands defined in site.yml - please move them to commands.yml.")
# Read in the commands file
if os.path.isfile(commands_file):
if 'commands' in settings:
maker.warning("Commands defined in commands.yml override the commands defined in site.yml")
f = open(commands_file)
commands = yaml.safe_load(f)
f.close()
commands['test'] = {"test": "test"}
# Add and overwrite commands with local_commands
if 'local_commands' in settings[site]:
commands.update(settings[site]['local_commands'])
if do_build:
# Execute the command(s).
if command in commands:
command_set = commands[command]
for step in command_set:
maker.execute(step)
else:
maker.notice("No such command defined as '" + command + "'")
except Exception, errtxt:
print "\033[91m** BUILD ERROR: \033[0m%s" % (errtxt)
exit(1)
# Entry point.
if __name__ == "__main__":
main(sys.argv[1:])