Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
executable file 694 lines (583 sloc) 18.4 KB
#!/usr/bin/env python
# *****************************************************************************
# build.sh by the Wunderful People at Wunderkraut
#
# https://github.com/wunderkraut/build.sh
# *****************************************************************************
import getopt
import sys
import yaml
import os
import subprocess
import shutil
import hashlib
import datetime
import shlex
import stat
import re
import gzip
import tarfile
# Build scripts version string.
build_sh_version_string = "build.sh 1.0"
build_sh_skip_backup = False
build_sh_disable_cache = False
# Site.make item (either a project/library from the site.make)
class MakeItem:
def __init__(self, type, name):
self.type = type
self.name = name
self.version = 'UNDEFINED'
if self.type == 'libraries':
self.project_type = 'library'
else:
self.project_type = 'module'
self.download_args = {}
# Parse a line from site.make for this project/lib
def parse(self, line):
# Download related items
type = re.compile("^[^\[]*\[[^\]]*\]\[download\]\[([^\]]*)\]\s*=\s*(.*)$")
t = type.match(line)
if t:
self.download_args[t.group(1)] = t.group(2)
# Version number
version = re.compile("^.*\[version\]\s*=\s*(.*)$")
v = version.match(line)
if v:
self.version = v.group(1)
# Project type
type = re.compile("^[^\[]*\[[^\]]*\]\[type\]\s*=\s*(.*)$")
t = type.match(line)
if t:
self.project_type = t.group(1)
# Validate site.make item, returns a string describing the issue or False if no issues
def validate(self):
if 'type' in self.download_args:
version = re.compile(".*[0-9]+\.[0-9]+.*")
if self.download_args['type'] == 'git':
if 'tag' not in self.download_args and 'revision' not in self.download_args:
return "No revision or tag defined for a git download"
elif self.download_args['type'] == 'file' and 'url' in self.download_args and not version.match(self.download_args['url']):
return "URL does not seem to have a version number in it (" + self.download_args['url'] + ")"
elif 'dev' in self.version:
return "Development version in use (" + self.version + ")"
return False
# BuildError exception class.
class BuildError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
# Maker class.
class Maker:
def __init__(self, settings):
self.composer = settings.get('composer', 'composer')
self.drush = settings.get('drush', 'drush')
self.type = settings.get('type', 'drush make')
self.drupal_subpath = settings.get('drupal_subpath', '')
self.temp_build_dir_name = settings['temporary']
self.temp_build_dir = os.path.abspath(self.temp_build_dir_name)
self.final_build_dir_name = settings['final']
self.final_build_dir = os.path.abspath(self.final_build_dir_name)
self.final_build_dir_bak = self.final_build_dir + "_bak"
self.old_build_dir = os.path.abspath(settings.get('previous', 'previous'))
self.profile_name = settings.get('profile', 'standard')
self.site_name = settings.get('site', 'A drupal site')
self.make_cache_dir = settings.get('make_cache', '.make_cache')
self.settings = settings
self.store_old_buids = True
self.linked = False
if self.type == 'drush make':
self.makefile = os.path.abspath(settings.get('makefile', 'conf/site.make'))
self.makefile_hash = hashlib.md5(open(self.makefile, 'rb').read()).hexdigest()
# See if drush is installed
if not self._which('drush'):
raise BuildError('Drush missing!?')
def test(self):
self._validate_makefile()
# Check if given program exists
# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
def _which(self, program):
import os
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
# Quickly validate the drush make file
def _validate_makefile(self):
f = open(self.makefile)
if f:
content = f.readlines()
projects = {}
prog = re.compile("^([^\[]*)\[([^\]]*)\]\[([^\]]*)\].*$")
for line in content:
m = prog.match(line)
if m:
name = m.group(2)
if name not in projects:
projects[name] = MakeItem(m.group(1), name)
projects[name].parse(line)
errors = False
for item in projects:
error = projects[item].validate()
if error:
errors = True
self.warning(projects[item].name + ': ' + error)
if errors:
raise BuildError("The make file is volatile - it is not ready for production use")
else:
self.notice("Everything looks good!")
# Run make
def make(self):
if self.type == 'drush make':
self._drush_make()
elif self.type == 'composer':
self._composer_make()
def _composer_make(self):
self._precheck()
self.link()
self._composer([
'-d=' + self.temp_build_dir,
'install'
]);
def _drush_make(self):
global build_sh_disable_cache
self._precheck()
self.notice("Building")
packaged_build = self.make_cache_dir + '/' + self.makefile_hash + '.tgz';
if not build_sh_disable_cache and os.path.exists(packaged_build):
# Existing build
self.notice("Make file unchanged - unpacking previous make")
tar = tarfile.open(packaged_build)
tar.extractall()
tar.close()
else:
if not self._drush(self._collect_make_args()):
raise BuildError("Make failed - check your makefile")
os.remove(self.temp_build_dir + "/sites/default/default.settings.php")
if not os.path.isdir(self.make_cache_dir):
os.makedirs(self.make_cache_dir)
tar = tarfile.open(packaged_build, "w:gz")
tar.add(self.temp_build_dir, arcname=self.temp_build_dir_name)
tar.close()
# f = open(self.temp_build_dir + "/buildhash", "w")
# f.write(self.makefile_hash)
# f.close()
# Remove default.settings.php
# Existing final build?
def hasExistingBuild(self):
return os.path.isdir(self.final_build_dir)
# Backup current final build
def backup(self, params):
global build_sh_skip_backup
if build_sh_skip_backup:
self.notice("Skipping backup!")
return
if not params:
params = {}
self.notice("Backing up current build")
if self.hasExistingBuild():
self._backup(params)
def cleanup(self):
import time
compare = time.time() - (60*60*24)
for f in os.listdir(self.old_build_dir):
fullpath = os.path.join(self.old_build_dir, f)
if os.stat(fullpath).st_mtime < compare:
if os.path.isdir(fullpath):
self.notice("Removing old build " + f)
shutil.rmtree(fullpath)
elif os.path.isfile(fullpath):
self.notice("Removing old build archive " + f)
os.remove(fullpath)
# Purge current final build
def purge(self):
self.notice("Purging current build")
if self.hasExistingBuild():
self._wipe()
# Link
def link(self):
# Link and copy required files
self._link()
self._copy()
self.linked = True
# Finalize new build to be the final build
def finalize(self):
self.notice("Finalizing new build")
if os.path.isdir(self.final_build_dir):
self._unlink()
self._ensure_writable(self.final_build_dir)
os.rename(self.final_build_dir, self.final_build_dir_bak)
# Make sure linking has happened
if not self.linked:
self.link()
os.rename(self.temp_build_dir, self.final_build_dir)
if os.path.isdir(self.final_build_dir_bak):
shutil.rmtree(self.final_build_dir_bak)
# Print notice
def notice(self, *args):
print "\033[92m** BUILD NOTICE: \033[0m" + ' '.join(str(a) for a in args)
# Print errror
def error(self, *args):
print "\033[91m** BUILD ERROR: \033[0m" + ' '.join(str(a) for a in args)
# Print warning
def warning(self, *args):
print "\033[93m** BUILD WARNING: \033[0m" + ' '.join(str(a) for a in args)
# Run install
def install(self):
if not self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
"site-install",
self.profile_name,
"install_configure_form.update_status_module='array(FALSE,FALSE)'"
"--account-name=admin",
"--account-pass=admin",
"--site-name=" + self.site_name,
"-y"
]):
raise BuildError("Install failed.");
# Update existing final build
def update(self):
if self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
'updatedb',
'--y'
]):
self.notice("Update process completed")
else:
self.warning("Unable to update")
# Ask user for verification
def verify(self, text):
if text:
response = raw_input(text)
else:
response = raw_input("Type yes to verify that you know what you are doing: ")
if response.lower() != "yes":
raise BuildError("Cancelled by user")
# Execute a shell command
def shell(self, command):
if isinstance(command, list):
for step in command:
value = os.system(command) == 0
if not value:
return False
return True
else:
return os.system(command) == 0
def append(self, command):
files = command.split(">")
if len(files) > 1:
with open(files[1].strip(), "a") as target:
target.write(open(files[0].strip(), "rb").read())
else:
raise BuildError("Append commands syntax is: source > target")
# Execute given step
def execute(self, step):
command = False
if isinstance(step, dict):
step, command = step.popitem()
if step == 'make':
self.make()
elif step == 'backup':
self.backup(command)
elif step == 'purge':
self.purge()
elif step == 'finalize':
self.finalize()
elif step == 'install':
self.install()
elif step == 'update':
self.update()
elif step == 'cleanup':
self.cleanup()
elif step == 'append':
self.append(command)
elif step == 'verify':
self.verify(command)
elif step == 'shell':
self.shell(command)
elif step == 'link':
self.link()
elif step == 'test':
self.test()
else:
print "Unknown step " + step
# Collect make args
def _collect_make_args(self):
return [
"--strict=0",
"--concurrency=20",
"-y",
"make",
self.makefile,
self.temp_build_dir
]
# Handle link
def _link(self):
if not "link" in self.settings:
return
for tuple in self.settings['link']:
source, target = tuple.items()[0]
target = self.temp_build_dir + "/" + target
if source.endswith('*'):
path = source[:-1]
paths = [path + name for name in os.listdir(path) if os.path.isdir(path + name)]
for source in paths:
self._link_files(source, target + "/" + os.path.basename(source))
else:
self._link_files(source, target)
# Handle unlink
def _unlink(self):
if not "link" in self.settings:
return
for tuple in self.settings['link']:
source, target = tuple.items()[0]
target = self.final_build_dir + "/" + target
if source.endswith('*'):
path = source[:-1]
paths = [path + name for name in os.listdir(path) if os.path.isdir(path + name)]
for source in paths:
self._unlink_files(target + "/" + os.path.basename(source))
else:
self._unlink_files(target)
# Handle copy
def _copy(self):
if not "copy" in self.settings:
return
for tuple in self.settings['copy']:
source, target = tuple.popitem()
target = self.temp_build_dir + "/" + target
self._copy_files(source, target)
# Execute a composer command
def _composer(self, args, quiet = False):
if quiet:
FNULL = open(os.devnull, 'w')
return subprocess.call([self.composer] + args, stdout=FNULL, stderr=FNULL) == 0
return subprocess.call([self.composer] + args) == 0
# Execute a drush command
def _drush(self, args, quiet = False):
if quiet:
FNULL = open(os.devnull, 'w')
return subprocess.call([self.drush] + args, stdout=FNULL, stderr=FNULL) == 0
return subprocess.call([self.drush] + args) == 0
# Ensure directories exist
def _precheck(self):
# Remove old build it if exists
if os.path.isdir(self.temp_build_dir):
shutil.rmtree(self.temp_build_dir)
if not os.path.isdir(self.old_build_dir):
os.mkdir(self.old_build_dir)
# TarFile exclude callback for _backup function
def _backup_exlude(self, file):
for exclude in self._build_exclude_files:
if file.endswith(exclude):
return True
return False
# Backup existing final build
def _backup(self, params):
if 'skip-database' in params:
self.notice("Database dump skipped as requested")
else:
dump_file = self.final_build_dir + '/db.sql'
gzdump_file = self.final_build_dir + '/db.sql.gz'
if self._drush([
"--root=" + format(self.final_build_dir + self.drupal_subpath),
'sql-dump',
'--result-file=' + dump_file
], True):
self.notice("Database dump taken")
else:
self.warning("No database dump taken")
name = datetime.datetime.now()
name = name.isoformat()
backup_file = self.old_build_dir + "/" + name + ".tgz"
if 'ignore' in params:
self._build_exclude_files = params['ignore']
else:
self._build_exclude_files = {}
tar = tarfile.open(backup_file, "w:gz", dereference=True)
tar.add(self.final_build_dir, arcname=self.final_build_dir_name, exclude=self._backup_exlude)
tar.close()
# Wipe existing final build
def _wipe(self):
if self._drush([
'--root=' + format(self.final_build_dir + self.drupal_subpath),
'sql-drop',
'--y'
], True):
self.notice("Tables dropped")
else:
self.notice("No tables dropped")
self._ensure_writable(self.final_build_dir)
shutil.rmtree(self.final_build_dir)
# Ensure we have write access to the given dir
def _ensure_writable(self, path):
for root, dirs, files in os.walk(path):
for momo in dirs:
file = os.path.join(root, momo)
mode = os.stat(file).st_mode
os.chmod(file, mode|stat.S_IWRITE)
for momo in files:
file = os.path.join(root, momo)
mode = os.stat(file).st_mode
os.chmod(file, mode|stat.S_IWRITE)
def _ensure_container(self, filepath):
# Ensure target directory exists
target_container = os.path.dirname(filepath)
if not os.path.exists(target_container):
self.notice("Created directory " + target_container)
os.makedirs(target_container)
# Symlink file from source to target
def _link_files(self, source, target):
self._ensure_container(target)
if os.path.exists(source) and not os.path.exists(target):
source = os.path.relpath(source, os.path.dirname(target))
os.symlink(source, target)
else:
raise BuildError("Can't link " + source + " to " + target + ". Make sure that the source exists.")
# Unlink file from target
def _unlink_files(self, target):
self._ensure_container(target)
if os.path.exists(target):
os.unlink(target)
# Copy file from source to target
def _copy_files(self, source, target):
self._ensure_container(target)
if os.path.exists(source) and not os.path.exists(target):
if os.path.isdir(source):
shutil.copytree(source, target)
else:
shutil.copyfile(source, target)
else:
raise BuildError("Can't copy " + source + " to " + target + ". Make sure that the source exists.")
# Print help function
def help():
print 'build.sh [options] [command] [site]'
print '[command] is one of the commands defined in the configuration file'
print '[site] defines the site to build, defaults to default'
print 'Options:'
print ' -h --help'
print ' Print this help'
print ' -c --config'
print ' Configuration file to use, defaults to conf/site.yml'
print ' -o --commands'
print ' Configuration file to use, defaults to conf/commands.yml'
print ' -s --skip-backup'
print ' Do not take backups, ever'
print ' -d --disable-cache'
print ' Do not use caches'
print ' -v --version'
print ' Print version information'
# Print version function.
def version():
print build_sh_version_string
# Program main:
def main(argv):
# Default configuration file to use:
config_file = 'conf/site.yml'
commands_file = 'conf/commands.yml'
do_build = True
# Parse options:
try:
opts, args = getopt.getopt(argv, "hdc:o:vst", ["help", "config=", "commands=", "version", "test", "skip-backup", "disable-cache"])
except getopt.GetoptError:
help()
return
for opt, arg in opts:
if opt in ('-h', "--help"):
help()
return
elif opt in ("-c", "--config"):
config_file = arg
elif opt in ("-o", "--commands"):
commands_file = arg
elif opt in ("-s", "--skip-backup"):
global build_sh_skip_backup
build_sh_skip_backup = True
elif opt in ("-d", "--disable-cache"):
global build_sh_disable_cache
build_sh_disable_cache = True
elif opt in ("-v", "--version"):
version()
return
try:
# Get the settings file YAML contents.
f = open(config_file)
settings = yaml.safe_load(f)
f.close()
try:
command = args[0]
except IndexError:
help()
return
# Default site is "default"
site = 'default'
try:
site = args[1]
except IndexError:
if 'WKV_SITE_ENV' in os.environ:
site = os.environ['WKV_SITE_ENV']
else:
site = 'default'
sites = []
sites.append(site)
for site in sites:
# Copy defaults.
site_settings = settings["default"].copy()
if not site in settings:
new_site = False
for site_name in settings:
if 'aliases' in settings[site_name]:
if isinstance(settings[site_name]['aliases'], basestring):
site_aliases = [ settings[site_name]['aliases'] ]
else:
site_aliases = settings[site_name]['aliases']
if site in site_aliases:
new_site = site_name
break
if not new_site:
raise BuildError("The site " + site + " is not defined")
site = new_site
# If not the default site, update it with defaults.
if site != "default":
site_settings.update(settings[site])
# Create the site maker based on the settings
maker = Maker(site_settings)
maker.notice("Using configuration " + site)
commands = {}
if 'commands' in settings:
commands = settings['commands']
maker.warning("There are commands defined in site.yml - please move them to commands.yml.")
# Read in the commands file
if os.path.isfile(commands_file):
if 'commands' in settings:
maker.warning("Commands defined in commands.yml override the commands defined in site.yml")
f = open(commands_file)
commands = yaml.safe_load(f)
f.close()
commands['test'] = {"test": "test"}
# Add and overwrite commands with local_commands
if 'local_commands' in settings[site]:
commands.update(settings[site]['local_commands'])
if do_build:
# Execute the command(s).
if command in commands:
command_set = commands[command]
for step in command_set:
maker.execute(step)
else:
maker.notice("No such command defined as '" + command + "'")
except Exception, errtxt:
print "\033[91m** BUILD ERROR: \033[0m%s" % (errtxt)
exit(1)
# Entry point.
if __name__ == "__main__":
main(sys.argv[1:])