Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 587 lines (529 sloc) 19.8 KB
#!/bin/bash
# This script download all the template library subsets from Git repositories
# on GitHub and gather them in a directory whose layout is compliant with
# SCDB cfg/ subdirectory.
#
# Written by Michel Jouvin <jouvin@lal.in2p3.fr>, 6/6/2014
#
# Variables describing repositories to fetch to build the template library
# For each repo xxx, the following variables must/may exist:
# - xxx_git_repo: name of the Git repo (at quattor_git_url url)
# - xxx_branch_pattern: a regexp to select the "branch" to fetch
# ("branch" is matched against the branch or tag
# according to xxx_use_tags)
# - xxx_use_tags (optional): use list of existing tags rather than list of branches
# (D: use_tags_default)
# - xxx_tags_ignore_pattern: ignore branch pattern when use_tags is true
# (incompatible with xxx_ignore_version)
# - xxx_ignore_version (optional): retrieve all branches/tags even if a specific
# version is requested (D: ignore_version_default)
# - xxx_dest_dir: directory where to put the repo contents, under template library root
# (%BRANCH% is replaced by the "branch" name, %TAG% by the
# Quattor version specified in the tag)
# - xxx_rename_master (optional): "branch name" to use if the branch is master
git_url_root=https://github.com
quattor_git_url=${git_url_root}/quattor
git_repo_list='core examples grid os standard monitoring'
# Do not change the following defaults except if you know what you are doing...
use_tags_default=1
ignore_version_default=0
tags_ignore_pattern_pattern=master
pull_request=''
use_ci_work_dir=0
core_git_repo=template-library-core
core_branch_pattern='^(template-library-)*'
core_dest_dir=quattor/%TAG%
# With core repo, always checkout all tags matching core_branch_pattern,
# whatever is done for other repos.
core_use_tags=1
core_ignore_version=1
# Rename master branch from -core repo
# Set to an empty string or comment out to disable renaming
# Can be used for each repository but generally used only with -core
core_rename_master=dev
examples_git_repo=template-library-examples
examples_branch_pattern=master
# examples provides sites/ and clusters/ directory directly under the template library root
examples_dest_dir=
grid_git_repo=template-library-grid
grid_branch_pattern=.*
grid_dest_dir=grid/%BRANCH%
os_git_repo=template-library-os
os_branch_pattern=.*
os_dest_dir=os/%BRANCH%
standard_git_repo=template-library-standard
standard_branch_pattern=master
standard_dest_dir=standard
monitoring_git_repo=template-library-monitoring
monitoring_branch_pattern=master
monitoring_dest_dir=standard/monitoring
# Other initializations
# If a branch name matches one of the pattern, it will be ignored
# HEAD added to workaround a bug in the release procedure when producing 14.5
ignore_branch_patterns='\.obsolete$ ^HEAD$'
ignore_version_patterns='13.1.[1-3] 13.[2-9] 13.12 14.[2-8]'
template_lib_root=${TMPDIR:-/tmp}/quattor-template-library
git_clone_root=${template_lib_root}/.gitrepos
list_branches=0
remove_tl_dir=0
verbose=0
add_legacy=0
add_rc_templates=0
add_spma_templates=0
usage () {
echo "usage: `basename $0` [-F] [--debug] [-d template_lib_root] [options...] quattor_version"
echo ""
echo " Valid options are:"
echo ""
echo " --continuous-integration: use working copy contents instead of the GitHub repository"
echo " -d template_library_root: directory where to create the template library."
echo " (D: ${template_lib_root})"
echo " --debug : verbose output"
echo " -F : remove {template_lib_root} if it already exists."
echo " -l : list available branches matching the selected version."
echo " --pull-request repository:user:branch[:target] : merge the GitHub pull request specifed by the"
echo " source branch repository:user:branch into repository:quattor:target."
echo " target can be a branch or a tag. If not specified, it defaults to"
echo " quattor_version or master when quattor_version=HEAD. Note that using"
echo " a version other than HEAD generally produces unpredictable results."
echo " --release-candidates: download release candidates in template-library-core (disabled by default)."
echo " --spma : download SPMA-based OS templates (disabled by default)."
echo ""
echo " quattor_version must be a released Quattor version (git tag) or HEAD."
echo ""
exit 1
}
# Redirect stdout and stderr to /dev/null, except if --debug
silent_command () {
exec 3>&1 4>&2
[ ${verbose} -eq 0 ] && exec &>/dev/null
$*
status=$?
exec 1>&3 2>&4
return $status
}
while [ -n "$(echo $1 | grep '^-')" ]
do
case $1 in
--add-legacy)
add_legacy=1
;;
--continuous*)
use_ci_work_dir=1
;;
-d)
shift
template_lib_root=$1
;;
--debug)
verbose=1
;;
-l)
list_branches=1
;;
-F)
remove_tl_dir=1
;;
--help)
usage
;;
--pull-request)
shift
pull_request=$1
;;
--release*)
add_rc_templates=1
;;
--spma)
add_spma_templates=1
;;
*)
echo "Unkown option ($1)"
usage
;;
esac
shift
done
if [ -n "$1" ]
then
quattor_version=$1
else
echo "Quattor version to checkout is required (use 'HEAD' for the most recent revision)"
exit 1
fi
# --pull-request value must match the following format: repository:user:branch[:target_branch].
# - repository must match one of the repository managed by this script (and the user's fork must
# have the same name).
# - user: the GitHub user who contributed the PR (will be used to select the fork to use)
# - branch: the branch to import from the fork
# - target_branch: the branch in the Quattor repo which is in the target for the PR. It is
# optional and default to master (it is in fact required if the target is not master)
# Check that the value is syntactically valid.
if [ -n "${pull_request}" ]
then
# --pull-request is incompatible with --continuous-integration
if [ ${use_ci_work_dir} -eq 1 ]
then
echo "--pull-request and --continuous-integration are incompatible"
usage
fi
pr_repo=$(echo ${pull_request} | awk -F: '{print $1}')
if [ -z "${pr_repo}" ]
then
echo "--pull-request value invalid (repository missing)"
usage
fi
pr_user=$(echo ${pull_request} | awk -F: '{print $2}')
if [ -z "${pr_user}" ]
then
echo "--pull-request value invalid (user missing)"
usage
fi
pr_branch=$(echo ${pull_request} | awk -F: '{print $3}')
if [ -z "${pr_branch}" ]
then
echo "--pull-request value invalid (branch missing)"
usage
fi
pr_target=$(echo ${pull_request} | awk -F: '{print $4}')
if [ -z "${pr_target}" ]
then
if [ "${quattor_version}" = "HEAD" ]
then
pr_target=master
else
pr_target=${quattor_version}
fi
elif [ -n "$(echo ${pr_target} | egrep '^template-library-')" ]
then
# Old tag name format: renormalize to the new format without the template-library- prefix
[ ${verbose} -eq 1 ] && echo "pr_target: template-library- prefix removed (legacy tag format)"
pr_target=$(echo ${pr_target} | sed 's/^template-library-//')
fi
[ ${verbose} -eq 1 ] && echo "Pull request to import: repository=${pr_repo}, user=${pr_user}, branch=${pr_branch}, target=${pr_target}"
fi
# --continuous-integration: check environment
# NOTE: currently only Travis is supported
if [ ${use_ci_work_dir} -eq 1 ]
then
if [ -n "${TRAVIS_BUILD_DIR}" ]
then
ci_work_dir=${TRAVIS_BUILD_DIR}
else
echo "Not in a Travis build environment: --continuous-integration invalid"
usage
fi
# $TRAVIS_REPO_SLUG contains the GitHub user/repo (e.g. quattor/template-library-core)
pr_repo=$(echo $TRAVIS_REPO_SLUG | sed -e 's%^quattor/%%')
if [ -z "${pr_repo}" ]
then
echo "Current PR repository could not be retrieved from TRAVIS_REPO_SLUG env var"
exit 2
fi
# TRAVIS_BRANCH contains the PR branch target
if [ -n "${TRAVIS_BRANCH}" ]
then
pr_target=${TRAVIS_BRANCH}
else
echo "TRAVIS_BRANCH not defined: not a Travis build environment, --continuous-integration invalid"
exit 2
fi
fi
if [ ${add_legacy} -ne 1 ]
then
ignore_branch_patterns="${ignore_branch_patterns} legacy$"
fi
if [ ${add_spma_templates} -ne 1 ]
then
ignore_branch_patterns="${ignore_branch_patterns} -spma$"
fi
if [ ${add_rc_templates} -ne 1 ]
then
ignore_version_patterns="${ignore_version_patterns} -rc[0-9]+$"
fi
# Check (or remove) the template library destination directory.
if [ -d ${template_lib_root} ]
then
if [ ${remove_tl_dir} -eq 0 ]
then
echo "Directory $template_lib_root already exists. Remove it or use -F"
exit
else
echo "Removing ${template_lib_root}..."
rm -Rf ${template_lib_root}
fi
fi
mkdir -p ${template_lib_root}
# Check (or remove+create) if the destination directory for Git clones exists
if [ -d ${git_clone_root} ]
then
if [ ${remove_tl_dir} -eq 0 ]
then
echo "Directory ${git_clone_root} already exists. Remove it or use -F"
exit
else
echo "Removing ${git_clone_root}..."
rm -Rf ${git_clone_root}
fi
fi
mkdir ${git_clone_root}
final_status=0 # Assume success
for repo in ${git_repo_list}
do
repo_name_variable=${repo}_git_repo
branch_variable=${repo}_branch_pattern
dest_dir_variable=${repo}_dest_dir
rename_master_variable=${repo}_rename_master
use_tags_variable=${repo}_use_tags
ignore_version_variable=${repo}_ignore_version
tags_ignore_pattern_variable=${repo}_tags_ignore_pattern
repo_name=${!repo_name_variable}
repo_url=${quattor_git_url}/${repo_name}.git
repo_dir=${git_clone_root}/${repo_name}
branch_pattern=${!branch_variable}
# If quattor_version=HEAD, always use the HEAD revision of the branches
# matching the branch pattern
if [ "${quattor_version}" = "HEAD" ]
then
use_tags_default=0
fi
use_tags=${!use_tags_variable}
if [ -z ${use_tags} ]
then
use_tags=${use_tags_default}
fi
ignore_version=${!ignore_version_variable}
if [ -z ${ignore_version} ]
then
ignore_version=${ignore_version_default}
fi
tags_ignore_pattern=${!tags_ignore_pattern_variable}
if [ -z ${tags_ignore_pattern} ]
then
if [ -z "$(echo ${branch_pattern} | egrep -- ${tags_ignore_pattern_pattern})" ]
then
tags_ignore_pattern=0
else
tags_ignore_pattern=1
fi
fi
git_clone_dir=${git_clone_root}/${repo}
if [ $?{!rename_master_variable} ]
then
master_dir_name=${!rename_master_variable}
fi
echo Cloning Git repository ${repo_url} in ${repo_dir}...
export GIT_WORK_TREE=${repo_dir}
export GIT_DIR=${repo_dir}/.git
silent_command git clone --no-checkout ${repo_url} ${GIT_DIR}
if [ $? -ne 0 ]
then
echo "Error cloning Git repository ${repo_url}"
exit 10
fi
# In fact branch_pattern can be a regexp matched against existing branch names
if [ ${use_tags} -eq 1 ]
then
if [ -n "${quattor_version}" -a ${ignore_version} -eq 0 ]
then
if [ ${tags_ignore_pattern} -eq 1 ]
then
branch_pattern="${quattor_version}$"
else
branch_pattern="${branch_pattern}-${quattor_version}$"
fi
fi
[ ${verbose} -eq 1 ] && echo "Using tags rather than branches for repository ${repo} (branch pattern=${branch_pattern})"
branch_list=$(git tag | egrep -- "${branch_pattern}")
# If version is HEAD, need to add master branch to all the other tags
if [ "${quattor_version}" = "HEAD" ]
then
branch_list="${branch_list} master"
fi
else
branch_list=$(git branch -r | egrep "origin/${branch_pattern}" | grep -v HEAD)
fi
[ ${verbose} -eq 1 -o ${list_branches} -eq 1 ] && echo -e "Branches/tags found in ${repo}:\n${branch_list}"
[ ${list_branches} -eq 1 ] && continue
# Will be set to 1 if at least
branch_found=0
for remote_branch in ${branch_list}
do
# Remove origin/ from the branch name
branch_or_tag=$(echo ${remote_branch} | sed -e 's#^.*origin/##')
# when using tags, branch_dir contains the branch name retrieved from the tag and
# tag_dir contains the tag version derived from the tag name with the prefix
# (e.g. template-library-) removed. Define a non empty default value.
# Also define the branch/tag name to use to compare with pr_target (--pull-request):
# when using a branch, use branch name (branch_or_tag) and when using a tag, use
# tag_dir before the rename applied to master.
tag_dir="undefined_tag"
if [ ${use_tags} -eq 0 ]
then
pr_actual_target=${branch_or_tag}
if [ "${branch_or_tag}" = "master" -a $?{master_dir_name} ]
then
branch_dir=${master_dir_name}
else
branch_dir=${branch_or_tag}
fi
else
branch_dir=$(echo ${branch_or_tag} | sed -e 's%-\([0-9\.]\+\)\+\(-[0-Z]\+\)*$%%')
tag_dir=$(echo ${branch_or_tag} | sed -e "s%^.*${branch_dir}-%%")
pr_actual_target=${tag_dir}
# master is normally not a tag name but master branch can be added to
# other tags when the version is HEAD (for example for template-library-core)
if [ "${tag_dir}" = "master" -a $?{master_dir_name} ]
then
tag_dir=${master_dir_name}
fi
# If branch_dir = template_library, reset it to tag_dir (old tag format)
if [ "${branch_dir}" == "template-library" ]
then
branch_dir=${tag_dir}
fi
# If branch_dir is not tag_dir or master, indicates a repository without a master
# branch (e.g. template-library-grid). In this case, pr_actual_target must be
# redefined as branch_dir-tag_dir, as the tag name is prefixed by the branch name (branch_dir).
if [ "${branch_dir}" != "master" -a "${branch_dir}" != "${tag_dir}" ]
then
pr_actual_target="${branch_dir}-${tag_dir}"
fi
fi
[ ${verbose} -eq 1 ] && echo branch_dir=${branch_dir}, tag_dir=${tag_dir}, pr_acctual_target=${pr_actual_target}
# Check if the branch should be ignored
# Ignore pattern can be against branch name or version
ignore_branch=0
for ignore_pattern in ${ignore_branch_patterns}
do
[ ${verbose} -eq 1 ] && echo "Branch=${remote_branch}, ignore_pattern = >>${ignore_pattern}<<"
if [ -n "$(echo ${branch_dir} | egrep -- ${ignore_pattern})" -a "${branch_dir}" != "${quattor_version}" ]
then
[ ${verbose} -eq 1 ] && echo "Branch ${remote_branch} ignored"
ignore_branch=1
break
fi
done
# Always go through version patterns, even if branch should be ignored based on branch pattern: harmless
for ignore_pattern in ${ignore_version_patterns}
do
[ ${verbose} -eq 1 ] && echo "Branch=${remote_branch}, version=${tag_dir}, ignore_version_pattern = >>${ignore_pattern}<<"
if [ -n "$(echo ${tag_dir} | egrep -- ${ignore_pattern})" -a "${tag_dir}" != "${quattor_version}" ]
then
[ ${verbose} -eq 1 ] && echo "Branch ${remote_branch} (version ${tag_dir}) ignored"
ignore_branch=1
break
fi
done
if [ ${ignore_branch} -eq 1 ]
then
continue
fi
branch_found=1
# If --continuous-inegration and the current repo/target matches the CI context, just define src_dir
# to be what has been checked out as part of the CI.
[ ${verbose} -eq 1 ] && echo "use_ci_work_dir=${use_ci_work_dir}, repo_name=${repo_name}, pr_repo=${pr_repo}, pr_target=${pr_target}, pr_actual_target=${pr_actual_target}"
if [ ${use_ci_work_dir} -eq 1 -a "${pr_repo}" = "${repo_name}" -a "${pr_target}" = "${pr_actual_target}" ]
then
src_dir=${ci_work_dir}
else
src_dir=${repo_dir}
# Checkout the appropriate branch or tag
silent_command git checkout ${branch_or_tag}
# Import pull request changes if requested and if the PR target matches the current repository/branch
if [ -n "${pull_request}" ]
then
[ ${verbose} -eq 1 ] && echo "Checking if pull request applies to current branch/tag (pr_repo=${pr_repo}, pr_target=${pr_target}, pr_actual_target=${pr_actual_target})..."
if [ "${pr_repo}" = "${repo_name}" -a "${pr_target}" = "${pr_actual_target}" ]
then
echo "Merging pull request from ${pr_user}/${pr_branch} into branch/tag ${branch_or_tag}..."
pr_remote=${git_url_root}/${pr_user}/${pr_repo}
silent_command git remote add ${pr_user} ${pr_remote}
if [ $? -ne 0 ]
then
echo "Error adding remote ${pr_remote} to repository ${repo_name}"
exit 11
fi
# git pull has a problem if GIT_WORK_TREE is defined (Git 1.7.1) but is not the current directory.
# As a workaround, cd into $GIT_WORK_TREE before the command and restore previous directory
# after the pull.
silent_command pushd $GIT_WORK_TREE
silent_command git pull ${pr_user} ${pr_branch}
if [ $? -ne 0 ]
then
echo "Error merging ${pr_user}/${pr_branch} into branch/tag ${branch_or_tag} of repository ${repo_name}"
exit 11
fi
silent_command popd
else
if [ ${verbose} -eq 1 ]
then
if [ "${pr_repo}" != "${repo_name}" ]
then
echo "Pull request doesn't match current repository (${repo_name}). Ignored"
elif [ "${pr_branch}" != "${branch_or_tag}" ]
then
echo "Pull request doesn't match current branch/tag (${branch_or_tag}) in repository ${repo_name}. Ignored"
fi
fi
fi
fi
fi
# Copy repository contents to the appropriate target directory.
# Either %BRANCH% or %TAG% can be specified: no attempt to check that both are not use at the same time...
dest_dir=${template_lib_root}/$(echo ${!dest_dir_variable} | sed -e "s#%BRANCH%#${branch_dir}#" | sed -e "s#%TAG%#${tag_dir}#")
# os repository is a special case: '-spma' suffix must be removed to build the destination directory
if [ ${repo} = "os" ]
then
dest_dir=$(echo ${dest_dir} | sed -e 's/-spma//')
fi
echo Copying Git branch ${branch_or_tag} contents to ${dest_dir}...
mkdir -p ${dest_dir}
cp -R ${src_dir}/* ${dest_dir}
# Remove potentially untracked files to prevent errors at next checkout
silent_command git clean -f -d
if [ $? -ne 0 ]
then
echo "Error removing untracked files from branch ${branch_or_tag} in repository ${repo}"
exit 11
fi
done # branch loop
# Display an error if no branch matching the selection were found but continue.
# Probably a wrong tag has been used...
if [ ${branch_found} -eq 0 ]
then
final_status=100
echo "Error: no branch found matching Quattor version ${quattor_version}"
fi
# Except if --debug has been specified, remove Git clone created
if [ ${verbose} -eq 0 -a -d ${repo_dir} ]
then
rm -Rf ${repo_dir}
fi
done # repository loop
# If version request is HEAD, update the cluster.build.properties file in each
# cluster example to use the master branch of template-library-core rather than
# the last release.
if [ ${list_branches} -eq 0 -a "${quattor_version}" = "HEAD" ]
then
property_files=$(find ${template_lib_root}/clusters -name cluster.build.properties)
if [ -n "${property_files}" ]
then
if [ $?{core_rename_master} ]
then
quattor_dev_version=${core_rename_master}
else
quattor_dev_version=master
fi
echo "Updating cluster.build.properties files to use Quattor version ${quattor_dev_version} ($(echo ${property_files}))..."
sed -i "s%\squattor/\S\+\s% quattor/${quattor_dev_version} %" ${property_files}
else
echo "Warning: no cluster.build.properties file found. Examples may be incomplete"
fi
fi
# Delete directory used for storing Git clones if empty
if [ -z "$(ls ${git_clone_root})" ]
then
rmdir ${git_clone_root}
fi
exit ${final_status}