Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

943 lines (854 sloc) 33.321 kb
#!/bin/bash
# Library of useful functions for building Crowbar.
# This is sourced by build_crowbar.sh and any other interested parties.
# If we have already been sourced in the current env, don't source us again.
[[ $CROWBAR_BUILD_SOURCED = true ]] && exit 0
[[ $DEBUG ]] && {
set -x
export PS4='${BASH_SOURCE}@${LINENO}(${FUNCNAME[0]}): '
}
# We might use lots and lots of open files. Bump our open FD limits.
ulimit -Sn unlimited
# Hashes to hold our "interesting" information.
# Key = barclamp name
# Value = whatever interesting thing we are looking for.
declare -A BC_DEPS BC_GROUPS BC_PKGS BC_EXTRA_FILES BC_OS_DEPS BC_GEMS
declare -A BC_REPOS BC_PPAS BC_RAW_PKGS BC_BUILD_PKGS BC_QUERY_STRINGS
declare -A BC_SMOKETEST_DEPS BC_SMOKETEST_TIMEOUTS BC_BUILD_CMDS
declare -A BC_SUPERCEDES BC_SRC_PKGS
# Build OS independent query strings.
BC_QUERY_STRINGS["deps"]="barclamp requires"
BC_QUERY_STRINGS["groups"]="barclamp member"
BC_QUERY_STRINGS["extra_files"]="extra_files"
BC_QUERY_STRINGS["build_cmd"]="build_cmd"
BC_QUERY_STRINGS["os_support"]="barclamp os_support"
BC_QUERY_STRINGS["gems"]="gems pkgs"
BC_QUERY_STRINGS["test_deps"]="smoketest requires"
BC_QUERY_STRINGS["test_timeouts"]="smoketest timeout"
BC_QUERY_STRINGS["supercedes"]="barclamp supercedes"
# By default, do not try to update the cache or the metadata.
# These will be unset if --update-cache is passed to the build.
ALLOW_CACHE_UPDATE=false
ALLOW_CACHE_METADATA_UPDATE=false
get_barclamp_info() {
local bc yml_file line query newdeps dep d i
local new_barclamps=()
# Pull in interesting information from all our barclamps
cd "$CROWBAR_DIR"
for bc in barclamps/*; do
[[ -d "$bc" ]] || continue
bc=${bc##*/}
debug "Reading metadata for $bc barclamp."
is_barclamp "$bc" || {
echo "$bc is not a barclamp, skipping."
continue
}
yml_file="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
[[ $bc = crowbar ]] || BC_DEPS["$bc"]+="crowbar "
for query in "${!BC_QUERY_STRINGS[@]}"; do
while read line; do
[[ $line = nil ]] && continue
case $query in
deps) is_in "$line "${BC_DEPS["$bc"]} || \
BC_DEPS["$bc"]+="$line ";;
groups) is_in "$line" ${BC_GROUPS["$bc"]} ||
BC_GROUPS["$line"]+="$bc ";;
pkgs|os_pkgs) is_in "$line" ${BC_PKGS["$bc"]} || \
BC_PKGS["$bc"]+="$line ";;
src_pkgs) is_in "$line" ${BC_SRC_PKGS["$bc"]} || \
BC_SRC_PKGS["$bc"]+="$line ";;
extra_files) BC_EXTRA_FILES["$bc"]+="$line\n";;
os_support) BC_OS_SUPPORT["$bc"]+="$line ";;
gems) BC_GEMS["$bc"]+="$line ";;
repos|os_repos) BC_REPOS["$bc"]+="$line\n";;
ppas|os_ppas) [[ $PKG_TYPE = debs ]] || \
die "Cannot declare a PPA for $PKG_TYPE!"
BC_REPOS["$bc"]+="ppa $line\n";;
build_pkgs|os_build_pkgs) BC_BUILD_PKGS["$bc"]+="$line ";;
raw_pkgs|os_raw_pkgs|pkg_sources|os_pkg_sources) BC_RAW_PKGS["$bc"]+="$line ";;
test_deps) BC_SMOKETEST_DEPS["$bc"]+="$line ";;
os_build_cmd|build_cmd) [[ ${BC_BUILD_CMDS["$bc"]} ]] && \
die "Only one os_build_cmd stanza per OS per barclamp allowed!"
BC_BUILD_CMDS["$bc"]="$line";;
test_timeouts) BC_SMOKETEST_TIMEOUTS["$bc"]+="$line ";;
supercedes)
[[ ${BC_SUPERCEDES[$line]} ]] && \
die "$line is already superceded by ${BC_SUPERCEDES[$line]}!"
BC_SUPERCEDES["$line"]="$bc";;
*) die "Cannot handle query for $query."
esac
done < <("$CROWBAR_DIR/parse_yml.rb" \
"$yml_file" \
${BC_QUERY_STRINGS["$query"]} 2>/dev/null)
done
done
cd -
debug "Analyzing barclamp group membership"
# If any barclamps need group expansion, do it.
for bc in "${!BC_DEPS[@]}"; do
newdeps=''
for dep in ${BC_DEPS["$bc"]}; do
if [[ $dep = @* ]]; then
[[ ${BC_GROUPS["${dep#@}"]} ]] || \
die "$bc depends on group ${dep#@}, but that group does not exist!"
for d in ${BC_GROUPS["${dep#@}"]}; do
newdeps+="$d "
done
else
newdeps+="$dep "
fi
done
BC_DEPS["$bc"]="$newdeps"
done
# Group-expand barclamps if needed, and unset groups after they are expanded
for i in "${!BARCLAMPS[@]}"; do
bc="${BARCLAMPS[$i]}"
if [[ $bc = @* ]]; then
[[ ${BC_GROUPS["${bc#@}"]} ]] || \
die "No such group ${bc#@}!"
BARCLAMPS+=(${BC_GROUPS["${bc#@}"]})
unset BARCLAMPS[$i]
else
is_barclamp "$bc" || die "$bc is not a barclamp!"
fi
done
BARCLAMPS=("${BARCLAMPS[@]//@*}")
# Pull in dependencies for the barclamps.
# Everything depends on the crowbar barclamp, so include it first.
new_barclamps=("crowbar")
while [[ t = t ]]; do
for bc in "${BARCLAMPS[@]}"; do
if [[ ${BC_SUPERCEDES[$bc]} ]]; then
debug "$bc is superceded by ${BC_SUPERCEDES[$bc]}. Skipping."
continue
fi
for dep in ${BC_DEPS["$bc"]}; do
dep="${BC_SUPERCEDES[$dep]:-$dep}"
is_barclamp "$bc" || die "$bc depends on $dep, which is not a barclamp!"
is_in "$dep" "${new_barclamps[@]}" && continue
new_barclamps+=("$dep")
done
is_barclamp "$bc" || die "$bc is not a barclamp!"
is_in "$bc" "${new_barclamps[@]}" || new_barclamps+=("$bc")
done
[[ ${BARCLAMPS[*]} = ${new_barclamps[*]} ]] && break
BARCLAMPS=("${new_barclamps[@]}")
done
}
[[ $CROWBAR_BUILD_PID ]] || export CROWBAR_BUILD_PID=$$
export CLEANUP_LOCK="$CROWBAR_DIR/.cleanup.lock"
cleanup_cmds=()
git_managed_cache() [[ -d $CACHE_DIR/.git ]]
with_build_lock() {
flock -n 65 || die "Could not grab build lock!"
"$@"
} 65>/tmp/.build_crowbar.lock
flat_checkout() [[ -d $CROWBAR_DIR/releases ]]
# Our general cleanup function. It is called as a trap whenever the
# build script exits, and it's job is to make sure we leave the local
# system in the same state we cound it, modulo a few calories of wasted heat
# and a shiny new .iso.
cleanup() {
flock -n 70 || exit 1
# Clean up any stray mounts we may have left behind.
# The paranoia with the grepping is to ensure that we do not
# inadvertently umount everything.
if [[ $BASH_SUBSHELL -gt 0 || $BASHPID != $CROWBAR_BUILD_PID ]]; then
kill -INT $CROWBAR_BUILD_PID
exit 1
fi
for c in "${cleanup_cmds[@]}"; do
$c || res=1
done
GREPOPTS=()
[[ $CACHE_DIR ]] && GREPOPTS=(-e "$CACHE_DIR")
[[ $IMAGE_DIR && $CACHE_DIR =~ $IMAGE_DIR ]] && GREPOPTS+=(-e "$IMAGE_DIR")
[[ $BUILD_DIR && $CACHE_DIR =~ $BUILD_DIR ]] && GREPOPTS+=(-e "$BUILD_DIR")
[[ $CROWBAR_DIR && -d $CROWBAR_DIR/testing ]] && GREPOPTS+=(-e "$CROWBAR_DIR/testing")
[[ $CHROOT && $CACHE_DIR =~ $CHROOT ]] && GREPOPTS+=(-e "$CHROOT")
if [[ $GREPOPTS ]]; then
while read dev fs type opts rest; do
sudo umount -d -l "$fs"
done < <(tac /proc/self/mounts |grep "${GREPOPTS[@]}")
fi
# If the build process spawned a copy of webrick, make sure it is dead.
[[ $webrick_pid && -d /proc/$webrick_pid ]] && kill -9 $webrick_pid
# clean up after outselves from merging branches, if needed.
[[ $CI_BARCLAMP ]] && {
if ! in_repo git submodule update -N "barclamps/$CI_BARCLAMP"; then
in_ci_barclamp git checkout -f master
fi
if ! in_ci_barclamp git branch -D ci-throwaway-branch; then
in_ci_barclamp git checkout -f master
in_ci_barclamp git branch -D ci-throwaway-branch
fi
}
cd "$CROWBAR_DIR"
if [[ $THROWAWAY_BRANCH ]]; then
# Check out the branch we started the build process, and then
# nuke whatever throwaway branch we may have created.
git checkout -f "${CURRENT_BRANCH#refs/heads/}" &>/dev/null
git branch -D "$THROWAWAY_BRANCH" &>/dev/null
fi
# If we saved unadded changes, resurrect them.
[[ $THROWAWAY_STASH ]] && git stash apply "$THROWAWAY_STASH" &>/dev/null
# Nuke any wild caches.
for f in "${CACHE_DIR%/*}/.crowbar_temp_cache"*; do
[[ -d $f ]] || continue
sudo rm -rf "$f"
done
if [[ -d $CACHE_DIR && -d $CACHE_DIR/.git ]]; then
cd "$CACHE_DIR"
if ! in_cache git diff-index --cached --quiet HEAD; then
in_cache git commit -m "Updated by build_crowbar.sh @ $(date) for ${OS_TOKEN}"
echo "The crowbar build cache has been updated, and the updates have"
echo "been comitted back to the cache. Please push any changes."
fi
fi
wait
flock -u 70
rm "$CROWBAR_DIR/".*.lock
exit $res
} 70> "$CLEANUP_LOCK"
# Arrange for cleanup to be called at the most common exit points.
trap cleanup 0 INT QUIT TERM
# Test to see if $1 is in the rest of the args.
is_in() {
local t="(^| )$1( |\$)"
shift
[[ $* =~ $t ]]
}
# Run a command in our chroot environment.
in_chroot() { sudo -H /usr/sbin/chroot "$CHROOT" /bin/bash -l -c "$*"; }
# A little helper function for doing bind mounts.
bind_mount() {
[[ -d $2 ]] || mkdir -p "$2"
grep -q "$2" /proc/self/mounts || sudo mount --bind "$1" "$2"
}
# Write a string with \n translated to a real line break.
write_lines() { [[ $1 ]] && printf "%b" "$1"; }
# Read base repository information from the *.list files.
read_base_repos() {
for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
[[ -f $pkgfile ]] || continue
while read line; do
line=${line%%#*}
[[ $line ]] || continue
if [[ $line = repository* ]]; then
REPOS+=("${line#* }")
else
echo "$line in $pkgfile has been superceded by crowbar.yml metadata." >&2
die "Please migrate $line into the appropriate barclamp config file."
fi
done <"$pkgfile"
done
}
# Worker function for all_deps
__all_deps() {
local dep
if [[ ${BC_DEPS["$1"]} ]]; then
for dep in ${BC_DEPS["$1"]}; do
is_in "$dep" "${deps[@]}" && continue
__all_deps "$dep"
done
fi
is_in "$1" "${deps[@]}" || deps+=("$1")
return 0
}
# Given a barclamp, echo all of the dependencies of that barclamp
all_deps() {
local deps=() dep
__all_deps "$1"
echo "${deps[*]}"
}
# A couple of utility functions for comparing version numbers.
num_re='^[0-9]+$'
__cmp() {
[[ $1 || $2 ]] || return 255 # neither 1 nor 2 or set, we are done
[[ $1 && ! $2 ]] && return 2 # 1 is set and 2 is not, 1 > 2
[[ ! $1 && $2 ]] && return 0 # 2 is set and 1 is not, 1 < 2
local a="$1" b="$2"
if [[ $a =~ $num_re && $b =~ $num_re ]]; then #both numbers, numeric cmp.
# make sure leading zeros do not confuse us
a=${a##0} b=${b##0}
((${a:=0} > ${b:=0})) && return 2
(($a < $b)) && return 0
return 1
else # string compare
[[ $a > $b ]] && return 2
[[ $a < $b ]] && return 0
return 1
fi
}
# Compare a version string, handling mixed numeric and alpha sequences
# as appropriate.
vercmp(){
# $1 = version string of first package
# $2 = version string of second package
# Returns 0 if $1 > $2, 1 otherwise.
# urldecode the version strings.
local a="$(printf "%b" "${1//%/\\x}")"
local b="$(printf "%b" "${2//%/\\x}")"
# prepend a version epoch if one is not already there.
[[ $a =~ ^[0-9]+: ]] || a="0:$a"
[[ $b =~ ^[0-9]+: ]] || b="0:$b"
local ver1=()
local ver2=()
local i=0
# split the version strings into arrays using the usual field splitters
IFS=':.-_ +' read -rs -a ver1 <<< "$a"
IFS=':.-_ +' read -rs -a ver2 <<< "$b"
for ((i=0;;i++)); do
__cmp "${ver1[$i]}" "${ver2[$i]}"
case $? in
2) return 0;;
0) return 1;;
255) return 1;;
esac
done
}
# Index the pool of packages in the CD.
index_cd_pool() {
# Scan through our pool to find pkgs we can easily omit.
local pkgname='' pkg='' cache="$CACHE_DIR/$OS_TOKEN/iso-packages"
if [[ $ISO_LIBRARY/$ISO -nt $cache ]]; then
mkdir -p "${cache%/*}"
> "$cache"
while read pkg; do
[[ -f $pkg ]] && is_pkg "$pkg" || continue
pkgname="$(pkg_name "$pkg")"
CD_POOL["$pkgname"]="${pkg}"
echo "CD_POOL[\"$pkgname\"]=\"${pkg}\"" >> "$cache"
done < <(find "$(find_cd_pool)" -type f)
else
. "$cache"
fi
}
# Make a chroot environment for package-fetching purposes.
make_chroot() {
[[ -f $CHROOT/etc/resolv.conf ]] && return 0
local bc repo
debug "Making utility chroot"
sudo mkdir -p "$CHROOT/$CHROOT_PKGDIR"
sudo mkdir -p "$CHROOT/$CHROOT_GEMDIR"
__make_chroot
if [[ $http_proxy || $https_proxy || $no_proxy ]]; then
local __f
__f=$(mktemp /tmp/proxy-XXXXX.sh) || \
die "Canot make utility chroot -- error adding proxies."
[[ $http_proxy ]] && echo "http_proxy=\"$http_proxy\"" >> "$__f"
[[ $https_proxy ]] && echo "https_proxy=\"$https_proxy\"" >> "$__f"
[[ $no_proxy ]] && echo "no_proxy=\"$no_proxy\"" >> "$__f"
sudo cp "$__f" "$CHROOT/etc/environment"
[[ $http_proxy ]] && echo "export http_proxy" >> "$__f"
[[ $https_proxy ]] && echo "export https_proxy" >> "$__f"
[[ $no_proxy ]] && echo "export no_proxy" >> "$__f"
sudo cp "$__f" "$CHROOT/etc/profile.d/proxy.sh"
fi
in_chroot ln -s /proc/self/mounts /etc/mtab
if [[ $ALLOW_CACHE_UPDATE = true ]]; then
read_base_repos
# Add our basic repositories
add_repos "${REPOS[@]}" || \
die "Could not add base repositories for $OS_TOKEN chroot!"
# Add the repos from the barclamps.
# We do it here because importing the metadata takes forever
# if we refresh on every barclamp.
for bc in "${BARCLAMPS[@]}"; do
while read repo; do
add_repos "$repo"
done < <(write_lines "${BC_REPOS[$bc]}")
done
fi
add_offline_repos
chroot_update
}
stage_pkgs() {
# $1 = cache to copy from.
# $2 = location to copy to
local pkg pkgname pkg_t
local -A to_copy STAGED_POOL
while read pkg; do
# If it is not a package, skip it.
is_pkg "$pkg" || continue
pkgname="$(pkg_name "$pkg")"
# Check to see if it is in the CD pool.
pkg_t="${CD_POOL["$pkgname"]}"
# If it is, and the one in the pool is not older than this one,
# skip it.
if [[ $pkg_t && -f $pkg_t ]] && ( ! pkg_cmp "$pkg" "$pkg_t" ); then
#debug "Skipping copy of ${pkg##*/}, it is on the install media"
# if we are shrinking our ISO, make sure this one is in.
[[ $SHRINK_ISO = true ]] && INSTALLED_PKGS["$pkgname"]="true"
continue
fi
# Now check to see if we have already staged it
pkg_t="${STAGED_POOL["$pkgname"]}"
if [[ $pkg_t && -f $pkg_t ]]; then
# We have already staged it. Check to see if ours is newer than
# the one already staged.
if pkg_cmp "$pkg" "$pkg_t"; then
# We are newer. Delete the old one, copy us,
# and update $STAGED_POOL
#debug "Replacing ${pkg_t##*/} with ${pkg##*/}"
[[ -f "$pkg_t" ]] && rm -f "$pkg_t"
[[ ${to_copy["$pkg_t"]} ]] && unset to_copy["$pkg_t"]
to_copy["$pkg"]="true"
STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
fi
else
# We have not seen this package before. Copy it.
to_copy["$pkg"]="true"
cp "$pkg" "$2"
STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
fi
done < <(find "$1" -type f)
[[ ${!to_copy[*]} ]] && cp "${!to_copy[@]}" "$2"
}
cache_add() {
# $1 = file to add.
# $2 = location to store it in the cache
cp "$1" "$2" || \
die "Cannot save $1 in $2!"
debug "cache_add: Saved $1 in $2"
if [[ $CURRENT_CACHE_BRANCH ]]; then
CACHE_NEEDS_COMMIT=true
in_cache git add "${2#${CACHE_DIR}/}"
fi
}
cache_rm() {
if [[ $CURRENT_CACHE_BRANCH ]]; then
CACHE_NEEDS_COMMIT=true
in_cache git rm -f "${1#${CACHE_DIR}/}"
fi
rm -f "$1"
}
make_barclamp_pkg_metadata() {
[[ $ALLOW_CACHE_UPDATE != true && \
$ALLOW_CACHE_METADATA_UPDATE != true ]] && return 0
[[ -d $CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs ]] || return 0
if [[ $force_update != true ]]; then
__barclamp_pkg_metadata_needs_update "$1" || return 0
fi
[[ $ALLOW_CACHE_METADATA_UPDATE = false ]] && \
die "Need to update cache metadata for $1, but --no-metadata-update passed."
debug "Updating package cache metadata for $1"
make_chroot
[[ $OS_METADATA_PKGS ]] && {
chroot_install $OS_METADATA_PKGS
unset OS_METADATA_PKGS
}
sudo mount --bind "$(readlink -f "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs")" \
"$CHROOT/mnt"
__make_barclamp_pkg_metadata "$1"
sudo umount "$CHROOT/mnt"
}
install_build_packages() {
chroot_install ${BC_BUILD_PKGS["$1"]}
}
update_barclamp_src_pkg_cache() {
[[ ${BC_SRC_PKGS["$1"]} ]] || return 0
local CHROOT_PKGDIR="tmp/$1"
in_chroot mkdir -p "/$CHROOT_PKGDIR"
chroot_fetch_source ${BC_SRC_PKGS["$1"]}
local p
for p in "$CHROOT/$CHROOT_PKGDIR/"*; do
[[ -f $p ]] || continue
cache_add "$p" "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/${p##*/}"
done
}
# Update the package cache for a barclamp.
update_barclamp_pkg_cache() {
# $1 = barclamp we are working with
local bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" pkg dest bc
local -A pkgs
# Wipe out the packages already in the chroot package directory.
while read pkg; do
is_pkg "$pkg" && sudo rm -f "$pkg"
done < <(find "$CHROOT/$CHROOT_PKGDIR" -type f)
for bc in $(all_deps "$1"); do
[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] || continue
sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
"$CHROOT/$CHROOT_PKGDIR"
done
# Remember what packages we already have.
while read pkg; do
is_pkg "$pkg" || continue
pkgs["$pkg"]="true"
done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
[[ ${BC_BUILD_PKGS["$1"]} ]] && install_build_packages "$1"
chroot_fetch ${BC_PKGS["$1"]} || \
die "Could not fetch packages required by barclamp $1"
mkdir -p "$bc_cache"
while read pkg; do
is_pkg "$CHROOT/$CHROOT_PKGDIR/$pkg" || continue
[[ ${pkgs["$pkg"]} = true ]] && continue
if [[ ${pkg%/*} != '.' ]]; then
[[ -d $bc_cache/${pkg%/*} ]] || mkdir -p "$bc_cache/${pkg%/*}"
[[ -f $bc_cache/${pkg##*/} ]] && cache_rm "$bc_cache/${pkg##*/}"
fi
cache_add "$CHROOT/$CHROOT_PKGDIR/$pkg" "$bc_cache/${pkg//%3a/:}"
done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
local force_update=true
update_barclamp_src_pkg_cache "$1"
make_barclamp_pkg_metadata "$1"
}
# Update the gem cache for a barclamp
update_barclamp_gem_cache() {
local -A gems
local gemname gemver gemopts bc gem
local bc_cache="$CACHE_DIR/barclamps/$1/gems"
install_rubygems || die "Could not install gem, cannot update gem cache!"
CHROOT_GEMDIR="$(in_chroot /usr/bin/gem environment gemdir | grep -v home)"
debug "update_barclamp_gem_cache: starting to ${1} to $bc_cache"
# Wipe out the caches.
debug "destroy that cruft"
( cd "$CHROOT/$CHROOT_GEMDIR" && sudo rm -rf * )
debug "$(ls -al ${CHROOT}/${CHROOT_GEMDIR})"
# if we have deb or gem caches, copy them back in.
# Make sure we copy the caches for all our dependent barclamps.
in_chroot mkdir -p "${CHROOT}/${CHROOT_GEMDIR}" || die "could not make ${CHROOT}/${CHROOT_GEMDIR}"
debug "copy from $CACHE_DIR/barclamps/$bc/gems/. to $CHROOT/$CHROOT_GEMDIR"
for bc in $(all_deps "$1"); do
debug "are there any gems? sudo ls $CACHE_DIR/barclamps/$bc/gems/."
debug "$(sudo ls ${CACHE_DIR}/barclamps/${bc}/gems/.)"
[[ -d "$CACHE_DIR/barclamps/$bc/gems/." ]] && \
sudo cp -va "$CACHE_DIR/barclamps/$bc/gems/." \
"$CHROOT/$CHROOT_GEMDIR"
debug "did it make it into chroot?"
debug "$(in_chroot ls -al /${CHROOT_GEMDIR})"
done
debug "create a hash of gems"
while read gem; do
debug "gem ${gem} is from ${CHROOT}/${CHROOT_GEMDIR}"
debug "gem ${gem} sure does end in .gem"
[[ $gem = *.gem ]] || continue
gems["$gem"]="true"
done < <(sudo find "$CHROOT/$CHROOT_GEMDIR" -type f)
# install any build dependencies we need.
# Grab the gems needed for this barclamp.
debug "installing gems needed in this barclamp $1"
for gem in ${BC_GEMS["$1"]}; do
debug "gem ${gem} is required by barclamp $1"
if [[ $gem =~ $GEM_RE ]]; then
gemname="${BASH_REMATCH[1]}"
gemver="${BASH_REMATCH[2]}"
else
gemname="$gem"
gemver=''
fi
gemopts=(install --no-ri --no-rdoc)
[[ $gemver ]] && gemopts+=(--version "'= ${gemver}'")
[[ $http_proxy ]] && gemopts+=(-p "$http_proxy")
in_chroot /usr/bin/gem "${gemopts[@]}" "$gemname"
done
debug "finished installing the gems"
# Save our updated gems and pkgs in the cache for later.
mkdir -p "$bc_cache" || die "Failed to make gem cache ${bc_cache}"
debug "made the gem cache ${bc_cache}"
debug "adding gems to cache"
debug "$(sudo find $CHROOT/$CHROOT_GEMDIR -name '*.gem')"
while read gem; do
debug "gem ${gem} is from $CHROOT/$CHROOT_GEMDIR"
[[ $gem = *.gem ]] || continue
[[ ${gems["$gem"]} = "true" ]] && continue
debug "gem hash ${gem} = ${gems[${gem}]} "
debug "update_barclamp_gem_cache: adding ${gem} to $bc_cache"
cache_add "$gem" "$bc_cache"
done < <(sudo find "$CHROOT/$CHROOT_GEMDIR" -type f -name '*.gem')
debug "done adding gems to cache"
}
# Fetch any raw packages we do not already have.
update_barclamp_raw_pkg_cache() {
local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
mkdir -p "$bc_cache"
# Fetch any raw_pkgs we were asked to.
for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
[[ -f $bc_cache/${pkg##*/} ]] && continue
echo "Caching $pkg:"
curl -L -o "$bc_cache/${pkg##*/}" "$pkg"
[[ $CURRENT_CACHE_BRANCH ]] && in_cache git add "$bc_cache/${pkg##*/}"
done
touch "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
}
# Fetch any bare files that we do not already have.
update_barclamp_file_cache() {
local dest pkg bc_cache="$CACHE_DIR/barclamps/$1/files"
# Fetch any extra_pkgs we need.
mkdir -p "$bc_cache"
while read pkg; do
dest=${pkg#* }
[[ $dest = $pkg ]] && dest=''
pkg=${pkg%% *}
[[ -f $bc_cache/files/$dest/${pkg##*/} ]] && continue
mkdir -p "$bc_cache/$dest"
echo "Caching $pkg:"
curl -L -o "$bc_cache/$dest/${pkg##*/}" "$pkg"
[[ $CURRENT_CACHE_BRANCH ]] && \
in_cache git add "$bc_cache/$dest/${pkg##*/}"
done < <(write_lines "${BC_EXTRA_FILES[$1]}")
}
# Check to see if the barclamp package cache needs update.
barclamp_pkg_cache_needs_update() {
local pkg pkgname arch bcs=() bc ret=1
local -A pkgs
[[ $need_update = true || ${FORCE_BARCLAMP_UPDATE["$1"]} = true ]] && return 0
[[ -d $CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs ]] && \
touch "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs"
# First, check to see if we have all the packages we need.
for bc in $(all_deps "$1"); do
[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs" ]] && \
bcs+=("$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs")
done
if [[ ${bcs[*]} ]]; then
while read pkg; do
is_pkg "$pkg" || continue
pkgname="$(pkg_name "$pkg")"
#debug "$pkgname is cached"
pkgs["$pkgname"]="$pkg"
done < <(find "${bcs[@]}" -type f)
fi
for pkg in ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]}; do
[[ $pkg ]] || continue
for arch in "${PKG_ALLOWED_ARCHES[@]}"; do
[[ ${pkgs["$pkg-$arch"]} ]] && continue 2
if [[ ${CD_POOL["$pkg-$arch"]} ]]; then
INSTALLED_PKGS["$pkg-$arch"]="true"
continue 2
fi
done
debug "Package $pkg is not cached, and $1 needs it."
ret=0
done
return $ret
}
# Check to see if the barclamp gem cache needs an update.
barclamp_gem_cache_needs_update() {
local pkg pkgname bc ret=1
local -A pkgs
# Second, check to see if we have all the gems we need.
for pkg in ${BC_GEMS["$1"]}; do
for bc in $(all_deps "$1"); do
local bc_cache="$CACHE_DIR/barclamps/$bc/gems"
mkdir -p "$bc_cache"
[[ $(find "$bc_cache" \
-name "$pkg*.gem" -type f) = *.gem ]] && continue 2
done
debug "Gem $pkg is not cached, and $1 needs it."
ret=0
done
return $ret
}
# CHeck to see if we are missing any raw packages.
barclamp_raw_pkg_cache_needs_update() {
local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" ret=1
mkdir -p "$bc_cache"
# Third, check to see if we have all the raw_pkgs we need.
for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
[[ -f $bc_cache/${pkg##*/} ]] && continue
debug "Raw package $pkg is not cached, and $1 needs it."
ret=0
done
return $ret
}
# Check to see if we are missing any raw files.
barclamp_file_cache_needs_update() {
local pkg dest bc_cache="$CACHE_DIR/barclamps/$1/files" ret=1
mkdir -p "$bc_cache"
# Fourth, check to make sure we have all the extra_pkgs we need.
while read pkg; do
dest=${pkg#* }
[[ $dest = $pkg ]] && dest=''
pkg=${pkg%% *}
[[ -f $bc_cache/$dest/${pkg##*/} ]] && continue
debug "File $pkg is not cached, and $1 needs it."
ret=0
done < <(write_lines "${BC_EXTRA_FILES[$1]}")
return $ret
}
# Some helper functions
# Print a message to stderr and exit. cleanup will be called.
die() { printf "$(date '+%F %T %z'): %s\n" "$@" >&2; res=1; exit 1; }
# Print a message to stderr and keep going.
debug() { [[ $VERBOSE ]] && echo "$(date '+%F %T %z'): $*" >&2; }
# Clean up any cruft that we might have left behind from the last run.
clean_dirs() {
local d=''
for d in "$@"; do
( mkdir -p "$d"
cd "$d"
sudo rm -rf --one-file-system * )
done
}
# Verify that the passed name is really a branch in the git repo.
branch_exists() { git show-ref --quiet --verify --heads -- "refs/heads/$1"; }
to_empty_branch() {
if branch_exists empty-branch; then
git checkout -q empty-branch
return $?
fi
if [[ -d .git ]]; then
git symbolic-ref HEAD refs/heads/empty-branch
rm -f .git/index
elif [[ -f .git ]]; then
git checkout --orphan empty-branch
git rm -r --cached .
fi
git clean -f -x -d
echo "This branch intentionally left blank" >README.empty-branch
git add README.empty-branch
git commit -m "Created empty branch"
}
# Run a git command in the crowbar repo.
in_repo() ( cd "$CROWBAR_DIR"; "$@")
# Get the head revision of a git repository.
get_rev() (
cd "$1"
if [[ -d .git || -f .git ]]; then
git rev-parse HEAD
else
echo "Not a Git Repository"
fi
)
# Run a git command in the build cache, assuming it is a git repository.
in_cache() (
cd "$CACHE_DIR"
"$@"
)
# Check to see if something is a barclamp.
is_barclamp() [[ -f "$CROWBAR_DIR/barclamps/$1/crowbar.yml" ]]
in_barclamp() {
( cd "$CROWBAR_DIR/barclamps/$1"
shift
"$@")
}
# Build our ISO image.
build_iso() (
cd "$BUILD_DIR"
rm -f isolinux/boot.cat
find -name '.svn' -type d -exec rm -rf '{}' ';' 2>/dev/null >/dev/null
find . -type f -not -name isolinux.bin -not -name sha1sums \
-not -name crowbar.json -not -path '*/.git/*' -print0 | \
xargs -0 -- sha1sum -b >sha1sums
mkdir -p "$ISO_DEST"
# Save the sha1sums and the build-info files along side the iso.
cp sha1sums build-info "$ISO_DEST"
if ! [[ $NO_GENERATE_ISO && $NO_GENERATE_ISO = true ]]; then
mkisofs -r -V "${VERSION:0:30}" -cache-inodes -J -l -quiet \
-b isolinux/isolinux.bin -c isolinux/boot.cat -joliet-long \
-no-emul-boot --boot-load-size 4 -boot-info-table \
-o "$ISO_DEST/$BUILT_ISO" "$IMAGE_DIR" "$BUILD_DIR"
fi
)
# Have the smoketest framework do its thing with the ISO we just made.
test_iso() {
run_test "$@" || \
die "$(date '+%F %T %z'): Smoketest of $ISO_DEST/$BUILT_ISO failed."
}
get_repo_cfg() { in_repo git config --get "$1"; }
git_config_has() { git config --get "$1" &>/dev/null; }
current_build() { get_repo_cfg 'crowbar.build'; }
build_exists() [[ -f $CROWBAR_DIR/releases/$1/barclamp-crowbar ]]
barclamp_exists_in_build() {
local build=${1%/*} bc=${1##*/}
[[ -f $CROWBAR_DIR/releases/$build/barclamp-$bc ]]
}
build_cfg_dir() {
local d="${1:-$(current_build)}"
build_exists "$d" || return 1
echo "$CROWBAR_DIR/releases/$d"
}
release_exists() [[ -d $CROWBAR_DIR/releases/$1/master ]]
# Get the current release we are working on, which is a function of
# the currently checked-out branch.
current_release() {
local rel
rel=$(current_build) || \
die "current_release: Cannot get current build information!"
echo "${rel%/*}"
}
release_cfg_dir() {
local d="${1:-$(current_release)}"
release_exists "$d" || return 1
echo "$CROWBAR_DIR/releases/$d"
}
# Find all barclamps for whatever.
barclamp_finder() {
# $1 = directory under $CROWBAR_DIR/releases to look in.
# $2 = regex to use as a filter.
# $2 = Match in the RE to return. Defaults to 1
local b
while read b; do
[[ $b =~ $2 ]] || continue
printf '%s\n' "${BASH_REMATCH[${3:-1}]}"
done < <(find "$CROWBAR_DIR/releases/$1" -name 'barclamp-*') |sort -u
}
barclamps_in_build() {
flat_checkout || die "Cannot get list of barclamps, must flatten build first!"
local build bc
build="${1:-$(current_build)}"
barclamp_finder "$build" '/barclamp-(.+)$'
}
barclamps_in_release() {
local release="${1:-$(current_release)}"
release_exists "$release" || return 1
barclamp_finder "$release" '/barclamp-(.+)$'
}
builds_in_release() {
local release="${1:-$(current_release)}"
release_exists "$release" || return 1
barclamp_finder "$release" "releases/.+/([^/]+)/barclamp-crowbar"
}
all_barclamps() {
local bc
local -A barclamps
barclamp_finder '' '/barclamp-(.+)$'
}
# Given a build, give us the branch the barclamps will use.
build_branch() {
# $1 = build
case $1 in
development/*) echo "master" ;;
stable/*) echo "stable/${1%/*}/master";;
feature/*/*) echo "${1%/*}/master";;
*) echo "release/${1%/*}/master";;
esac
}
crowbar_version() {
local bc build br
build=${1:-$(current_build)}
commits=0
build_exists "$build" || die "$build is not a build!"
br="$(build_branch "$build")"
for bc in $(barclamps_in_build); do
commits=$((commits + $(in_barclamp "$bc" git rev-list --count --sparse --no-merges "$br")))
done
echo "${build//\//_}.$commits"
}
all_releases() { barclamp_finder '' 'releases/(.+)/master/barclamp-crowbar$'; }
if [[ $http_proxy ]]; then
export USE_PROXY=1
raw_proxy="${http_proxy#*://}"
raw_proxy="${raw_proxy%/}"
proxy_re='^(.+):([0-9]+)$'
hostsplit_re='(.*)@(.*)'
userpass_re='(.*):(.*)'
if [[ $raw_proxy =~ $proxy_re ]]; then
export PROXY_PORT="${BASH_REMATCH[2]}"
raw_proxy="${BASH_REMATCH[1]}"
fi
if [[ $raw_proxy =~ $hostsplit_re ]]; then
raw_proxy="${BASH_REMATCH[2]}"
export PROXY_USER="${BASH_REMATCH[1]}"
if [[ ${BASH_REMATCH[1]} =~ $userpass_re ]]; then
export PROXY_PASSWORD="${BASH_REMATCH[2]}"
export PROXY_USER="${BASH_REMATCH[1]}"
fi
fi
export PROXY_HOST="$raw_proxy"
[[ $no_proxy ]] || no_proxy="localhost,localhost.localdomain,127.0.0.0/8,$PROXY_HOST"
else
unset USE_PROXY PROXY_HOST PROXY_PORT PROXY_USER PROXY_PASS
fi
export PATH="$PATH:$CROWBAR_DIR:$CROWBAR_DIR/extra:$CROWBAR_DIR/change-image/dell:$CROWBAR_DIR/test_framework"
CROWBAR_BUILD_SOURCED=true
Jump to Line
Something went wrong with that request. Please try again.