Permalink
Browse files

Add support in the build system for downloading raw files and package…

…s driven by crowbar.yml.
  • Loading branch information...
1 parent c04a527 commit 55ff5477ccd0637e96993144a581fecc2071ff4a @VictorLowther VictorLowther committed Sep 22, 2011
Submodule ganglia updated 1 files
+11 −3 crowbar.yml
View
@@ -135,7 +135,7 @@ trap cleanup 0 INT QUIT TERM
# Crowbar with.
PKGS=()
GEMS=()
-AMIS=("http://uec-images.ubuntu.com/releases/11.04/release/ubuntu-11.04-server-uec-amd64.tar.gz")
+
# Some helper functions
@@ -344,7 +344,7 @@ fi
# Directories where we cache our pkgs, gems, and ami files
[[ $PKG_CACHE ]] || PKG_CACHE="$CACHE_DIR/$OS_TOKEN/pkgs"
[[ $GEM_CACHE ]] || GEM_CACHE="$CACHE_DIR/gems"
- [[ $AMI_CACHE ]] || AMI_CACHE="$CACHE_DIR/amis"
+ [[ $FILE_CACHE ]] || FILE_CACHE="$CACHE_DIR/files"
# Directory where we will look for our package lists
[[ $PACKAGE_LISTS ]] || PACKAGE_LISTS="$BUILD_DIR/extra/packages"
@@ -437,8 +437,9 @@ fi
debug "Copying pkgs, gems, and amis"
copy_pkgs "$IMAGE_DIR" "$PKG_CACHE" "$BUILD_DIR/extra/pkgs"
cp -r "$GEM_CACHE" "$BUILD_DIR/extra"
- cp -r "$AMI_CACHE/." "$BUILD_DIR/ami/."
-
+ cp -r "$FILE_CACHE" "$BUILD_DIR/extra"
+ # Make sure we still provide the legacy ami location
+ (cd "$BUILD_DIR"; ln -sf extra/files/ami ami)
# Store off the version
echo "$VERSION" >> "$BUILD_DIR/dell/Version"
@@ -182,17 +182,8 @@ update_caches() {
"WEBrick::HTTPServer.new(:BindAddress=>\"127.0.0.1\",:Port=>54321,:DocumentRoot=>\".\").start" &>/dev/null ) &
webrick_pid=$!
make_redhat_chroot
- # Copy in our current cached packages and fix up ownership.
- # This prevents excessive downloads.
- for d in "$PKG_CACHE"/*; do
- [[ -d $d ]] || continue
- in_chroot mkdir -p "/var/cache/yum/${d##*/}/packages"
- in_chroot chmod 777 "/var/cache/yum/${d##*/}/packages"
- sudo cp -a "$d/"* "$CHROOT/var/cache/yum/${d##*/}/packages"
- done
- in_chroot chown -R root:root "/var/cache/yum/"
- # Once the current caches are copied in, set up repos for any other
- # packages we want to grab.
+
+ # set up repos for any other packages we want to grab.
for repo in "${REPOS[@]}"; do
rtype="${repo%% *}"
rdest="${repo#* }"
@@ -204,6 +195,17 @@ update_caches() {
esac
done
+
+ # Copy in our current cached packages and fix up ownership.
+ # This prevents excessive downloads.
+ for d in "$PKG_CACHE"/*; do
+ [[ -d $d ]] || continue
+ in_chroot mkdir -p "/var/cache/yum/${d##*/}/packages"
+ in_chroot chmod 777 "/var/cache/yum/${d##*/}/packages"
+ sudo cp -a "$d/"* "$CHROOT/var/cache/yum/${d##*/}/packages"
+ done
+ in_chroot chown -R root:root "/var/cache/yum/"
+
fix_repo_proxy
# Copy in our gems.
in_chroot mkdir -p "/usr/lib/ruby/gems/1.8/cache/"
@@ -358,7 +360,7 @@ copy_pkgs() {
# This function checks to see if we need or asked for a cache update, and performs
# one if we do.
maybe_update_cache() {
- local pkgfile deb rpm pkg_type rest _pwd
+ local pkgfile deb rpm pkg_type rest _pwd l t
debug "Processing package lists"
# Download and stash any extra files we may need
# First, build our list of repos, ppas, pkgs, and gems
@@ -368,18 +370,34 @@ maybe_update_cache() {
yml="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
[[ -f $yml ]] || continue
echo "Processing $yml"
- for t in repos pkgs; do
+ for t in repos raw_pkgs pkgs; do
while read l; do
echo "Found $t $l"
case $t in
repos) REPOS+=("$l");;
pkgs) PKGS+=("$l");;
+ raw_pkgs) [[ -f $PKG_CACHE/raw_downloads/${l##*/} ]] && continue
+ mkdir -p "$PKG_CACHE/raw_downloads"
+ curl -s -S -o "$PKG_CACHE/raw_downloads/${l##*/}" "$l";;
esac
done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" rpms "$t" 2>/dev/null)
done
while read l; do
GEMS+=("$l")
done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" gems pkgs 2>/dev/null)
+ # l = the full URL to download
+ # t = the location on the ISO it should wind up in
+ while read l t; do
+ case $t in
+ # If we were not given a destination, just stick it in the file cache
+ '') t="$FILE_CACHE";;
+ # Anything else will wind up in a directory under $FILE_CACHE
+ *) mkdir -p "$FILE_CACHE/$t"
+ t="$FILE_CACHE/$t";;
+ esac
+ [[ -f "$t/$l" ]] || wget -q --continue "$l" -O "$t/${l##*/}";;
+ done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" extra_files 2>/dev/null)
+
done
for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
@@ -6,7 +6,7 @@
# Base repository for Chef and friends
repository rpm http://rbel.frameos.org/rbel5
-repository bare rbel5-ruby-test 10 http://rbel.frameos.org/ruby-test/el5/x86_64
+# repository bare rbel5-ruby-test 10 http://rbel.frameos.org/ruby-test/el5/x86_64
# Base repository for EPEL on RHEL5, we will need this.
repository rpm http://ftp.cse.buffalo.edu/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
@@ -185,16 +185,33 @@ maybe_update_cache() {
yml="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
[[ -f $yml ]] || continue
echo "Processing $yml"
- for t in repos pkgs ppas; do
+ for t in repos raw_pkgs pkgs ppas; do
while read l; do
echo "Found $t $l"
case $t in
repos) echo "$l" >> "$BUILD_DIR/extra/sources.list";;
pkgs) PKGS+=("$l");;
ppas) PPAS+=("$l");;
+ raw_pkgs) [[ -f $PKG_CACHE/raw_downloads/${l##*/} ]] && continue
+ mkdir -p "$PKG_CACHE/raw_downloads"
+ curl -s -S -o "$PKG_CACHE/raw_downloads/${l##*/}" "$l";;
esac
done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" debs "$t" 2>/dev/null)
done
+
+ # l = the full URL to download
+ # t = the location on the ISO it should wind up in
+ while read l t; do
+ case $t in
+ # If we were not given a destination, just stick it in the file cache
+ '') t="$FILE_CACHE";;
+ # Anything else will wind up in a directory under $FILE_CACHE
+ *) mkdir -p "$FILE_CACHE/$t"
+ t="$FILE_CACHE/$t";;
+ esac
+ [[ -f "$t/$l" ]] || wget -q --continue "$l" -O "$t/${l##*/}";;
+ done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" extra_files 2>/dev/null)
+
while read l; do
GEMS+=("$l")
done < <("$CROWBAR_DIR/parse_yml.rb" "$yml" gems pkgs 2>/dev/null)

0 comments on commit 55ff547

Please sign in to comment.