tools - Add scripts for snapshot generation.

tools/snapshots/README

@@ -0,0 +1,20 @@
+
+    Generate daily HEAD snapshots.  Basically setup the config file and
+    these files in ~root/snapshots and add a root cron job:
+
+    23 2 * * *      cd /root/snapshots; lockf -ks -t 0 .buildlock ./docron
+
+    The snapshots can be made available via rsyncd using a rsyncd.conf
+    file similar to this:
+
+uid = nobody
+gid = nobody
+use chroot = yes
+
+[snapshots]
+        path = /archive/snapshots
+
+    Another machine can mirror the snapshots using something like this:
+
+    lockf -ks -t 0 /root/adm/.rsynclock2 /usr/pkg/bin/rsync -aHS	\
+	snapshotbox.domain::snapshots/ /archive/snapshots/

tools/snapshots/config

@@ -0,0 +1,10 @@
+# Config file for builder
+#
+
+set build = /build/snapshots
+set dflygit = /repository/git/dragonfly.git
+set pkgsrcgit = /repository/git/pkgsrc.git
+set dfly_branch = master
+set pkgsrc_branch = vendor
+set snapshots = /archive/snapshots
+set relsname = DEV

tools/snapshots/dobuild

@@ -0,0 +1,117 @@
+#!/bin/csh
+#
+# Create a working chroot for snapshot building
+#
+
+source config
+
+set ncpu = `sysctl -n hw.ncpu`
+set pushdsilent
+set xdate = "date"
+
+echo "`$xdate` - SNAPSHOTS DOBUILD BEGIN"
+
+# Options
+#
+set nrelease_args = "release"
+
+foreach arg ( $argv )
+    switch ( $arg )
+    case "realquick":
+	set quick = 1
+	set realquick = 1
+	set nrelease_args = "realquick"
+	breaksw
+    case "quick":
+	set quick = 1
+	set nrelease_args = "quick"
+	breaksw
+    default:
+	echo "`$xdate` - Unknown option $arg"
+	echo "`$xdate` - SNAPSHOTS DOBUILD END"
+	exit 1
+    endsw
+end
+
+echo "`$xdate` - Setting up chroot environment"
+if ( ! -e $build/root/dev/null ) then
+    mount_devfs dummy $build/root/dev
+endif
+
+umount $build/root/usr/pkgsrc/distfiles >& /dev/null
+mount_null $build/distfiles $build/root/usr/pkgsrc/distfiles
+
+setenv PKGSRC_PATH /usr/pkgsrc
+
+echo "`$xdate` - Build is running nrelease, tail -f $build/nrelease-build.out"
+chroot $build/root csh -c "cd /usr/src/nrelease; make -DNOPROFILE -DWITHOUT_SRCS img $nrelease_args" >& $build/nrelease-build.out
+
+if ( $? > 0 ) then
+    echo "`$xdate` - NRelease build had a problem, see $build/nrelease-build.out"
+    echo "`$xdate` - SNAPSHOTS DOBUILD END"
+    exit 1
+endif
+
+# Figure out the filename
+#
+set arch = `uname -p`
+if ( "$arch" == "i386" ) then
+set kobjs = $build/root/usr/obj/usr/src/sys/GENERIC
+else
+set kobjs = $build/root/usr/obj/usr/src/sys/X86_64_GENERIC
+endif
+
+if ( ! -f $kobjs/vers.txt ) then
+    echo "`$xdate` - nrelease build did not generate $kobjs/vers.txt"
+    echo "`$xdate` - SNAPSHOTS DOBUILD END"
+    exit 1
+endif
+
+# Store the snapshots.  Make sure the files are inaccessible until
+# completely written out.
+#
+set basename = `cat $kobjs/vers.txt`
+mkdir -p $snapshots/$arch/images
+mkdir -p $snapshots/$arch/kernels
+set filepath = $snapshots/$arch
+umask 077
+bzip2 < $build/root/usr/release/dfly.iso > $filepath/images/$basename.iso.bz2
+bzip2 < $build/root/usr/release/dfly.img > $filepath/images/$basename.img.bz2
+(cd $build/root/usr/release/root; tar cf - boot | bzip2 > $filepath/kernels/$basename.boot.tar.bz2)
+chmod 644 $filepath/images/$basename.iso.bz2
+chmod 644 $filepath/images/$basename.img.bz2
+chmod 644 $filepath/kernels/$basename.boot.tar.bz2
+sync
+pushd $filepath/images
+md5 $basename.iso.bz2 >> CHECKSUM.MD5
+md5 $basename.img.bz2 >> CHECKSUM.MD5
+popd
+
+pushd $filepath/kernels
+md5 $basename.boot.tar.bz2 >> CHECKSUM.MD5
+popd
+
+# Only adjust the 'latest' softlink shortcuts for
+# development snapshots.  Release shortcuts have
+# to be done manually.
+#
+if ( "$relsname" == "DEV" ) then
+    pushd $filepath
+    rm -f DragonFly-${arch}-LATEST-ISO.iso.bz2.new
+    rm -f DragonFly-${arch}-LATEST-IMG.img.bz2.new
+    rm -f DragonFly-${arch}-LATEST-BOOT.tar.bz2.new
+    ln -s images/$basename.iso.bz2 DragonFly-${arch}-LATEST-ISO.iso.bz2.new
+    ln -s images/$basename.img.bz2 DragonFly-${arch}-LATEST-IMG.img.bz2.new
+    ln -s kernels/$basename.boot.tar.bz2 DragonFly-${arch}-LATEST-BOOT.tar.bz2.new
+    mv -f DragonFly-${arch}-LATEST-ISO.iso.bz2.new \
+	    DragonFly-${arch}-LATEST-ISO.iso.bz2
+    mv -f DragonFly-${arch}-LATEST-IMG.img.bz2.new \
+	    DragonFly-${arch}-LATEST-IMG.img.bz2
+    mv -f DragonFly-${arch}-LATEST-BOOT.tar.bz2.new \
+	    DragonFly-${arch}-LATEST-BOOT.tar.bz2
+    popd
+endif
+
+sync
+
+echo "`$xdate` - SNAPSHOTS DOBUILD END"

tools/snapshots/doclean

@@ -0,0 +1,219 @@
+#!/bin/csh
+#
+# Create a working chroot for snapshot building
+#
+
+source config
+
+set ncpu = `sysctl -n hw.ncpu`
+set pushdsilent
+set xdate = "date"
+
+echo "`$xdate` - SNAPSHOTS DOCLEAN BEGIN"
+
+# Options
+#
+foreach arg ( $argv )
+    switch ( $arg )
+    case "realquick":
+	set quick = 1
+	set realquick = 1
+	breaksw
+    case "quick":
+	set quick = 1
+	breaksw
+    default:
+	echo "Unknown option $arg"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endsw
+end
+
+# Remove environment
+#
+if ( "$build" == "" ) then
+    echo "build variable not set"
+    echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+    exit 1
+endif
+
+if ( -e $build/root/dev/null ) then
+    echo "`$xdate` - Unmount devfs from $build/root/dev"
+    umount $build/root/dev
+    if ( $? > 0 ) then
+	echo "`$xdate` - Unable to umount $build/root/dev"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endif
+endif
+
+# remount the nullfs mount read-only just in case the umount fails.
+#
+umount $build/root/usr/pkgsrc/distfiles >& /dev/null
+mount -u -o ro $build/root/usr/pkgsrc/distfiles >& /dev/null
+
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Destroying root environment at $build/root"
+    sleep 5
+    rm -rf $build/root >& /dev/null
+    if ( -d $build/root ) then
+	chflags -R noschg $build/root
+    endif
+    rm -rf $build/root
+endif
+mkdir -p $build/distfiles
+mkdir -p $build/root
+mkdir -p $build/root/usr
+
+echo "`$xdate` - Log files are in $build/*.out"
+
+# Ignore quick/realquick options if there is no prior
+# installation
+#
+if ( ! -e $build/root/etc/networks ) then
+    unset realquick
+endif
+if ( ! -d $build/root/usr/src ) then
+    unset quick
+    unset realquick
+endif
+if ( ! -d $build/root/usr/pkgsrc ) then
+    unset quick
+    unset realquick
+endif
+
+# Clone the repos, create trakcing branches as necessary, checkout,
+# and update (using fetch)
+#
+# DragonFly sources
+#
+pushd $build/root/usr
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Cloning $dflygit"
+    rm -rf src
+    git clone -n $dflygit src
+endif
+popd
+pushd $build/root/usr/src
+if ( "`git branch | fgrep $dfly_branch`" == "" ) then
+    echo "`$xdate` - Branching-src $dfly_branch"
+    git branch $dfly_branch origin/$dfly_branch
+endif
+echo "`$xdate` - Updating src"
+git checkout $dfly_branch
+git pull
+popd
+
+# Package sources
+#
+pushd $build/root/usr
+if ( $?quick == 0 ) then
+    echo "`$xdate` - Cloning $pkgsrcgit"
+    rm -rf pkgsrc
+    git clone -n $pkgsrcgit pkgsrc
+endif
+popd
+pushd $build/root/usr/pkgsrc
+if ( "`git branch | fgrep $pkgsrc_branch`" == "" ) then
+    echo "`$xdate` - Branching-pkgsrc $pkgsrc_branch"
+    git branch $pkgsrc_branch origin/$pkgsrc_branch
+endif
+echo "`$xdate` - Updating pkgsrc"
+git checkout $pkgsrc_branch
+git pull
+popd
+
+# Install/upgrade environment
+#
+pushd $build/root/usr/src
+
+if ( $?realquick ) then
+    echo "`$xdate` - Not running build/quickworld - realquick mode"
+else
+if ( $?quick ) then
+    echo "`$xdate` - Starting quickworld, tail -f $build/build.out"
+    make -j $ncpu quickworld >& $build/build.out
+    if ( $? ) then
+	echo "`$xdate` - primary environment quickworld failed"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endif
+else
+    echo "`$xdate` - Starting buildworld, tail -f $build/build.out"
+    make -j $ncpu buildworld >& $build/build.out
+    if ( $? ) then
+	echo "`$xdate` - primary environment buildworld failed"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endif
+endif
+endif
+
+if ( $?realquick == 0 ) then
+    echo "`$xdate` - Installing the world, tail -f $build/install.out"
+    make installworld DESTDIR=$build/root >& $build/install.out
+    pushd etc
+    make distribution DESTDIR=$build/root >& $build/install.out
+    popd
+    echo "`$xdate` - Upgrading the world, tail -f $build/upgrade.out"
+    make upgrade DESTDIR=$build/root >& $build/upgrade.out
+endif
+
+popd
+
+# Setup the chroot environment, including packages nrelease needs to
+# build.
+#
+echo "`$xdate` - Setting up chroot environment in $build/root"
+echo "`$xdate` - Mounting devfs"
+if ( ! -e $build/root/dev/null ) then
+    mount_devfs dummy $build/root/dev
+endif
+echo "`$xdate` - Mounting distfiles"
+mount_null $build/distfiles $build/root/usr/pkgsrc/distfiles
+
+#mkdir -m 1777 $build/root/tmp
+#mkdir -m 1777 $build/root/var/tmp
+
+if ( $?realquick == 0 ) then
+    echo "`$xdate` - Setting up ldd"
+    chroot $build/root /etc/rc.d/ldconfig start
+endif
+cp /etc/resolv.conf $build/root/etc/
+
+if ( ! -e $build/root/usr/pkg/bin/bmake ) then
+    echo "`$xdate` - Bootstrapping pkgsrc, tail -f $build/bootstrap.out"
+    chroot $build/root csh -c "cd /usr/pkgsrc/bootstrap; rm -rf /usr/obj/bootstrap; ./bootstrap --workdir=/usr/obj/bootstrap" >& $build/bootstrap.out
+    if ( $? > 0 ) then
+	echo "`$xdate` - pkgsrc bootstrap had problems"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endif
+endif
+
+if ( ! -e $build/root/usr/pkg/bin/mkisofs ) then
+    echo "`$xdate` - Setting up cdrecord and friends, tail -f $build/nrelease1.out"
+    setenv PKGSRC_PATH /usr/pkgsrc
+    chroot $build/root csh -c "cd /usr/src/nrelease; make fetch pkgsrc_cdrecord" >& $build/nrelease1.out
+    if ( $? > 0 ) then
+	echo "`$xdate` - nrelease had problems"
+	echo "`$xdate` - SNAPSHOTS DOCLEAN END"
+	exit 1
+    endif
+endif
+
+# The nrelease build needs scmgit as well.  Try to get the binary package
+# first and then build whatever else is needed from source.  These packages
+# are not part of the nrelease snapshot build, they are used by nrelease
+# to do the build.
+#
+if ( ! -e $build/root/usr/pkg/bin/git ) then
+    echo "`$xdate` - Setting up scmgit from pkgsrc, tail -f $build/nrelease2.out"
+    chroot $build/root csh -c "pkg_radd scmgit"
+    if ( ! -e $build/root/usr/pkg/bin/git ) then
+	    echo "`$xdate` - Binary package not found, building from source"
+	    chroot $build/root csh -c "cd /usr/pkgsrc/devel/scmgit; bmake update"
+    endif
+endif
+
+echo "`$xdate` - SNAPSHOTS DOCLEAN END"

tools/snapshots/docron

@@ -0,0 +1,9 @@
+#!/bin/csh
+#
+
+./doclean quick
+if ( $? > 0 ) then
+    echo "doclean script failed, aborting build"
+    exit 1
+endif
+./dobuild