build-vm

#
# VM specific functions for the build script
#
################################################################
#
# Copyright (c) 1995-2014 SUSE Linux Products GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################

# defaults for vm_img_mkfs
vm_img_mkfs_ext4_options='-O ^has_journal,^huge_file,^resize_inode,sparse_super'
vm_img_mkfs_ext4_extra='-E lazy_itable_init,discard'
vm_img_mkfs_ext4="mkfs.ext4 -m 0 -q -F $vm_img_mkfs_ext4_options"
vm_img_tunefs_ext4='tune2fs -c 0'
vm_img_mkfs_ext3='mkfs.ext3 -m 0 -q -F'
vm_img_tunefs_ext3='tune2fs -c 0 -o journal_data_writeback'
vm_img_mkfs_ext2='mkfs.ext2 -m 0 -q -F'
vm_img_tunefs_ext2='tune2fs -c 0'
vm_img_mkfs_reiserfs='mkreiserfs -q -f'
vm_img_mkfs_btrfs='mkfs.btrfs'
vm_img_mkfs_xfs='mkfs.xfs -f'

# guest visible swap device
VM_SWAPDEV=/dev/hda2

VM_TYPE=
VM_IMAGE=
VM_SWAP=
VM_KERNEL=
VM_INITRD=
VM_WORKER=
VM_SERVER=
VM_MEMSIZE=
VM_NETOPT=()
VM_NETDEVOPT=()
VM_DEVICEOPT=()
VM_TELNET=
VM_CONSOLE_INPUT=
VM_USER=
VMDISK_ROOTSIZE=4096
VMDISK_SWAPSIZE=1024
VMDISK_FILESYSTEM=
VMDISK_MOUNT_OPTIONS=__default
VMDISK_CLEAN=
VM_VOLUME_NAME=
VM_VOLUME_SWAP=

# zvm specific?
VM_WORKER_NR=

# kvm specific?
HUGETLBFSPATH=

# emulator specific?
EMULATOR_SCRIPT=

for i in ec2 emulator kvm lxc openstack qemu uml xen zvm docker pvm; do
    . "$BUILD_DIR/build-vm-$i"
done

VM_WATCHDOG=
VM_WATCHDOG_PID=

# the following functions just call the corresponding vm versions
vm_kill() {
    vm_kill_$VM_TYPE "$@"
}

vm_verify_options() {
    vm_verify_options_$VM_TYPE "$@"
}

vm_attach_root() {
    vm_attach_root_$VM_TYPE "$@"
}

vm_attach_swap() {
    vm_attach_swap_$VM_TYPE "$@"
}

vm_detach_root() {
    vm_detach_root_$VM_TYPE "$@"
}

vm_detach_swap() {
    vm_detach_swap_$VM_TYPE "$@"
}

vm_fixup() {
    vm_fixup_$VM_TYPE "$@"
}

vm_startup() {
    vm_startup_$VM_TYPE "$@"
}

vm_kill() {
    vm_kill_$VM_TYPE "$@"
}

vm_cleanup() {
    kill_watchdog
    vm_cleanup_$VM_TYPE "$@"
}

vm_parse_options() {
    case ${PARAM/#--/-} in
      -vm-emulator-script|-emulator-script)
	needarg
	EMULATOR_SCRIPT="$ARG"
	shift
      ;;
      -xen|-kvm|-uml|-qemu|-emulator)
	VM_TYPE=${PARAM##*-}
	test -z "$VM_IMAGE" && VM_IMAGE=1
	if test -n "$ARG" ; then
	    VM_IMAGE="$ARG"
	    shift
	fi
      ;;
      -zvm|-lxc)
        VM_TYPE=${PARAM##*-}
	shift
      ;;
      -vm-type)
	needarg
	VM_TYPE="$ARG"
	case "$VM_TYPE" in
	    lxc|docker) ;;
	    ec2|xen|kvm|uml|qemu|emulator|openstack|zvm|pvm)
		test -z "$VM_IMAGE" && VM_IMAGE=1
	    ;;
	    none|chroot) VM_TYPE= ;;
	    *)
		cleanup_and_exit 1 "VM '$VM_TYPE' is not supported"
	    ;;
	esac
	shift
      ;;
      -vm-worker)
        needarg
        VM_WORKER="$ARG"
        shift
      ;;
      -vm-worker-nr|-vm-worker-no)
        needarg
        VM_WORKER_NR="$ARG"
        shift
      ;;
      -vm-server|-vm-region)
	needarg
	VM_SERVER="$ARG"
	shift
      ;;
      -vm-volumes)
	needarg
	VM_VOLUME_NAME="$ARG"
	shift
        ARG="$1"
        test "$ARG" = "${ARG#-}" || ARG=
	needarg
	VM_VOLUME_SWAP="$ARG"
	shift
      ;;
      -vm-disk)
	needarg
	VM_IMAGE="$ARG"
	shift
      ;;
      -vm-swap|-xenswap|-swap)
	needarg
	VM_SWAP="$ARG"
	shift
      ;;
      -vm-memory|-xenmemory|-memory)
	needarg
	VM_MEMSIZE="$ARG"
	shift
      ;;
      -vm-kernel)
	needarg
	VM_KERNEL="$ARG"
	shift
      ;;
      -vm-initrd)
	needarg
	VM_INITRD="$ARG"
	shift
      ;;
      -vm-disk-size|-vmdisk-rootsize)
	needarg
	VMDISK_ROOTSIZE="$ARG"
	shift
      ;;
      -vm-swap-size|-vmdisk-swapsize)
	needarg
	VMDISK_SWAPSIZE="$ARG"
	shift
      ;;
      -vm-disk-filesystem|-vmdisk-filesystem)
	needarg
	VMDISK_FILESYSTEM="$ARG"
	shift
      ;;
      -vm-disk-mount-options|-vmdisk-mount-options)
       needarg
       # options needs to be quoted to handle argument which might start with "-o ..."
       VMDISK_MOUNT_OPTIONS=$(echo $ARG | sed 's/^\"\(.*\)\"$/\1/g')
       shift
      ;;
      -vm-disk-clean|-vmdisk-clean)
	# delete old root/swap to get rid of the old blocks
        VMDISK_CLEAN=true
      ;;
      -vm-hugetlbfs|-hugetlbfs)
	needarg
	HUGETLBFSPATH="$ARG"
	shift
      ;;
      -vm-watchdog)
	VM_WATCHDOG=true
      ;;
      -vm-user)
        needarg
        VM_USER="$ARG"
        shift
      ;;
      -vm-enable-console)
        VM_CONSOLE_INPUT=true
      ;;
      -vm-telnet)
        needarg
        VM_TELNET="$ARG"
        shift
      ;;
      -vm-net)
        needarg
        VM_NETOPT=("${VM_NETOPT[@]}" "$ARG")
        shift
      ;;
      -vm-netdev)
        needarg
        VM_NETDEVOPT=("${VM_NETDEVOPT[@]}" "$ARG")
        shift
      ;;
      -vm-device)
        needarg
        VM_DEVICEOPT=("${VM_DEVICEOPT[@]}" "$ARG")
        shift
      ;;
      -*)
	return 1
      ;;
    esac
    nextargs=("$@")
    return 0
}


#
# shutdown the system from inside the VM
#
vm_shutdown() {
    test -n "$VM_WATCHDOG" && echo "### WATCHDOG MARKER START ###"
    cd /
    test -n "$1" || set 1
    if test -n "$VM_SWAP" -a -e "$VM_SWAP" ; then
	swapoff "$VM_SWAP" 2>/dev/null
	echo -n "BUILDSTATUS$1" >"$VM_SWAP"
    fi
    exec >&0 2>&0	# so that the logging tee finishes
    sleep 1		# wait till tee terminates
    test "$VM_TYPE" = lxc -o "$VM_TYPE" = docker && exit $1
    kill -9 -1        # goodbye cruel world
    if ! test -x /sbin/halt ; then
	test -e /proc/sysrq-trigger || mount -n -tproc none /proc
	sync
	sleep 2	# like halt does
	if test -e /proc/sysrq-trigger; then
	    echo o > /proc/sysrq-trigger
	    sleep 5 # wait for sysrq to take effect
	else
	    echo "Warning: VM doesn't support sysrq and /sbin/halt not installed"
	fi
    else
	sync	# halt from systemd is not syncing anymore.
	halt -f -p
    fi
    echo "Warning: clean shut down of the VM didn't work"
    exit $1	# init died...
}

vm_img_create() {
    local img="$1"
    local size="$2"

    if test -e "${img}" ; then
	local origsize=$(cat "${img}.size" 2> /dev/null)
	if test -z "$origsize" -o "$origsize" != "$size" ; then
	    echo "Resizing $img (${size}M)"
	fi
    else
	echo "Creating $img (${size}M)"
	rm -f "${img}.size"
    fi

    mkdir -p "${img%/*}" || cleanup_and_exit 3
    # truncate file to the desired size
    dd if=/dev/zero of="$img" bs=1M count=0 seek="$size" || cleanup_and_exit 3
    echo "$size" > "${img}.size"
    # allocate blocks
    if type -p fallocate > /dev/null ; then
        fallocate -p -l "${size}M" "$img" 2> /dev/null
        errout=$( fallocate -l "${size}M" "$img" 2>&1 )
        if test $? != 0; then
            echo $errout
            if test "${errout/Operation not supported/}" = "$errout"; then
                # Do not fail on not support file systems, eg ext2 or ext3
                cleanup_and_exit 3
            fi
        fi
    fi
}

vm_img_wipe() {
    vm_wipe_$VM_TYPE "$@"

    if test -n "$VM_IMAGE" -a ! -b "$VM_IMAGE" ; then
       rm -f "$VM_IMAGE"
    fi
    if test -n "$VM_SWAP" -a ! -b "$VM_SWAP" ; then
       rm -f "$VM_SWAP"
    fi
}

vm_img_mkfs() {
    local fs="$1"
    local img="$2"
    local mkfs tunefs
    eval "mkfs=\"\$vm_img_mkfs_${fs}\""
    eval "mkfs_exta_options=\"\$vm_img_mkfs_${fs}_extra\""
    eval "tunefs=\"\$vm_img_tunefs_${fs}\""

    if test -z "$mkfs"; then
	cleanup_and_exit 3 "filesystem \"$fs\" is not supported"
    fi

    echo "Creating $fs filesystem on $img"
    export MKE2FS_SYNC=0
    if ! $mkfs $mkfs_exta_options "$img"; then
        if test -z "$mkfs_exta_options"; then
            cleanup_and_exit 3
        else
            echo "Format call failed, trying again without extra options..."
            $mkfs "$img" || cleanup_and_exit 3
        fi
    fi
    if test -n "$tunefs" ; then
	$tunefs "$img" || cleanup_and_exit 3
    fi
}

background_monitor_process() {
    max_disk=0
    max_mem=0
    while sleep 5; do
	test -e /.build/_statistics.exit  && exit 0

	# memory usage
	if test -e /proc/meminfo ; then
	    memtotal=0
	    while read key value unit; do
		case $key in
		    MemTotal:|SwapTotal:) memtotal=$(( $memtotal + $value )) ;;
		    MemFree:|SwapFree:|SwapCached:|Cached:|Buffers:) memtotal=$(( $memtotal - $value )) ;;
		esac
	    done < /proc/meminfo
	    if test ${memtotal} -gt $max_mem ; then
		max_mem="${memtotal}"
		echo -n $(( $max_mem / 1024 )) > /.build/_statistics.memory.new && mv /.build/_statistics.memory.new /.build/_statistics.memory
	    fi
	fi

	# disk storage usage
	if type -p df >& /dev/null; then
	    c=(`df -m / 2>/dev/null | tail -n 1`)

	    if test ${c[2]} -gt $max_disk ; then
		max_disk="${c[2]}"
		echo -n $max_disk > /.build/_statistics.df.new && mv /.build/_statistics.df.new /.build/_statistics.df
	    fi
	fi
    done
}

background_watchdog() {
    WATCHDOG_START=
    WATCHDOG_TIMEOUT=300
    while sleep 5 ; do
	WATCH=`grep -a "### WATCHDOG MARKER" "$LOGFILE" | tail -n 1`
	case $WATCH in
	    *WATCHDOG\ MARKER\ START*) test -n "$WATCHDOG_START" || WATCHDOG_START=`date +%s` ;;
	    *WATCHDOG\ MARKER\ END*) WATCHDOG_START= ;;
	esac
	if test -n "$WATCHDOG_START" ; then
	    NOW=`date +%s`
	    ELAPSED=$((NOW-WATCHDOG_START))
	    if test $ELAPSED -gt $WATCHDOG_TIMEOUT ; then
		# kill the VM
		 echo "### WATCHDOG TRIGGERED, KILLING VM ###"
	    fuser -k -TERM "$VM_IMAGE"
	    exit 0
       fi
    fi
    done
}

start_watchdog() {
    local wf=$(mktemp)
    ( background_watchdog & echo $! > "$wf" )
    read VM_WATCHDOG_PID < "$wf"
    rm -f "$wf"
}

kill_watchdog() {
    test -n "$VM_WATCHDOG_PID" && kill "$VM_WATCHDOG_PID"
    VM_WATCHDOG_PID=
}

vm_set_personality_syscall() {
    local archname
    archname=`perl -V:archname 2>/dev/null`
    archname="${archname#archname=?}"
    case "$archname" in
	x86_64*) PERSONALITY_SYSCALL=135 ;;
	alpha*) PERSONALITY_SYSCALL=324 ;;
	sparc*) PERSONALITY_SYSCALL=191 ;;
	ia64*) PERSONALITY_SYSCALL=1140 ;;
	i?86*|ppc*|aarch64*|arm*|sh4|cris|m68k*|s390*|unicore32|microblaze)   PERSONALITY_SYSCALL=136 ;;
	*) cleanup_and_exit 1 "Unknown architecture personality: '$archname'" ;;
    esac
}

# used before calling kvm or xen
linux64() {
    perl -e 'syscall('$PERSONALITY_SYSCALL', 0); exec(@ARGV) || die("$ARGV[0]: $!\n")' "$@"
}

vm_detect_2nd_stage() {
    if test ! -e /.build/build.data -o -n "$BUILD_IGNORE_2ND_STAGE" ; then
	return 1
    fi
    . /.build/build.data
    if test -z "$VM_TYPE" ; then
	return 1
    fi
    BUILD_OPTIONS_PARSED=true
    if test $$ -eq 1 || test $$ -eq 2 ; then
	# ignore special init signals if we're init
	# we're using ' ' instead of '' so that the signal handlers
	# are reset in the child processes
	trap ' ' HUP TERM
	$0 "$@"
	cleanup_and_exit $?
    fi

    test -n "$VM_WATCHDOG" -a -z "$PERSONALITY_SET" && echo "### WATCHDOG MARKER END ###"
    echo "2nd stage started in virtual machine"
    BUILD_ROOT=/
    BUILD_DIR=/.build
    echo "machine type: `uname -m`"
    if test "$PERSONALITY" != 0 -a -z "$PERSONALITY_SET" ; then
	export PERSONALITY_SET=true
	echo "switching personality to $PERSONALITY..."
	# this is 32bit perl/glibc, thus the 32bit syscall number
	exec perl -e 'syscall(136, '$PERSONALITY') == -1 && warn("personality: $!\n");exec "/.build/build" || die("/.build/build: $!\n")'
    fi
    RUNNING_IN_VM=true
    test -e /proc/version || mount -orw -n -tproc none /proc
    if test "$VM_TYPE" != lxc -a "$VM_TYPE" != docker ; then
	mount -n ${VMDISK_MOUNT_OPTIONS},remount,rw /
    fi
    umount /run >/dev/null 2>&1
    # mount /sys
    if ! test -e /sys/block; then
	mkdir -p /sys
	mount -orw -n -tsysfs sysfs /sys
        # Docker already has sysfs mounted ro elsewhere,
        # need to remount rw explicitly.
        mount -o remount,rw sysfs /sys
    fi
# qemu inside of xen does not work, check again with kvm later before enabling this
#    if test -e /dev/kqemu ; then
#        # allow abuild user to run qemu
#        chmod 0666 /dev/kqemu
#    fi

    if test -n "$VM_SWAP" ; then
	for i in 1 2 3 4 5 6 7 8 9 10 ; do
	    test -e "$VM_SWAP" && break
	    test $i = 1 && echo "waiting for $VM_SWAP to appear"
	    echo -n .
	    sleep 1
	done
	test $i = 1 || echo
	# recreate the swap device manually if it didn't exist for some
	# reason, hardcoded to hda2 atm
	if ! test -b "$VM_SWAP" ; then
	    rm -f "$VM_SWAP"
	    umask 027
	    mknod "$VM_SWAP" b 3 2
	    umask 022
	fi
	# Do not rely on external system writing the signature, it might differ...
	mkswap "$VM_SWAP"
	swapon -v "$VM_SWAP" || exit 1
    fi
    HOST="$MYHOSTNAME"

    # repair dracut damage, see bsc#922676
    test -L /var/run -a ! -e /var/run && rm -f /var/run
    test -L /var/lock -a ! -e /var/lock && rm -f /var/lock

    # fork a process monitoring max filesystem usage during build
    if test "$DO_STATISTICS" = 1 ; then
	rm -f /.build/_statistics.exit
        ( background_monitor_process & )
    fi

    if test ! -e /dev/.udev ; then
        echo "WARNING: udev not running, creating extra device nodes"
        test -e /dev/fd || ln -sf /proc/self/fd /dev/fd
        test -e /etc/mtab || ln -sf /proc/mounts /etc/mtab
    fi

    # set date to build start on broken systems (now < build start)
    if test $(date '+%s') -lt $(date -r /.build/.date '+%s') ; then
        echo -n "WARNING: system has a broken clock, setting it to a newer time: "
        date -s `cat /.build/.date`
    fi

    return 0
}

vm_set_filesystem_type() {
    if test -z "$VMDISK_FILESYSTEM" -a -n "$BUILD_DIST" ; then 
	VMDISK_FILESYSTEM=`queryconfig --dist "$BUILD_DIST" --configdir "$CONFIG_DIR" --archpath "$BUILD_ARCH" buildflags vmfstype`
    fi   
    test -n "$VMDISK_FILESYSTEM" || VMDISK_FILESYSTEM=ext3
}

vm_set_mount_options() {
    if test "$VMDISK_MOUNT_OPTIONS" = __default; then
	if test "$VMDISK_FILESYSTEM" = reiserfs ; then
	    VMDISK_MOUNT_OPTIONS='-o data=writeback,commit=150,noatime'
	elif test "$VMDISK_FILESYSTEM" = btrfs ; then
	    VMDISK_MOUNT_OPTIONS='-o nobarrier,noatime'
	elif test "$VMDISK_FILESYSTEM" = "ext4" ; then
	    VMDISK_MOUNT_OPTIONS='-o noatime'
	elif test "$VMDISK_FILESYSTEM" = "ext3" ; then
	    VMDISK_MOUNT_OPTIONS='-o data=writeback,nobarrier,commit=150,noatime'
	elif test "$VMDISK_FILESYSTEM" = "ext2" ; then
	    VMDISK_MOUNT_OPTIONS='-o noacl,noatime'
	elif test "$VMDISK_FILESYSTEM" = "xfs" ; then
	    VMDISK_MOUNT_OPTIONS='-o noatime'
	else
	    VMDISK_MOUNT_OPTIONS='-o noatime'
	fi
    fi
}

#
# create file system and swap space, mount file system to $BUILD_ROOT
#
vm_setup() {
    vm_set_filesystem_type
    vm_set_mount_options
    if test "$VM_IMAGE" = 1 ; then
	VM_IMAGE="$BUILD_ROOT.img"
	if test -z "$VM_SWAP" -a "$VM_TYPE" != emulator; then
	    VM_SWAP="$BUILD_ROOT.swap"
	fi
	echo "VM_IMAGE: $VM_IMAGE, VM_SWAP: $VM_SWAP"
    else
	echo "VM_IMAGE: $VM_IMAGE, VM_SWAP: $VM_SWAP"
        vm_attach_root
    fi
    # this should not be needed, but sometimes a xen instance got lost
    test "$VM_TYPE" = xen && vm_purge_xen
    if test -n "$VMDISK_CLEAN" ; then
	# delete old root/swap to get rid of the old blocks
	if test -n "$VM_IMAGE" -a -f "$VM_IMAGE" ; then
	    echo "Deleting old $VM_IMAGE"
	    rm -rf "$VM_IMAGE"
	fi
	if test -n "$VM_SWAP" -a -f "$VM_SWAP" ; then
	    echo "Deleting old $VM_SWAP"
	    rm -rf "$VM_SWAP"
	fi
    fi
    test -b "$VM_IMAGE" || vm_img_create "$VM_IMAGE" "$VMDISK_ROOTSIZE"
    if test -z "$CLEAN_BUILD" ; then
        vm_img_mkfs "$VMDISK_FILESYSTEM" "$VM_IMAGE"
    fi
    if test -n "$VM_SWAP" -a ! -b "$VM_SWAP" ; then
	vm_img_create "$VM_SWAP" "$VMDISK_SWAPSIZE"
    fi
    if test ! -e "$VM_IMAGE" ; then
	cleanup_and_exit 3 "you need to create $VM_IMAGE first"
    fi
    if test -n "$CLEAN_BUILD" ; then
	vm_img_mkfs "$VMDISK_FILESYSTEM" "$VM_IMAGE" || cleanup_and_exit 3
    fi
    # now mount root/swap
    mkdir_build_root
    if test -w /root ; then
	if test -b $VM_IMAGE ; then
	    # mount device directly
	    mount $VMDISK_MOUNT_OPTIONS $VM_IMAGE $BUILD_ROOT || cleanup_and_exit 3
	else
	    mount ${VMDISK_MOUNT_OPTIONS},loop $VM_IMAGE $BUILD_ROOT || cleanup_and_exit 3
	fi
    else
	if ! mount $BUILD_ROOT; then
	    echo "mounting the build root failed. An fstab entry is probably missing or incorrect."
	    echo "/etc/fstab should contain an entry like this:"
	    echo "$VM_IMAGE $BUILD_ROOT auto noauto,user,loop 0 0"
	    cleanup_and_exit 3
	fi
    fi
    if test -n "$VM_SWAP" ; then
	vm_attach_swap
	dd if=/dev/zero of="$VM_SWAP" bs=1024 count=1 conv=notrunc 2>/dev/null
	vm_detach_swap
        # mkswap happens inside of the vm
    fi
}

vm_update_hostarch() {
    local kernel="$vm_kernel"
    local hostarchfile
    local newhostarch
    if test -z "$VM_KERNEL" -a -e "$BUILD_ROOT/.build.kernel.$VM_TYPE" ; then
	kernel="$BUILD_ROOT/.build.kernel.$VM_TYPE"
	hostarchfile="$BUILD_ROOT/.build.hostarch.$VM_TYPE"
    elif test -n "$kernel" -a -e "$kernel" -a -e "$kernel.hostarch" ; then
	hostarchfile="$kernel.hostarch"
    fi
    if test -n "$hostarchfile" -a -e "$hostarchfile"; then
	newhostarch=`cat "$hostarchfile"`
    elif test -n "$kernel" -a -e "$kernel" ; then
	case `objdump -f "$kernel" | sed -ne 's/.*file format //p'` in
	    elf64-powerpcle) newhostarch=ppc64le ;;
	    elf64-powerpc) newhostarch=ppc64 ;;
	esac
    fi
    if test -n "$newhostarch" -a "$newhostarch" != "$BUILD_HOST_ARCH" ; then
	echo "setting hostarch to $newhostarch"
	BUILD_HOST_ARCH="$newhostarch"
	# update BUILD_INITVM_ARCH
	build_host_arch
    fi
}

#
# prepare for vm startup
#
vm_first_stage() {
    vm_set_personality_syscall
    rm -rf "$BUILD_ROOT/.build"
    mkdir -p "$BUILD_ROOT/.build"
    TIME_PREINSTALL=
    if test "$DO_INIT" = true ; then
	# do first stage of init_buildsystem
	rm -f $BUILD_ROOT/.build.success
	set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$RECIPEFILE" $ADDITIONAL_PACKS
	echo "$* ..."
	start_time=`date +%s`
	"$@" || cleanup_and_exit 1
	check_exit
	TIME_PREINSTALL=$(( `date +%s` - $start_time ))
	unset start_time
	if test ! -w /root ; then
	    # remove setuid bit if files belong to user to make e.g. mount work
	    find $BUILD_ROOT/{bin,sbin,usr/bin,usr/sbin} -type f -uid $UID -perm +4000 -print0 | xargs -0 --no-run-if-empty chmod -s
	fi
	copy_oldpackages
    fi

    # start up VM, rerun ourself
    cp -a $BUILD_DIR/. $BUILD_ROOT/.build
    if ! test "$MYSRCDIR" = $BUILD_ROOT/.build-srcdir ; then
	rm -rf "$BUILD_ROOT/.build-srcdir"
	mkdir "$BUILD_ROOT/.build-srcdir"
	if test "$BUILDTYPE" = kiwi ; then
	    cp -pRL "$MYSRCDIR"/* $BUILD_ROOT/.build-srcdir
	else
	    cp -p "$MYSRCDIR"/* $BUILD_ROOT/.build-srcdir
	fi
	MYSRCDIR=$BUILD_ROOT/.build-srcdir
    else
	# cwd is at $BUILD_ROOT/.build-srcdir which we want to
	# umount later so step aside
	cd "$SRCDIR"
    fi

    # do vm specific fixups
    vm_fixup

    # update the hostarch
    if test -n "$VM_IMAGE" ; then
	vm_update_hostarch
    fi

    # the watchdog needs a log file
    test -n "$LOGFILE" || VM_WATCHDOG=
    # put our config into .build/build.data
    Q="'\''"
    echo "RECIPEFILE='${RECIPEFILE//"'"/$Q}'" > $BUILD_ROOT/.build/build.data
    echo "BUILD_JOBS='${BUILD_JOBS//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "BUILD_ARCH='${BUILD_ARCH//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "BUILD_RPMS='${BUILD_RPMS//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    case $BUILD_DIST in
	*/*)
	    cp $BUILD_DIST $BUILD_ROOT/.build/build.dist
	    BUILD_DIST=/.build/build.dist
	    ;;
    esac
    echo "BUILD_DIST='${BUILD_DIST//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "RELEASE='${RELEASE//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "BUILD_DEBUG='${BUILD_DEBUG//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "SIGNDUMMY='${SIGNDUMMY//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "DO_LINT='${DO_LINT//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "DO_CHECKS='${DO_CHECKS//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "NOROOTFORBUILD='${NOROOTFORBUILD//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "CREATE_BASELIBS='$CREATE_BASELIBS'" >> $BUILD_ROOT/.build/build.data
    echo "REASON='${REASON//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "CHANGELOG='${CHANGELOG//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "INCARNATION='${INCARNATION//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "DISTURL='${DISTURL//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "DO_INIT='${DO_INIT//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "DO_INIT_TOPDIR='${DO_INIT_TOPDIR//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "KIWI_PARAMETERS='${KIWI_PARAMETERS//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "VM_TELNET='${VM_TELNET//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    echo "VM_CONSOLE_INPUT='${VM_CONSOLE_INPUT//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    test -n "$VM_SWAP" && echo "VM_SWAP='${VM_SWAPDEV//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    test -n "$VMDISK_MOUNT_OPTIONS" && echo "VMDISK_MOUNT_OPTIONS='${VMDISK_MOUNT_OPTIONS//"'"/$Q}'" >> $BUILD_ROOT/.build/build.data
    PERSONALITY=0
    test -n "$PERSONALITY_SYSCALL" && PERSONALITY=`perl -e 'print syscall('$PERSONALITY_SYSCALL', 0)."\n"'`
    test "$PERSONALITY" = -1 && PERSONALITY=0	# syscall failed?
    case $(uname -m) in
	ppc|ppcle|s390) PERSONALITY=8 ;;	# ppc/s390 kernel never tells us if a 32bit personality is active, assume we run on 64bit
	aarch64) test "$BUILD_ARCH" != "${BUILD_ARCH#armv[567]}" && PERSONALITY=8 ;; # workaround, to be removed
    esac
    test "$VM_TYPE" = lxc -o "$VM_TYPE" = docker && PERSONALITY=0
    echo "PERSONALITY='$PERSONALITY'" >> $BUILD_ROOT/.build/build.data
    echo "MYHOSTNAME='`hostname`'" >> $BUILD_ROOT/.build/build.data
    echo -n "definesnstuff=(" >> $BUILD_ROOT/.build/build.data
    shellquote "${definesnstuff[@]}" >> $BUILD_ROOT/.build/build.data
    echo ")" >> $BUILD_ROOT/.build/build.data
    echo -n "repos=(" >> $BUILD_ROOT/.build/build.data
    shellquote "${repos[@]}" >> $BUILD_ROOT/.build/build.data
    echo ")" >> $BUILD_ROOT/.build/build.data
    echo "VM_TYPE='$VM_TYPE'" >> $BUILD_ROOT/.build/build.data
    echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data
    echo "DO_STATISTICS='$DO_STATISTICS'" >> $BUILD_ROOT/.build/build.data
    echo "TIME_PREINSTALL='$TIME_PREINSTALL'" >> $BUILD_ROOT/.build/build.data
    echo "VM_WATCHDOG='$VM_WATCHDOG'" >> $BUILD_ROOT/.build/build.data
    echo "BUILDENGINE='$BUILDENGINE'" >> $BUILD_ROOT/.build/build.data
    echo "CCACHE='$CCACHE'" >> $BUILD_ROOT/.build/build.data
    echo "ABUILD_TARGET='$ABUILD_TARGET'" >> $BUILD_ROOT/.build/build.data
    # fallback time for broken hosts
    date '+@%s' > $BUILD_ROOT/.build/.date
    # we're done with the root file system, unmount
    umount -n $BUILD_ROOT/proc/sys/fs/binfmt_misc 2> /dev/null || true
    umount -n $BUILD_ROOT/proc 2> /dev/null || true
    umount -n $BUILD_ROOT/dev/pts 2> /dev/null || true
    umount -n $BUILD_ROOT/dev/shm 2> /dev/null || true
    umount -n $BUILD_ROOT/mnt 2> /dev/null || true

    vm_init_script="/.build/build"
    if check_use_emulator ; then
	vm_init_script="/.build/$INITVM_NAME"
    fi
    if test -n "$VM_IMAGE" ; then
	# copy out kernel & initrd (if they exist) during unmounting VM image
	KERNEL_TEMP_DIR=
	if test -z "$VM_KERNEL" -a -e "$BUILD_ROOT/.build.kernel.$VM_TYPE" ; then
	    KERNEL_TEMP_DIR=`mktemp -d`
	    cp "$BUILD_ROOT/.build.kernel.$VM_TYPE" "$KERNEL_TEMP_DIR/kernel"
	    if test -e  "$BUILD_ROOT/.build.initrd.$VM_TYPE" ; then
	        cp "$BUILD_ROOT/.build.initrd.$VM_TYPE" "$KERNEL_TEMP_DIR/initrd"
	    fi
	fi
	check_exit
	# needs to work otherwise we have a corrupted file system
	if ! umount $BUILD_ROOT; then
	    rm -rf "$KERNEL_TEMP_DIR"
	    cleanup_and_exit 3
	fi
	# copy back the kernel and set it for VM
	if test -n "$KERNEL_TEMP_DIR" ; then
	    mkdir -p "$BUILD_ROOT/boot"
	    mv "$KERNEL_TEMP_DIR/kernel" "$BUILD_ROOT/boot/kernel"
	    vm_kernel="$BUILD_ROOT/boot/kernel"
	    if test -e "$KERNEL_TEMP_DIR/initrd" ; then
	        mv "$KERNEL_TEMP_DIR/initrd" "$BUILD_ROOT/boot/initrd"
	        test -z "$VM_INITRD" && vm_initrd="$BUILD_ROOT/boot/initrd"
	    fi
	    rmdir "$KERNEL_TEMP_DIR"
	fi
    fi
    vm_detach_root

    # start watchdog if requested
    if test -n "$VM_WATCHDOG" ; then
	start_watchdog
	echo "### WATCHDOG MARKER START ###"
    fi

    echo "booting $VM_TYPE..."
    vm_startup

    # kill watchdog again
    if test -n "$VM_WATCHDOG" ; then
	echo "### WATCHDOG MARKER END ###"
	kill_watchdog
    fi

    vm_attach_root
    if test -n "$VM_SWAP" ; then
	vm_attach_swap
	BUILDSTATUS=`dd if="$VM_SWAP" bs=12 count=1 2>/dev/null`
	case $BUILDSTATUS in
	  BUILDSTATUS[02])
	    mkdir -p $BUILD_ROOT/.build.packages
	    cd $BUILD_ROOT/.build.packages || cleanup_and_exit 1
	    echo "build: extracting built packages..."
	    extractbuild --disk "$VM_IMAGE" --input "$VM_SWAP" --skip 512 -v || cleanup_and_exit 3
	    if test "$DO_STATISTICS" = 1 ; then
		mkdir -p OTHER
		TIME_TOTAL=$(( `date +%s` - $TIME_START_TIME ))
		echo "TIME_total: $TIME_TOTAL"  >> OTHER/_statistics
	    fi
	    cleanup_and_exit ${BUILDSTATUS#BUILDSTATUS}
	    ;;
	  BUILDSTATUS*)
	    cleanup_and_exit ${BUILDSTATUS#BUILDSTATUS}
	    ;;
	  *)
	    echo "No buildstatus set, either the base system is broken (kernel/initrd/udev/glibc/bash/perl)"
	    echo "or the build host has a kernel or hardware problem..."
	    cleanup_and_exit 3
	    ;;
	esac
	cleanup_and_exit 1
    fi
}

vm_save_statistics() {
    echo "... saving statistics"
    local sys_mounted otherdir
    otherdir="$BUILD_ROOT$TOPDIR/OTHER"
    test -n "$TIME_PREINSTALL" && echo "TIME_preinstall: $TIME_PREINSTALL"  >> $otherdir/_statistics
    test -n "$TIME_INSTALL" && echo "TIME_install: $TIME_INSTALL"  >> $otherdir/_statistics
    if test -e /.build/_statistics.df ; then
	echo -n "MAX_mb_used_on_disk: " >> $otherdir/_statistics
	cat /.build/_statistics.df >> $otherdir/_statistics
	echo "" >> $otherdir/_statistics
	rm /.build/_statistics.df
    fi
    if test -e /.build/_statistics.memory ; then
	echo -n "MAX_mb_used_memory: " >> $otherdir/_statistics
	cat /.build/_statistics.memory >> $otherdir/_statistics
	echo "" >> $otherdir/_statistics
	rm /.build/_statistics.memory
    fi
    if ! test -e /sys/block; then
	mkdir -p /sys
	mount -n sys /sys -t sysfs
	sys_mounted=1
    fi
    device="hda1"
    test -e /dev/sda && device="sda"
    test -e /dev/vda && device="vda"
    test -e /dev/dasda && device="dasda" # in z/VM
    test -e /dev/nfhd0 && device="nfhd0" # in aranym
    if test -e /sys/block/${device}/stat ; then
	disk=(`cat /sys/block/${device}/stat`)
	test "0${disk[0]}" -gt 0 && echo "IO_requests_read: ${disk[0]}"  >> $otherdir/_statistics
	test "0${disk[2]}" -gt 0 && echo "IO_sectors_read: ${disk[2]}"   >> $otherdir/_statistics
	test "0${disk[4]}" -gt 0 && echo "IO_requests_write: ${disk[4]}" >> $otherdir/_statistics
	test "0${disk[6]}" -gt 0 && echo "IO_sectors_write: ${disk[6]}"  >> $otherdir/_statistics
    else
	echo "ERROR: no root disk device found, yet another new device name?"
	ls -l /sys/block/
    fi
    test -n "$sys_mounted" && umount /sys
}

# args: resultdirs
vm_wrapup_build() {
    test "$DO_STATISTICS" = 1 && vm_save_statistics
    if test -n "$VM_SWAP"; then
        echo "... saving built packages"
        swapoff "$VM_SWAP"
	pushd "$BUILD_ROOT$TOPDIR" >/dev/null
	find "$@" -print0 | computeblocklists --padstart 512 --padend 512 -v --manifest - -0 > "$VM_SWAP"
	popd >/dev/null
    fi
}

vm_setup_network() {
    if test -x /sbin/ip ; then 
	ip addr add 127.0.0.1/8 dev lo
	ip addr add ::1/128 dev lo
	ip link set lo up
    else
	ifconfig lo 127.0.0.1 up
	ifconfig lo add ::1/128
    fi
    if test -n "$VM_TELNET"; then 
	VM_TELNET_DEVICE=$( cd /sys/class/net/; echo * )
	VM_TELNET_DEVICE=${VM_TELNET_DEVICE#lo }
	VM_TELNET_DEVICE=${VM_TELNET_DEVICE%% *}
	if test -z "$VM_TELNET_DEVICE" ; then 
	    cleanup_and_exit 1 "ERROR: no network device found for telnet server"
	fi
	if test -x /sbin/ip ; then 
	    ip addr add 10.0.2.15/8 dev ${VM_TELNET_DEVICE}
	    ip addr add ::1/24 dev ${VM_TELNET_DEVICE}
	    ip link set ${VM_TELNET_DEVICE} up
	elif test -x /sbin/ifconfig ; then 
	    ifconfig ${VM_TELNET_DEVICE} 10.0.2.15 up
	    ifconfig ${VM_TELNET_DEVICE} add ::1/24
	else
	    cleanup_and_exit 1 "ERROR: neither /sbin/ifconfig nor /sbin/ip is installed, please specify correct package via -x option"
	fi
    fi
    if test -n "$MYHOSTNAME" ; then 
	hostname "$MYHOSTNAME"
    fi
    if test -n "$VM_TELNET"; then 
	echo WARNING: telnet option used, setting up telnet server ${VM_TELNET_DEVICE}
	if test -x /usr/sbin/in.telnetd; then 
	    ( /usr/sbin/in.telnetd -L /.build/telnet_login_wrapper -debug 23 & )
	else
	    cleanup_and_exit 1 "ERROR: /usr/sbin/in.telnetd is not installed, please specify correct package via -x option"
	fi
    fi
}