ra-tester-build-vms

#!/bin/bash

# This file is part of ratester.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA.

# ---- User config
# number of nodes/VM to build
NODES=3

# path of the cloud image to use for creating VMs that will host the cluster
IMG=

# Name prefix for the VMs to be created.
VMPREFIXNAME=ratester

# hypervisor node that will host the VMs
HYPERVISOR=localhost

# libvirt networks to use for the VM. If those networks do not exist,
# the script will create them.
# A few notes on network properties if you intent to provide yours:
#   . A first network SSHNW is both used for SSH connection and for
#     fencing requests. SSHNW provides relies on DHCP to give guests
#     their IP address, and provide NAT'd access to external world.
#     No cluster service runs on it.
#   . A second network CLUSTERNW is where all cluster services run
#     (pacemaker, corosync, resource's own communication...)
#     CLUSTERNW has DHCP disabled to avoid issue with cluster
#     manager when leases expire. IP are generated by ra-tester and
#     assigned automatically via cloud-init
CLUSTERNW=ratester
CLUSTERNWCIDR_DEFAULT=172.16.0.0/12
CLUSTERNWCIDR=
CLUSTER6PREFIX=
SSHNW=ratesterssh

# name of the storage pool that holds ra-tester VM's images
VMPOOLNAME=ratester

# Optional SSH public key to install on all created VM
ADDITIONALSSHKEY= #$HOME/.ssh/id_rsa.pub

# Optional packages to install on all created VM
ADDITIONALPKGS= #emacs-nox,vim

# Additional libvirt customize setup
ADDITIONALCMDS= #

# whether we install cluster packages
CLUSTERPKGS=1
# whether we enable SELinux on hosts or not
SELINUX=0
# shall we allow recreating VM if they already exist?
TEARDOWN=0
# keep tmp files for debugging problems
SAVETEMPS=0

VFLAGS=

# ---- Obscure config

VMRAM=${VMRAM:-2048}
VMCPU=${VMCPU:-2}
VMDISK=${VMDISK:-}
QEMUURI=${QEMUURI:-qemu:///system}
CMDTIMEOUT=${CMDTIMEOUT:-300}
GUESTFS_OPTS=${GUESTFS_OPTS:-}
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
SSH_CONTROL_MASTER_OPTS="-o ControlMaster=auto -o ControlPath=~/.ssh/ratester-%r@%h:%p -o ControlPersist=30m -o CheckHostIP=no"
LOGINKEYNAME=login-key
FENCEKEYNAME=fence-key
FENCEXVMKEYNAME=ratester_fence_xvm.key


function ssh_virt_inspector {
    $SSH_HYPERVISOR "env $GUESTFS_OPTS virt-inspector $1 '$2'"
}

function ssh_virt_customize {
    CMD=""; while (( "$#" )); do
	case "$1" in
	    \'*\'|\"*\") CMD="$CMD $1";;
	    *\ *) CMD="$CMD \""${1//\"/\\\"}"\"";;
            *) CMD="$CMD $1";;
	esac
	shift;
    done
    $SSH_HYPERVISOR "env $GUESTFS_OPTS virt-customize $CMD"
}

function ssh_virsh {
    $SSH_HYPERVISOR "virsh $*"
}

function ssh_virt_df {
    $SSH_HYPERVISOR "env $GUESTFS_OPTS virt-df $*"
}

function ssh_virt_resize {
    $SSH_HYPERVISOR "env $GUESTFS_OPTS virt-resize $*"
}

function ssh_virt_filesystems {
    $SSH_HYPERVISOR "env $GUESTFS_OPTS virt-filesystems $*"
}

function ssh_virt_rescue_grub {
    local device=$1
    local image=$2
    python <<EOF
import pexpect
import sys
child = pexpect.spawn('${SSH_HYPERVISOR_TTY} "env ${GUESTFS_OPTS} virt-rescue -a ${image} -i"', encoding='utf-8')
child.logfile_read = sys.stdout
child.expect('<rescue>')
child.sendline('chroot /sysroot grub-install ${device}')
child.expect('<rescue>')
child.sendline('exit')
child.expect(pexpect.EOF)
EOF
}

function configure_ssh_commands {
    SSH_HYPERVISOR="ssh $SSH_CONTROL_MASTER_OPTS $HYPERVISOR"
    SSH_HYPERVISOR_TTY="ssh $SSH_CONTROL_MASTER_OPTS -t $HYPERVISOR"
    SSH_PROXY_CMD="ssh $SSH_CONTROL_MASTER_OPTS -W %h:%p $HYPERVISOR"
}

VIRSH=ssh_virsh
VIRT_INSPECTOR=ssh_virt_inspector
VIRT_CUSTOMIZE=ssh_virt_customize

# Generated based on user config
SSH_HYPERVISOR=
SSH_HYPERVISOR_TTY=
SSH_PROXY_CMD=
SSHMAC=
CLUSTERMAC=
CLUSTERNWGW=
VMSTORAGEPATH=
VMTMPPATH=
VMBASEIMG=

# Distro-specific settings
PKG_FORMAT=
DISTRO=
DISTRO_PKGS=distrib-pkgs.yml


usage()
{
    cat <<EOF
Create and set up KVM-based virtual machines for ra-tester
Usage: $(basename $0) --img QCOW2_IMAGE_PATH --name VM_PREFIX_NAME [options ...]

Options:
   --hypervisor HOSTNAME        create the VMs on host HOSTNAME (default: localhost)
   --img QCOW2_IMAGE_PATH       QCOW2 cloud image to be used as a base for creating VMs
   --name VM_PREFIX_NAME        Name prefix for the VMs to be created (default: ratester)
   --nodes NUMBER               NUMBER of VMs to be created (default: 3)
   --cluster-nw NW_NAME         Libvirt IPv4 network for pacemaker and resources (no DHCP)
   --cluster-nw-cidr NW_CIDR    Libvirt IPv4 network CIDR for pacemaker and resources (no DHCP)
   --cluster-prefix-ipv6 IP     Libvirt IPv6 network prefix (/64) for pacemaker and resources (no DHCP)
   --ssh-nw NW_NAME             Secondary network, for debugging purpose
   --opt-ssh-key SSH_KEY        Optional public key to inject into created VMs
   --opt-pkgs PKG1,PKG2,...     Additional packages to install on created VMs
   --opt-cmd RUN-CMD            Additional shell command to run in the VMs before setup
   --skip-install-cluster       Do not install cluster packages (pacemaker, pcs, agents)
   --package-mapping FILE       YAML file that remaps package names based on distro
   --teardown                   Delete VMs and associated storage if they already exist
   --selinux                    Configure VMs to enable SELinux
   --verbose                    run all virsh command in verbose mode
   --save-temps                 Keep intermediate images and files

Exemple:
   $(basename $0) --img ./CentOS-7-x86_64-GenericCloud.qcow2 --name ratester
EOF
}

distro_pkg()
{
    local pkg=$1
    cat $DISTRO_PKGS | python -c "import yaml,sys; j=yaml.safe_load(sys.stdin); r=j.get('$PKG_FORMAT-$DISTRO',{}).get('$pkg',False); print(r if r else j.get('$PKG_FORMAT',{}).get('$pkg','$pkg'))"
}

cleanup_tmp_files()
{
    if [ $SAVETEMPS -ne 1 ]; then
        $SSH_HYPERVISOR rm -f ${VMTMPPATH}/ratmp-* ${VMTMPPATH}/*-data* ${VMTMPPATH}/*.rules
        $SSH_HYPERVISOR rm -f ${VMTMPPATH}/$LOGINKEYNAME ${VMTMPPATH}/$LOGINKEYNAME.pub ${VMTMPPATH}/$FENCEKEYNAME ${VMTMPPATH}/$FENCEKEYNAME.pub ${VMTMPPATH}/$FENCEXVMKEYNAME
    fi
}

on_error()
{
    echo "ERROR: failure occurred in $(basename $0):$1. Aborting" >&2
    cleanup_tmp_files
}

define_networks()
{
    trap 'on_error $LINENO' ERR

    if ! $VIRSH -c ${QEMUURI} net-list | grep -q "${CLUSTERNW}\\s"; then
        echo "Creating cluster network \"$CLUSTERNW\" (no dhcp)"
        CLUSTERNW=$CLUSTERNW \
        CLUSTERNWGW=$CLUSTERNWGW \
        CLUSTERNWMASK=$CLUSTERNWMASK \
        CLUSTER6NWGW=$CLUSTER6NWGW \
        envsubst <templates/cluster-net.xml.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/ratmp-cluster-net.xml >/dev/null
        $VIRSH -c ${QEMUURI} net-define ${VMTMPPATH}/ratmp-cluster-net.xml
        $VIRSH -c ${QEMUURI} net-autostart ${CLUSTERNW}
        $VIRSH -c ${QEMUURI} net-start ${CLUSTERNW}
    fi
    if ! $VIRSH -c ${QEMUURI} net-list | grep -q "${SSHNW}\\s"; then
        echo "Creating cluster network \"$SSHNW\" (dhcp, external access)"
        SSHNW=$SSHNW envsubst <templates/ssh-net.xml.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/ratmp-ssh-net.xml >/dev/null
        $VIRSH -c ${QEMUURI} net-define ${VMTMPPATH}/ratmp-ssh-net.xml
        $VIRSH -c ${QEMUURI} net-autostart ${SSHNW}
        $VIRSH -c ${QEMUURI} net-start ${SSHNW}
    fi
}

define_pool()
{
    trap 'on_error $LINENO' ERR

    if ! $VIRSH -c ${QEMUURI} pool-list | grep -q ratester; then
        echo "Creating storage pool for ra-tester VMs"
        VMPOOLNAME=$VMPOOLNAME \
        VMSTORAGEPATH=$VMSTORAGEPATH \
        envsubst <templates/pool.xml.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/ratmp-pool.xml >/dev/null
        $VIRSH -c ${QEMUURI} pool-define ${VMTMPPATH}/ratmp-pool.xml
        $VIRSH -c ${QEMUURI} pool-autostart ratester
        $VIRSH -c ${QEMUURI} pool-start ratester
    fi
}

generate_fence_config_and_key()
{
    trap 'on_error $LINENO' ERR

    if [ ! -f fence_virt.conf ]; then
        echo "Generating configuration for multicast-based fencing"
        local iface=$($VIRSH -c ${QEMUURI} net-dumpxml $SSHNW | sed -ne "s/.*bridge.*name='\([^']*\)' .*/\1/p")
        QEMUURI=$QEMUURI \
        IFACE=$iface \
        FENCEXVMKEYNAME=$FENCEXVMKEYNAME \
        envsubst <templates/fence_virt.conf.in | $SSH_HYPERVISOR tee fence_virt.conf >/dev/null
    fi

    if [ ! -f $FENCEXVMKEYNAME ]; then
        echo "Generating authorization key for multicast-based fencing"
        dd if=/dev/urandom of=$FENCEXVMKEYNAME bs=4096 count=1
    fi
}

ip_from_cidr()
{
    local cidr=$1
    local offset=$2
    python -c "from netaddr import *; print(IPNetwork('$cidr')[$offset])"
}

mask_from_cidr()
{
    local cidr=$1
    python -c "from netaddr import *; print(IPNetwork('$cidr').netmask)"
}

cluster_gw()
{
    local cidr=$1
    ip_from_cidr $cidr 1
}

mac_template()
{
    local oui=$(echo $1 | md5sum | head -c +4)
    local nic=$(echo $2 | md5sum | head -c +4)
    local sub=$3
    echo ee$oui$nic$sub | sed -e 's/\(..\)/\1:/g' -e 's/:$//'
}

node_mac()
{
    local mac=$1
    local nodenum=$2
    echo "$(echo $mac | head -c +16)$nodenum"
}

node_dhcp_ip()
{
    trap 'on_error $LINENO' ERR

    local nodenum=$1
    echo $($VIRSH -c ${QEMUURI} net-dhcp-leases $SSHNW | awk "/"$(node_mac $SSHMAC $nodenum)"/ {print \$5}" | head -1 | sed 's%/.*%%')
}

node_static_ip()
{
    trap 'on_error $LINENO' ERR
    local gw=$($VIRSH -c ${QEMUURI} net-dumpxml $CLUSTERNW | grep netmask | sed -ne "s/.*ip.*address='\([^']*\)'.*/\1/p")
    local mask=$($VIRSH -c ${QEMUURI} net-dumpxml $CLUSTERNW | grep netmask | sed -ne "s/.*netmask='\([^']*\)'.*/\1/p")
    local hashdata=$1
    local nodenum=$2
    python -c "import hashlib, struct; from netaddr import *; ipn=IPNetwork('$gw/$mask'); h=struct.unpack('I',hashlib.md5(b'$hashdata').digest()[:4])[0]; subip=h<<3&((1<<(32-ipn.prefixlen))-1); print(ipn[subip+$nodenum])"
}

default_prefix_ipv6()
{
    echo fd00$(echo ra-tester | md5sum | head -c +12) | sed -e 's/\(....\)/\1:/g' -e 's/:$//'
}

node_static_ipv6()
{
    trap 'on_error $LINENO' ERR

    local mac_prefix=$1
    local node=$2
    echo $CLUSTER6PREFIX"::"$(echo $mac_prefix | sed -e 's/://g' -e 's/.$//' -e 's/\(....\)/\1:/g')$node
}

prepare_base_image()
{
    trap 'on_error $LINENO' ERR

    local etc_hosts=$(for i in `seq 1 $NODES`; do echo -n "$(node_static_ip $VMPREFIXNAME $i) $VMPREFIXNAME$i.ratester $VMPREFIXNAME$i\n"; done)
    local etc_hosts6=$(for i in `seq 1 $NODES`; do echo -n "$(node_static_ipv6 $CLUSTERMAC $i) $VMPREFIXNAME$i.v6.ratester $VMPREFIXNAME$i.v6\n"; done)

    echo "Creating temporary base image based on ${IMG}"
    $SSH_HYPERVISOR cp ${IMG} ${VMBASEIMG}

    if [ -n "${VMDISK}" ]; then
	echo "Setting temporary base image's disk size to ${VMDISK}"
	$SSH_HYPERVISOR qemu-img resize ${VMBASEIMG} ${VMDISK}
	partition=$(ssh_virt_df ${IMG} --csv | sed '1d' | sort -r -t, -nk3,3 | awk -F, '{print $2;exit}')
	echo "Expanding partition '${partition}' in temporary base image to match new disk image size"
	ssh_virt_resize --output-format qcow2 --expand $partition ${IMG} ${VMBASEIMG}
	# Some images (e.g. ubuntu focal) get their partition renamed
	# by the resize, so re-run grub to fix any boot issue
	device=$(ssh_virt_filesystems -a ${IMG} --blkdevs | head -1)
	ssh_virt_rescue_grub $device $VMBASEIMG
    fi

    echo "Determining package format in base image ${IMG}"
    local inspect=$($VIRT_INSPECTOR -a ${VMBASEIMG})
    DISTRO=$(echo "$inspect" | $VIRT_INSPECTOR --xpath '//operatingsystems/operatingsystem/distro/text()')
    PKG_FORMAT=$(echo "$inspect" | $VIRT_INSPECTOR --xpath '//operatingsystems/operatingsystem/package_format/text()')

    echo "Pushing generated SSH keys to the hypervisor's temporary storage directory"
    scp $LOGINKEYNAME $LOGINKEYNAME.pub $FENCEKEYNAME $FENCEKEYNAME.pub $FENCEXVMKEYNAME ${HYPERVISOR}:${VMTMPPATH}

    echo "Preparing temporary base image"
    # startup and access
    $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} \
        --run-command "\\\`which echo\\\` -e \"$etc_hosts\"\"$etc_hosts6\" >> /etc/hosts" \
        --timezone $(ls -l /etc/localtime | sed -s 's%.*/usr/share/zoneinfo/%%') \
        --run-command "echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config" \
        --mkdir "/root/.ssh" \
        --upload ${VMTMPPATH}/$LOGINKEYNAME:/root/.ssh/id_rsa \
        --upload ${VMTMPPATH}/$LOGINKEYNAME.pub:/root/.ssh/id_rsa.pub \
        --run-command "cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys" \
        --upload ${VMTMPPATH}/$FENCEKEYNAME:/root/.ssh/$FENCEKEYNAME \
        --upload ${VMTMPPATH}/$FENCEKEYNAME.pub:/root/.ssh/$FENCEKEYNAME.pub

    # optional pre-configure commands to be run, typically set by user to add repos
    if [ "$ADDITIONALCMDS" != "" ]; then
	IFS=$'\n';
	CMD=""
	for i in $(echo "$ADDITIONALCMDS" | sed 's/\\0000/\n/g'); do
	    case "$i" in
		\'*\'|\"*\") CMD="$CMD $i";;
		*\ *) CMD="$CMD \""${i//\"/\\\"}"\"";;
		*) CMD="$CMD $i";;
	    esac
	done
	unset IFS
	$VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} ${CMD}
    fi

    if [ "$PKG_FORMAT" = "deb" ]; then
	echo "Update packages on the temporary image"
	$VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} \
	    --run-command "apt-get update" --run-command "apt-get upgrade -y"
    fi

    # ensure lp#1577982 is fixed otherwise cloud-init fail to assign ip properly
    $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} \
        --run-command "sed -i \"s/'network-config': {}/'network-config': None/\" /usr/lib/python*/*-packages/cloudinit/sources/DataSourceNoCloud.py" \
	--install $(distro_pkg lsb-core),crudini

    if [ $CLUSTERPKGS -eq 1 ]; then
    $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} \
	--install pacemaker,$(distro_pkg pacemaker-remote),$(distro_pkg pacemaker-cts) \
	--install pcs,resource-agents \
        --install $(distro_pkg fence-virtd),$(distro_pkg fence-agents-virsh),$(distro_pkg fence-agents-xvm) \
        --run-command "systemctl enable pcsd" \
        --password hacluster:password:ratester \
        --mkdir /etc/cluster \
        --upload ${VMTMPPATH}/$FENCEXVMKEYNAME:/etc/cluster/fence_xvm.key
    fi

    # convenience packages
    if [ ! -z "$ADDITIONALPKGS" ]; then
        $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} --install $ADDITIONALPKGS
    fi

    # convenience ssh key
    if [ -f "$ADDITIONALSSHKEY" ]; then
        $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} --ssh-inject root:file:$ADDITIONALSSHKEY
    fi

    # SELinux settings
    if [ $SELINUX -eq 1 ]; then
        if [ "$PKG_FORMAT" == "rpm" ]; then
            $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} --selinux-relabel
        fi
    else
        $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMBASEIMG} \
           --run-command "if [ -f /etc/selinux/config ]; then sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config; fi"
    fi

}

setup_node()
{
    trap 'on_error $LINENO' ERR

    local nodenum=$1

    local IMGNAME=${VMPREFIXNAME}${nodenum}.qcow2
    local ISONAME=${VMPREFIXNAME}${nodenum}.iso

    if $VIRSH -c ${QEMUURI} dominfo ${VMPREFIXNAME}${nodenum} &>/dev/null; then
        if $VIRSH -c ${QEMUURI} domstate ${VMPREFIXNAME}${nodenum} | grep -q running; then
            echo "Stopping previously defined VM ${VMPREFIXNAME}${nodenum}..."
            $VIRSH -c ${QEMUURI} destroy ${VMPREFIXNAME}${nodenum}
        fi
        echo "Deleting previously defined VM ${VMPREFIXNAME}${nodenum}..."
        $VIRSH -c ${QEMUURI} undefine ${VMPREFIXNAME}${nodenum} --remove-all-storage
    fi
    if $VIRSH -c ${QEMUURI} vol-info --pool ${VMPOOLNAME} ${IMGNAME} &>/dev/null; then
       $VIRSH -c ${QEMUURI} vol-delete --pool ${VMPOOLNAME} ${IMGNAME} 2>/dev/null
    fi
    if $VIRSH -c ${QEMUURI} vol-info --pool ${VMPOOLNAME} ${ISONAME} &>/dev/null; then
       $VIRSH -c ${QEMUURI} vol-delete --pool ${VMPOOLNAME} ${ISONAME} 2>/dev/null
    fi

    echo "Creating storage volume info for ${VMPREFIXNAME}${nodenum}"
    $VIRSH -c ${QEMUURI} vol-create-as ${VMPOOLNAME} ${IMGNAME} 10GB --format qcow2
    $VIRSH -c ${QEMUURI} vol-create-as ${VMPOOLNAME} ${ISONAME} 500KB --format raw

    echo "Forcing order of network interface cards for ${VMPREFIXNAME}${nodenum}"
    SSHMAC=$(node_mac $SSHMAC $nodenum) \
    CLUSTERMAC=$(node_mac $CLUSTERMAC $nodenum) \
    envsubst <templates/udev-90-ratester-nic-order.rules.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/90-ratester-nic-order.rules >/dev/null

    echo "Forcing DHCP to use MAC address as ClientId for ${VMPREFIXNAME}${nodenum}"
    envsubst <templates/00-ratester.network.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/00-ratester.network >/dev/null

    echo "Creating image for ${VMPREFIXNAME}${nodenum}"
    # we could let cloudinit set hostname, but doing it here
    # has the proper hostname propagated to dnsmasq's DHCP on the host
    # NOTE: do a special dance to prevent --hostname to update /etc/hosts
    # NOTE2: remove the machine-id to force virt-customize to re-generate
    # unique machine id for this VM
    $VIRT_CUSTOMIZE ${VFLAGS} -a ${VMTMPPATH}/ratmp-base.qcow2 \
        --upload ${VMTMPPATH}/90-ratester-nic-order.rules:/etc/udev/rules.d \
        --upload ${VMTMPPATH}/00-ratester.network:/etc/systemd/network \
        --move /etc/hosts:/etc/hosts.tmp \
        --hostname ${VMPREFIXNAME}${nodenum} \
        --move /etc/hosts.tmp:/etc/hosts \
        --run-command "rm -f /etc/machine-id && systemd-machine-id-setup"
    $VIRSH -c ${QEMUURI} vol-upload --pool ${VMPOOLNAME} ${IMGNAME} ${VMTMPPATH}/ratmp-base.qcow2

    echo "Creating cloud-init metadata for ${VMPREFIXNAME}${nodenum}"
    DOMNAME=${VMPREFIXNAME}${nodenum} \
    CLUSTERIP=$(node_static_ip $VMPREFIXNAME $nodenum) \
    CLUSTERIPV6=$(node_static_ipv6 $CLUSTERMAC $nodenum) \
    CLUSTERNWGW=$CLUSTERNWGW \
    CLUSTERNWMASK=$CLUSTERNWMASK \
    envsubst <templates/cloudinit-meta-data.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/meta-data >/dev/null
    $SSH_HYPERVISOR cp ${VMTMPPATH}/meta-data /tmp/dciabrin-meta-data-$DOMNAME
    DOMNAME=${VMPREFIXNAME}${nodenum} \
    envsubst <templates/cloudinit-user-data.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/user-data >/dev/null
    $SSH_HYPERVISOR cp ${VMTMPPATH}/user-data /tmp/dciabrin-user-data-$DOMNAME
    $SSH_HYPERVISOR genisoimage -output ${VMTMPPATH}/ratmp-${ISONAME} -volid cidata -joliet -rock ${VMTMPPATH}/user-data ${VMTMPPATH}/meta-data
    $VIRSH -c ${QEMUURI} vol-upload --pool ${VMPOOLNAME} ${ISONAME} ${VMTMPPATH}/ratmp-${ISONAME}

    echo "Creating ${VMPREFIXNAME}${nodenum} and waiting for its IP to be assigned by DHCP"
    DOMNAME=${VMPREFIXNAME}${nodenum} \
    VMRAM=${VMRAM} \
    VMCPU=${VMCPU} \
    IMGNAME=${IMGNAME} \
    ISONAME=$ISONAME \
    VMPOOLNAME=$VMPOOLNAME \
    SSHNW=$SSHNW \
    SSHMAC=$(node_mac $SSHMAC $nodenum) \
    CLUSTERNW=$CLUSTERNW \
    CLUSTERMAC=$(node_mac $CLUSTERMAC $nodenum) \
    envsubst <templates/domain.xml.in | $SSH_HYPERVISOR tee ${VMTMPPATH}/ratmp-${VMPREFIXNAME}${nodenum}.xml >/dev/null

    $VIRSH -c ${QEMUURI} define ${VMTMPPATH}/ratmp-${VMPREFIXNAME}${nodenum}.xml
    $VIRSH -c ${QEMUURI} start ${VMPREFIXNAME}${nodenum}
    # timeout $CMDTIMEOUT sh -c "while ! ($VIRSH -c ${QEMUURI} net-dhcp-leases $SSHNW | grep -q $(node_mac $SSHMAC $nodenum)); do sleep 2; done"
    t=$CMDTIMEOUT
    while ! ($VIRSH -c ${QEMUURI} net-dhcp-leases $SSHNW | grep -q $(node_mac $SSHMAC $nodenum)); do
	sleep 2
	t=$((t-2))
	if [ $t -eq 0 ]; then exit 124; fi
    done
}

wait_for_node()
{
    trap 'on_error $LINENO' ERR

    local nodenum=$1
    local nodeip=$(node_dhcp_ip $nodenum)

    # the node may reboot when first started, so loop here
    # until ssh is successful, rather than just ping
    echo "Wait until we can connect to ${VMPREFIXNAME}${nodenum}"
    timeout $CMDTIMEOUT sh -c "while ! ssh -q -i $LOGINKEYNAME $SSH_OPTS -o ControlMaster=no -o ProxyCommand='$SSH_PROXY_CMD -o ControlMaster=no' root@$nodeip true 2>/dev/null; do sleep 2; done"
    if [ $? -ne 0 ]; then
        echo "ERROR: Failed to get SSH connection to domain ${VMPREFIXNAME}${nodenum}" >&2
        exit 1
    fi

    # tmp for opensuse: fix reboot
    if [ "$PKG_FORMAT" == "rpm" -a "$DISTRO" == "opensuse" ]; then
        echo "Update grub to fix reboot on ${VMPREFIXNAME}${nodenum}"
        ssh -i $LOGINKEYNAME $SSH_OPTS -o ProxyCommand="$SSH_PROXY_CMD" root@$nodeip "grub2-mkconfig > /boot/grub2/grub.cfg"
    fi

    # cloud-init and nocloud ds: ensure that cluster nw nic is brought up
    echo "Ensure reboot succeeds and cluster network is up"
    ssh -i $LOGINKEYNAME $SSH_OPTS -o ProxyCommand="$SSH_PROXY_CMD" root@$nodeip "reboot && cat" || true
    local clustermac=$(node_mac $CLUSTERMAC $nodenum)
    timeout $CMDTIMEOUT sh -c "while ! ssh -q -i $LOGINKEYNAME $SSH_OPTS -o ControlMaster=no -o ProxyCommand='$SSH_PROXY_CMD -o ControlMaster=no' root@$nodeip \"ip link | grep -B1 $clustermac | head -1 | grep -q 'state UP'\" 2>/dev/null; do sleep 2; done"
    if [ $? -ne 0 ]; then
        echo "ERROR: Failed to get verify cluster network status on domain ${VMPREFIXNAME}${nodenum}" >&2
        exit 1
    fi
}

wait_for_pcsd()
{
    trap 'on_error $LINENO' ERR

    local nodenum=$1
    local nodeip=$(node_dhcp_ip $nodenum)

    # SSH is not enough, pcsd has to be up and running to be able
    # to create a cluster from a single node
    echo "Wait until pcsd is up and running ${VMPREFIXNAME}${nodenum}"
    timeout $CMDTIMEOUT sh -c "while ! ssh -q -i $LOGINKEYNAME $SSH_OPTS -o ProxyCommand='$SSH_PROXY_CMD' root@$nodeip \"netstat -tnlp | grep -q ':2224\\s'\" 2>/dev/null; do sleep 2; done"
    if [ $? -ne 0 ]; then
        echo "ERROR: Failed to get pcsd status on domain ${VMPREFIXNAME}${nodenum}" >&2
        exit 1
    fi
}

ensure_teardown_if_vm_exist()
{
    for nodenum in `seq 1 $NODES`; do
        if $VIRSH -c ${QEMUURI} dominfo ${VMPREFIXNAME}${nodenum} &>/dev/null; then
            if [ $TEARDOWN -eq 0 ]; then
                echo "ERROR: VM ${VMPREFIXNAME}${nodenum} already exists. Use option --teardown to allow ra-tester to recreate VMs" >&2
                exit 1
            else
                return 0
            fi
        fi
    done
}

generate_ssh_keys()
{
    trap 'on_error $LINENO' ERR

    if [ ! -f $LOGINKEYNAME ]; then
        echo "Creating SSH keypair for logging in"
        ssh-keygen -t rsa -f $LOGINKEYNAME -N "" -C 'ratester login key'
    fi
    if [ ! -f $FENCEKEYNAME ]; then
        echo "Creating SSH keypair for node fencing"
        ssh-keygen -t rsa -f $FENCEKEYNAME -N "" -C 'ratester fence key'
        cat $FENCEKEYNAME.pub >> $HOME/.ssh/authorized_keys
    fi
}

generate_ssh_config()
{
    trap 'on_error $LINENO' ERR

    echo "Generating SSH config for logging in"

    VMPREFIXNAME=${VMPREFIXNAME} \
    LOGINKEYNAME=${PWD}/${LOGINKEYNAME} \
    envsubst <templates/ssh_config.in >ssh_config_${VMPREFIXNAME}
    if [ "x$HYPERVISOR" != "xlocalhost" ]; then
        echo -e "     ProxyCommand ssh -o ControlMaster=auto -o ControlPath=~/.ssh/ratester-hypervisor-$HYPERVISOR:%p -o ControlPersist=30m -o CheckHostIP=no -W %h:%p $HYPERVISOR" >>ssh_config_${VMPREFIXNAME}
    fi
    for i in `seq 1 $NODES`; do
        local node=${VMPREFIXNAME}$i
        local nodeip=$(node_dhcp_ip $i)
        echo -e "\nHost ${node}\n\tHostName ${nodeip}" >>ssh_config_${VMPREFIXNAME}
    done
}

populate_ssh_known_hosts()
{
    trap 'on_error $LINENO' ERR

    echo "Pre-populate SSH known hosts for cross-VM connections"

    for i in `seq 1 $NODES`; do
	local node=${VMPREFIXNAME}$j
        local nodeip=$(node_dhcp_ip $i)
	for j in `seq 1 $NODES`; do
            local target_node=${VMPREFIXNAME}$j
	    ssh -i $LOGINKEYNAME $SSH_OPTS -o ProxyCommand="$SSH_PROXY_CMD" root@$nodeip "ssh -o StrictHostKeyChecking=no $target_node true"
	done
    done
}

while [ "$1" != "" ]; do
    case "$1" in
        --hypervisor ) HYPERVISOR="$2"; shift 2;;
        --img ) IMG="$2"; shift 2;;
        --name ) VMPREFIXNAME="$2"; shift 2;;
        --nodes ) NODES="$2"; shift 2;;
        --cluster-nw ) CLUSTERNW="$2"; SSHNW="$2"; shift 2;;
        --cluster-nw-cidr ) CLUSTERNWCIDR="$2"; shift 2;;
        --cluster-prefix-ipv6 ) CLUSTER6PREFIX="$2"; shift 2;;
        --ssh-nw ) SSHNW="$2"; shift 2;;
        --vm-storage-path ) VMSTORAGEPATH="$2"; shift 2;;
        --opt-ssh-key ) ADDITIONALSSHKEY="$2"; shift 2;;
        --opt-pkgs ) ADDITIONALPKGS="$2"; shift 2;;
        --opt-cmd )
            if [ "$ADDITIONALCMDS" != "" ]; then
                ADDITIONALCMDS="$ADDITIONALCMDS\0000"
            fi
            ADDITIONALCMDS="$ADDITIONALCMDS--run-command\0000$2"; shift 2;;
        --skip-install-cluster ) CLUSTERPKGS=0; shift;;
        --package-mapping ) DISTRO_PKGS="$2"; shift 2;;
        --teardown ) TEARDOWN="1"; shift;;
        --selinux ) SELINUX=1; shift;;
        --verbose ) VFLAGS="$VFLAGS -v"; shift;;
        --save-temps ) SAVETEMPS=1; shift;;
        -h|--help ) usage; exit 0;;
        * ) echo "ERROR: unknown parameter \"$1\"" >&2
            usage
            exit 1;;
    esac
done

# various prechecks

if [ "$IMG" == "" ]; then
    echo "ERROR: no qcow2 image provided to build VM. Use --img QCOW2_IMAGE_PATH" >&2
    exit 1
fi

configure_ssh_commands
if ! $VIRSH -c ${QEMUURI} list &>/dev/null; then
    echo "ERROR: Cannot connect to hypervisor at ${QEMUURI}" >&2
    echo "Check that user belongs to group 'kvm' or has proper sudo settings" >&2
    exit 1
fi

# make sure the storage path is writable
if [ "x$VMSTORAGEPATH" = "x" ]; then
    if [ "x$HYPERVISOR" != "xlocalhost" ]; then
        VMSTORAGEPATH=/var/lib/ratester/vm/tmp
    else
        VMSTORAGEPATH=$(cd `dirname $0` && pwd)/vm
    fi
fi
VMTMPPATH=$VMSTORAGEPATH/tmp
VMBASEIMG=$VMTMPPATH/ratmp-base.qcow2

for path in "$VMSTORAGEPATH" "$VMTMPPATH"; do
    if ! $SSH_HYPERVISOR "test -d \"$path\""; then
        echo "Creating path $path on hypervisor ($HYPERVISOR)"
        cmd='mkdir -p "'$path'" && chown $(id -u) "'$path'"'
        if ! $SSH_HYPERVISOR "bash -c '$cmd'"; then
            echo "Requesting permission to create directory"
            $SSH_HYPERVISOR -t "sudo bash -c '$cmd'"
        fi
    fi
done

set -e

# Creation and provisioning
# -------------------------


SSHMAC=$(mac_template ratester $VMPREFIXNAME 00)
CLUSTERMAC=$(mac_template ratester $VMPREFIXNAME f0)
CLUSTERNWCIDR=${CLUSTERNWCIDR:-$CLUSTERNWCIDR_DEFAULT}
CLUSTERNWGW=$(cluster_gw $CLUSTERNWCIDR)
CLUSTERNWMASK=$(mask_from_cidr $CLUSTERNWCIDR)
CLUSTER6PREFIX=${CLUSTER6PREFIX:-$(default_prefix_ipv6)}
CLUSTER6NWGW=${CLUSTER6PREFIX}"::1"


# Pre-check: ensure ra-tester can delete VMs if they already exist
ensure_teardown_if_vm_exist

# Generate ratester keys, keep if already present
generate_ssh_keys

# # Create networks in libvirt, reuse if existing
define_networks

# Create storage pool in libvirt, reuse if existing
define_pool

# Generate fencing config for fence_xvm
generate_fence_config_and_key

# Create the and customize the base image
prepare_base_image

# Create VMs from the base image and finish provisioning
for i in `seq 1 $NODES`; do setup_node $i; done

# Dump connection setting
generate_ssh_config

# Wait for VMs availability
for i in `seq 1 $NODES`; do wait_for_node $i; done

# Delete all previous control master sockets
rm -rf ~/.ssh/ratester-*

# Pre-populate SSH known hosts for cross-VM connections
populate_ssh_known_hosts

if [ $CLUSTERPKGS -eq 1 ]; then
    # Wait for pcsd availability
    for i in `seq 1 $NODES`; do wait_for_pcsd $i; done
fi

cleanup_tmp_files

cat <<EOF
Cluster bootstrapped. You may now log in with:
   ssh -F ssh_config_${VMPREFIXNAME} ${VMPREFIXNAME}{x}

You're almost ready to run ra-tester!
------------------------------------------------------
* Finish fencing configuration manually
If you intend to use SSH-based fencing (fence_virsh):
   . Please add SSH public key ${FENCEKEYNAME}.pub to your .ssh/authorized_keys
   . you'll need to run ra-tester with "--stonith 1"

If you intend to use Multicast-based fencing (fence_xvm):
   . Please copy the generated fence_virt.conf to /etc/fence_virt.conf
   . Please copy the generated xvm fence key ${FENCEXVMKEYNAME} to /etc/cluster/
   . you'll need to run ra-tester with "--stonith xvm"
EOF