test-ha-sr2

#!/bin/sh

# Test repeatedly blocking and unblocking iSCSI statefile access to check that HA starts then stops using
# survival rule 2. See CA-25817

set -e

# Given either '1' or '2', block for up to 120s for the pool to stabilise
# using the specified 'survival rule'
wait_for_sr() {
  local rule=$1
  for((retry=1;retry<120;retry++)); do
    finished=1
    IFS=","; for host in $(xe host-list params=uuid --minimal); do
      x=$(xe host-param-get uuid=${host} param-name=ha-statefiles)
      if [ ${rule} -eq 2 ]; then
        if [ -n "${x}" ]; then
          # Someone has statefile access -> not SR 2
          finished=0
        fi
      else
        if [ -z "${x}" ]; then
	  # Someone doesn't have statefile access -> not SR 1
          finished=0
        fi
      fi
    done
    if [ ${finished} -eq 1 ]; then
      return 0
    else
      echo -n .
      sleep 1s
    fi
  done
  echo Failed
  return 1
}

# Make a DSA key if none exists and add it to the authorized_keys on this host
if [ ! -e ${HOME}/.ssh/id_dsa ]; then
    echo "Generating SSH DSA key"
    ssh-keygen -t dsa
    cat ${HOME}/.ssh/id_dsa.pub >> ${HOME}/.ssh/authorized_keys
fi

# Be lazy and copy the .ssh/ dir to every host
echo "Copying ssh key to every host"
IFS=","; for address in $(xe host-list params=address --minimal); do
    echo "Copying to ${address}"
    scp -r ${HOME}/.ssh root@${address}:
done

# Helper function to run a remote command with logging
remote_exec(){
    local address=$1
    local cmd=$2
    date=$(date -u +"%Y%m%dT%H:%M:%SZ")
    echo "[${date}] ssh ${address} ${cmd}"
    ssh ${address} ${cmd}
}

# Helper function to run a given command on all hosts
remote_exec_all(){
    local cmd=$1
    IFS=","; for address in $(xe host-list params=address --minimal); do
	remote_exec ${address} ${cmd}
    done
}

# Make sure we're running normally before starting the test
echo -n "Waiting for Survival Rule 1: "
wait_for_sr 1
echo OK

while [ /bin/true ]; do
    # Block statefile everywhere
    remote_exec_all "iptables -I OUTPUT -p tcp --dport 3260 -j DROP"

    echo -n "Waiting for Survival Rule 2: "
    wait_for_sr 2
    echo OK
    # Unblock statefile everywhere
    remote_exec_all "iptables -F"
    echo -n "Waiting for Survival Rule 1: "
    wait_for_sr 1
    echo OK
done