Skip to content
This repository has been archived by the owner on Sep 26, 2019. It is now read-only.

Commit

Permalink
SuperHAProxy
Browse files Browse the repository at this point in the history 
SuperHAProxy is a script around HAProxy to add support for live
migration and easy handling of multiple instances. Live migration is
added using the CRIU system. Very lightweight containers are used to
isolate the proxy instances from each other to enable live migration
to efficiently transfer the proxies and to ensure instances can be
running different versions to support proper rolling upgrades.

This script can be used stand alone, or as a building block to
provide multiserver proxy pools that are high performance and support
a workflow where the servers can be rolling upgraded live without
connection loss.

Change-Id: Id3150c5d5de6c3f5552f27f79cbcf0debe8adb1f
  • Loading branch information
@kfox1111
kfox1111 committed Mar 30, 2016
1 parent 529e1db commit bd711d6
Showing 1 changed file with 346 additions and 0 deletions.
346 changes: 346 additions & 0 deletions multi/superhaproxy
View file
@@ -0,0 +1,346 @@
#!/bin/bash

#Document the bridge setup....
#ovs-vsctl set bridge shabr stp_enable=false

#FIXME not all of them work... hardcoding for now.
#mirror=$(curl -s http://nl.alpinelinux.org/alpine/MIRRORS.txt | shuf | head -n 1)
mirror="http://dl-6.alpinelinux.org/alpine/"
#FIXME write some logic to detect this.
version=2.6.5-r1
statedir=/var/lib/superhaproxy
wrapperurl='http://git.haproxy.org/?p=haproxy-1.6.git;a=blob_plain;f=src/haproxy-systemd-wrapper.c;hb=HEAD'
#FIXME make this configurable
bridge=shabr

function init_config {
name="$1"
ip=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy ip)
subnet=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy subnet)
gateway=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy gateway)
mtu=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy mtu)
}

function get_pid_file {
echo "$statedir/containers/$1/container.pid"
}

function get_pid {
echo "$(< "$statedir/containers/$1/container.pid")"
}

function get_dump_dir {
echo "$statedir/dumps/$1"
}

function get_container_dir {
echo "$statedir/containers/$1"
}

if [ "x$1" == "x" ]
then
echo "Usage:"
echo " init"
echo " list"
echo " create"
echo " show"
echo " start"
echo " stop"
echo " reload"
echo " pid"
echo " pstree"
echo " shell"
echo " hatop"
echo " dump local"
echo " restore local"
exit -1
fi

if [ "x$1" == "xinit" ]
then
mkdir -p $statedir
if [ ! -d $statedir/alpine-tools ]
then
mkdir -p $statedir/alpine-tools
pushd $statedir/alpine-tools
curl ${mirror}/latest-stable/main/x86_64/apk-tools-static-${version}.apk | tar -zxf -
popd
fi
if [ ! -d $statedir/rootimg ]
then
mkdir -p $statedir/rootimg
$statedir/alpine-tools/sbin/apk.static -X ${mirror}/latest-stable/main -U --allow-untrusted --root $statedir/rootimg --initdb add alpine-base haproxy
#FIXME this makes way too big a binary. Remove once alpine provides the wrapper
curl -s "$wrapperurl" -o $statedir/wrapper.c
gcc --static -o $statedir/rootimg/usr/sbin/haproxy-systemd-wrapper $statedir/wrapper.c
#FIXME criu doesn't support checkpinting the chroot yet.
sed -i '/chroot/d' $statedir/rootimg/etc/haproxy/haproxy.cfg
fi
mkdir -p $statedir/containers
mkdir -p $statedir/dumps
mkdir -p $statedir/action-scripts
exit 0
fi

if [ "x$1" == "xlist" ]
then
ls $statedir/containers/ | cat
exit 0
fi

if [ "x$1" == "xcreate" ]
then
shift
ip=""
name=""
subnet="255.255.255.0"
gateway=""
mtu=9000
while getopts ":i:m:n:s:g:" opt; do
case ${opt} in
i )
ip="$OPTARG"
;;
m )
mtu="$OPTARG"
;;
s )
subnet="$OPTARG"
;;
n )
name="$OPTARG"
;;
\? ) echo "Usage: superhaproxy create [-m mtu] [-s subnetmask] [-g gatewayip] -i ip_address -n name"
exit -1
;;
esac
done
if [ "x$name" == "x" ]
then
echo "You must specify a name with -n"
exit -1
fi
if [ "x$ip" == "x" ]
then
echo "You must specify an ip with -i"
exit -1
fi
cp -a $statedir/rootimg "$statedir/containers/$name"
touch "$statedir/containers/$name/container.ini"
crudini --set "$statedir/containers/$name/container.ini" superhaproxy ip "$ip"
crudini --set "$statedir/containers/$name/container.ini" superhaproxy mtu "$mtu"
crudini --set "$statedir/containers/$name/container.ini" superhaproxy subnet "$subnet"
crudini --set "$statedir/containers/$name/container.ini" superhaproxy gateway "$gateway"
exit 0
fi

if [ "x$1" == "xshow" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
init_config "$name"
echo "IP: $ip"
echo "Subnet Mask: $subnet"
if [ "x$gateay" != "x" ]
then
echo "Gateway: $gateway"
fi
echo "MTU: $mtu"
exit 0
fi

if [ "x$1" == "xstart" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
init_config "$name"
container="$(get_container_dir "$name")"
#FIXME ensure escaping is correct.
unshare --net --mount --pid --fork -- bash -c "/usr/bin/setsid -- /bin/bash -c 'mount --make-rprivate /; mount --bind $container /tmp; cd /tmp; mkdir -p old; pivot_root . old; mount --bind /old/dev /dev; mount /proc /proc -t proc; umount -l old; exec /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid </dev/null >/dev/null 2>&1'" &
sleep 1
awk '{print $1}' /proc/$!/task/$!/children > "$container/container.pid"
P="$(get_pid "$name")"
ovs-vsctl del-port $bridge "sha$(get_pid "$name")" > /dev/null 2>&1
ip link add sha$P type veth peer name shai$P
ip link set dev sha$P mtu "$mtu" up
ip link set shai$P netns $P name eth0
nsenter -t $P -n ip addr add "$ip/$subnet" dev eth0
nsenter -t $P -n ip link set dev eth0 mtu "$mtu" up
ovs-vsctl add-port $bridge sha$P
exit $?
fi

if [ "x$1" == "xpid" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
get_pid $name
exit 0
fi

if [ "x$1" == "xpstree" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
pstree -p $(get_pid "$name")
exit 0
fi

if [ "x$1" == "xstop" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
kill $(get_pid "$name")
ovs-vsctl del-port $bridge "sha$(get_pid "$name")"
exit 0
fi

if [ "x$1" == "xshell" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
nsenter -n -m -p -t $(get_pid "$name") /bin/busybox sh
exit 0
fi

if [ "x$1" == "xhatop" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
hatop -s "$(get_container_dir "$name")/var/lib/haproxy/stats"
exit 0
fi

if [ "x$1" == "xreload" ]
then
name="$2"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
kill -USR2 $(get_pid "$name")
exit 0
fi

if [ "x$1" == "xdump" ]
then
subcmd="$2"
if [ "x$subcmd" != "xlocal" ]
then
echo "only local is supported at the moment"
exit -1
fi
name="$3"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
if [ "x$subcmd" == "xlocal" ]
then
dumpdir=$(get_dump_dir "$name")
rm -rf "$dumpdir"
mkdir -p "$dumpdir"
criu dump -D "$dumpdir" -t "$(get_pid "$name")" --tcp-established --shell-job --ext-mount-map /dev:dev
exit $?
fi
exit 0
fi

if [ "x$1" == "xrestore" ]
then
subcmd="$2"
if [ "x$subcmd" != "xlocal" ]
then
echo "only local is supported at the moment"
exit -1
fi
name="$3"
if [ "x$name" == "x" ]
then
echo "You must specify a name"
exit -1
fi
if [ "x$subcmd" == "xlocal" ]
then
tmpid=$$
pidfile=$(get_pid_file "$name")
as="$statedir/action-scripts/$name.sh"
cat > "$as" <<EOF
#!/bin/bash
if [ "x\${CRTOOLS_SCRIPT_ACTION}" == "xpost-restore" ]
then
P=\$(cat "$pidfile")
ip link set dev sha$tmpid name "sha\$P"
ip link set dev "sha\$P" mtu 9000 up
ovs-vsctl add-port $bridge "sha\$P"
fi
EOF
chmod +x "$as"
dumpdir=$(get_dump_dir "$name")
container="$(get_container_dir "$name")"
if [ ! -d "$dumpdir" ]
then
echo "Dump does not exist"
exit -1
fi
rm -f "$(get_pid_file "$name")"
ovs-vsctl del-port $bridge "sha$(get_pid "$name")" > /dev/null 2>&1
mount --bind "$container" "$container"
criu restore -d -D "$dumpdir" --shell-job --tcp-established --ext-mount-map dev:/dev --root "$container" --veth-pair eth0="sha$tmpid" --action-script "$as" --pidfile "$(get_pid_file "$name")"
res=$?
umount "$container"
exit $res
fi
exit 0
fi

#migrate
#rsync -avz --delete -e ssh /var/lib/superhaproxy/containers/foo 192.168.0.20:/var/lib/superhaproxy/containers/
# procedure:
# * initial rsync of container
# * dump on local host
# * second rsync of container
# * rsync of images
# * restore on remote host
# * On success
# * rm container and dump on localhost
# * On failure
# * If autofailback
# * Restore container local
# * on restore failure
# * Try starting remote, if works, remove local container/images all done.
# * If failed to start remote, try and start local
# * If state all still local, remove remote data.

echo "Unknown command: $1"
exit -1

0 comments on commit bd711d6

Please sign in to comment.