Merge branch 'master' into antidote/txn_retries

rebar.config

@@ -40,7 +40,8 @@
     ]},
     {test, [
         {deps, [
-            rand_str
+            {rand_str, "~>1.0"},
+            {cmd, "~>1.0"}
         ]}
     ]}
 ]}.

scripts/aws/1-setup-vms.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# author goncalotomas
+# This script prepares several amazon virtual machines for later use.
+# You should pass in a list of (public) IP addresses as arguments to the script,
+# as well as the following environment variables:
+# PRIVATEKEY: used to ssh into the amazon virtual machines. Every machine is
+# assumed to be accessible using one key.
+
+set -e # Any subsequent(*) commands which fail will cause the shell script
+       # to exit immediately
+
+# args checking
+if [[ $# -lt 2 ]]; then
+    echo "Error: usage $0 <private_key_file> <IP_ADDR_LIST> ..."
+    exit 2
+fi;
+
+if [[ ! -e $1 ]]; then
+    echo "Error: $1: no such file"
+    exit 2
+fi;
+
+# env
+PRIVATEKEY=$1
+KEY_FILE_NAME=$(basename $PRIVATEKEY)
+
+IP_ADDR_LIST=$(echo $* | cut -d' ' -f2-)
+
+INSTALL_SOFTWARE_SCRIPT="./src/bin/worker-setup-machine.sh"
+REMOTE_INSTALL_SOFTWARE_SCRIPT="~/worker-setup-machine.sh"
+
+SSH_USERNAME=ubuntu
+SSH_OPTIONS="-i $PRIVATEKEY -o StrictHostKeyChecking=no"
+
+echo "[SCRIPT] RUNNING SCRIPT TO SETUP MULTIPLE REMOTE MACHINES..."
+
+# copy scripts to remote machines and add execute permission
+echo "[SCRIPT] 1/2: COPYING REQUIRED SCRIPTS TO REMOTE MACHINES..."
+for IP_ADDR in $IP_ADDR_LIST; do
+    scp $SSH_OPTIONS $INSTALL_SOFTWARE_SCRIPT $SSH_USERNAME@$IP_ADDR:$REMOTE_INSTALL_SOFTWARE_SCRIPT
+    ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR chmod u+x $REMOTE_INSTALL_SOFTWARE_SCRIPT
+done
+
+# install required software on remote machines
+echo "[SCRIPT] 2/2: INSTALLING NECESSARY SOFTWARE AND REPOSITORIES ON REMOTE MACHINES..."
+for IP_ADDR in $IP_ADDR_LIST; do
+   ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR nohup $REMOTE_INSTALL_SOFTWARE_SCRIPT &
+   pids="$pids $!"
+done
+
+echo "[SCRIPT] Waiting for SSH processes to finish their work..."
+for pid in $pids; do
+    wait $pid || let "RESULT=1"
+done
+
+if [ "$RESULT" == "1" ]; then
+    echo "[SCRIPT] Something went wrong in installing all the software!"
+    exit 1
+else
+    echo "[SCRIPT] Done. All remote machines have the required software stack and repositories."
+fi

scripts/aws/2-start-antidote-nodes.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# author goncalotomas
+# This script starts N replicas of antidote on previously configured remote vms.
+# You should pass in a list of (public) IP addresses as arguments to the script,
+# as well as the following environment variables:
+# PRIVATEKEY: used to ssh into the amazon virtual machines. Every machine is
+# assumed to be accessible using one key.
+# GITBRANCH: antidote branch to start up
+# CLEANMAKE = TRUE/FALSE: make relclean && make rel or not
+
+set -e # Any subsequent(*) commands which fail will cause the shell script
+       # to exit immediately
+
+# args checking
+if [[ $# -lt 2 ]]; then
+    echo "Error: usage $0 <private_key_file> <IP_ADDR_LIST> ..."
+    exit 2
+fi;
+
+if [[ ! -e $1 ]]; then
+    echo "Error: $1: no such file"
+    exit 2
+fi;
+if [ -z "$CLEANMAKE" ]; then
+    CLEANMAKE=TRUE
+fi
+if [ -z "$GITBRANCH" ]; then
+    GITBRANCH="build-local-cluster-aws"
+fi
+
+# env
+PRIVATEKEY=$1
+KEY_FILE_NAME=$(basename $PRIVATEKEY)
+
+IP_ADDR_LIST=$(echo $* | cut -d' ' -f2-)
+
+SSH_OPTIONS="-i $PRIVATEKEY -o StrictHostKeyChecking=no"
+SSH_USERNAME=ubuntu
+
+echo "[SCRIPT] RUNNING SCRIPT TO START MULTIPLE ANTIDOTE REPLICAS..."
+
+ANTIDOTE_SCRIPT="./src/bin/worker-start-antidote.sh"
+REMOTE_ANTIDOTE_SCRIPT="/home/ubuntu/worker-start-antidote.sh"
+
+# copy scripts to remote machines and add execute permission
+echo "[SCRIPT] COPYING REQUIRED SCRIPTS TO REMOTE MACHINES..."
+for IP_ADDR in $IP_ADDR_LIST; do
+    scp $SSH_OPTIONS $ANTIDOTE_SCRIPT $SSH_USERNAME@$IP_ADDR:$REMOTE_ANTIDOTE_SCRIPT
+    ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR chmod u+x $REMOTE_ANTIDOTE_SCRIPT
+done
+
+echo "[SCRIPT] COPIED ALL WORKER SCRIPTS."
+
+echo "[SCRIPT] STARTING REMOTE ANTIDOTE NODES..."
+for IP_ADDR in $IP_ADDR_LIST; do
+    Command="ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR GITBRANCH=${GITBRANCH} CLEANMAKE=${CLEANMAKE} IP=${IP_ADDR} $REMOTE_ANTIDOTE_SCRIPT"
+    echo "[SCRIPT] Starting antidote on node ${IP_ADDR}, using the following command:"
+    echo "[SCRIPT] ${Command}"
+    eval $Command &
+done
+
+# making sure the message is visible, antidote takes a few seconds to turn on
+sleep 10
+echo "[SCRIPT] Done. Antidote has been launched on the specified replicas."

scripts/aws/3-create-antidote-cluster.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# author goncalotomas
+# This script joins N already running replicas of antidote into a cluster (DC).
+# You should pass in a list of (public) IP addresses as arguments to the script,
+# as well as the following environment variables:
+# PRIVATEKEY: used to ssh into the amazon virtual machines. Every machine is
+# assumed to be accessible using one key.
+
+set -e # Any subsequent(*) commands which fail will cause the shell script
+       # to exit immediately
+
+# args checking
+if [[ $# -lt 2 ]]; then
+    echo "Error: usage $0 <private_key_file> <IP_ADDR_LIST> ..."
+    exit 2
+fi;
+
+if [[ ! -e $1 ]]; then
+    echo "Error: $1: no such file"
+    exit 2
+fi;
+if [ -z "$CLEANMAKE" ]; then
+    CLEANMAKE=TRUE
+fi
+if [ -z "$GITBRANCH" ]; then
+    GITBRANCH="build-local-cluster"
+fi
+
+# env
+PRIVATEKEY=$1
+KEY_FILE_NAME=$(basename $PRIVATEKEY)
+
+SSH_USERNAME=ubuntu
+SSH_OPTIONS="-i $PRIVATEKEY -o StrictHostKeyChecking=no"
+
+IP_ADDR_LIST=$(echo $* | cut -d' ' -f2-)
+
+for IP_ADDR in $IP_ADDR_LIST; do
+    ACCUM="$ACCUM antidote@$IP_ADDR"
+    REQUESTER=$IP_ADDR
+done
+
+echo "[SCRIPT] RUNNING SCRIPT TO JOIN MULTIPLE ANTIDOTE REPLICAS IN A CLUSTER..."
+
+echo "[SCRIPT] RUNNING THE JOIN CLUSTER SCRIPT FROM $REQUESTER..."
+
+JOIN_CLUSTER_SCRIPT="./src/bin/join_antidote_cluster.erl"
+REMOTE_JOIN_CLUSTER_SCRIPT="/home/ubuntu/join_antidote_cluster.erl"
+
+scp $SSH_OPTIONS $JOIN_CLUSTER_SCRIPT $SSH_USERNAME@$REQUESTER:$REMOTE_JOIN_CLUSTER_SCRIPT
+ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR chmod u+x $REMOTE_JOIN_CLUSTER_SCRIPT
+Command="ssh $SSH_OPTIONS $SSH_USERNAME@$REQUESTER $REMOTE_JOIN_CLUSTER_SCRIPT $ACCUM"
+echo "Requesting antidote cluster join on node $REQUESTER, using the following command:"
+echo "${Command}"
+eval $Command
+
+# cluster creation may take a while
+echo "[SCRIPT] Done. The specified antidote replicas are now joined in a cluster."

scripts/aws/3a-create-multi-dc-cluster.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# author goncalotomas
+# This script joins N already running replicas of antidote into a cluster (DC).
+# You should pass in a list of (public) IP addresses as arguments to the script,
+# as well as the following environment variables:
+# PRIVATEKEY: used to ssh into the amazon virtual machines. Every machine is
+# assumed to be accessible using one key.
+
+set -e # Any subsequent(*) commands which fail will cause the shell script
+       # to exit immediately
+
+# args checking
+if [[ $# -lt 2 ]]; then
+    echo "Error: usage $0 <private_key_file> <IP_ADDR_LIST> ..."
+    exit 2
+fi;
+
+if [[ ! -e $1 ]]; then
+    echo "Error: $1: no such file"
+    exit 2
+fi;
+if [ -z "$CLEANMAKE" ]; then
+    CLEANMAKE=TRUE
+fi
+if [ -z "$GITBRANCH" ]; then
+    GITBRANCH="build-local-cluster"
+fi
+
+# env
+PRIVATEKEY=$1
+KEY_FILE_NAME=$(basename $PRIVATEKEY)
+
+SSH_USERNAME=ubuntu
+SSH_OPTIONS="-i $PRIVATEKEY -o StrictHostKeyChecking=no"
+
+IP_ADDR_LIST=$(echo $* | cut -d' ' -f2-)
+
+for IP_ADDR in $IP_ADDR_LIST; do
+    ACCUM="$ACCUM antidote@$IP_ADDR"
+    REQUESTER=$IP_ADDR
+done
+
+echo "[SCRIPT] RUNNING SCRIPT TO JOIN MULTIPLE ANTIDOTE CLUSTERS..."
+
+echo "[SCRIPT] RUNNING THE JOIN CLUSTER SCRIPT FROM $REQUESTER..."
+
+JOIN_CLUSTER_SCRIPT="./src/bin/join_dcs_script.erl"
+REMOTE_JOIN_CLUSTER_SCRIPT="/home/ubuntu/join_dcs_script.erl"
+
+scp $SSH_OPTIONS $JOIN_CLUSTER_SCRIPT $SSH_USERNAME@$REQUESTER:$REMOTE_JOIN_CLUSTER_SCRIPT
+ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR chmod u+x $REMOTE_JOIN_CLUSTER_SCRIPT
+Command="ssh $SSH_OPTIONS $SSH_USERNAME@$REQUESTER $REMOTE_JOIN_CLUSTER_SCRIPT $ACCUM"
+echo "Requesting antidote cluster join on node $REQUESTER, using the following command:"
+echo "${Command}"
+eval $Command
+
+# cluster creation may take a while
+echo "[SCRIPT] Done. The specified antidote replicas are now joined in a cluster."

scripts/aws/4-start-fmke-nodes.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# author goncalotomas
+# This script starts N replicas of FMKe on previously configured remote vms.
+# You should pass in a list of (public) IP addresses as arguments to the script,
+# as well as the following environment variables:
+# PRIVATEKEY: used to ssh into the amazon virtual machines. Every machine is
+# assumed to be accessible using one key.
+# GITBRANCH: FMKe branch to start up
+# ANTIDOTE_ADDRESS: list of CSV antidote IP addresses
+# ANTIDOTE_PB_PORT: list of CSV antidote PB ports (must be 1-to-1 with previous list)
+
+set -e # Any subsequent(*) commands which fail will cause the shell script
+       # to exit immediately
+
+USAGE="Usage: ANTIDOTE_ADDRESS=<CSV IP LIST> ANTIDOTE_PB_PORT=<CSV PORT LIST> $0 <private_key_file> <FMK_IP_ADDR_LIST>"
+
+# args checking
+if [[ $# -lt 2 ]]; then
+    echo $USAGE
+    exit 2
+fi;
+
+if [[ ! -e $1 ]]; then
+    echo "Error: $1: no such file"
+    exit 2
+fi;
+if [ -z "$ANTIDOTE_ADDRESS" ]; then
+    echo $USAGE
+    exit 2
+fi
+if [ -z "$ANTIDOTE_PB_PORT" ]; then
+    echo $USAGE
+    exit 2
+fi
+
+if [ -z "$CLEANMAKE" ]; then
+    CLEANMAKE=TRUE
+fi
+if [ -z "$GITBRANCH" ]; then
+    GITBRANCH="perf-and-errors"
+fi
+
+# env
+PRIVATEKEY=$1
+KEY_FILE_NAME=$(basename $PRIVATEKEY)
+
+IP_ADDR_LIST=$(echo $* | cut -d' ' -f2-)
+
+
+echo "[SCRIPT] RUNNING SCRIPT TO START MULTIPLE FMKe REPLICAS..."
+
+SSH_USERNAME=ubuntu
+SSH_OPTIONS="-i $PRIVATEKEY -o StrictHostKeyChecking=no"
+
+FMK_SCRIPT="./src/bin/worker-start-fmk.sh"
+REMOTE_FMK_SCRIPT="/home/ubuntu/worker-start-fmk.sh"
+
+# copy scripts to remote machines and add execute permission
+echo "[SCRIPT] COPYING REQUIRED SCRIPTS TO REMOTE MACHINES..."
+for IP_ADDR in $IP_ADDR_LIST; do
+    scp $SSH_OPTIONS $FMK_SCRIPT $SSH_USERNAME@$IP_ADDR:$REMOTE_FMK_SCRIPT
+    ssh $SSH_OPTIONS $SSH_USERNAME@$IP_ADDR chmod u+x $REMOTE_FMK_SCRIPT
+done
+echo "[SCRIPT] COPIED ALL WORKER SCRIPTS."
+
+ANTIDOTE_ADDRESS_ARR=(`echo ${ANTIDOTE_ADDRESS}`);
+ANTIDOTE_ADDRESS_ARR_SIZE=${#ANTIDOTE_ADDRESS_ARR[@]}
+ANTIDOTE_PORT_ARR=(`echo ${ANTIDOTE_PB_PORT}`);
+IP_ARR=(`echo ${IP_ADDR_LIST}`);
+
+for index in "${!IP_ARR[@]}"; do
+    ## ASSIGN EACH FMK TO EACH ANTIDOTE IN A ROUND ROBIN FASHION
+    ## SO THAT WE ALLOW THE CASE WHERE #FMKNODES > #ANTIDOTENODES
+    Command="ssh $SSH_OPTIONS $SSH_USERNAME@${IP_ARR[$index]} GITBRANCH=${GITBRANCH} CLEANMAKE=${CLEANMAKE} IP=${IP_ARR[$index]} ANTIDOTE_ADDRESS=${ANTIDOTE_ADDRESS_ARR[$(($index % $ANTIDOTE_ADDRESS_ARR_SIZE))]} ANTIDOTE_PB_PORT=${ANTIDOTE_PORT_ARR[$(($index % $ANTIDOTE_ADDRESS_ARR_SIZE))]} ${REMOTE_FMK_SCRIPT}"
+    echo "[SCRIPT] Starting antidote on node ${IP_ARR[$index]}, using the following command:"
+    echo "[SCRIPT] ${Command}"
+    eval $Command &
+done
+
+sleep 10 # fmk takes a while to boot up.
+echo "[SCRIPT] Done. FMKe has been launched on the specified replicas."