Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #2 from raineralves/newfeatures

Full Changelog:

* Update to newer Route53 API: switch from "2010-10-01" to "2012-02-29"
* Implement HTTP retries (applies to webserver probe as well as API communication)
* Switch from wget & lynx to curl, which is more robust and less error-prone.
* Improve logging on multi-line output (requires GNU awk)
* Validate AWS credentials once per hour. This ensures our credentials are always working, even if we rarely send an update to Route53.
* Security: Enforce strict file permission to avoid leaking AWS credentials (detect if script is chmod'ed to 700)
* Security: Switch from SHA1 to SHA256 when submitting updates to AWS
* Improve detection of DNS resolution problems
* Improve AWS signature generation on non-Linux platforms (use 'printf' instead of 'echo', which was causing problems on OSX)
* "base64" is no longer required (switched to "openssl enc -base64")
* Improve detection of status changes and avoid sending API updates when not needed (also added a "--force" argument to force an update at any time).
* Lots of logging improvements: show current production hosts when updating Route53, show how many hosts are up/down/disabled when no update is needed, show http code returned by the API, and more.
* Initial work on email notifications (see documentation on the "mailNotification" function)
* Initial work on multi-site probing (see "proberesult" file)
  • Loading branch information...
commit af7a68289ce235740b0384a60adf5c004e20b38a 2 parents 29e8256 + ba211e4
@ebrandi authored
Showing with 329 additions and 117 deletions.
  1. +329 −117 route53-failover.sh
View
446 route53-failover.sh
@@ -39,6 +39,8 @@ logfile=$script_path/$(basename $0 .sh).log.$(date +"%Y-%m")
lockfile=$script_path/$(basename $0 .sh).lock
test_file=status
test_string="Error 200 OK"
+connect_timeout=2
+retries=3
###############################################################
# You should not need to change anything bellow this point #
@@ -48,22 +50,46 @@ test_string="Error 200 OK"
set -o nounset # avoid breaking everything in case of an uninitialised variable
set -o pipefail # always set exit code to 1 when a piped subcommand fails
+# Our logging function requires GNU awk [`gawk(1)'].
+# The code bellow tries to find a suitable binary on non-Linux platforms.
+echo | awk '{ print systime() }' >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+ gawk=$(which awk)
+else
+ echo | gawk '{ print systime() }' >/dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ gawk=$(which gawk)
+ else
+ echo && echo "Error: Please install GNU awk (gawk)"
+ echo && exit 1
+ fi
+fi
+
# Logging function
log() {
- echo "[$(date +"%D %T")] $2" | tee -a $logfile
+ echo "$2" | $gawk '{ print "[" strftime("%Y-%m-%d %H:%M:%S") "]" "\t" $0; }' | tee -a $logfile
if [[ $1 == "error" ]]; then
- exit 1
+ echo && exit 1
fi
}
# Make sure some tools are installed correctly
-for i in dig lynx wget awk base64 diff openssl lockfile xmllint; do
- which $i >/dev/null || log error "Error: Please install $i before proceeding"
+for i in dig awk curl diff openssl lockfile xmllint; do
+ which $i >/dev/null 2>&1 || log error "Error: Please install $i before proceeding"
done
+# Enforce script permission to 700 for improved security (avoid leaking AWS credentials)
+if [[ -z "$(find $script_path/$(basename $0) -perm 700)" ]]; then
+ log error "Error: This script should NOT be accessible to other users since it contains sensitive information, please chmod it to 700"
+fi
+
# Test if this script has write permission on $script_path
-if [ ! -w $script_path ]; then
- log error "Error: I don't have write permission on $script_path, please fix and try again"
+if [ -z $script_path ]; then
+ log error "Error: Please set the \$script_path variable"
+else
+ if [ ! -w $script_path ]; then
+ log error "Error: I don't have write permission on $script_path, please fix and try again"
+ fi
fi
# Create lockfile and avoid more than one script execution ('lockfile(1)' is used to avoid race conditions)
@@ -74,17 +100,123 @@ fi
# Remove lockfile if some other error causes the script to exit
trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT
+# Initialize the mail notification variables
+for i in $(seq 1 8); do
+ mailNotificationStatus[i]=0
+done
+
# Set variables with DNS Record Values to create DELETE API request
-AuthServer=$(dig NS $Domain | awk "/^$Domain/ { print \$5 }" | head -1) || log error "Error retrieving domain info, check dns resolution"
+AuthServer=$(dig NS $Domain | awk "/^$Domain/ { print \$5 }" | head -1) || (mailNotificationStatus[1]=1 && log error "Error retrieving domain info, check dns resolution")
+
+# Test DNS resolution and check if our domain is actually hosted on Route53
+if [ -z $AuthServer ]; then
+ mailNotificationStatus[1]=1 && log error "Error retrieving domain info, check dns resolution"
+else
+ if [ "$(echo $AuthServer | grep awsdns)" == "" ]; then
+ log error "Error: Your domain is not hosted on Route 53, exiting..."
+ fi
+fi
+
+# Function: Generate AWS signature
+awssignature() {
+ if [ -z $AWSZoneID ] || [ -z $AWSAccesskeyID ] || [ -z $AWSSecretAPIKey ]; then
+ log error "Error: Please provide a valid set of AWS credentials"
+ else
+ AWSCurrentDate="$(curl -sS -I --connect-timeout $connect_timeout --retry $retries --retry-delay 5 --stderr /dev/null https://route53.amazonaws.com/date | grep Date | sed 's/.*Date: //' | tr -d '\r')" || (mailNotificationStatus[2]=1 && log error "Error retrieving current date from AWS")
+ AWSSignature=$(printf "$AWSCurrentDate" | openssl dgst -binary -sha256 -hmac $AWSSecretAPIKey | openssl enc -base64) || (mailNotificationStatus[2]=1 && log error "Error generating AWS signature")
+ AWSDateHeader="Date: $AWSCurrentDate"
+ AWSAuthHeader="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$AWSAccesskeyID,Algorithm=HmacSHA256,Signature=$AWSSignature"
+ fi
+}
+
+# Function: Submit Route53 update
+submitroute53() {
+ awssignature # call our signature generation function
+ AWSResult=$(curl -sS -w ";;%{http_code}" --connect-timeout $connect_timeout --retry $retries --retry-delay 5 -H "$AWSDateHeader" -H "$AWSAuthHeader" -H "Content-Type: text/xml; charset=UTF-8" -d "$AWSChangeset" https://route53.amazonaws.com/2012-02-29/hostedzone/$AWSZoneID/rrset)
+ AWSResultHTTPCode=$(echo $AWSResult | awk -F ";;" "{ print \$2 }")
+
+ case "$AWSResultHTTPCode" in
+ 000) echo "error" > $script_path/awsresult || log error "Error manipulating temporary files"
+ mailNotificationStatus[3]=1
+ log error "Error: Connection timeout while submitting Route53 update to AWS, aborting..."
+ ;;
+ 200) echo "ok" > $script_path/awsresult || log error "Error manipulating temporary files"
+ log info "Successfully updated \"$Hostname.$Domain\" on Route53"
+ echo
+ log info "*** Current production hosts:"
+ if [[ -n $NewRecordSorted ]]; then
+ for i in $NewRecordSorted; do
+ log info "$i"
+ done
+ else
+ log info "$fail_host [failover activated]"
+ fi
+ ;;
+ 400) echo "error" > $script_path/awsresult || log error "Error manipulating temporary files"
+ mailNotificationStatus[3]=1
+ log error "Error: \"Bad request\" while updating \"$Hostname.$Domain\" on Route53, double-check AWS credentials. Aborting..."
+ ;;
+ 403) echo "error" > $script_path/awsresult || log error "Error manipulating temporary files"
+ mailNotificationStatus[3]=1
+ log error "Error: \"Access forbidden\" while updating \"$Hostname.$Domain\" on Route53, double-check AWS credentials. Aborting..."
+ ;;
+ *) echo "error" > $script_path/awsresult || log error "Error manipulating temporary files"
+ mailNotificationStatus[3]=1
+ log error "Error: Unknown error while submitting Route53 update to AWS, aborting..."
+ ;;
+ esac
+
+ echo
+ log info "*** API Output:"
+ AWSResult=$(echo $AWSResult | awk -F ";;" "{ print \$1 }" | xmllint --format -)
+ log info "$AWSResult"
+ echo
+ log info "*** Changeset submited:"
+ AWSChangeset=$(echo $AWSChangeset | xmllint --format -)
+ log info "$AWSChangeset"
+ echo
+}
+
-if [ "$(echo $AuthServer | grep awsdns)" == "" ]; then
- echo "Error: Your domain is not hosted on Route 53, exiting..."
- exit 1
+# Validate AWS credentials (once per hour)
+# This validation makes sure your AWS credentials are always working,
+# even if we rarely update our zone records.
+
+touch $script_path/awslastvalidation || log error "Error manipulating temporary files"
+
+if [[ ! "$(date +"%H")" = "$(cat $script_path/awslastvalidation)" ]]; then
+
+ awssignature # call our signature generation function
+ AWSResult=$(curl -sS -w ";;%{http_code}" --connect-timeout $connect_timeout --retry $retries --retry-delay 5 -H "$AWSDateHeader" -H "$AWSAuthHeader" -H "Content-Type: text/xml; charset=UTF-8" https://route53.amazonaws.com/2012-02-29/hostedzone?marker=$AWSZoneID)
+ AWSResultHTTPCode=$(echo $AWSResult | awk -F ";;" "{ print \$2 }")
+
+ case "$AWSResultHTTPCode" in
+ 000) mailNotificationStatus[2]=1
+ log error "Error: Connection timeout while validating AWS credentials, aborting..."
+ ;;
+ 200) if [[ -n "$(echo "$AWSResult" | awk -F ";;" "{ print \$1 }" | xmllint --format - | grep $AWSZoneID)" ]]; then
+ log info "Validating AWS credentials... OK"
+ echo "$(date +"%H")" > $script_path/awslastvalidation || log error "Error manipulating temporary files"
+ else
+ mailNotificationStatus[2]=1
+ log error "Error: AWS credentials are OK, but you don't have access to the $AWSZoneID zoneset"
+ fi
+ ;;
+ 400) mailNotificationStatus[2]=1
+ log error "Error: \"Bad request\" while validating AWS credentials. Aborting..."
+ ;;
+ 403) mailNotificationStatus[2]=1
+ log error "Error: \"Access forbidden\" while validating AWS credentials. Aborting..."
+ ;;
+ *) mailNotificationStatus[2]=1
+ log error "Error: Unknown error while validating AWS credentials, aborting..."
+ ;;
+ esac
fi
-OldType=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$4 }" | head -1) || log error "Error while running dig"
-OldTTL=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$2 }" | head -1) || log error "Error while running dig"
-OldRecord=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$5 }" | sed s/\ //g) || log error "Error while running dig"
+OldType=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$4 }" | head -1) || (mailNotificationStatus[1]=1 && log error "Error while running dig")
+OldTTL=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$2 }" | head -1) || (mailNotificationStatus[1]=1 && log error "Error while running dig")
+OldRecord=$(dig @$AuthServer A $Hostname.$Domain | awk "/^$Hostname.$Domain/ { print \$5 }" | sed s/\ //g) || (mailNotificationStatus[1]=1 && log error "Error while running dig")
# Create temporary files needed by this script
@@ -92,18 +224,35 @@ touch $script_path/ips.tmp.old || log error "Error manipulating temporary files"
touch $script_path/ips.tmp || log error "Error manipulating temporary files"
mv -f $script_path/ips.tmp $script_path/ips.tmp.old || log error "Error manipulating temporary files"
touch $script_path/ips.tmp || log error "Error manipulating temporary files"
+mkdir $script_path/probe/ >/dev/null 2>&1
+touch $script_path/probe/proberesult.old || log error "Error manipulating temporary files"
+touch $script_path/probe/proberesult || log error "Error manipulating temporary files"
+mv -f $script_path/probe/proberesult $script_path/probe/proberesult.old || log error "Error manipulating temporary files"
+touch $script_path/probe/proberesult || log error "Error manipulating temporary files"
+touch $script_path/awsresult || log error "Error manipulating temporary files"
# Connect to webserver and search for a specific string to
# check if webserver are up and running for each address
# listed in ips.master file. Than print multiple lines
# for each IP address based in fixed weight seted in ips.master
+HostsUpAmount=0
+HostsDownAmount=0
+HostsDisabledAmount=$(cat $script_path/ips.master | egrep "^#.*[0-9]" | wc -l | cut -d ' ' -f 8)
+
for i in $(cat $script_path/ips.master | grep -v "#")
do
ip=$(echo $i | awk -F":" '{print $2}')
- condition=$(lynx -connect_timeout=1 -dump http://$ip/$test_file 2>&1 | grep "$test_string" | wc -l | cut -d ' ' -f 8)
- if [ "$condition" -eq "1" ]
+ webserverprobe=$(curl -sS -w ";;%{http_code}" --connect-timeout $connect_timeout --retry $retries --retry-delay 5 http://$ip/$test_file 2>&1)
+ webserverprobeHTTPcode=$(echo $webserverprobe | awk -F ";;" "{ print \$2 }")
+ webserverprobecondition=$(echo "$webserverprobe" | grep "$test_string" | wc -l | cut -d ' ' -f 8)
+ echo "# Timestamp: $(date +%s) [$(date)]" > $script_path/probe/proberesult
+ echo "# Format: <Webserver IP>:<Returned HTTP Code>:<String Found>" >> $script_path/probe/proberesult
+ echo "# HTTP Code \"000\" means timeout or connection refused" >> $script_path/probe/proberesult
+ echo "$ip:$webserverprobeHTTPcode:$webserverprobecondition" >> $script_path/probe/proberesult
+ if [ "$webserverprobecondition" -eq "1" ]
then
+ HostsUpAmount=$(( $HostsUpAmount + 1 ))
peso=$(echo $i | awk -F":" '{print $1}')
counter=1
while [ $counter -le $peso ]
@@ -111,6 +260,8 @@ do
echo $ip >> $script_path/ips.tmp
counter=$(( $counter + 1 ))
done
+ else
+ HostsDownAmount=$(( $HostsDownAmount + 1 ))
fi
done
@@ -119,17 +270,49 @@ done
if [ -s "$script_path/ips.tmp" ]
then
- # If file ips.tmp has any content, compare it with previous version of this file.
- # If both files has same content, script will quit
- # If not ot procede with route 53 update using IPs from ips.tmp file
+ # Let's decide if we need to update our zone or not
+ #
+ # Script will quit if ALL of the following conditions are met:
+ # 1) ips.tmp has the same content as the previous version (ips.tmp.old)
+ # 2) last AWS update was successfull
+ # 3) current records returned by `dig' perfectly match our probe result
+ #
+ # If any of the above is false, we procede with Route53 update.
+
+ OldRecordSorted=$(echo "$OldRecord" | sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n)
+ NewRecordSorted=$(cat $script_path/ips.tmp | sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n)
+
+ if [[ "$OldRecordSorted" == "$NewRecordSorted" ]] && [[ ! "${1:-unset}" = "--force" ]]; then
+ echo
+ log info "Update not needed [up:$HostsUpAmount][down:$HostsDownAmount][disabled:$HostsDisabledAmount]"
+ echo
+ echo "You may force an update to Route53 by using the '--force' argument"
+ echo
+ exit 0
+ fi
- if $(diff $script_path/ips.tmp.old $script_path/ips.tmp >/dev/null)
+ if [[ -z $(diff $script_path/ips.tmp.old $script_path/ips.tmp >/dev/null) ]] && [[ -n $(grep ok $script_path/awsresult) ]] && [[ "$OldRecordSorted" == "$NewRecordSorted" ]] && [[ ! "${1:-unset}" = "--force" ]]
then
- echo
- log info "Update is not necessary, nothing changed since last execution"
- echo
- else
- echo
+ if [ "$OldType" == "CNAME" ]; then
+ echo
+ log info "Update not needed [failover activated]"
+ else
+ echo
+ log info "Update not needed [up:$HostsUpAmount][down:$HostsDownAmount][disabled:$HostsDisabledAmount]"
+ fi
+ echo
+ echo "You may force an update to Route53 by using the '--force' argument"
+ echo
+
+ else
+
+ if [[ "$OldType" == "CNAME" ]] && [[ -n "$NewRecordSorted" ]]; then
+ echo
+ log info "Disabling failover state, returning to normal operation"
+ mailNotificationStatus[7]=1
+ echo
+ fi
+
# Show which hosts have been added or removed
# TODO: Send this information by email
addHosts[0]="$(diff -u $script_path/ips.tmp.old $script_path/ips.tmp | sort -u | awk "/^\+[0-9]+/ {printf \$0 \" \";}" | sed -e s/\+//g -e s/\ $//)"
@@ -157,9 +340,12 @@ then
increasedHostWeightCount=0
if [ -n "${addHosts[0]}" ]; then
# which hosts had its weight increased? grab this info and notify the user
+ mailNotificationStatus[5]=0
for i in $(seq 1 $(($j-1))); do
if [ ${addHosts[i*1000+4]} -eq 0 ]; then
showAddedHosts=$showAddedHosts", "${addHosts[i*1000+1]}
+ mailNotificationStatus[5*10+i]=${addHosts[i*1000+1]} # ip address for each added host
+ mailNotificationStatus[5]=$((${mailNotificationStatus[5]}+1)) # increase host added count
else
increasedHostWeight=$increasedHostWeight"| "${addHosts[i*1000+1]}"|"${addHosts[i*1000+2]}"|"${addHosts[i*1000+3]}
increasedHostWeightCount=$(($increasedHostWeightCount+1))
@@ -187,8 +373,8 @@ then
# new weight for this host
removeHosts[j*1000+3]="$(grep $i $script_path/ips.master | grep -v '^\#' | awk -F":" '{print $1}')"
if [ "${removeHosts[j*1000+3]}" == "" ]; then
- # host was removed, new weight is 0
- removeHosts[j*1000+3]=0
+ # host was removed, new weight is 0
+ removeHosts[j*1000+3]=0
fi
# check if previous and new weight are equal
if [[ ${removeHosts[j*1000+2]} -ne ${removeHosts[j*1000+3]} ]] && [[ ${removeHosts[j*1000+2]} -ne 0 ]] && [[ ${removeHosts[j*1000+3]} -ne 0 ]] ; then
@@ -205,9 +391,12 @@ then
decreasedHostWeightCount=0
if [ -n "${removeHosts[0]}" ]; then
# which hosts had its weight decreased? grab this info and notify the user
+ mailNotificationStatus[4]=0
for i in $(seq 1 $(($j-1))); do
if [ ${removeHosts[i*1000+4]} -eq 0 ]; then
showRemovedHosts=$showRemovedHosts", "${removeHosts[i*1000+1]}
+ mailNotificationStatus[4*10+i]=${removeHosts[i*1000+1]} # ip address for each removed host
+ mailNotificationStatus[4]=$((${mailNotificationStatus[4]}+1)) # increase host removed count
else
decreasedHostWeight=$decreasedHostWeight"| "${removeHosts[i*1000+1]}"|"${removeHosts[i*1000+2]}"|"${removeHosts[i*1000+3]}
decreasedHostWeightCount=$(($decreasedHostWeightCount+1))
@@ -248,60 +437,50 @@ then
NewRecord=$(cat $script_path/ips.tmp | awk '{print "<ResourceRecord><Value>"$1"</Value></ResourceRecord>"}')
# Create Route 53 Changeset data set
- Changeset=""
- Changeset=$Changeset"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
- Changeset=$Changeset"<ChangeResourceRecordSetsRequest xmlns=\"https://route53.amazonaws.com/doc/2010-10-01/\">"
- Changeset=$Changeset"<ChangeBatch><Comment>Update $Hostname.$Domain</Comment><Changes>"
+ AWSChangeset=""
+ AWSChangeset=$AWSChangeset"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ AWSChangeset=$AWSChangeset"<ChangeResourceRecordSetsRequest xmlns=\"https://route53.amazonaws.com/doc/2012-02-29/\">"
+ AWSChangeset=$AWSChangeset"<ChangeBatch><Comment>Update $Hostname.$Domain</Comment><Changes>"
# Delete previous DNS records
- Changeset=$Changeset"<Change>"
- Changeset=$Changeset"<Action>DELETE</Action>"
- Changeset=$Changeset"<ResourceRecordSet>"
- Changeset=$Changeset"<Name>$Hostname.$Domain.</Name>"
- Changeset=$Changeset"<Type>$OldType</Type>"
- Changeset=$Changeset"<TTL>$OldTTL</TTL>"
- Changeset=$Changeset"<ResourceRecords>"
+ AWSChangeset=$AWSChangeset"<Change>"
+ AWSChangeset=$AWSChangeset"<Action>DELETE</Action>"
+ AWSChangeset=$AWSChangeset"<ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"<Name>$Hostname.$Domain.</Name>"
+ AWSChangeset=$AWSChangeset"<Type>$OldType</Type>"
+ AWSChangeset=$AWSChangeset"<TTL>$OldTTL</TTL>"
+ AWSChangeset=$AWSChangeset"<ResourceRecords>"
for i in $OldRecord; do
- Changeset=$Changeset"<ResourceRecord><Value>$i</Value></ResourceRecord>"
+ AWSChangeset=$AWSChangeset"<ResourceRecord><Value>$i</Value></ResourceRecord>"
done
- Changeset=$Changeset"</ResourceRecords>"
- Changeset=$Changeset"</ResourceRecordSet>"
- Changeset=$Changeset"</Change>"
+ AWSChangeset=$AWSChangeset"</ResourceRecords>"
+ AWSChangeset=$AWSChangeset"</ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"</Change>"
# Create new DNS records
- Changeset=$Changeset"<Change>"
- Changeset=$Changeset"<Action>CREATE</Action>"
- Changeset=$Changeset"<ResourceRecordSet>"
- Changeset=$Changeset"<Name>$Hostname.$Domain.</Name>"
- Changeset=$Changeset"<Type>A</Type>"
- Changeset=$Changeset"<TTL>$ttl</TTL>"
- Changeset=$Changeset"<ResourceRecords>"
- Changeset=$Changeset"`echo $NewRecord | sed s/\ //g`"
- Changeset=$Changeset"</ResourceRecords>"
- Changeset=$Changeset"</ResourceRecordSet>"
- Changeset=$Changeset"</Change>"
+ AWSChangeset=$AWSChangeset"<Change>"
+ AWSChangeset=$AWSChangeset"<Action>CREATE</Action>"
+ AWSChangeset=$AWSChangeset"<ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"<Name>$Hostname.$Domain.</Name>"
+ AWSChangeset=$AWSChangeset"<Type>A</Type>"
+ AWSChangeset=$AWSChangeset"<TTL>$ttl</TTL>"
+ AWSChangeset=$AWSChangeset"<ResourceRecords>"
+ AWSChangeset=$AWSChangeset"`echo $NewRecord | sed s/\ //g`"
+ AWSChangeset=$AWSChangeset"</ResourceRecords>"
+ AWSChangeset=$AWSChangeset"</ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"</Change>"
# Close Route 53 changeset data set
- Changeset=$Changeset"</Changes>"
- Changeset=$Changeset"</ChangeBatch>"
- Changeset=$Changeset"</ChangeResourceRecordSetsRequest>"
+ AWSChangeset=$AWSChangeset"</Changes>"
+ AWSChangeset=$AWSChangeset"</ChangeBatch>"
+ AWSChangeset=$AWSChangeset"</ChangeResourceRecordSetsRequest>"
# Submit Route 53 changeset data set
- CurrentDate=$(wget --no-check-certificate -q -S https://route53.amazonaws.com/date -O /dev/null 2>&1 | grep Date | sed 's/.*Date: //') || log error "Error retrieving current date from AWS"
- Signature=$(echo -n $CurrentDate | openssl dgst -binary -sha1 -hmac $SecretAPIKey | base64) || log error "Error generating AWS signature"
- DateHeader="Date: "$CurrentDate
- AuthHeader="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$AccesskeyID,Algorithm=HmacSHA1,Signature=$Signature"
- Result=$(wget --no-check-certificate -nv --header="$DateHeader" --header="$AuthHeader" --header="Content-Type: text/xml; charset=UTF-8" --post-data="$Changeset" -O /dev/stdout -o /dev/stdout https://route53.amazonaws.com/2010-10-01/hostedzone/$ZoneID/rrset | grep -v WARNING | grep -v locally)
- log info "API Output:"
- log info "$Result"
- echo
- log info "Changeset submited:"
- Changeset=$(echo $Changeset | xmllint --format -)
- log info "$Changeset"
- echo
- fi
- else
+ submitroute53
+
+ fi
+ else
# File ips.tmp is empty, you dont have any webserver UP
# script will change DNS to failover your website to your backup site
@@ -309,81 +488,114 @@ then
# First let's make sure the failover host is OK
# This avoids a false positive if the probe machine has connectivity problems
- condition=$(lynx -connect_timeout=1 -dump http://$fail_host/$test_file 2>&1 | grep "$test_string" | wc -l | cut -d ' ' -f 8)
+ condition=$(curl -sS -w ";;%{http_code}" --connect-timeout $connect_timeout --retry $retries --retry-delay 5 http://$fail_host/$test_file 2>&1 | grep "$test_string" | wc -l | cut -d ' ' -f 8)
if [ "$condition" -ne "1" ]; then
+ mailNotificationStatus[8]=1
log error "Error: failover host is also down, I don't know what else to do. Make sure my network connectivity is OK. Exiting..."
fi
# Failover host is OK, let's proceed with the update
- if $(diff $script_path/ips.tmp.old $script_path/ips.tmp >/dev/null)
+ if [[ -z $(diff $script_path/ips.tmp.old $script_path/ips.tmp >/dev/null) ]] && [[ -n $(grep ok $script_path/awsresult) ]] && [[ "$OldType" == "CNAME" ]] && [[ ! "${1:-unset}" = "--force" ]]
then
echo
- log info "Update is not necessary, nothing changed since last execution"
+ log info "Update not needed [failover activated]"
echo
+
else
- echo
- log info "Activating FAILOVER host"
+
+ if [[ ! "$OldType" == "CNAME" ]]; then
+ echo
+ log info "All hosts are down, enabling failover state"
+ mailNotificationStatus[6]=1
+ echo
+ fi
+
log info "Starting update process..."
echo
+ NewRecordSorted=""
+
# Create Route 53 Changeset data set
- Changeset=""
- Changeset=$Changeset"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
- Changeset=$Changeset"<ChangeResourceRecordSetsRequest xmlns=\"https://route53.amazonaws.com/doc/2010-10-01/\">"
- Changeset=$Changeset"<ChangeBatch><Comment>Update $Hostname.$Domain</Comment><Changes>"
+ AWSChangeset=""
+ AWSChangeset=$AWSChangeset"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ AWSChangeset=$AWSChangeset"<ChangeResourceRecordSetsRequest xmlns=\"https://route53.amazonaws.com/doc/2012-02-29/\">"
+ AWSChangeset=$AWSChangeset"<ChangeBatch><Comment>Update $Hostname.$Domain</Comment><Changes>"
# Delete previous DNS records
- Changeset=$Changeset"<Change>"
- Changeset=$Changeset"<Action>DELETE</Action>"
- Changeset=$Changeset"<ResourceRecordSet>"
- Changeset=$Changeset"<Name>$Hostname.$Domain.</Name>"
- Changeset=$Changeset"<Type>$OldType</Type>"
- Changeset=$Changeset"<TTL>$OldTTL</TTL>"
- Changeset=$Changeset"<ResourceRecords>"
+ AWSChangeset=$AWSChangeset"<Change>"
+ AWSChangeset=$AWSChangeset"<Action>DELETE</Action>"
+ AWSChangeset=$AWSChangeset"<ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"<Name>$Hostname.$Domain.</Name>"
+ AWSChangeset=$AWSChangeset"<Type>$OldType</Type>"
+ AWSChangeset=$AWSChangeset"<TTL>$OldTTL</TTL>"
+ AWSChangeset=$AWSChangeset"<ResourceRecords>"
for i in $OldRecord; do
- Changeset=$Changeset"<ResourceRecord><Value>$i</Value></ResourceRecord>"
+ AWSChangeset=$AWSChangeset"<ResourceRecord><Value>$i</Value></ResourceRecord>"
done
- Changeset=$Changeset"</ResourceRecords>"
- Changeset=$Changeset"</ResourceRecordSet>"
- Changeset=$Changeset"</Change>"
+ AWSChangeset=$AWSChangeset"</ResourceRecords>"
+ AWSChangeset=$AWSChangeset"</ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"</Change>"
# Create new DNS records
- Changeset=$Changeset"<Change>"
- Changeset=$Changeset"<Action>CREATE</Action>"
- Changeset=$Changeset"<ResourceRecordSet>"
- Changeset=$Changeset"<Name>$Hostname.$Domain.</Name>"
- Changeset=$Changeset"<Type>CNAME</Type>"
- Changeset=$Changeset"<TTL>$ttl</TTL>"
- Changeset=$Changeset"<ResourceRecords>"
- Changeset=$Changeset"<ResourceRecord>"
- Changeset=$Changeset"<Value>$fail_host</Value>"
- Changeset=$Changeset"</ResourceRecord>"
- Changeset=$Changeset"</ResourceRecords>"
- Changeset=$Changeset"</ResourceRecordSet>"
- Changeset=$Changeset"</Change>"
+ AWSChangeset=$AWSChangeset"<Change>"
+ AWSChangeset=$AWSChangeset"<Action>CREATE</Action>"
+ AWSChangeset=$AWSChangeset"<ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"<Name>$Hostname.$Domain.</Name>"
+ AWSChangeset=$AWSChangeset"<Type>CNAME</Type>"
+ AWSChangeset=$AWSChangeset"<TTL>$ttl</TTL>"
+ AWSChangeset=$AWSChangeset"<ResourceRecords>"
+ AWSChangeset=$AWSChangeset"<ResourceRecord>"
+ AWSChangeset=$AWSChangeset"<Value>$fail_host</Value>"
+ AWSChangeset=$AWSChangeset"</ResourceRecord>"
+ AWSChangeset=$AWSChangeset"</ResourceRecords>"
+ AWSChangeset=$AWSChangeset"</ResourceRecordSet>"
+ AWSChangeset=$AWSChangeset"</Change>"
# Close Route 53 changeset data set
- Changeset=$Changeset"</Changes>"
- Changeset=$Changeset"</ChangeBatch>"
- Changeset=$Changeset"</ChangeResourceRecordSetsRequest>"
+ AWSChangeset=$AWSChangeset"</Changes>"
+ AWSChangeset=$AWSChangeset"</ChangeBatch>"
+ AWSChangeset=$AWSChangeset"</ChangeResourceRecordSetsRequest>"
# Submit Route 53 changeset data set
- CurrentDate=$(wget --no-check-certificate -q -S https://route53.amazonaws.com/date -O /dev/null 2>&1 | grep Date | sed 's/.*Date: //') || log error "Error retrieving current date from AWS"
- Signature=$(echo -n $CurrentDate | openssl dgst -binary -sha1 -hmac $SecretAPIKey | base64) || log error "Error generating AWS signature"
- DateHeader="Date: "$CurrentDate
- AuthHeader="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$AccesskeyID,Algorithm=HmacSHA1,Signature=$Signature"
- Result=$(wget --no-check-certificate -nv --header="$DateHeader" --header="$AuthHeader" --header="Content-Type: text/xml; charset=UTF-8" --post-data="$Changeset" -O /dev/stdout -o /dev/stdout https://route53.amazonaws.com/2010-10-01/hostedzone/$ZoneID/rrset | grep -v WARNING | grep -v locally)
- log info "API Output:"
- log info "$Result"
- echo
- log info "Changeset submited:"
- Changeset=$(echo $Changeset | xmllint --format -)
- log info "$Changeset"
+ submitroute53
+
echo
fi
- fi
+fi
+
+mailNotification() {
+
+# During script execution we set special status codes (described bellow)
+# depending on which problems were encountered.
+# These status codes are later processed to decide which notifications should be sent.
+#
+# mailNotificationStatus[1] = DNS resolution problems
+# mailNotificationStatus[2] = Failed to generate AWS signature or validate AWS credentials
+# mailNotificationStatus[3] = Failed to submit AWS zoneset update
+# mailNotificationStatus[4] = Hosts down
+# mailNotificationStatus[5] = Hosts up
+# mailNotificationStatus[6] = Failover activated (problems)
+# mailNotificationStatus[7] = Failover disabled (back to normal)
+# mailNotificationStatus[8] = All hosts down, failover also down
+#
+# For status 4 and 5 (hosts up/down) we also store the amount of affected hosts.
+#
+# For example:
+#
+# mailNotificationStatus[4]=2 means we have 2 hosts down.
+# The IP address for the affected hosts will be stored on mailNotificationStatus[4*10+1] and mailNotificationStatus[4*10+2]
+# The reason for each problem will be stored on mailNotificationStatus[4*100+1] and mailNotificationStatus[4*100+2]
+# The string test for each host will be stored on mailNotificationStatus[4*1000+1] and mailNotificationStatus[4*1000+2]
+#
+# mailNotificationStatus[5]=3 means 3 hosts are back online (up)
+# The IP address for each host back online will be stored on mailNotificationStatus[5*10+1] until mailNotificationStatus[5*10+3]
+#
+
+echo
+
+}
# Remove lockfile
rm -f "$lockfile"
Please sign in to comment.
Something went wrong with that request. Please try again.