Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

executable file 127 lines (99 sloc) 2.751 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
#!/bin/sh
export WHOAMI=`basename "$0"`
HERE=`pwd`
ACTIONBASEDIR="$HERE/actiondir"
DATADIR="$HERE/datadir"
NUMBER="$1"

function usage() {
    echo -n "Enter one numeric agument between 1 and "
    wc -l < tunnel_hosts.txt
    exit 1
}

function log() {
echo `date "+%F+%T" | tr -d '\n'` $WHOAMI[$$] "$1" >> $LOGFILE
}

function fail() {
if [ -n "$1" ]; then
log "----------------------------------------------------------"\
"$1" \
"----------------------------------------------------------"
fi

if [ -n "$DEST" ]; then # Adding the current dest for processing
./next_dest "$DEST"
fi

if [ -f "$AUTOSSH_PIDFILE" ]; then
log "Attempting to kill $AUTOSSH_PIDFILE"
kill `cat $AUTOSSH_PIDFILE`
fi

echo "$1" >> $ACTIONBASEDIR/fail-$NUMBER.log
if [ -n "$FBASE" ]; then
touch "$FBASE.fail"
fi
exit 1
}

function proc_params() {
[[ -n "$(echo "$NUMBER" | grep -E "^[0-9]+$")" ]] || usage

SOX="$(sed -n "${NUMBER}p" tunnel_hosts.txt)"
[[ -n "$SOX" ]] || usage

export LOGFILE="$ACTIONBASEDIR/app-$NUMBER.log"

export HOST="$(echo $SOX | cut -d: -f1)"
export PORT="$(echo $SOX | cut -d: -f2)"

export SPORT=$((10000+$NUMBER)) # Socks port
export CPORT=$((22000+$NUMBER*29-5)) # Check port
}

function check_if_sport_used() {
if [ -n "`lsof -i TCP:${SPORT}`" ]; then
fail "Port ${SPORT} is busy:\n`lsof -i TCP:${SPORT}`"
fi
}

function start_autossh() {
export AUTOSSH_LOGFILE=$LOGFILE
export AUTOSSH_PIDFILE=$ACTIONBASEDIR/autossh-$NUMBER.pid

autossh -M${CPORT} -f -ND${SPORT} -i${HERE}/crawler_key -p${PORT} ${HOST}
RETCODE=$?
if [ "$RETCODE" != "0" ]; then
fail "General autossh fail (ret code = $RETCODE).\n"\
"Please have a look at $AUTOSSH_LOGFILE"
fi
}

function check_autossh_pid() {
if [ ! -f "$AUTOSSH_PIDFILE" ]; then
fail "Autossh pidfile '${AUTOSSH_PIDFILE}' not found"
fi
}

function check_if_display_ok() {
if [ -z "$DISPLAY" ]; then
fail "Bad display: $DISPLAY"
fi
}


[[ "$#" = 1 ]] || usage

trap fail SIGINT SIGTERM

proc_params
check_if_sport_used
check_if_display_ok
start_autossh

while true; do
check_autossh_pid

export DEST="$(./next_dest)"
TIME="$(date '+%F+%T')"
DATEHOUR=$(echo $TIME | perl -pe 's/(.*)\+(\d+).*/\1_\2/') # 2010-01-01_13

mkdir -p "${DATADIR}/${DATEHOUR}"
export FBASE="${DATADIR}/${DATEHOUR}/${DEST}_${TIME}-${NUMBER}"
touch "${FBASE}.started"

from="`echo $DEST | cut -d- -f1`"
to="`echo $DEST | cut -d- -f2`"
check_days=180
socks_host="127.0.0.1"
socks_port="$SPORT"
export from to check_days socks_host socks_port DISPLAY

java -jar ryanaid.jar > "$FBASE.flights" 2> "$FBASE.inf"

RET=$?
if [ "$RET" = "0" ]; then # all fine
rm "${FBASE}.started"
else
touch "${FBASE}.fail"
./next_dest "${DEST}"
fi
export FBASE=

sleep 5
done
Something went wrong with that request. Please try again.