start_megaglest_gameserver

#!/bin/sh
#
# Start a headless game server (and keep it running)
#
# Using this script, a headless game server is started, waiting for players to
# connect (if your firewall configuration permits). Players connect and start a
# game. As soon as the game ends, the server will quit, and this script will
# start up a new server again. This is a stability measure to rule out
# the unlikely case where side effects (such as memory leaks or corruption)
# could drain on system resources.
#
# For this to work, Internet originated traffic must be able to reach the
# server on the following ports:
# TCP port 61357: game protocol port
# TCP port 61358: FTP control port
# TCP ports 61359 to 61366: FTP data ports
#
# Once publishing to the master server succeeded (this can be verified at
# master server's site) you may connect to your headless game server
# using a copy of game you have installed on a Desktop computer. The first
# user connecting to a headless server controls it. If this user disconnects,
# the next user who connects (or had already connected) takes control.
#
# Please read https://docs.megaglest.org/MG/Dedicated_Server for more information
#
# ----------------------------------------------------------------------------
# 2013 Written by Tom Reynolds <tomreyn[at]megaglest.org>
# 2015 Rewritten by filux <heross(@@)o2.pl>
# Copyright (c) 2013-2017 under GNU GPL v3.0+
# ----------------------------------------------------------------------------
LANG=C
trap "kill -- -$$" HUP INT QUIT TERM EXIT

KERNEL="$(uname -s | tr '[A-Z]' '[a-z]')"
if [ "$KERNEL" = "linux" ] || [ "$(echo "$(readlink -f "$0" >/dev/null 2>&1; echo $?)" | grep '^[0-9]$')" -eq "0" ]; then
    GAMEDIR="$(dirname "$(readlink -f "$0")")"
else
    GAMEDIR="$(cd "$(dirname "$0")"; pwd)"
fi
HOME_DIR="$HOME"
B_SCRIPT_DIR="$GAMEDIR"

# -------
# useful for mods
BASIC_SCRIPT="start_megaglest"
FAKE_EXC_BINARY_MAC="MegaGlest"
EXC_BINARY="megaglest"
EXC_BINARY_DISTRO="megaglest"
SHORT_GAME_NAME="megaglest"
# -------
PORTSTART=62001
NUM_OM_SERVERS=2
NUM_OA_SERVERS=1
SERVER_HARDW_MAX_LOAD_PERC="100"
# Log file location (beware, this can grow large)
# LOG_SERVER=/dev/null

if [ "$(which curl 2>/dev/null)" = "" ]; then
    echo "WARNING: Downloading tool 'curl' DOES NOT EXIST on this system, please install it." >&2
fi
if [ "$KERNEL" = "darwin" ]; then
    if [ -e "$GAMEDIR/../../MacOS/$FAKE_EXC_BINARY_MAC" ]; then
	BASIC_SCRIPT="$FAKE_EXC_BINARY_MAC"; B_SCRIPT_DIR="$(cd "${GAMEDIR}/../../MacOS/"; pwd)"
    elif [ -e "$GAMEDIR/$FAKE_EXC_BINARY_MAC.sh" ]; then
	BASIC_SCRIPT="$FAKE_EXC_BINARY_MAC.sh"
    fi
fi
if [ "$(which lscpu 2>/dev/null)" != "" ]; then NUMCORES="$(lscpu -p | grep -cv '^#')"
    elif [  "$(which sysctl 2>/dev/null)" != "" ]; then NUMCORES="$(sysctl -n hw.ncpu)"; fi
if [ "$NUMCORES" = "" ]; then NUMCORES=1; fi
echo "Notice: Detected processor with $NUMCORES cores." >&2
case $NUMCORES in
    1) MAX_LOAD_MULT="0.65";; 2) MAX_LOAD_MULT="0.75";; 3) MAX_LOAD_MULT="0.82";;
    4) MAX_LOAD_MULT="0.88";; 5) MAX_LOAD_MULT="0.92";; *) MAX_LOAD_MULT="0.95";;
esac
if [ "$SERVER_HARDW_MAX_LOAD_PERC" -gt "100" ]; then SERVER_HARDW_MAX_LOAD_PERC=100; fi
MAX_LOAD="$(echo "$NUMCORES $MAX_LOAD_MULT $SERVER_HARDW_MAX_LOAD_PERC" | awk '{print $1*$2*($3/100)}')"

if [ "$1" != "" ] && [ "$(echo "$1" | grep '^[0-9]\+$')" != "" ]; then SERVERCOUNT="$1"
    else SERVERCOUNT=0; fi
if [ "$2" != "" ] && [ "$(echo "$2" | grep '[0-9.:]\+' | grep -v '[A-Za-z]')" != "" ]; then
    SERVER_GREP_IP="$(echo "$2" | sed 's/\./\\./g')"; else SERVER_GREP_IP=""; fi
if [ "$3" != "" ] && [ "$(echo "$3" | grep '^[0-9]\+$')" != "" ]; then PORTSTART="$3"; fi
if [ "$4" != "" ]; then SERVERTITLE="$4"; fi
PORT="$(($PORTSTART + (($SERVERCOUNT - 1) * 11)))"; STATUSPORT="$(($PORT - 1))"
PORT_FD="$(echo "$PORT" | cut -c1-2)"; PORT_NLD="$((${#PORT} - ${#PORT_FD}))"
PORT_FDX="$PORT_FD"; c=1; while [ "$c" -le "$PORT_NLD" ]; do PORT_FDX="$(echo "${PORT_FDX}X")"; c="$(($c + 1))"; done

IDLE_TIME_MOD=12
if [ "$NUM_OA_SERVERS" -ge "$NUM_OM_SERVERS" ]; then NUM_OS_MOD="$NUM_OA_SERVERS"
    else NUM_OS_MOD="$NUM_OM_SERVERS"; fi
if [ "$NUM_OS_MOD" -lt "3" ]; then NUM_OS_SMOD="$(($NUM_OS_MOD + 1))"
    else NUM_OS_SMOD="$(($NUM_OS_MOD + ($NUM_OS_MOD / 3)))"; fi
if [ "$SERVERCOUNT" -ge "1" ]; then
    IDLE_TIME_MOD="$((18 + ($NUM_OS_MOD * 2) - ($SERVERCOUNT * 2)))"
    if [ "$IDLE_TIME_MOD" -lt "2" ]; then IDLE_TIME_MOD=2
	elif [ "$IDLE_TIME_MOD" -gt "24" ]; then IDLE_TIME_MOD=24; fi
fi
MAX_IDLE_TIME="$(($IDLE_TIME_MOD * 3600))"

SERVER_SCRIPT="$(basename "$0")"
if [ -f "$GAMEDIR/glest.ini" ]; then
    GLEST_INI="$(cat "$GAMEDIR/glest.ini" | sed -e 's:\$HOME:'"$HOME_DIR"':g')"
    LOG_DIR="$(echo "$GLEST_INI" | grep '^LogPath=' | awk -F '=' '{print $2}')"
    mkdir -p "$LOG_DIR"
    # e.g. on macos are problems with more advanced using awk ^
    if [ "$LOG_DIR" != "" ] && [ "$(echo "$LOG_DIR" | grep '/$')" ]; then LOG_DIR="${LOG_DIR%?}"; fi
    MASTER_SERVER="$(echo "$GLEST_INI" | grep '^Masterserver=' | awk -F '=' '{print $2}')"
    if [ "$(echo "$MASTER_SERVER" | grep '/$')" ]; then CLEAR_M_SERVER="${MASTER_SERVER}showServersForGlest.php"
	else CLEAR_M_SERVER="$MASTER_SERVER/showServersForGlest.php"; fi
    if [ "$SERVERCOUNT" -gt "0" ]; then
	USER_DATA_DIR="$(echo "$GLEST_INI" | grep '^UserData_Root=' | awk -F '=' '{print $2}')"
	if [ "$USER_DATA_DIR" != "" ] && [ "$(echo "$USER_DATA_DIR" | grep '/$')" ]; then
	    USER_DATA_DIR="${USER_DATA_DIR%?}"
	fi
	if [ "$SERVERTITLE" = "" ] && [ -f "$USER_DATA_DIR/glestuser.ini" ]; then
	    GLEST_USER_INI="$(cat "$USER_DATA_DIR/glestuser.ini")"
	    SERVERNAME="$(echo "$GLEST_USER_INI" | grep '^NetPlayerName=' | awk -F '=' '{print $2}')"
	    if [ "$SERVERNAME" = "" ]; then SERVERNAME="Unknown"; fi
	fi
    fi
elif [ "$SERVER_GREP_IP" != "" ]; then
    echo "WARNING: file 'glest.ini' not found, maybe '$SERVER_SCRIPT' script is placed in the wrong location." >&2
fi

EXC_BINARY_FPID="$EXC_BINARY"; EXC_BINARY_DISTRO_W="$(which "$EXC_BINARY_DISTRO" 2>/dev/null)"
if [ -e "$B_SCRIPT_DIR/$BASIC_SCRIPT" ]; then SERVER_EXEC="$B_SCRIPT_DIR/$BASIC_SCRIPT"
    elif [ -e "$GAMEDIR/$EXC_BINARY" ]; then SERVER_EXEC="$GAMEDIR/$EXC_BINARY"
    else SERVER_EXEC="$EXC_BINARY_DISTRO_W"; EXC_BINARY_FPID="$EXC_BINARY_DISTRO"; fi
if [ "$SERVERCOUNT" -eq "0" ]; then
    #if [ "$SERVER_EXEC" != "$EXC_BINARY_DISTRO_W" ]; then ulimit -c unlimited; fi
    USED_PORTS=""; LOG_FILE="server.log"
    RestartMarker="${SHORT_GAME_NAME}-server-restart.log"
else
    USED_PORTS="--use-ports=$PORT,$PORT,$STATUSPORT "
    RestartMarker="${SHORT_GAME_NAME}-server-restart-${PORT}.log"
    WaitMarker="${SHORT_GAME_NAME}-server-wait-${PORT_FDX}.log"
    LOG_FILE="server_${SERVERCOUNT}.log"
    SERVERNAME="$SERVERNAME-${SERVERCOUNT}"
fi
SingleSMarker=".${SHORT_GAME_NAME}-server-single.log"
if [ "$LOG_DIR" != "" ]; then
    if [ ! -f "$LOG_DIR/$SingleSMarker" ] && [ "$SERVERCOUNT" -le "1" ]; then
	echo "#" > "$LOG_DIR/$SingleSMarker"; sleep 1s
    elif [ -f "$LOG_DIR/$SingleSMarker" ] && [ "$SERVERCOUNT" -gt "1" ]; then
	sleep 1s; rm -f "$LOG_DIR/$SingleSMarker"
    fi
fi
if [ "$SERVERTITLE" = "" ] && [ "$SERVERNAME" != "" ]; then SERVERTITLE="$SERVERNAME"; fi
if [ "$SERVERCOUNT" -gt "0" ]; then
    echo "Info: Server '$SERVERTITLE' nr. $SERVERCOUNT ($2 ; $PORT)." >&2
    sleep "$(($SERVERCOUNT * 5))"s
fi
SER_PARAMETERS="--headless-server-mode=vps,exit"
#^ parameters without spaces and numbers inside
SER_GREP_PARAMETERS="$(echo "${USED_PORTS}$SER_PARAMETERS" | sed 's/--/\\--/g')"

if [ "$LOG_SERVER" != "" ]; then :
elif [ "$LOG_DIR" != "" ]; then
    mkdir -p "$LOG_DIR"
    LOG_SERVER="$LOG_DIR/$LOG_FILE"
else
    LOG_SERVER=/dev/null
fi
cd "$GAMEDIR"

AVG_LOAD="unknown"; SER_SITUATION="unknown"; CHECK_AVG_LOAD_M=0
if [ -e "/proc/loadavg" ]; then CHECK_AVG_LOAD_M=1
    elif [  "$(which sysctl 2>/dev/null)" != "" ]; then CHECK_AVG_LOAD_M=2; fi
while true; do
    if [ "$LOG_SERVER" != "/dev/null" ] && [ -f "$LOG_SERVER" ] && [ "$(wc -c < "$LOG_SERVER")" -gt "250000" ]; then
	mv -f "$LOG_SERVER" "$LOG_SERVER.1"; fi
    if [ -e "core" ]; then mv -f "core" "core.1"; fi
    date >> "$LOG_SERVER"
    while true; do
	if [ "$CHECK_AVG_LOAD_M" -eq "1" ]; then
	    AVG_LOAD="$(awk '{print $2}' /proc/loadavg)"
	elif [  "$CHECK_AVG_LOAD_M" -eq "2" ]; then
	    AVG_LOAD="$(sysctl -n vm.loadavg | awk -F '{' '{print $2}'  | awk '{print $2}')"
	fi
	if [ "$AVG_LOAD" != "unknown" ]; then
		SER_SITUATION="$(echo "$AVG_LOAD $MAX_LOAD" | awk '{if ($1 > $2) print "highload"}')"
	fi
	if [ "$SER_SITUATION" = "highload" ]; then
	    if [ "$OLD_SER_SITUATION_STATUS" = "" ]; then
		echo "WARNING: Detected high load on the server." >&2
		OLD_SER_SITUATION_STATUS="yes"
	    fi
	    sleep 2.5m; sleep "$((RANDOM % 30))"s
	else
	    OLD_SER_SITUATION_STATUS=""; NUM_O_FREE_SER=""
	    if [ "$LOG_DIR" != "" ] && [ ! -f "$LOG_DIR/$SingleSMarker" ]; then
		if [ "$MASTER_SERVER" != "" ] && [ "$SERVER_GREP_IP" != "" ]; then
		    if [ "$WaitMarker" != "" ] && [ ! -f "$LOG_DIR/$WaitMarker" ]; then
			echo "#" > "$LOG_DIR/$WaitMarker"; sleep 1s
		    fi
		    if [ -f "$LOG_DIR/$WaitMarker" ]; then
			InWaitMarker="$(cat "$LOG_DIR/$WaitMarker" | tail -1 | grep '^[0-9]*$')"
			if [ "$InWaitMarker" = "" ] || \
			    [ "$(find "$LOG_DIR/" -maxdepth 1 -name "$WaitMarker" -mmin +30)" ]; then
			    InWaitMarker=100; NUM_O_FREE_SER=103
			fi
			if [ "$SERVERCOUNT" -lt "$InWaitMarker" ]; then
			    echo "$SERVERCOUNT" >> "$LOG_DIR/$WaitMarker"; InWaitMarker="$SERVERCOUNT"
			fi
		    fi
		fi
		if [ -f "$LOG_DIR/$WaitMarker" ] && [ "$NUM_O_FREE_SER" != "103" ] && \
		    [ "$(($(date +%s)-$(date +%s -r "$LOG_DIR/$WaitMarker")))" -le "30" ]; then
		    NUM_O_FREE_SER=102
		elif [ "$MASTER_SERVER" != "" ] && [ "$SERVER_GREP_IP" != "" ]; then
		    if [ "$SERVERCOUNT" -ne "$InWaitMarker" ]; then
			NUM_O_FREE_SER=101
		    elif [ "$OLD_InWaitMarker" = "" ] || [ "$InWaitMarker" -ne "$OLD_InWaitMarker" ]; then
			NUM_O_FREE_SER=100; OLD_InWaitMarker="$InWaitMarker"
		    fi
		    if [ "$NUM_O_FREE_SER" = "" ]; then
			FIND_ALL_SER="$(curl -s -L "$CLEAR_M_SERVER")"; M_SERVER_STATUS="$?"
			FIND_OUR_SER="$(echo "$FIND_ALL_SER" | grep '^\([^|]*|\)\{4\}'"$SERVER_GREP_IP"'|' \
			    | grep '^\([^|]*|\)\{11\}'"$PORT_FD"'[0-9]\{'"$PORT_NLD"'\}|')"
			NUM_O_O_SER="$(echo "$FIND_OUR_SER" | wc -l)"
			if [ "$NUM_O_O_SER" -eq "0" ]; then M_SERVER_STATUS=111; fi
			if [ "$NUM_O_O_SER" != "" ] && [ "$NUM_O_O_SER" -lt "$NUM_OM_SERVERS" ]; then :
			elif [ "$SERVERCOUNT" -le "$NUM_OS_SMOD" ]; then
			    NUM_O_FREE_SER="$(echo "$FIND_OUR_SER" | grep '^\([^|]*|\)\{10\}0|' -c)"
			else
			    NUM_O_FREE_SER="$(echo "$FIND_OUR_SER" | grep '^\([^|]*|\)\{13\}0|' -c)"
			fi
		    fi
		fi
	    fi
	    if [ "$NUM_O_FREE_SER" != "" ] && [ "$NUM_O_FREE_SER" -ge "$NUM_OA_SERVERS" ]; then
		if [ "$OLD_NUM_OFS_STATUS" = "" ] && [ "$NUM_O_FREE_SER" -ne "100" ]; then
		    echo "Notice: Waiting for situation when server may be really needed." >&2
		    OLD_NUM_OFS_STATUS="yes"
		fi
	    else
		if [ "$M_SERVER_STATUS" = "" ] || [ "$M_SERVER_STATUS" -eq "0" ] || [ "$SERVERCOUNT" -le "1" ]; then
		    OLD_NUM_OFS_STATUS=""
		    if [ -f "$LOG_DIR/$WaitMarker" ]; then echo "#" > "$LOG_DIR/$WaitMarker"; fi
		    break
		else
		    echo "WARNING: Detected some problems with connection to master server." >&2
		    M_SERVER_STATUS=""
		fi
	    fi
	    sleep 32s
	fi
    done

    if [ "$SERVERCOUNT" -eq "1" ] && [ ! -e "$GAMEDIR/${SHORT_GAME_NAME}-mini-update.sh" ] && \
	[ -e "$GAMEDIR/../${SHORT_GAME_NAME}-dev_version-update.sh" ]; then
	if [ ! -f "$GAMEDIR/${SHORT_GAME_NAME}-dev_version-update-done.log" ]; then
	    echo "... dev_version update:" >&2
	    "$GAMEDIR/../${SHORT_GAME_NAME}-dev_version-update.sh" "$GAMEDIR"; sleep 1s
	    echo "#" > "$GAMEDIR/${SHORT_GAME_NAME}-dev_version-update-done.log"; sleep 1s
	    #"$GAMEDIR/$SERVER_SCRIPT" "$@" &
	    #break
	else
	    rm -f "$GAMEDIR/${SHORT_GAME_NAME}-dev_version-update-done.log"
	fi
    fi
    echo 'Starting server...' | tee -a "$LOG_SERVER"

    if [ "$MASTER_SERVER" != "" ] && [ "$SERVER_GREP_IP" != "" ]; then
	( check_nr=0; force_restart=0; loop_start="yes"; BeginTime="$(date +"%s")"; sleep 4m
	while true; do
	    SER_PID="$(ps -ef | grep "[ \t/]$EXC_BINARY_FPID $SER_GREP_PARAMETERS" | grep -v 'grep' | awk '{print $2}')"
	    if [ "$loop_start" = "yes" ]; then SER_PID_S="$SER_PID"; loop_start="no"; fi
	    CheckTime="$(date +"%s")"; DiffTime="$(($CheckTime-$BeginTime))"
	    if [ "$SER_PID" != "" ] && [ "$SER_PID" = "$SER_PID_S" ]; then
		FIND_SER="$(curl -s -L "$CLEAR_M_SERVER" | grep '^\([^|]*|\)\{4\}'"$SERVER_GREP_IP"'|' \
		    | grep '^\([^|]*|\)\{11\}'"$PORT"'|')"
		if [ "$FIND_SER" = "" ]; then
		    check_nr="$(($check_nr + 1))"
		    if [ "$OLD_FIND_SER_STATUS" = "" ]; then
			echo "WARNING: The master server doesn't see this server." >&2
			OLD_FIND_SER_STATUS="yes"
		    elif [ "$check_nr" -eq "120" ]; then
			echo "ERROR: The master server doesn't see this server for over 4 hours." >&2; force_restart=1
		    fi
		else
		    OLD_FIND_SER_STATUS=""
		    SER_STATUS="$(echo "$FIND_SER" | awk -F '|' '{print $14}')"
		    if [ "$SER_STATUS" = "" ] || [ "$(echo "$SER_STATUS" | grep '^[0-9]\+$')" = "" ]; then
			echo "ERROR: Server status cannot be determined." >&2
		    fi
		fi
		if [ "$SER_STATUS" != "2" ] && [ "$DiffTime" -gt "$MAX_IDLE_TIME" ]; then
		    echo "Notice: The server is working continuously for over $IDLE_TIME_MOD hour(s)." >&2; force_restart=1
		elif [ "$SER_STATUS" != "" ] && [ "$SER_STATUS" -eq "2" ]; then
		    if [ "$OLD_PROG_STATUS" = "" ]; then
			echo "Status: Game in progress..." >&2
			OLD_PROG_STATUS="yes"; BeginTime2="$(date +"%s")"
		    fi
		    DiffTime2="$(($CheckTime-$BeginTime2))"
		    if [ "$DiffTime2" -gt "14400" ]; then
			echo "WARNING: The game is in progress for over 4 hours." >&2; force_restart=1
		    else
			sleep 8m
		    fi
		elif [ "$SER_STATUS" != "" ] && [ "$SER_STATUS" -eq "1" ]; then
		    if [ "$OLD_READY_STATUS" = "" ]; then
			OLD_READY_STATUS="yes"; BeginTime3="$(date +"%s")"
		    fi
		    DiffTime3="$(($CheckTime-$BeginTime3))"
		    if [ "$DiffTime3" -gt "3600" ]; then
			echo "WARNING: The game seems to waiting for start for over an hour." >&2; force_restart=1
		    fi
		else
		    OLD_READY_STATUS=""
		fi
		if [ "$SER_STATUS" != "" ] && [ "$SER_STATUS" -ne "2" ]; then OLD_PROG_STATUS=""; fi
		if [ "$force_restart" -eq "1" ]; then
		    if [ "$check_nr" -lt "120" ]; then check_nr=120; fi
		    if [ "$LOG_DIR" != "" ]; then echo "#" > "$LOG_DIR/$RestartMarker"; sleep 2s; fi
		    if [ "$check_nr" -ge "123" ]; then kill -9 "$SER_PID"
			else kill "$SER_PID"; fi
		elif [ "$FIND_SER" != "" ]; then
		    check_nr=0
		fi
		sleep 2m
	    else
		break
	    fi
	done
	) &
    fi
    if [ "$SERVERTITLE" != "" ]; then
	"$SERVER_EXEC" ${USED_PORTS}"$SER_PARAMETERS" --server-title="Headless (${SERVERTITLE})" >> "$LOG_SERVER" 2>&1
    else
	"$SERVER_EXEC" ${USED_PORTS}"$SER_PARAMETERS" >> "$LOG_SERVER" 2>&1
    fi
    if [ "$?" -ne "0" ]; then
	if [ ! -f "$LOG_DIR/$RestartMarker" ]; then
	    echo 'ERROR: Server has quit unexpectedly.' >> "$LOG_SERVER"
	    echo 'ERROR: Server has quit unexpectedly.' >&2
	    if [ "$SERVERCOUNT" -eq "0" ]; then
		echo '       Please inspect '"$LOG_SERVER"'.' >&2
		exit 1
	    fi
	    sleep 5s
	    if [ "$SERVERCOUNT" -eq "1" ] && [ -e "$GAMEDIR/${SHORT_GAME_NAME}-mini-update.sh" ] && \
		[ "$SERVER_GREP_IP" != "" ] && [ "$MINI_UPDATE_USED" = "" ]; then
		echo "... attempt to perform mini update, which may solve tiny problems:" >&2
		"$GAMEDIR/${SHORT_GAME_NAME}-mini-update.sh"; sleep 1s
		MINI_UPDATE_USED="yes"
	    fi
	else
	    echo 'Notice: Server was restarted by script.' >> "$LOG_SERVER"
	    if [ "$LOG_DIR" != "" ]; then rm -f "$LOG_DIR/$RestartMarker"; fi
	fi
	sleep 2s
    else
	echo 'Server has quit.' | tee -a "$LOG_SERVER"
    fi
done