mobro.sh

#!/bin/bash

# ====================================================================================================================
# Modbros Monitoring Service (MoBro) - Raspberry Pi image
# Copyright (C) 2021 ModBros
# Contact: mod-bros.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
# ====================================================================================================================

# Directories
readonly TMP_DIR='/tmp'
readonly RESOURCES_DIR='/home/modbros/mobro-raspberrypi/resources'
readonly CONFIG_DIR='/home/modbros/mobro-raspberrypi/config'
readonly SCRIPT_DIR='/home/modbros/mobro-raspberrypi/scripts'

# Files
readonly MOBRO_CONFIG_FILE="$CONFIG_DIR/mobro_config"
readonly VERSION_FILE="$CONFIG_DIR/version"

# mobro partition files
readonly SKIP_FLAG="/mobro/skip_service"
readonly CONNECTED_HOST="/mobro/connected_host"
readonly MOBRO_CONFIG_BOOT_FILE="/mobro/mobro_config"
readonly MOBRO_CONFIGTXT_BOOT_FILE="/mobro/mobro_configtxt"

# TMP files
readonly MOBRO_FOUND_FLAG="$TMP_DIR/mobro_found"
readonly LOG_FILE="$TMP_DIR/mobro_log"
readonly HOSTS_FILE="$TMP_DIR/mobro_hosts"
readonly SSIDS_FILE="$TMP_DIR/mobro_ssids"

# Scripts
readonly APPLY_CONFIG_SCRIPT="$SCRIPT_DIR/apply_new_config.sh"
readonly FS_MOUNT_SCRIPT="$SCRIPT_DIR/fsmount.sh"

# Resources
readonly IMAGE_SPLASH="$RESOURCES_DIR/splashscreen.png"
readonly IMAGE_FOUND="$RESOURCES_DIR/found.png"
readonly IMAGE_NOTFOUND="$RESOURCES_DIR/notfound.png"
readonly IMAGE_CONNECTWIFI="$RESOURCES_DIR/wifi_connect.png"
readonly IMAGE_DISCOVERY="$RESOURCES_DIR/discovery.png"
readonly IMAGE_HOTSPOT="$RESOURCES_DIR/hotspot.png"
readonly IMAGE_HOTSPOTCREATION="$RESOURCES_DIR/creatinghotspot.png"
readonly IMAGE_WIFIFAILED="$RESOURCES_DIR/wifi_failed.png"
readonly IMAGE_WIFISUCCESS="$RESOURCES_DIR/wifi_success.png"
readonly IMAGE_NOWIFIINTERFACE="$RESOURCES_DIR/wifi_nointerface.png"
readonly IMAGE_ETHSUCCESS="$RESOURCES_DIR/eth_success.png"
readonly IMAGE_ETHFAILED="$RESOURCES_DIR/eth_failed.png"
readonly IMAGE_ETHNOLINK="$RESOURCES_DIR/eth_nolink.png"
readonly IMAGE_ETHCONNECT="$RESOURCES_DIR/eth_connect.png"
readonly IMAGE_CONFIGURATION="$RESOURCES_DIR/applyconfig.png"

# Ports
readonly MOBRO_PORT='42100'             # port of the MoBro desktop application

# Global Constants
readonly AP_SSID='MoBro_Configuration'  # ssid of the created access point
readonly LOOP_INTERVAL=60               # in seconds
readonly AP_RETRY_WAIT=20               # how long to wait for AP to start/stop before issuing command again (in s)
readonly AP_FAIL_WAIT=90                # how long to wait until AP creation/stopping is considered failed -> reboot (in s)
readonly STARTUP_CONNECTION_WAIT=45     # seconds to wait for network connection on startup

# Global Vars
LOOP_COUNTER=0                          # counter variable for main loop iterations
CURR_URL=''                             # save current chromium Url
CURR_IMAGE=''                           # save currently displayed image
NETWORK_MODE=''                         # save current network mode (eth|wifi|usb)
SCREENSAVER=0                           # flag if screensaver is active
LAST_CONNECTED=$(date +%s)              # timestamp (epoch seconds) of last successful connection check
NO_SCREEN=0                             # flag whether a screen has been found
PI_MODEL=$(cat /proc/device-tree/model) # the Raspberry Pi model (full string)
VERSION=$(cat $VERSION_FILE)            # the current version of this image

# ====================================================================================================================
# Functions
# ====================================================================================================================

log() {
    local temp date throttle
    temp=$(sudo vcgencmd measure_temp)
    date=$(date "+%d.%m.%y %T")
    throttle=$(sudo vcgencmd get_throttled)
    echo "$date [$LOOP_COUNTER][${temp:5:-4}][${throttle:10}][$1] $2" >>$LOG_FILE
}

prop() {
    grep "$1=" "$2" | cut -d '=' -f2
}

begins_with() {
    case $2 in "$1"*) true;; *) false;; esac;
}

get_network_ip() {
  ifconfig "$1" | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p'
}

wait_window() {
    until [[ $(xdotool search --onlyvisible --class "$1" | wc -l) -gt 0 ]]; do
        log "helper" "waiting for $1 to become available.."
        sleep 3
    done
}

stop_process() {
    log "helper" "stopping process: $1"
    sudo pkill "$1" 2>>$LOG_FILE
    for i in {4..0}; do
        if [[ $(pgrep -fc "$1") -eq 0 ]]; then
            return
        fi
        sleep 2
    done
    log "helper" "killing process: $1"
    sudo pkill -9 "$1" 2>>$LOG_FILE
    sleep_pi 1 2
}

stop_chrome() {
    stop_process "chromium"
    CURR_URL=''
}

stop_feh() {
    stop_process "feh"
    CURR_IMAGE=''
}

sleep_pi() {
    if [[ $(nproc --all) -gt 1 ]]; then
        sleep "$1"
    else
        sleep "$2"
    fi
}

wait_endless() {
    while true; do
        sleep 60
    done
}

show_image() {
    if [[ $NO_SCREEN == 1 ]]; then
        return
    fi
    if [[ $(pgrep -fc chromium) -gt 0 ]]; then
        stop_chrome
        sleep_pi 1 2
    fi
    if [[ $(pgrep -fc feh) -gt 0 ]]; then
        if [[ $CURR_IMAGE == "$1" ]]; then
            # already showing requested image
            return
        fi
        stop_feh
    fi
    CURR_IMAGE="$1"
    log "feh" "switching to image $1"
    feh \
        --fullscreen \
        --hide-pointer \
        --no-menus \
        --scale-down \
        --auto-zoom \
        --image-bg "white" \
        "$1" &>>$LOG_FILE &

    sleep "${2:-1}"
}

show_page_chrome() {
    if [[ $NO_SCREEN == 1 ]]; then
        return
    fi
    if [[ $(pgrep -fc chromium) -gt 0 ]]; then
        if [[ $CURR_URL == "$1" ]]; then
            # already showing requested page
            return
        fi
        stop_chrome
    fi
    stop_feh
    CURR_URL="$1"
    log "chromium" "switching to: '$1'"
    # check-for-update-interval flag can be removed once chromium is fixed
    # temporary workaround for: https://www.raspberrypi.org/forums/viewtopic.php?f=63&t=264399
    chromium-browser "$1" \
        --noerrdialogs \
        --incognito \
        --check-for-update-interval=2592000 \
        --kiosk \
        --disk-cache-size=1000\
        &>>$LOG_FILE &
    wait_window "chromium"
}

show_mobro() {
    local uuid resolution url
    resolution=$(xdpyinfo | awk '/dimensions/{print $2}') # current display resolution
    uuid=$(grep Serial /proc/cpuinfo | cut -d ' ' -f 2) # get pi serial number as unique ID
    url="http://$1:$MOBRO_PORT?version=$VERSION&uuid=$uuid&resolution=$resolution&device=pi&name=$PI_MODEL"
    SCREENSAVER=0
    LAST_CONNECTED=$(date +%s)
    show_page_chrome "$url"
}

show_screensaver() {
    if [[ $SCREENSAVER == 0 ]]; then
        log "screensaver" "activating screensaver: $1"
    fi
    SCREENSAVER=1
    if begins_with "http" "$1"; then
        show_page_chrome "$1"
    else
        show_page_chrome "http://localhost/screensavers/$1"
    fi
}

search_ssids() {
    log "search_ssids" "scanning for wireless networks"
    sudo iwlist wlan0 scan | grep -i essid: | sed 's/^.*"\(.*\)"$/\1/' 2>>$LOG_FILE 1>$SSIDS_FILE
    log "search_ssids" "detected wifi networks in range:"
    cat $SSIDS_FILE &>>$LOG_FILE
}

access_point() {
    log "create_access_point" "creating access point"

    show_image $IMAGE_HOTSPOTCREATION

    # scan for available wifi networks
    # (we can't do that once we the access point is up)
    search_ssids

    # create access point
    create_access_point_call
    sleep_pi 2 5

    local ap_create_counter ap_retry ap_fail
    ap_create_counter=1
    ap_retry=$((AP_RETRY_WAIT / 5))
    ap_fail=$((AP_FAIL_WAIT / 5))
    until is_access_point_open; do
        ap_create_counter=$((ap_create_counter + 1))
        if [[ $ap_create_counter -gt $ap_fail ]]; then
            log "create_access_point" "failed to create access point multiple times - rebooting.."
            sudo shutdown -r now
        fi
        if [[ $((ap_create_counter % ap_retry)) -eq 0 ]]; then
            log "create_access_point" "failed to create access point - trying again"
            create_access_point_call
        fi
        log "create_access_point" "waiting for access point.."
        sleep 5
    done

    log "create_access_point" "access point up"
    show_image $IMAGE_HOTSPOT

    # wait for user to configure the pi
    # on configuration apply the pi will be rebooted automatically
    wait_endless
}

create_access_point_call() {
    sudo create_ap \
        -w 2 \
        -m none \
        --isolate-clients \
        --freq-band 2.4 \
        --driver nl80211 \
        --no-virt \
        --daemon \
        -g 192.168.4.1 \
        wlan0 $AP_SSID \
        &>>$LOG_FILE
}

is_access_point_open() {
    create_ap --list-running | grep wlan0 -q
}

set_mobro_found() {
    echo "1" >$MOBRO_FOUND_FLAG
    local found_host
    found_host=$(sed -n 1p <$CONNECTED_HOST)
    if [[ "$found_host" != "$1" ]]; then
        echo "$1" >$CONNECTED_HOST
    fi
}

is_mobro_found() {
    if [[ $(cat $MOBRO_FOUND_FLAG) -eq 1 ]]; then
        return 0
    else
        return 1
    fi
}

try_ip_static() {
    local response_code
    response_code=$(curl -o /dev/null --silent --connect-timeout 5 --retry 2 --write-out '%{http_code}' "$1:$MOBRO_PORT/discover")
    if [[ $response_code == 403 || $response_code == 200 ]]; then
        # 403 -> mobro is there but wrong connection key.
        # since we're going for static ip, we don't need to match the key
        set_mobro_found "$1"
        return 0
    else
        return 1
    fi
}

try_ip() {
    # $1 = IP, $2 = key
    if [[ $(curl -o /dev/null --silent --max-time 5 --connect-timeout 3 --write-out '%{http_code}' "$1:$MOBRO_PORT/discover?key=$2") -eq 200 ]]; then
        # 200 -> mobro is there and we're using the correct connection key
        set_mobro_found "$1"
        return 0
    else
        return 1
    fi
}

check_screensaver() {
    # check if screensaver is configured
    local screensaver
    screensaver=$(prop 'display_screensaver' $MOBRO_CONFIG_FILE)
    if [[ -z $screensaver || $screensaver == "disabled" ]]; then
        # no screensaver configured
        return
    fi

    if [[ $screensaver == "custom" ]]; then
        screensaver=$(prop 'display_screensaver_url' $MOBRO_CONFIG_FILE)
        if ! begins_with "http" "$screensaver"; then
            screensaver="https://$screensaver";
        fi
    fi

    # screensaver configured -> check delay
    local delay
    delay=$(prop 'display_delay' $MOBRO_CONFIG_FILE)
    if [[ -n $delay ]]; then
        local diff now
        now=$(date +%s)
        diff=$((now - LAST_CONNECTED))
        delay=$((delay * 60))
        log "screensaver" "delay: $diff/$delay"
        if [[ $diff -gt $delay ]]; then
            # delay reached -> show
            show_screensaver "$screensaver"
        fi
    else
        # no delay -> just show screensaver
        show_screensaver "$screensaver"
    fi
}

service_discovery() {

    echo "0" >$MOBRO_FOUND_FLAG
    if [[ $SCREENSAVER == 0 ]]; then
        show_image $IMAGE_DISCOVERY 2
    fi

    local mode ip key interface
    mode=$(prop 'discovery_mode' $MOBRO_CONFIG_FILE)
    key=$(prop 'discovery_key' $MOBRO_CONFIG_FILE)
    ip=$(prop 'discovery_ip' $MOBRO_CONFIG_FILE)
    case $NETWORK_MODE in
    "wifi") interface='wlan0' ;;
    "eth") interface='eth0' ;;
    "usb") interface='usb0' ;;
    esac

    # check if static ip is configured
    if [[ $mode == "manual" ]]; then
        log "service_discovery" "configured to use static ip. trying: $ip"
        if try_ip_static "$ip"; then
            log "service_discovery" "MoBro application found on static IP $ip"
            set_mobro_found "$ip"
            show_image $IMAGE_FOUND 5
            show_mobro "$ip"
        else
            log "service_discovery" "no MoBro application found on static ip $ip"
            if [[ $SCREENSAVER == 0 ]]; then
                show_image $IMAGE_NOTFOUND 5
                check_screensaver
            fi
        fi
        return
    fi

    log "service_discovery" "configured to automatic discovery using connection key"
    # check previous host if configured
    ip=$(sed -n 1p <$CONNECTED_HOST)
    if ! [[ -z $ip || -z $key ]]; then
        log "service_discovery" "checking previous host on $ip with key '$key'"
        try_ip "$ip" "$key"
    fi

    if ! is_mobro_found; then
        # search available IPs on the network via arp scan
        log "service_discovery" "performing arp scan"
        sudo arp-scan \
            --interface="$interface" \
            --localnet \
            --retry=3 \
            --timeout=500 \
            --backoff=2 | grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" \
            2>>$LOG_FILE 1>>"$HOSTS_FILE"

        while read -r ip; do
            log "service_discovery" "trying IP: $ip with key: $key"
            try_ip "$ip" "$key" &
            pids[$j]=$! # remember pids of started sub processes
            if is_mobro_found; then
                break # no need to continue if found
            fi
        done <$HOSTS_FILE

        for pid in ${pids[*]}; do
            # wait for all started checks to finish
            wait $pid
        done
        unset pids
    fi

    if ! is_mobro_found; then
        # fallback: get current ip of pi to try all host in range
        log "service_discovery" "fallback"
        local pi_ip
        pi_ip=$(get_network_ip "$interface")
        if [[ -n $pi_ip ]]; then
            local pi_ip_1 pi_ip_2 pi_ip_3 pi_ip_4
            pi_ip_1=$(echo "$pi_ip" | cut -d . -f 1)
            pi_ip_2=$(echo "$pi_ip" | cut -d . -f 2)
            pi_ip_3=$(echo "$pi_ip" | cut -d . -f 3)
            pi_ip_4=0
            log "service_discovery" "trying all IPs in range $pi_ip_1.$pi_ip_2.$pi_ip_3.X"

            local num_cores
            num_cores=$(nproc --all)
            while [[ $pi_ip_4 -lt 255 ]]; do
                for j in $(seq $((5 * num_cores))); do
                    try_ip "$pi_ip_1.$pi_ip_2.$pi_ip_3.$pi_ip_4" "$key" &
                    pids[$j]=$! # remember pids of started sub processes
                    if [[ pi_ip_4 -ge 255 ]]; then
                        break
                    fi
                    pi_ip_4=$((pi_ip_4 + 1))
                done

                # wait for all started checks to finish
                for pid in ${pids[*]}; do
                    wait $pid
                done
                unset

                if is_mobro_found; then
                    break # no need to continue if found
                fi
            done
        fi
    fi

    if is_mobro_found; then
        # found MoBro
        local found_host
        found_host=$(sed -n 1p <$CONNECTED_HOST)
        log "service_discovery" "MoBro application found on IP $found_host"
        show_image $IMAGE_FOUND 5
        show_mobro "$found_host"
    else
        # couldn't find application -> delete IPs
        log "service_discovery" "no MoBro application found"
        truncate -s 0 $HOSTS_FILE
        if [[ $SCREENSAVER == 0 ]]; then
            show_image $IMAGE_NOTFOUND 5
            check_screensaver
        fi
    fi
}

background_check() {
    # check if host is still reachable
    local ip
    ip=$(sed -n 1p <$CONNECTED_HOST)
    if [[ -z $ip ]]; then
        log "background_check" "no previous host found. starting discovery"
        service_discovery
        return
    fi

    # try multiple times to make sure we didn't just drop connection for a few seconds
    local response_code
    response_code=$(curl -o /dev/null --silent --connect-timeout 10 --retry 3 --write-out '%{http_code}' "$ip:$MOBRO_PORT/discover")
    if [[ $response_code == 403 || $response_code == 200 ]]; then
        # reachable
        log "background_check" "MoBro on $ip reachable"
        show_mobro "$ip"
        return
    fi

    # we lost connection
    log "background_check" "MoBro on $ip no longer reachable (code $response_code)"
    if [[ $SCREENSAVER == 0 ]]; then
        show_image $IMAGE_NOTFOUND 10
    fi

    # check if we need to show screensaver
    check_screensaver

    # start searching again
    service_discovery
}

wait_wifi_connection() {
    local try_seconds=0
    until [[ $try_seconds -ge $1 ]]; do
        if [[ $(iwgetid wlan0 --raw) ]]; then
            # wifi connected
            return 0
        fi
        sleep 5
        try_seconds=$((try_seconds + 5))
    done
    return 1
}

wait_ip() {
    local try_seconds=0
    until [[ $try_seconds -ge $2 ]]; do
        if check_network_ip_valid "$1"; then
            return 0
        fi
        sleep 5
        try_seconds=$((try_seconds + 5))
    done
    return 1
}

check_network_link() {
    local operstate
    operstate=$(cat "/sys/class/net/$1/operstate")
    if [[ $operstate == "up" ]]; then
        return 0;
    fi
    return 1;
}

check_network_ip_valid() {
    local ip
    ip=$(get_network_ip "$1")
    log "check_network_ip_valid" "$1: $ip"
    if [[ -n $ip ]]; then
        if ! begins_with "169.254." "$ip"; then
            return 0
        fi
        # The 169.254.x.x range is used when no DHCP server is found on the local network
        # or it cannot issue a new IP address to the requesting device
    fi
    return 1
}

get_overlay_now() {
    grep -q "boot=overlay" /proc/cmdline
}

overlay_disabled() {
    if [[ $(prop 'advanced_fs_dis_overlayfs' "$MOBRO_CONFIG_FILE") == "1" ]]; then
        return 0
    else
        return 1
    fi
}

config_boot() {
    if ! [ -s "$MOBRO_CONFIG_BOOT_FILE" ]; then
       log "config_boot" "skipping - no config to apply"
       return
    fi

    if get_overlay_now; then
        log "config_boot" "OverlayFS still active. disabling and rebooting"
        {
          sudo /bin/bash "$FS_MOUNT_SCRIPT" --rw root
          sudo shutdown -r now
        } &>>$LOG_FILE
    fi

    show_image $IMAGE_CONFIGURATION
    log "config_boot" "applying new configuration"
    # note: applying the configuration will trigger a reboot
    sudo /bin/bash "$APPLY_CONFIG_SCRIPT" "$MOBRO_CONFIG_BOOT_FILE" "$MOBRO_CONFIGTXT_BOOT_FILE"
}

# ====================================================================================================================
# Startup Sequence
# ====================================================================================================================

if [[ -f "$SKIP_FLAG" ]]; then
    # do not process if the service skip flag is present
    # but start networking so we can connect via SSH
    log "startup" "skip flag for service is set"
    sudo systemctl start dhcpcd.service
    wait_endless
fi

log "startup" "starting service"

# create files
for arg in LOG_FILE MOBRO_FOUND_FLAG SSIDS_FILE HOSTS_FILE CONNECTED_HOST; do
    eval value=\$$arg
    sudo touch "$value"
    sudo chown modbros "$value"
done

log "startup" "Pi Model: $PI_MODEL"
log "startup" "Version: $VERSION"

log "startup" "Current configuration:"
cat -n "$MOBRO_CONFIG_FILE" | sed '/network_pw/d' &>>$LOG_FILE

# env vars
export DISPLAY=:0

# reset flag
echo "0" >$MOBRO_FOUND_FLAG

# disable dnsmasq and hostapd
log "startup" "disabling services: dnsmasq, hostapd"
{
    sudo systemctl stop dnsmasq
    sudo systemctl disable dnsmasq.service
    sudo systemctl stop hostapd
    sudo systemctl disable hostapd.service
} &>>$LOG_FILE

# start x
log "startup" "starting x server"
sudo xinit \
    /bin/sh -c "exec /usr/bin/matchbox-window-manager -use_titlebar no -use_cursor no" \
    -- -nocursor \
    &>>$LOG_FILE &

# wait for x server
sleep 2
while ! xset q &>/dev/null; do
    log "startup" "waiting for X.."
    if [[ $(grep -c "no screens found" $LOG_FILE) -gt 0 ]]; then
        NO_SCREEN=1
        stop_process "xinit"
        break
    fi
    sleep 5
done
sleep_pi 2 5

# check if we need to apply config
config_boot

if overlay_disabled; then
    if get_overlay_now; then
        log "startup" "OverlayFS disabled but still active. disabling and rebooting"
        sudo /bin/bash "$FS_MOUNT_SCRIPT" --rw root
        sudo shutdown -r now
    fi
else
    if ! get_overlay_now; then
        log "startup" "OverlayFS enabled but not active. enabling and rebooting"
        sudo /bin/bash "$FS_MOUNT_SCRIPT" --ro root
        sudo shutdown -r now
    fi
fi

# from this point onwards, we require networking -> start dhcpcd
log "startup" "starting dhcpcd service"
sudo systemctl start dhcpcd.service

# handle case if no connected display is found
if [[ $NO_SCREEN == 1 ]]; then
    log "startup" "no screen found - waiting for configuration"
    access_point
    wait_endless
fi

# disabling screen blanking
log "startup" "disable blank screen"
{
    sudo xset s off
    sudo xset -dpms
    sudo xset s noblank
} &>>$LOG_FILE

# show background
show_image $IMAGE_SPLASH 7

# determine and set network mode
NETWORK_MODE=$(prop 'network_mode' $MOBRO_CONFIG_FILE)
if [[ -z "$NETWORK_MODE" ]]; then
    log "startup" "no network mode in config, trying to determine connection"
    if check_network_link "eth0"; then
        NETWORK_MODE='eth'
    else
        NETWORK_MODE='wifi'
    fi
fi
log "startup" "network mode set to $NETWORK_MODE"

NETWORK_INTERFACE=''
case $NETWORK_MODE in
wifi) NETWORK_INTERFACE='wlan0' ;;
eth) NETWORK_INTERFACE='eth0' ;;
usb) NETWORK_INTERFACE='usb0' ;;
esac

case $NETWORK_MODE in
wifi)
    # check for wifi interface
    if [[ $(ifconfig -a | grep wlan -c) -lt 1 ]]; then
        log "startup" "no wifi interface detected, aborting"
        show_image $IMAGE_NOWIFIINTERFACE
        wait_endless
    fi
    # unblock the wifi interface
    log "startup" "unblocking wifi interface"
    sudo rfkill unblock 0 &>>$LOG_FILE

    # check if wifi is configured
    # (skip if no ssid is set - e.g. first boot)
    ssid=$(prop 'network_ssid' $MOBRO_CONFIG_FILE)
    if [[ -z "$ssid" ]]; then
        log "startup" "no previous network configuration found"
        access_point
    fi

    show_image $IMAGE_CONNECTWIFI 2

    log "startup" "waiting for wifi connection to '$ssid'..."
    if ! wait_wifi_connection $STARTUP_CONNECTION_WAIT; then
        log "startup" "failed to connect to '$ssid', restarting wlan0, reconfiguring wpa and trying again"
        {
            sudo ifconfig wlan0 down
            sudo ifconfig wlan0 up
            sudo wpa_cli -i wlan0 reconfigure
        } &>>$LOG_FILE
        if ! wait_wifi_connection $STARTUP_CONNECTION_WAIT; then
            log "startup" "couldn't connect to wifi network '$ssid'"
            show_image $IMAGE_WIFIFAILED 10
            access_point
        fi
    fi

    log "startup" "connected to wifi network '$ssid'"
    show_image $IMAGE_WIFISUCCESS 3
    ;;

eth|usb)
    # check for network link
    while ! check_network_link $NETWORK_INTERFACE; do
        show_image $IMAGE_ETHNOLINK
        sleep 2
    done

    show_image $IMAGE_ETHCONNECT 2

    if ! check_network_ip_valid $NETWORK_INTERFACE; then
        log "startup" "$NETWORK_INTERFACE not connected correctly, re-negotiate and try again"
        {
            sudo systemctl restart dhcpcd.service
            sudo ethtool -r eth0
            sleep 5
        } &>>$LOG_FILE

        if ! (wait_ip $NETWORK_INTERFACE $STARTUP_CONNECTION_WAIT); then
            log "startup" "couldn't connect to $NETWORK_MODE network"
            show_image $IMAGE_ETHFAILED 20
            sudo shutdown -r now
        fi
    fi

    show_image $IMAGE_ETHSUCCESS 3
    ;;
esac

# search network for application
log "startup" "connected to $NETWORK_INTERFACE"
service_discovery

# ====================================================================================================================
# Main Loop
# ====================================================================================================================

log "main" "Entering main loop"
while true; do
    sleep $LOOP_INTERVAL
    LOOP_COUNTER=$((LOOP_COUNTER + 1))

    case $NETWORK_MODE in
    wifi)
        while ! wait_wifi_connection 15; do
          show_image $IMAGE_CONNECTWIFI
        done
        background_check
        ;;
    eth|usb)
        background_check
        ;;
    esac

done

log "main" "unexpected shutdown"