checks.sh

#!/bin/bash
set -o pipefail

DIAGNOSE_VERSION=4.23.1
# Don't run anything before this source as it sets PATH here
# shellcheck disable=SC1091
source /etc/profile

# Newer OS versions do not set REGISTRY_ENDPOINT via
# /usr/sbin/resin-vars; that's because the information provided by
# balena cloud is meant to be the source of truth. Thus, we add the
# REGISTRY_ENDPOINT argument for this script, and will supply that
# argument in the proxy when this script is called. To avoid having
# them overwritten by older versions of resin-vars, we save the CLI
# arguments and pop them back out after sourcing resin-vars.

# Set aside arguments, come back to them later
current_args=( "$@" )
set --
# We still need to include resin-vars on legacy systems
if [ -f /usr/sbin/resin-vars ]; then
# shellcheck disable=SC1091
  source /usr/sbin/resin-vars
else
# shellcheck disable=SC1091
  source /usr/sbin/balena-config-vars
fi
# Restore arguments passed in originally
set -- "${current_args[@]}"
# shellcheck disable=SC1091
source /etc/os-release

# Determine whether we're using the older 'rce'-aliased docker or not.
# stolen directly from the proxy:
# (https://github.com/balena-io/resin-proxy/blob/master/src/common/host-scripts.ts#L28)
X=/usr/bin/
ENG=rce
[ -x $X$ENG ] || ENG=docker
[ -x $X$ENG ] || ENG=balena
[ -x $X$ENG ] || ENG=balena-engine

TIMEOUT=10
TIMEOUT_CMD="timeout --kill-after=$(( TIMEOUT * 2 )) ${TIMEOUT}"
mountpoint="/var/lib/${ENG}"

external_fqdn="0.resinio.pool.ntp.org"
IPV4_ADDRESS="1.1.1.1" # https://1.1.1.1/dns/
IPV6_ADDRESS="2606:4700:4700::6400" # https://developers.cloudflare.com/1.1.1.1/ipv6-networks
IPV4_ENDPOINT="ipv4.google.com"
IPV6_ENDPOINT="ipv6.google.com"

# workaround for self-signed certs, waiting for https://github.com/balena-os/meta-balena/issues/1398
TMPCRT=$(mktemp)
echo "${BALENA_ROOT_CA}" | base64 -d > "${TMPCRT}"
cat /etc/ssl/certs/ca-certificates.crt >> "${TMPCRT}"

low_mem_threshold=10 #%
low_disk_threshold=10 #%
wifi_threshold=40 #%, very handwavy
expansion_threshold=80 #%
slow_disk_write=1000 #ms
temperature_threshold=80000 #millidegree C

GOOD="true"
BAD="false"

mapfile -t USERVICES < <(${TIMEOUT_CMD} "${ENG}" ps --format "{{.Names}}" | awk '/(resin|balena)_supervisor/{next;}{print}' 2> /dev/null)

# Helper functions
function help()
{
	cat << EOF
Script to run checks on balenaOS devices

Options:
	-h, --help
		Display this help and exit.

	--balenaos-registry
		Upstream registry to use for host OS apps.
EOF
}

function announce_version()
{
	jq -n --arg dv "${DIAGNOSE_VERSION}" '{"diagnose_version":$dv}'
}

function get_meminfo_field()
{
	awk '/^'"$1"':/{print $2}' /proc/meminfo
}

function log_status()
{
	# success (g) ${1}
	# function (f) ${2}
	# status (s) ${3}
	jq -cn --argjson g "${1}" --arg f "${2}" --arg s "${3}" '[{"name":$f,"success":$g,"status":$s}]'
}

function test_upstream_dns()
{
	# force dnsmasq to print statistics to journal to read them back
	systemctl kill -s USR1 dnsmasq
	for i in $(journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show -p InvocationID --value dnsmasq.service)" | tail -n +2 | awk '{$1=$2=$3=""; print $0}' | grep -E -o '([12]?[0-9]?[0-9]\.[12]?[0-9]?[0-9]\.[12]?[0-9]?[0-9]\.[12]?[0-9]?[0-9])|(([a-f0-9:]+:+)+[a-f0-9]+)' | sort -u); do
		# shellcheck disable=SC2001
		for j in "${external_fqdn}" "$(echo "${API_ENDPOINT}" | sed -e 's@^https*://@@')"; do
			if ! nslookup "${j}" "${i}" > /dev/null 2>&1; then
				echo "${FUNCNAME[0]}: DNS lookup failed for ${j} via upstream: ${i}"
			fi
		done;
	done
}

function test_wifi()
{
	local wifi_active_interfaces
	wifi_active_interfaces=$(${TIMEOUT_CMD} nmcli --terse --fields DEVICE,TYPE,STATE device status | \
		awk -F: '/wifi:connected$/{print $1}')
	if [[ -n ${wifi_active_interfaces} ]]; then
		local -i wifi_strength
		wifi_strengths=$(${TIMEOUT_CMD} nmcli --terse --fields DEVICE,ACTIVE,SIGNAL device wifi list)
		for ifname in ${wifi_active_interfaces}; do
			wifi_strength=$(echo "${wifi_strengths}" | awk -F: '/^'"${ifname}"':yes:/{print $3}')
			if (( wifi_strength < wifi_threshold )) && (( wifi_strength > 0 )); then
				echo "${FUNCNAME[0]}: Configured wifi interface ${ifname} has a weak signal (<${wifi_threshold}%)"
			fi
		done
	fi
}

function test_ping()
{
	local -i lost_packets
	lost_packets=$(${TIMEOUT_CMD} ping -c 3 "${external_fqdn}" 2>/dev/null | awk -F, '/loss/{print $7}')
	if (( lost_packets > 0 )); then
		echo "${FUNCNAME[0]}: Packets lost while pinging ${external_fqdn}"
	fi
}

function test_balena_api()
{
	# can we reach the API?
	local api_ret
	local -i api_retval
	api_ret=$(CURL_CA_BUNDLE=${TMPCRT} ${TIMEOUT_CMD} curl -qs "${API_ENDPOINT}/ping")
	api_retval=$?
	if [[ "${api_ret}" != "OK" ]]; then
		# from man curl:
		# EXIT CODES
		# 60	 Peer certificate cannot be authenticated with known CA certificates.
		if [ "${api_retval}" -eq 60 ]; then
			echo "${FUNCNAME[0]}: There may be a firewall blocking traffic to ${API_ENDPOINT} (SSL errors)"
			return
		fi
		echo "${FUNCNAME[0]}: Could not contact ${API_ENDPOINT}"
	fi
}

function test_balena_registry()
{
	# can we authenticate with the registry? TODO: this isn't really a good check for auth, just for connectivity
	# TODO: old OSes fail because no --password-stdin, so can't use that.
	if ! ${TIMEOUT_CMD} ${ENG} login "${REGISTRY_ENDPOINT}" -u "d_${UUID}" \
		--password "${DEVICE_API_KEY}" > /dev/null 2>&1; then
		echo "${FUNCNAME[0]}: Could not communicate with ${REGISTRY_ENDPOINT} for authentication"
	fi
	${TIMEOUT_CMD} ${ENG} logout "${REGISTRY_ENDPOINT}" > /dev/null
}

function test_ipv4_stack()
{
	# Check if device expects a functioning IPv4 stack
	if ! ip -4 route get ${IPV4_ADDRESS} &>/dev/null; then
		# Don't check if IPv4 stack works since device isn't configured to use it
		return
	fi 
	# Check if we can reach an IPv4 HTTP service since device is configured to do so 
	local -i res
	res=$(CURL_CA_BUNDLE=${TMPCRT} ${TIMEOUT_CMD} curl -qs "https://${IPV4_ENDPOINT}" 1>/dev/null 2>&1; echo $?)
	if test "$res" != "0"; then
		# from man curl:
		# EXIT CODES
		# 60	 Peer certificate cannot be authenticated with known CA certificates.
		if [ "${res}" -eq 60 ]; then
			echo "${FUNCNAME[0]}: There may be a firewall blocking traffic to https://${IPV4_ENDPOINT} (SSL errors)"
			return
		fi
		echo "${FUNCNAME[0]}: Could not contact https://${IPV4_ENDPOINT}"
	fi
}

function test_ipv6_stack()
{
	# Check if device expects a functioning IPv6 stack
	if ! ip -6 route get ${IPV6_ADDRESS} &>/dev/null; then
		# Don't check if IPv6 stack works since device isn't configured to use it
		return
	fi 
	# Check if we can reach an IPv6 HTTP service since device is configured to do so 
	local -i res
	res=$(CURL_CA_BUNDLE=${TMPCRT} ${TIMEOUT_CMD} curl -qs "https://${IPV6_ENDPOINT}" 1>/dev/null 2>&1; echo $?)
	if test "$res" != "0"; then
		# from man curl:
		# EXIT CODES
		# 60	 Peer certificate cannot be authenticated with known CA certificates.
		if [ "${res}" -eq 60 ]; then
			echo "${FUNCNAME[0]}: There may be a firewall blocking traffic to https://${IPV6_ENDPOINT} (SSL errors)"
			return
		fi
		echo "${FUNCNAME[0]}: Could not contact https://${IPV6_ENDPOINT}"
	fi
}

function test_write_latency()
{
	# from https://www.kernel.org/doc/Documentation/iostats.txt:
	#
	# Field  5 -- # of writes completed
	#	  This is the total number of writes completed successfully.
	# Field  8 -- # of milliseconds spent writing
	#	  This is the total number of milliseconds spent by all writes (as
	#	  measured from __make_request() to end_that_request_last()).
	local write_output
	write_output=$(awk -v limit=${slow_disk_write} '!/(loop|ram)/{if ($11/(($8>0)?$8:1)>limit){print $3": " $11/(($8>0)?$8:1) "ms / write, sample size " $8}}' /proc/diskstats)
	if [ -n "${write_output}" ]; then
		echo "${FUNCNAME[0]}" "Slow disk writes detected: ${write_output}"
	fi
}

function test_diskspace()
{
	# Last +0 forces the field to a number, stripping the '%' on the end.
	# Tested working on busybox.
	local -i used_percent
	local -i free_percent
	used_percent=$(df ${mountpoint} | tail -n 1 | awk '{print $5+0}')
	free_percent=$(( 100 - used_percent ))

	if (( free_percent < low_disk_threshold )); then
		echo "${FUNCNAME[0]}" "Low disk space: (df reports ${free_percent}% free)"
	fi
}

function test_disk_expansion()
{
	local -i expansion_perc
	# TODO: ceil is better, but floor is supported back to jq-1.5
	expansion_perc=$(lsblk -Jb -o NAME,SIZE,MOUNTPOINT | jq -S '.blockdevices[] | select(.children[]?.mountpoint == "/mnt/boot").size as $total |
		[.children[].size | tonumber] | 100 * add / ($total | tonumber) | floor')
	if (( expansion_perc < expansion_threshold )) ; then
		echo "${FUNCNAME[0]}" "Block media device may not have fully expanded (${expansion_perc}% of available space)"
	fi
}

function test_data_partition_mounted()
{
	data_mounted=$(awk '($2 == "/mnt/data") {print $4}' /proc/mounts)
	case $data_mounted in
		*rw*) return ;;
		*ro*) echo "${FUNCNAME[0]}" "Data partition not mounted read-write" ;;
		"") echo "${FUNCNAME[0]}" "Data partition not mounted" ;;
	esac
}

function is_valid_check()
{
	local list_type="${1}"
	shift
	local found=1 i
	read -r -a args <<< "$@"
	for i in ${args[*]}
	do
		if [ "${i}" == "${SLUG}" ] ; then
			found=0
			break
		fi
	done
	if [[ "${list_type}" == "WHITELIST" ]]; then
		return "${found}"
	elif [[ "${list_type}" == "BLACKLIST" ]]; then
		return $(( 1 - found ))
	fi
}

function run_tests()
{
	local original_check
	local subject
	original_check=${1}
	subject=${original_check//check_/}
	shift;

	local tests=("$@")

	local output
	output=""
	local cmd_out
	for i in "${tests[@]}"; do
		cmd_out=$(${i})
		if [ -n "${cmd_out}" ]; then
			output+=$'\n'$(printf '%s' "${cmd_out}")
		fi
	done
	if [ "$(echo "${output}" | wc -l)" -gt 1 ]; then
		log_status "${BAD}" "${original_check}" "Some ${subject} issues detected: ${output}"
	else
		log_status "${GOOD}" "${original_check}" "No ${subject} issues detected"
	fi
}

# Check functions
function check_networking()
{
	local tests=(
		test_upstream_dns
		test_wifi
		test_ping
		test_balena_api
		test_balena_registry
		test_ipv4_stack
		test_ipv6_stack
	)
	run_tests "${FUNCNAME[0]}" "${tests[@]}"
}

function check_localdisk()
{
	local tests=(
		test_write_latency
		test_diskspace
		test_disk_expansion
		test_data_partition_mounted
	)
	run_tests "${FUNCNAME[0]}" "${tests[@]}"
}

function check_under_voltage(){
	local SLUG_WHITELIST=('raspberrypi3-64' 'raspberrypi4-64' 'raspberry-pi' 'raspberry-pi2' 'raspberrypi3' 'fincm3' 'raspberrypi0-2w-64' 'npe-x500-m3' 'raspberrypi3-unipi-neuron' 'raspberrypi4-superhub' 'raspberrypi4-unipi-neuron' 'raspberrypi400-64' 'raspberrypicm4-ioboard' 'revpi-connect-4' 'revpi-connect-s' 'revpi-connect' 'revpi-core-3' 'rt-rpi-300' 'raspberrypi5')
	if is_valid_check WHITELIST "${SLUG_WHITELIST[*]}"; then
		local -i UNDERVOLTAGE_COUNT
		UNDERVOLTAGE_COUNT=$(dmesg | grep -E -c "Under-?voltage detected\!")
		if (( UNDERVOLTAGE_COUNT > 0 )); then
			log_status "${BAD}" "${FUNCNAME[0]}" "${UNDERVOLTAGE_COUNT} under-voltage events detected, check/change the power supply ASAP"
		else
			log_status "${GOOD}" "${FUNCNAME[0]}" "No under-voltage events detected"
		fi
	fi
}

function check_temperature(){
	local tests=(
		test_current_temperature
		test_throttling_dmesg
	)
	run_tests "${FUNCNAME[0]}" "${tests[@]}"
}

function test_current_temperature(){
	# see https://github.com/balena-io/device-diagnostics/issues/168
	SLUG_WHITELIST=('astro-tx2' 'blackboard-tx2' 'cti-rogue-xavier' 'jetson-nano-2gb-devkit' 'jetson-nano-emmc' 'jetson-nano' 'floyd-nano' \
		'jetson-tx1' 'jetson-tx2' 'jetson-xavier-nx-devkit-emmc' 'jetson-xavier-nx-devkit' 'jetson-xavier' \
		'jn30b-nano' 'n310-tx2' 'n510-tx2' 'nru120s-xavier' 'orbitty-tx2' 'photon-nano' 'photon-xavier-nx' \
		'spacely-tx2' 'skx2' 'apollo-tx2' 'j120-tx2' 'srd3-tx2')
	local -i temp
	for i in /sys/class/thermal/thermal* ; do
		if [ -e "$i/temp" ]; then
			temp=$(cat "$i/temp")
			if (( temp >= temperature_threshold )); then
				if is_valid_check WHITELIST "${SLUG_WHITELIST[*]}" && [ -e "$i/type" ] && grep -q "PMIC-Die" "$i/type"; then
					:
					# Ignore PMIC-Die because it reports a dummy value on Jetsons: https://forums.developer.nvidia.com/t/pmic-temperature-in-tegrastats-output/75582
				else
					echo "${FUNCNAME[0]}" "Temperature above $(( temperature_threshold / 1000 ))C detected ($i)"
				fi
			fi
		fi
	done
}


function test_throttling_dmesg(){
	# see https://github.com/balena-io/device-diagnostics/issues/183
	local -i TEMP_THROTTLING_COUNT
	TEMP_THROTTLING_COUNT=$(dmesg | grep -cE 'Temperature above threshold, cpu clock throttled')
	if (( TEMP_THROTTLING_COUNT > 0 )); then
		echo "${FUNCNAME[0]}" "${TEMP_THROTTLING_COUNT} cpu throttling events detected, check CPU temperature"
	fi

}

function check_balenaOS()
{
	# test resinOS 1.x based on matches like the following:
	# VERSION="1.24.0"
	# PRETTY_NAME="Resin OS 1.24.0"

	local variant_tag=$(echo "${VARIANT:-production}" | tr "[:upper:]" "[:lower:]")
	local variant_slug=$(echo "${variant_tag}" | sed "s/production/prod/" | sed "s/development/dev/")

	if grep -q -e '^VERSION="1.*$' -e '^PRETTY_NAME="Resin OS 1.*$' /etc/os-release; then
		log_status "${BAD}" "${FUNCNAME[0]}" "ResinOS 1.x is now completely deprecated"
	else
		local -i versions
		# We still need to try finding the hostApp release by filtering using the deprecated release_tags,
		# since the versioning format of balenaOS [2019.10.0.dev, 2022.01.0] was non-semver compliant
		# and they were not migrated to the release semver fields.
		versions=$(CURL_CA_BUNDLE=${TMPCRT} curl -qs --retry 3 --max-time 5 --retry-connrefused -X GET \
			-H "Content-Type: application/json" \
			-H "Authorization: Bearer ${DEVICE_API_KEY}" \
			"${API_ENDPOINT}/v6/release?\$select=id,semver&\$expand=release_tag&\$filter=(belongs_to__application/any(a:a/is_for__device_type/any(dt:dt/slug%20eq%20'${SLUG}')%20and%20a/is_host%20eq%20true))%20and%20is_invalidated%20eq%20false%20and%20(((semver%20eq%20%27${VERSION}%27)%20and%20(variant%20in%20(%27${variant_slug}%27,%20%27%27)))%20or%20((release_tag/any(rt:(rt/tag_key%20eq%20'version')%20and%20(rt/value%20eq%20'${VERSION}')))%20and%20((release_tag/any(rt:(rt/tag_key%20eq%20'variant')%20and%20(rt/value%20eq%20'${variant_tag}')))%20or%20not(release_tag/any(rt:rt/tag_key%20eq%20'variant')))))" \
			| jq -r "[.d[]] | length")
		if (( versions == 0 )); then
			log_status "${BAD}" "${FUNCNAME[0]}" "balenaOS 2.x detected, but this version is not currently available in ${API_ENDPOINT}"
		elif (( versions > 1 )); then
			log_status "${BAD}" "${FUNCNAME[0]}" "balenaOS 2.x detected, but more than one hostapp release available in ${API_ENDPOINT}"
		else
			log_status "${GOOD}" "${FUNCNAME[0]}" "Supported balenaOS 2.x detected"
		fi
	fi
}

function check_memory()
{
	local -i total_kb
	local -i avail_kb
	local avail_exists

	total_kb=$(get_meminfo_field MemTotal)
	avail_exists=$(get_meminfo_field MemAvailable)

	if [ -z "${avail_exists}" ]; then
		# For kernels that don't support MemAvailable.
		# Not as accurate, but a good approximation.
		avail_kb=$(( $(get_meminfo_field MemFree) + \
			$(get_meminfo_field Cached) + \
			$(get_meminfo_field Buffers) ))
	else
		avail_kb="${avail_exists}"
	fi

	local -i percent_avail
	percent_avail=$(( 100 * avail_kb / total_kb ))

	if (( percent_avail < low_mem_threshold )); then
		local -i total_mb
		local -i avail_mb
		total_mb=$(( total_kb / 1024 ))
		avail_mb=$(( avail_kb / 1024 ))
		local -i used_mb
		used_mb=$(( total_mb - avail_mb ))
		log_status "${BAD}" "${FUNCNAME[0]}" "Low memory: ${percent_avail}% (${avail_mb}MB) available, ${used_mb}MB/${total_mb}MB used"
	else
		log_status "${GOOD}" "${FUNCNAME[0]}" "${percent_avail}% memory available"
	fi
}

function check_timesync()
{
	local is_time_synced
	is_time_synced=$(timedatectl | awk -F": " '/(System clock|NTP) synchronized:/{print $2}')
	if [[ "${is_time_synced}" != "yes" ]]; then
		log_status "${BAD}" "${FUNCNAME[0]}" "Time is not being synchronized via NTP"
	else
		log_status "${GOOD}" "${FUNCNAME[0]}" "Time is synchronized"
	fi
}

function check_container_engine() {
	local tests=(
		test_container_engine_running_now
		test_container_engine_restarts
		test_container_engine_responding
		test_container_engine_migration_failed
	)
	run_tests "${FUNCNAME[0]}" "${tests[@]}"
}

function test_container_engine_running_now() {
	if (! systemctl is-active ${ENG} > /dev/null); then
		echo "${FUNCNAME[0]}" "Container engine ${ENG} is NOT running"
	fi
}

function test_container_engine_restarts() {
	local -i engine_restarts
	engine_restarts=$(systemctl show -p NRestarts ${ENG} | awk -F= '{print $2}')
	if (( engine_restarts > 0 )); then
		local start_timestamp
		start_timestamp=$(systemctl show -p ExecMainStartTimestamp ${ENG} | awk -F= '{print $2}')
		echo "${FUNCNAME[0]}" "Container engine ${ENG} has ${engine_restarts} \
restarts and may be crashlooping (most recent start time: ${start_timestamp})"
	fi
}

function test_container_engine_responding() {
	if ! ERRMSG=$(${TIMEOUT_CMD} "${ENG}" ps 2>&1); then
		echo "${FUNCNAME[0]}" "Error querying container engine: ${ERRMSG}"
	fi
}

function test_container_engine_migration_failed() {
    # needs to match the path here:
    # https://github.com/balena-os/meta-balena/blob/master/meta-balena-common/recipes-containers/balena/balena/balena.conf.storagemigration#L3
    local logpath="/mnt/state/balena-engine-storage-migration.log"
    if [ -f "${logpath}" ]; then
        echo "${FUNCNAME[0]}" "Storage migration to overlayfs failed. Check the logs at ${logpath}."
    fi
}

function check_supervisor()
{
	# TODO: grab healthcheck results here
	if [[ ! $(${TIMEOUT_CMD} ${ENG} ps --filter "name=resin_supervisor" --filter "name=balena_supervisor" -q) ]] ; then
		log_status "${BAD}" "${FUNCNAME[0]}" "Supervisor is NOT running"
	else
		if ! curl -qs --max-time 10 "localhost:${LISTEN_PORT}/v1/healthy" > /dev/null; then
			log_status "${BAD}" "${FUNCNAME[0]}" "Supervisor is running, but may be unhealthy"
		else
			log_status "${GOOD}" "${FUNCNAME[0]}" "Supervisor is running & healthy"
		fi
	fi
}

function check_os_rollback()
{
	local root="/mnt/state"
	local health_name="rollback-health-triggered"
	local altboot_name="rollback-altboot-triggered"
	local rolled_back
	for i in "${health_name}" "${altboot_name}"; do
		if [ -f "${root}/${i}" ]; then
			rolled_back="${i}"
			break
		fi
	done

	if [[ -n "${rolled_back}" ]]; then
		log_status "${BAD}" "${FUNCNAME[0]}" "OS rollback detected: ${rolled_back}"
	else
		log_status "${GOOD}" "${FUNCNAME[0]}" "No OS rollbacks detected"
	fi
}

function check_user_services()
{
	local status healthcheck_output inspect fail_streak last_code last_output pretty_name healthy
	if (( ${#USERVICES[@]} > 0 )); then
		for service in "${USERVICES[@]}"; do

			if ! inspect=$(${TIMEOUT_CMD} "${ENG}" inspect "${service}"); then
				status="User service '${service}' inspection timed out"
				log_status "${BAD}" "check_service_${service}" "${status}"
				continue
			fi

			pretty_name=$(echo -E "${inspect}" | jq -r '.[].Config.Labels."io.balena.service-name"')
			healthcheck_output=$(echo -E "${inspect}" | jq -r '.[].State.Health | select(. != null)')
			healthy=$(echo -E "${healthcheck_output}" | jq -r '.Status == "healthy"')

			if [ -n "${healthcheck_output}" ] && [ "${healthy}" = "false" ]; then
				# artificially limited to 100 chars
				last_output=$(echo -E "${healthcheck_output}" | jq -ar '.Log[-1].Output[:100]' | sed "s/\"/\'/g")
				last_code=$(echo -E "${healthcheck_output}" | jq -ar '.Log[-1].ExitCode')
				fail_streak=$(echo -E "${healthcheck_output}" | jq -r '.FailingStreak')

				status="User service '${pretty_name}' healthcheck failed, count: ${fail_streak}"
				status+=$'\n'$(printf 'exit code: %s, output: %s' "${last_code}" "${last_output}")

				log_status "${BAD}" "check_service_${pretty_name}" "${status}"
				continue
			fi

			restart_count=$(echo -E "${inspect}" | jq -r '.[].RestartCount')

			if [ "${restart_count}" -gt 0 ]; then
				status="User service '${pretty_name}' is restarting unexpectedly, count: ${restart_count}"
				log_status "${BAD}" "check_service_${pretty_name}" "${status}"
				continue
			fi

			status="User service '${pretty_name}' is running & healthy"
			log_status "${GOOD}" "check_service_${pretty_name}" "${status}"
			continue

		done
	fi
}

function run_checks()
{
	# TODO remove echo | jq
	echo -E \
	"$(check_balenaOS)" \
	"$(check_container_engine)" \
	"$(check_localdisk)" \
	"$(check_memory)" \
	"$(check_networking)" \
	"$(check_os_rollback)" \
	"$(check_supervisor)" \
	"$(check_temperature)" \
	"$(check_timesync)" \
	"$(check_under_voltage)" \
	"$(check_user_services)" \
	| jq -s 'add | {checks:.}'
}

# Parse arguments
while [[ $# -gt 0 ]]; do
	arg="$1"

	case $arg in
		-h|--help)
			help
			exit 0
			;;
		--balenaos-registry)
			if [ -z "$2" ]; then
				log ERROR "\"$1\" argument needs a value."
			fi
			REGISTRY_ENDPOINT=$2
			shift
			;;
	esac
	shift
done

jq --argjson a1 "$(announce_version)" --argjson a2 "$(run_checks)" -cn '$a1 + $a2'
rm -f "${TMPCRT}" > /dev/null 2>&1