monitor

#!/usr/bin/php -q
<?PHP
/* Copyright 2005-2023, Lime Technology
 * Copyright 2012-2023, Bergware International.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2,
 * as published by the Free Software Foundation.
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 */
?>
<?
$docroot = $docroot ?? $_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp';

// Multi-language handling
if (!function_exists('_')) {
  function _($text) {return $text;}
}
// Exit when settings are not yet initialized
if (!file_exists("/var/local/emhttp/var.ini")) exit;

$var   = @parse_ini_file("/var/local/emhttp/var.ini") ?: [];
$devs  = @parse_ini_file("/var/local/emhttp/devs.ini",true) ?: [];
$disks = @parse_ini_file("/var/local/emhttp/disks.ini",true) ?: [];

require_once "$docroot/webGui/include/Helpers.php";
require_once "$docroot/webGui/include/Preselect.php";
require_once "$docroot/webGui/include/CustomMerge.php";

extract(parse_plugin_cfg("dynamix",true));

$notify = "$docroot/webGui/scripts/notify";
$ram    = "/var/local/emhttp/monitor.ini";
$rom    = "/boot/config/plugins/dynamix/monitor.ini";
$saved  = @parse_ini_file($ram,true);
$high1  = _var($display,'critical',0);
$high2  = _var($display,'warning',0);
$server = strtoupper(_var($var,'NAME','tower'));
$pools  = pools_filter($disks);
$errors = [];
$top    = 120;

function check_temp(&$disk,$text,$info) {
  global $notify,$saved,$server,$display,$top;
  $name = _var($disk,'name');
  $temp = _var($disk,'temp','*');
  $max  = _var($disk,'maxTemp')>=0 ? $disk['maxTemp'] : (_var($display,'max')>=0 ? $display['max'] : 0);
  $hot  = _var($disk,'hotTemp')>=0 ? $disk['hotTemp'] : (_var($display,'hot')>=0 ? $display['hot'] : 0);
  $warn = exceed($temp,$max,$top) ? 'alert' : (exceed($temp,$hot,$top) ? 'warning' : false);
  $item = 'temp';
  $last = $saved[$item][$name] ?? 0;
  if ($warn) {
    if ($temp>$last) {
      exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text temperature")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text ".($warn=='alert'?'overheated (':'is hot (').my_temp($temp).")")." -d ".escapeshellarg("$info")." -i \"$warn\" 2>/dev/null");
      $saved[$item][$name] = max($max,$temp);
    }
  } else {
    if ($last && $temp<=$top) {
      exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal temperature")." -d ".escapeshellarg("$info")." 2>/dev/null");
      unset($saved[$item][$name]);
    }
  }
}
function check_smart(&$disk,$port,$text,$info) {
  global $notify,$saved,$server,$numbers;
  $name   = _var($disk,'name');
  $select = get_value($disk,'smSelect',0);
  $level  = get_value($disk,'smLevel',1);
  $events = explode('|',get_value($disk,'smEvents',$numbers));
  $type   = get_value($disk,'smType','');
  get_ctlr_options($type, $disk);
  $file = "/var/local/emhttp/smart/$name";
  exec("awk 'NR>7{print $1,$2,$4,$6,$9,$10}' ".escapeshellarg($file)." 2>/dev/null", $codes);
  $item = 'smart';
  foreach ($codes as $code) {
    if (!$code || !is_numeric($code[0])) continue;
    [$id,$class,$value,$thres,$when,$raw] = my_explode(' ',$code,7);
    $fail = strpos($when,'FAILING_NOW')!==false;
    if (!$fail && !in_array($id,$events)) continue;
    $word = str_replace(['_',' (-)'],[' ',''],strtolower("$class ($when)"));
    $ack = "$name.ack";
    switch ($select) {
    case 0:
      $attr = "$name.$id";
      $last = ($saved[$item][$attr] ?? 0)*$level;
      if ($raw>0 || $fail) {
        if ($raw>$last) {
          exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $raw")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
          $saved[$item][$attr] = $raw;
          unset($saved[$item][$ack]);
        }
      } else {
        if ($last>0) {
          exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info")." 2>/dev/null");
          unset($saved[$item][$attr]);
          unset($saved[$item][$ack]);
        }
      }
      break;
    case 1:
      $attr = "$name.{$id}n";
      $last = $saved[$item][$attr] ?? 255;
      if (($thres>0 && $value<=$thres*$level) || $fail) {
        if ($value*($value>$thres?$level:1)<$last) {
          exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $value")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
          $saved[$item][$attr] = $value;
          unset($saved[$item][$ack]);
        }
      } else {
        if ($last<255) {
          exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info")." 2>/dev/null");
          unset($saved[$item][$attr]);
          unset($saved[$item][$ack]);
        }
      }
      break;
    }
  }
}
function check_usage(&$disk,$used,$text,$info) {
  global $notify,$saved,$server,$display;
  if ($used == -1) return;
  $name = _var($disk,'name');
  $critical = _var($disk,'critical')>=0 ? $disk['critical'] : (_var($display,'critical')>=0 ? $display['critical'] : 0);
  $warning = _var($disk,'warning')>=0 ? $disk['warning'] : (_var($display,'warning')>=0 ? $display['warning'] : 0);
  $warn = exceed($used,$critical) ? 'alert' : (exceed($used,$warning) ? 'warning' : false);
  $item = 'used';
  $last = $saved[$item][$name] ?? 0;
  if ($warn) {
    if ($used>$last) {
      exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text disk utilization")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text is ".($warn=='alert'?'low on space':'high on usage')." ({$used}%)")." -d ".escapeshellarg("$info")." -i \"$warn\" 2>/dev/null");
      $saved[$item][$name] = max($critical,$used);
    }
  } else {
    if ($last && $used<=100) {
      exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal utilization level")." -d ".escapeshellarg("$info")." 2>/dev/null");
      unset($saved[$item][$name]);
    }
  }
}

// check array devices
foreach ($disks as $disk) {
  $name = _var($disk,'name');
  if ($name=='flash' || substr(_var($disk,'status'),-3)=='_NP') continue;
  $text = my_disk($name).(in_array($name,$pools)||$name=='parity'?' disk':'');
  $device = _var($disk,'device');
  $info = !empty($disk['id']) ? "{$disk['id']} ($device)" : "No device identification ($device)";
// process disk temperature notifications
  check_temp($disk,$text,$info);
// process disk SMART notifications
  check_smart($disk,port_name($disk['smDevice'] ?? $device),$text,$info);
// process disk usage notifications
  check_usage($disk,_var($disk,'fsSize',0)>0?100-round(100*_var($disk,'fsFree',0)/$disk['fsSize']):-1,$text,$info);
// process disk operation notifications
  $warn = strtok(_var($disk,'color'),'-');
  $item = 'disk';
  $last = $saved[$item][$name] ?? '';
  switch ($warn) {
  case 'red':
    if ($warn!=$last) {
      if (_var($var,'fsState')!='Stopped') {
        $status = strtolower(str_replace(['NP_','_'],['',' '],_var($disk,'status')));
        exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text error")." -s ".escapeshellarg("Alert [$server] - $text in error state ($status)")." -d ".escapeshellarg("$info")." -i \"alert\" 2>/dev/null");
      }
      $saved[$item][$name] = $warn;
    }
  break;
  case 'yellow':
    if ($warn!=$last) {
      if (_var($var,'fsState')!='Stopped') {
        $status = $name=='parity' ? "parity-sync in progress" : " is being reconstructed and is available for normal operation";
        exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text, $status")." -d ".escapeshellarg("$info")." 2>/dev/null");
      }
      $saved[$item][$name] = $warn;
    }
  break;
  default:
    if ($last) {
      if (_var($var,'fsState')!='Stopped') {
        exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal operation")." -d ".escapeshellarg("$info")." 2>/dev/null");
      }
      unset($saved[$item][$name]);
    }
  break;}
// count disk errors
  if (_var($disk,'numErrors',0)>0) $errors[] = "$text - $info (errors {$disk['numErrors']})";
// check file system of cache pool
  $item = 'pool';
  if (in_array($name,$pools) && strpos(_var($disk,'fsType'),'btrfs')!==false && _var($disk,'uuid')!=="") {
    $attr = 'missing';
    if (exec("/sbin/btrfs filesystem show "._var($disk,'uuid')." 2>/dev/null|grep -c 'missing'")>0) {
      if (empty($saved[$item][$attr])) {
        exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Warning [$server] - Cache pool BTRFS missing device(s)")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
        $saved[$item][$attr] = 1;
      }
    } elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
    $attr = "profile-$name";
    if (exec("/sbin/btrfs filesystem df /mnt/$name 2>/dev/null|grep -c '^Data'")>1) {
      if (empty($saved[$item][$attr])) {
        exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Warning [$server] - $pool pool BTRFS too many profiles (You can ignore this warning when a pool balance operation is in progress)")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
        $saved[$item][$attr] = 1;
      }
    } elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
  }
}

// check unassigned devices
foreach ($devs as $dev) {
  $name = _var($dev,'name','no name');
  $id   = _var($dev,'id');
  $port = port_name($name);
  $text = "device $name";
  $info = !empty($id) ? "$id ($name)": "No device identification ($name)";
// process disk temperature notifications
  check_temp($dev,$text,$info);
// process disk SMART notifications
  check_smart($dev,$port,$text,$info);
}

// report array read errors
$item = 'array';
$name = 'errors';
$last = $saved[$item][$name] ?? 0;
$warn = count($errors);
$info = "Array has $warn disk".($warn==1 ? "" : "s")." with read errors";
if ($warn>0) {
  if ($warn<>$last) {
    $message = implode('\n', $errors);
    exec("$notify -l '/Main' -e \"Unraid array errors\" -s ".escapeshellarg("Warning [$server] - array has errors")." -d ".escapeshellarg("$info")." -m ".escapeshellarg("$message")." -i \"warning\" 2>/dev/null");
    $saved[$item][$name] = $warn;
  }
} else {
  if ($last) {
    exec("$notify -l '/Main' -e \"Unraid array errors\" -s ".escapeshellarg("Notice [$server] - array turned good")." -d ".escapeshellarg("$info")." 2>/dev/null");
    unset($saved[$item][$name]);
  }
}

// process parity check, parity sync and data-rebuild notifications
$name = 'parity';
$last = $saved[$item][$name] ?? '';
if ($var['mdResyncPos']) {
  if (!$last) {
    $action = preg_split('/\s+/',$var['mdResyncAction']);
    switch ($action[0]) {
      case 'recon': $last = $action[1]=='P' ? 'Parity-Sync' : 'Data-Rebuild'; break;
      case 'check': $last = count($action)>1 ? 'Parity-Check' : 'Read-Check'; break;
      case 'clear': $last = 'Disk-Clear'; break;
      default     : $last = '';
    }
    $info = "Size: ".my_scale($var['mdResyncSize']*1024,$unit)." $unit";
    exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $last")." -s ".escapeshellarg("Notice [$server] - $last started")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
    $saved[$item][$name] = $last;
  }
} else {
  if ($last) {
    [$date,$duration,$speed,$status,$error,$action,$size] = last_parity_log();
    $info = $status==0 ? "Duration: ".my_check($duration, $speed) : ($status==-4 ? "Canceled" : "Error code: $status");
    $level = ($status==0 && $var['sbSyncErrs']==0) ? "normal" : "warning";
    exec("$notify -l '/Main' -e ".escapeshellarg("Unraid $last")." -s ".escapeshellarg("Notice [$server] - $last finished ($error errors)")." -d ".escapeshellarg("$info")." -i \"$level\" 2>/dev/null");
    unset($saved[$item][$name]);
  }
}

// check read-write status of USB flash drive
$name = 'flash';
$last = $saved[$item][$name] ?? '';
$warn = exec("grep -Pom1 '/boot \S+ \K\S{2}' /proc/mounts");
$info = "{$disks['flash']['id']} ({$disks['flash']['device']})";
if ($warn!="rw") {
  if ($warn!=$last) {
    exec("$notify -l '/Main' -e \"USB flash drive failure\" -s ".escapeshellarg("Alert [$server] - USB drive is not read-write")." -d ".escapeshellarg("$info")." -i \"alert\" 2>/dev/null");
    $saved[$item][$name] = $warn;
  }
} else {
  if ($last) {
    exec("$notify -l '/Main' -e \"USB flash drive operation\" -s ".escapeshellarg("Notice [$server] - USB drive returned to normal operation")." -d ".escapeshellarg("$info")." 2>/dev/null");
    unset($saved[$item][$name]);
  }
}

// check docker image disk utilization
system('mountpoint -q /var/lib/docker', $retval);
if ($retval===0 && exec("df /var/lib/docker|grep -Po '^/dev/\Kloop'")) {
  $item = 'system';
  $name = 'docker';
  $last = $saved[$item][$name] ?? '';
  if (file_exists("/boot/config/docker.cfg")) {
    $cfg  = parse_ini_file("/boot/config/docker.cfg");
    $info = "Docker utilization of image file {$cfg['DOCKER_IMAGE_FILE']}";
  } else
    $info = "Docker image file not specified";

  $warn = exec("df /var/lib/docker|awk '/^\//{print $5*1}'");
  if ($warn>=$high1 && $high1>0) {
    if ($warn>$last) {
      exec("$notify -l '/Docker' -e \"Docker critical image disk utilization\" -s ".escapeshellarg("Alert [$server] - Docker image disk utilization of {$warn}%")." -d ".escapeshellarg("$info")." -i \"alert\" 2>/dev/null");
      $saved[$item][$name] = $warn;
    }
  } elseif ($warn>=$high2 && $high2>0) {
    if ($warn>$last) {
      exec("$notify -l '/Docker' -e \"Docker high image disk utilization\" -s ".escapeshellarg("Warning [$server] - Docker image disk utilization of {$warn}%")." -d ".escapeshellarg("$info")." -i \"warning\" 2>/dev/null");
      $saved[$item][$name] = $warn;
    }
  } else {
    if ($last) {
      exec("$notify -l '/Docker' -e \"Docker image disk utilization\" -s ".escapeshellarg("Notice [$server] - Docker image disk utilization returned to normal level")." -d ".escapeshellarg("$info")." 2>/dev/null");
      unset($saved[$item][$name]);
    }
  }
}

// save new status
if ($saved) {
  $text = '';
  foreach ($saved as $item => $block) {
    if ($block) $text .= "[$item]\n";
    foreach ($block as $key => $value) $text .= "$key=\"$value\"\n";
  }
  if ($text) {
    if ($text != @file_get_contents($ram)) file_put_contents($ram, $text);
    if (!file_exists($rom) || exec("diff -q $ram $rom")) file_put_contents($rom, $text);
  } else {
    delete_file($ram,$rom);
  }
}
exit(0);
?>