- Added auto remediation to monitor function

- Updated DB for monitoring function updates
jessebuffington · Mar 17, 2019 · 6ea5e68 · 6ea5e68
1 parent c1fcdde
commit 6ea5e68
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 0 deletions.
diff --git a/INSTALL/initTables.sql b/INSTALL/initTables.sql
@@ -28,10 +28,12 @@ CREATE TABLE `app_status` (
   `name` varchar(45) NOT NULL,
   `status` varchar(8) NOT NULL DEFAULT 'Inactive',
   `enabled` varchar(1) NOT NULL DEFAULT '1',
+  `monRestart` varchar(1) NOT NULL DEFAULT '1',
   `description` varchar(45) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 
+
 CREATE TABLE `badWords` (
   `badWord` varchar(15) NOT NULL,
   `cost` int(11) NOT NULL DEFAULT '10',
@@ -445,3 +447,15 @@ INSERT INTO `site_config` (configName, configValue, comment) VALUES
 ('SITE_ACCESS_LOG_LIMIT','200',NULL),
 ('SITE_FQDN','**CONFIG-ME**',NULL),
 ('TELNET_PORT','**CONFIG-ME**',NULL);
+
+INSERT INTO `app_status` VALUES
+(1,'syncAllPlayers','Active','1','1',NULL),
+(2,'syncEntities','InActive','0','0',NULL),
+(3,'syncGameLog','InActive','0','0',NULL),
+(4,'syncGameTime','Active','1','1',NULL),
+(5,'syncGameVersion','Active','1','1',NULL),
+(6,'syncLandclaims','InActive','0','0',NULL),
+(7,'syncOnlinePlayers','Active','1','1',NULL),
+(8,'syncServerInfo','Active','1','1',NULL),
+(9,'insertPlayerHistory','Active','1','1',NULL),
+(10,'syncGameChat','Active','1','1',NULL);
diff --git a/includes/functions.php b/includes/functions.php
@@ -540,6 +540,57 @@ function monitorAppStatus() {
             }
           }
         }
+        $queryMonRestart = mysql_query('SELECT monRestart FROM app_status where name = "' . $appName . '" and enabled = "1"');
+        if (!$queryMonRestart) {
+          die('Invalid query: ' . mysql_error());
+          if(APP_LOG_LEVEL >= 1) {
+            $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'CRIT', 'monitorAppStatus', 'ERROR: COULD NOT CONNECT TO DB')";
+            if (!mysql_query($log)) {
+              die('Error: ' . mysql_error());
+            }
+          }
+        }
+        $monRestart = mysql_fetch_array($queryMonRestart);
+        if ($monRestart['monRestart'] == 0) {
+          echo Console::light_red('**Monitor detected that ' . $appName . ' is NOT configured to restart upon failure. Please review. -- SKIPPING') . "\n\n";
+          if(APP_LOG_LEVEL >= 2) {
+            $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'WARN', 'monitorAppStatus', '<b>**Monitor detected that " . $appName . " is NOT configured to restart upon failure. Please review. -- SKIPPING</b>')";
+            if(!mysql_query($log)) {
+              die('Error: ' . mysql_error());
+              if(APP_LOG_LEVEL >= 1) {
+                $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'CRIT', 'monitorAppStatus', 'ERROR: COULD NOT CONNECT TO DB')";
+                if (!mysql_query($log)) {
+                  die('Error: ' . mysql_error());
+                }
+              }
+            }
+          }
+        } else {
+          echo Console::light_green('**Monitor detected that process needs to be restarted upon failure -- Starting ' . $appName . '...') . "\n\n";
+          shell_exec('nohup php -f ' . APP_ROOT . 'lib/' . $appName . '.php >> ' . APP_ROOT . 'var/log/' . $appName . '_' . date('Ymd') . '.log 2>&1 &');
+          $sql = "update app_status set status = 'Active' where name = '" . $appName . "'";
+          if (!mysql_query($sql)) {
+            die('Error: ' . mysql_error());
+            if(APP_LOG_LEVEL >= 1) {
+              $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'CRIT', 'monitorAppStatus', 'ERROR: COULD NOT CONNECT TO DB')";
+              if (!mysql_query($log)) {
+                die('Error: ' . mysql_error());
+              }
+            }
+          }
+          if(APP_LOG_LEVEL >= 2) {
+            $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'WARN', 'monitorAppStatus', '**Monitor detected that process needs to be restarted upon failure -- " . $appName . " is <b>STARTING UP!</b>')";
+            if(!mysql_query($log)) {
+              die('Error: ' . mysql_error());
+              if(APP_LOG_LEVEL >= 1) {
+                $log = "insert into app_log (datetime, logLevel, runName, message) values ('" . date('Y-m-d H:i:s') . "', 'CRIT', 'monitorAppStatus', 'ERROR: COULD NOT CONNECT TO DB')";
+                if (!mysql_query($log)) {
+                  die('Error: ' . mysql_error());
+                }
+              }
+            }
+          }
+        }
       } else { // Update status of active apps
         echo $appName . " is " . Console::light_green('ACTIVE!') . "\n";
         $sql = "update app_status set status = 'Active' where name = '" . $appName . "'";