Post internal error / disable judgedaemon on low disk space.

Fixes #186.

Author: meisterT

judge/judgedaemon.main.php

@@ -400,6 +400,29 @@ function fetch_executable($workdirpath, $execid, $md5sum)
 $currentEndpoint = 0;
 while ( TRUE ) {
 
+	// Check for available disk space
+	$free_space = disk_free_space(JUDGEDIR);
+	$total_space = disk_total_space(JUDGEDIR);
+	$free_perc = (100.*$free_space) / $total_space;
+
+	$allowed_free_perc = dbconfig_get_rest('diskspace_error_perc');
+	$allowed_free_abs  = dbconfig_get_rest('diskspace_error_abs'); // in kB
+	if ( $free_perc <  $allowed_free_perc || $free_space < 1024*$allowed_free_abs ) {
+		$free_perc = sprintf("%01.2f%%", $free_perc);
+		$free_abs = sprintf("%01.2fGB", $free_space / (1024*1024*1024));
+		logmsg(LOG_ERR, "Low on disk space: $free_perc available, i.e. $free_abs");
+
+		$disabled = json_encode(array(
+			'kind' => 'judgehost',
+			'hostname' => $myhost));
+		$judgehostlog = read_judgehostlog();
+		$error_id = request('internal_error', 'POST',
+			'description=' . urlencode("low on disk space on $myhost") .
+			'&judgehostlog=' . urlencode(base64_encode($judgehostlog)) .
+			'&disabled=' . urlencode($disabled));
+		logmsg(LOG_ERR, "=> internal error " . $error_id);
+	}
+
 	// If all endpoints are waiting, sleep for a bit
 	$dosleep = TRUE;
 	foreach ($endpoints as $id=>$endpoint) {
@@ -451,11 +474,11 @@ function fetch_executable($workdirpath, $execid, $md5sum)
 
 	judge($row);
 
-    // Check if we were interrupted while judging, if so, exit(to avoid sleeping)
-    if ($exitsignalled) {
-        logmsg(LOG_NOTICE, "Received signal, exiting.");
+	// Check if we were interrupted while judging, if so, exit(to avoid sleeping)
+	if ($exitsignalled) {
+		logmsg(LOG_NOTICE, "Received signal, exiting.");
 		exit;
-    }
+	}
 
 	// restart the judging loop
 }
@@ -587,7 +610,7 @@ function judge($row)
 				'&description=' . urlencode("no test cases found") .
 				'&judgehostlog=' . urlencode(base64_encode($judgehostlog)) .
 				'&disabled=' . urlencode($disabled));
-			logmsg(LOG_WARNING, "No testcases found for p$row[probid] => internal error " . $error_id);
+			logmsg(LOG_ERR, "No testcases found for p$row[probid] => internal error " . $error_id);
 			break;
 		}
 		$tc = dj_json_decode($testcase);

lib/www/common.jury.php

@@ -625,6 +625,12 @@ function set_internal_error($disabled, $cid, $value) {
 				WHERE cid=%i AND probid=%i',
 				$value, $cid, $disabled['probid']);
 			break;
+		case 'judgehost':
+			$DB->q('RETURNAFFECTED UPDATE judgehost
+				SET active=%i
+				WHERE hostname=%s',
+				$value, $disabled['hostname']);
+			break;
 		default:
 			$api->createError("unknown internal error kind '" . $disabled['kind'] . "'");
 	}

sql/mysql_db_defaultdata.sql

@@ -39,7 +39,9 @@ INSERT INTO `configuration` (`name`, `value`, `type`, `description`) VALUES
 ('allow_registration', '0', 'bool', 'Allow users to register themselves with the system?'),
 ('judgehost_warning', '30', 'int', 'Time in seconds after a judgehost last checked in before showing its status as "warning".'),
 ('judgehost_critical', '120', 'int', 'Time in seconds after a judgehost last checked in before showing its status as "critical".'),
-('thumbnail_size', '128', 'int', 'Maximum width/height of a thumbnail for uploaded testcase images.');
+('thumbnail_size', '128', 'int', 'Maximum width/height of a thumbnail for uploaded testcase images.'),
+('diskspace_error_perc', '5', 'int', 'Minimum percentage of free disk space on judgehosts.'),
+('diskspace_error_abs', '1048576', 'int', 'Minimum absolute free disk space (in kB) on judgehosts.');
 
 --
 -- Dumping data for table `executable`

sql/upgrade/upgrade_5.1.0_5.2.0DEV.sql

@@ -78,6 +78,9 @@ UPDATE `judging` SET `judgehost` = 'host-created-by-SQL-upgrade' WHERE `judgehos
 --
 -- Add/remove sample/initial contents
 --
+INSERT INTO `configuration` (`name`, `value`, `type`, `description`) VALUES
+('diskspace_error_perc', '5', 'int', 'Minimum percentage of free disk space on judgehosts.'),
+('diskspace_error_abs', '1048576', 'int', 'Minimum absolute free disk space (in kB) on judgehosts.');
 
 --
 -- Finally remove obsolete structures after moving data

www/api/index.php

@@ -1299,8 +1299,11 @@ function internal_error_POST($args)
 	$disabled = dj_json_decode($args['disabled']);
 	// disable what needs to be disabled
 	set_internal_error($disabled, $args['cid'], 0);
-	$submitid = $DB->q('VALUE SELECT submitid FROM judging WHERE judgingid = %i', $args['judgingid']);
-	give_back_judging($args['judgingid'], $submitid);
+	if ( $disabled['kind'] == 'problem' ) {
+		// give back judging if we have to
+		$submitid = $DB->q('VALUE SELECT submitid FROM judging WHERE judgingid = %i', $args['judgingid']);
+		give_back_judging($args['judgingid'], $submitid);
+	}
 
 	return $errorid;
 }