Depending on the size of your grid there may be variable delay when

parsing the metrics XML. This may result in check_heartbeat alerting
errantly because check time may be far enough to hit the threshold.

Instead we'll just add last_reported_uptime_in_sec which is from the point of
view of the collection process and should be more consistent.

ganglia.php

@@ -163,6 +163,7 @@ function start_cluster ($parser, $tagname, $attrs)
             $metrics[$hostname]['last_reported']['NAME'] = "REPORTED";
             $metrics[$hostname]['last_reported']['VAL'] = uptime($cluster['LOCALTIME'] - $attrs['REPORTED']);
             $metrics[$hostname]['last_reported']['TYPE'] = "string";
+            $metrics[$hostname]['last_reported_uptime_in_sec']['VAL'] = $cluster['LOCALTIME'] - $attrs['REPORTED'];
             $metrics[$hostname]['last_reported_timestamp']['NAME'] = "REPORTED TIMESTAMP";
             $metrics[$hostname]['last_reported_timestamp']['VAL'] = $attrs['REPORTED'];
             $metrics[$hostname]['last_reported_timestamp']['TYPE'] = "uint32";

nagios/check_heartbeat.php

@@ -81,15 +81,14 @@
 # Host has been found in the Ganglia tree
 if ( $host_found == 1 ) {
   # Check for the existence of a metric
-  if ( isset($metrics[$fqdn]['last_reported_timestamp']['VAL']) ) {
-    $last_reported = $metrics[$fqdn]['last_reported_timestamp']['VAL'];
+  if ( isset($metrics[$fqdn]['last_reported_uptime_in_sec']['VAL']) ) {
+    $last_reported = $metrics[$fqdn]['last_reported_uptime_in_sec']['VAL'];
   } else {
     echo("UNKNOWN|" . $metric_name . " - Invalid metric request for this host. Please check metric exists.");
     exit(3);
   }
 
-  $time_diff = time() - $last_reported;   
-  if ( $time_diff < $threshold  ) {
+  if ( $metrics[$fqdn]['last_reported_uptime_in_sec']['VAL'] < $threshold  ) {
    print "OK|Last beacon received " . $metrics[$fqdn]['last_reported']['VAL'];
    exit (0);
   } else {