Merge pull request #9 from jenkinsci/better-safety

[FIXED JENKINS-28620] Add better safety to the AutoSamplingHistogram

src/main/java/jenkins/metrics/util/AutoSamplingHistogram.java

@@ -37,7 +37,11 @@
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.WeakHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
 
 /**
  * This is a {@link Histogram} that is derived from a {@link Gauge} by sampling it 4 times a minute.
@@ -46,7 +50,11 @@
  */
 public class AutoSamplingHistogram extends Histogram {
 
+    private static final Logger LOGGER = Logger.getLogger(AutoSamplingHistogram.class.getName());
+
     private final Gauge<? extends Number> source;
+
+    private volatile transient boolean badGauge;
 
     public AutoSamplingHistogram(Gauge<? extends Number> source) {
         this(source, new ExponentiallyDecayingReservoir());
@@ -59,11 +67,23 @@ public AutoSamplingHistogram(Gauge<? extends Number> source, Reservoir reservoir
     }
 
     public void update() {
-        Number value = source.getValue();
-        if (value instanceof Integer) {
-            update(value.intValue());
-        } else {
-            update(value.longValue());
+        try {
+            Number value = source.getValue();
+            if (value instanceof Integer) {
+                update(value.intValue());
+            } else if (value != null) {
+                update(value.longValue());
+            } else {
+                LOGGER.log(Level.FINE, "Gauge {0} returned null", source);
+            }
+            badGauge = false;
+        } catch (ClassCastException e) {
+            LogRecord lr = new LogRecord(badGauge ? Level.FINE : Level.WARNING,
+                    "Gauge {0} is supposed to return a subclass of java.lang.Number but didn't");
+            badGauge = true;
+            lr.setThrown(e);
+            lr.setParameters(new Object[]{source});
+            LOGGER.log(lr);
         }
     }
 
@@ -74,9 +94,19 @@ public MetricSet toMetricSet() {
         return new GaugeHistogramMetricSet(metrics);
     }
 
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("AutoSamplingHistogram{");
+        sb.append("source=").append(source);
+        sb.append('}');
+        return sb.toString();
+    }
+
     @Extension
     public static class PeriodicWorkImpl extends PeriodicWork {
-
+
+        private final Map<Histogram,Long> lastWarning = new WeakHashMap<Histogram, Long>();
+
         @Override
         public long getRecurrencePeriod() {
             return TimeUnit.SECONDS.toMillis(15);
@@ -87,7 +117,31 @@ protected synchronized void doRun() throws Exception {
             MetricRegistry registry = Metrics.metricRegistry();
             for (Histogram histogram : registry.getHistograms().values()) {
                 if (histogram instanceof AutoSamplingHistogram) {
-                    ((AutoSamplingHistogram) histogram).update();
+                    try {
+                        ((AutoSamplingHistogram) histogram).update();
+                    } catch (Exception e) {
+                        Long lw = lastWarning.get(histogram);
+                        final boolean warn = lw == null || lw + TimeUnit.HOURS.toMillis(1) < System.currentTimeMillis();
+                        LogRecord lr = new LogRecord(warn ? Level.WARNING : Level.FINE, 
+                                "Uncaught exception when calling update for {0}");
+                        lr.setParameters(new Object[]{histogram});
+                        lr.setThrown(e);
+                        LOGGER.log(lr);
+                        if (warn) {
+                            lastWarning.put(histogram, System.currentTimeMillis());
+                        }
+                    } catch (Error e) {
+                        LogRecord lr = new LogRecord(Level.WARNING, "Error encountered while attempting to update {0}");
+                        lr.setParameters(new Object[]{histogram});
+                        lr.setThrown(e);
+                        LOGGER.log(lr);
+                        throw e;
+                    } catch (Throwable t) {
+                        LogRecord lr = new LogRecord(Level.SEVERE, "Uncaught throwable when calling update for {0}");
+                        lr.setParameters(new Object[]{histogram});
+                        lr.setThrown(t);
+                        LOGGER.log(lr);
+                    }
                 }
             }
         }