(2.10) dcache-webadmin: add TimeoutCacheException to catch clause in …

…billing service

https://rb.dcache.org/r/7306/ attempted to rectify the exiting of
the plot loop in the webadmin billing plot service, but negelected
to handle the TimeoutCacheException.  As a result, a timeout on
the message reply, which should not cause the thread to exit, is
fatal to the plotting page (the plots no longer update).

This patch adds that exception to those which should not be
considered unexpected RuntimeExceptions.  For the timeout and noroute
exceptions, the thread should retry after a given timeout.

Target:   2.10
Acked-by: Tigran
Require-book:  no
Require-notes: yes

RELEASE NOTES:  Fixes bug which causes the billing plot thread to exit rather than retry because of a timeout.

modules/dcache-webadmin/src/main/java/org/dcache/webadmin/controller/IBillingService.java

@@ -62,6 +62,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import java.io.File;
 
 import diskCacheV111.util.ServiceUnavailableException;
+import diskCacheV111.util.TimeoutCacheException;
 
 import dmg.cells.nucleus.NoRouteToCellException;
 
@@ -87,5 +88,6 @@ public interface IBillingService {
 
     void initialize();
 
-    void refresh() throws NoRouteToCellException, ServiceUnavailableException;
+    void refresh() throws NoRouteToCellException,
+                    TimeoutCacheException, ServiceUnavailableException;
 }

modules/dcache-webadmin/src/main/java/org/dcache/webadmin/controller/impl/StandardBillingService.java

@@ -79,9 +79,9 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import java.util.concurrent.TimeUnit;
 
 import diskCacheV111.util.ServiceUnavailableException;
+import diskCacheV111.util.TimeoutCacheException;
 
 import dmg.cells.nucleus.NoRouteToCellException;
-
 import org.dcache.cells.CellStub;
 import org.dcache.services.billing.histograms.ITimeFrameHistogramFactory;
 import org.dcache.services.billing.histograms.ITimeFrameHistogramFactory.Style;
@@ -150,8 +150,9 @@ public final class StandardBillingService implements IBillingService, Runnable {
     private Thread refresher;
 
     public List<TimeFrameHistogramData> load(PlotType plotType,
-                    TimeFrame timeFrame) throws NoRouteToCellException,
-                                                ServiceUnavailableException {
+                    TimeFrame timeFrame) throws ServiceUnavailableException,
+                                                NoRouteToCellException,
+                                                TimeoutCacheException {
         logger.debug("remote fetch of {} {}", plotType, timeFrame);
         List<TimeFrameHistogramData> histograms = new ArrayList<>();
         try {
@@ -207,6 +208,10 @@ public List<TimeFrameHistogramData> load(PlotType plotType,
             if (cause != null) {
                 throw (NoRouteToCellException)cause;
             }
+            cause = Exceptions.findCause(ute, TimeoutCacheException.class);
+            if (cause != null) {
+                throw (TimeoutCacheException)cause;
+            }
             cause = ute.getCause();
             Throwables.propagateIfPossible(cause);
             throw new RuntimeException("Unexpected error: "
@@ -325,6 +330,7 @@ public void initialize() {
 
     @Override
     public void refresh() throws NoRouteToCellException,
+                                 TimeoutCacheException,
                                  ServiceUnavailableException{
         TimeFrame[] timeFrames = generateTimeFrames();
         for (int tFrame = 0; tFrame < timeFrames.length; tFrame++) {
@@ -356,6 +362,12 @@ public void run() {
                                         + "retrying every 10 seconds");
                     }
                     Thread.sleep(TimeUnit.SECONDS.toMillis(10));
+                } catch (TimeoutCacheException e) {
+                    if (rate.tryAcquire()) {
+                        logger.warn("Billing service currently unreachable; "
+                                        + "retrying after timeout ...");
+                    }
+                    Thread.sleep(timeout);
                 }
             }
         } catch (InterruptedException interrupted) {
@@ -395,6 +407,7 @@ public void shutDown() {
 
     private void generatePlot(PlotType type, TimeFrame timeFrame,
                     String fileName, String title) throws ServiceUnavailableException,
+                                                          TimeoutCacheException,
                                                           NoRouteToCellException {
         List<TimeFrameHistogramData> data = load(type, timeFrame);
         List<HistogramWrapper<?>> config = new ArrayList<>();