cells: Don't retry send during shutdown

The sendAndWaitToPermanent method allows blocking send to
cells we know are usually available. When unavailable we
optimistically wait till the cell comes back online or
the timeout has expired. This is an excellent strategy,
except if our cell is about to shut down.

This patch alters the logic so that a NoRouteToCellException
during cell shutdown causes the sendAndWaitToPermanent
method to return right away.

Target: trunk
Require-notes: no
Require-book: no
Acked-by: Dmitry Litvintsev <litvinse@fnal.gov>
Acked-by: Paul Millar <paul.millar@desy.de>
Patch: http://rb.dcache.org/r/5291/

Author: Gerd Behrmann

modules/cells/src/main/java/dmg/cells/nucleus/CellAdapter.java

@@ -77,6 +77,7 @@ public class   CellAdapter
     private final CellNucleus _nucleus;
     private final Gate        _readyGate = new Gate(false);
     private final Gate        _startGate = new Gate(false);
+    private final Gate        _shutdownGate = new Gate(false);
     private final Args        _args;
     private boolean     _useInterpreter = true;
     private boolean     _returnCommandException = true;
@@ -569,8 +570,11 @@ public CellMessage sendAndWaitToPermanent(CellMessage envelope,
             return sendAndWait(envelope, timeUntil(deadline));
         } catch (NoRouteToCellException e) {
             _log.warn("{}", e.toString());
+
             while (timeUntil(deadline) > RETRY_PERIOD) {
-                Thread.sleep(RETRY_PERIOD);
+                if (_shutdownGate.await(RETRY_PERIOD)) {
+                    break;
+                }
                 try {
                     return sendAndWait(envelope, timeUntil(deadline));
                 } catch (NoRouteToCellException ignored) {
@@ -844,6 +848,7 @@ public void prepareRemoval(KillEvent ce)
     {
         _log.info("CellAdapter : prepareRemoval : waiting for gate to open");
         _readyGate.check();
+        _shutdownGate.open();
         cleanUp();
         dumpPinboard();
         _log.info("CellAdapter : prepareRemoval : done");

modules/cells/src/main/java/dmg/util/Gate.java

@@ -6,22 +6,32 @@ public Gate(){}
    public Gate( boolean isOpen ){
       _isOpen = isOpen ;
    }
+
+   public synchronized boolean await(long millis) throws InterruptedException
+   {
+       long deadline = System.currentTimeMillis() + millis;
+       while (!_isOpen && deadline > System.currentTimeMillis()) {
+           wait(deadline - System.currentTimeMillis());
+       }
+       return _isOpen;
+   }
+
    public synchronized Object check(){
-      while( true ){  
+      while( true ){
          if( _isOpen ) {
              return this;
          }
          try{ wait() ; }catch( Exception ee ){}
       }
-   
+
    }
-   public synchronized void open(){ 
+   public synchronized void open(){
      _isOpen = true ;
      notifyAll() ;
-   } 
-   public synchronized void close(){ 
+   }
+   public synchronized void close(){
      _isOpen = false ;
      notifyAll() ;
-   } 
+   }
 
 }