doors: Fix timeout handling during retries

Motivation:

Doors will retry pool selection and mover startup several times (subject to a
door specific policy). A timeout limits the total duration and this timeout is
applied as an upper bound for pool selection and mover submission; yet it
wasn't applied for pnfs manager interactions that happen during the retry loop.

This could in some cases cause errors in pool manager about discarded messages
with a negative TTL.

Modification:

Applies the total timeout as an upper bound to the pnfs manager interaction.

Result:

Greatly reduced risk of ending up with a negative TTL.

Target: trunk
Require-notes: yes
Require-book: no
Request: 2.13
Acked-by: Paul Millar <paul.millar@desy.de>
Patch: https://rb.dcache.org/r/8492/

modules/dcache/src/main/java/diskCacheV111/util/PnfsHandler.java

@@ -216,14 +216,23 @@ public <T extends PnfsMessage> T request(T msg)
      * Sends a message to the pnfs manager and returns a promise of a future reply.
      */
     public <T extends PnfsMessage> ListenableFuture<T> requestAsync(T msg)
+    {
+        checkState(_cellStub != null, "Missing endpoint");
+        return requestAsync(msg, _cellStub.getTimeoutInMillis());
+    }
+
+    /**
+     * Sends a message to the pnfs manager and returns a promise of a future reply.
+     */
+    public <T extends PnfsMessage> ListenableFuture<T> requestAsync(T msg, long timeout)
     {
         checkState(_cellStub != null, "Missing endpoint");
 
         msg.setReplyRequired(true);
         if (_subject != null) {
             msg.setSubject(_subject);
         }
-        return _cellStub.send(msg);
+        return _cellStub.send(msg, timeout);
     }
 
     public PnfsCreateEntryMessage createPnfsDirectory(String path)

modules/dcache/src/main/java/org/dcache/util/Transfer.java

@@ -695,6 +695,11 @@ public final void readNameSpaceEntry(boolean allowWrite)
      * @oaram allowWrite whether the file may be opened for writing
      */
     public ListenableFuture<Void> readNameSpaceEntryAsync(boolean allowWrite)
+    {
+        return readNameSpaceEntryAsync(allowWrite, _pnfs.getPnfsTimeout());
+    }
+
+    private ListenableFuture<Void> readNameSpaceEntryAsync(boolean allowWrite, long timeout)
     {
         Set<FileAttribute> attr = EnumSet.of(PNFSID, TYPE, STORAGEINFO, SIZE);
         attr.addAll(_additionalAttributes);
@@ -714,7 +719,7 @@ public ListenableFuture<Void> readNameSpaceEntryAsync(boolean allowWrite)
         }
         request.setAccessMask(mask);
         request.setUpdateAtime(true);
-        ListenableFuture<PnfsGetFileAttributes> reply = _pnfs.requestAsync(request);
+        ListenableFuture<PnfsGetFileAttributes> reply = _pnfs.requestAsync(request, timeout);
 
         setStatusUntil("PnfsManager: Fetching storage info", reply);
 
@@ -1123,6 +1128,11 @@ private static long getTimeoutFor(CellStub stub, long deadline)
         return Math.min(subWithInfinity(deadline, System.currentTimeMillis()), stub.getTimeoutInMillis());
     }
 
+    private static long getTimeoutFor(PnfsHandler pnfs, long deadline)
+    {
+        return Math.min(subWithInfinity(deadline, System.currentTimeMillis()), pnfs.getPnfsTimeout());
+    }
+
     /**
      * Select a pool and start a mover. Failed attempts are handled
      * according to the {@link TransferRetryPolicy}. Note, that there
@@ -1148,7 +1158,7 @@ public ListenableFuture<Void> selectPoolAndStartMoverAsync(String queue, Transfe
         AsyncFunction<Void, Void> startMover =
                 ignored -> startMoverAsync(queue, getTimeoutFor(_pool, deadLine));
         AsyncFunction<Void, Void> readNameSpaceEntry =
-                ignored -> readNameSpaceEntryAsync(false);
+                ignored -> readNameSpaceEntryAsync(false, getTimeoutFor(_pnfs, deadLine));
 
         FutureFallback<Void> retry =
                 new FutureFallback<Void>()