dcache-frontend: invalidate transfer when killed mover not found

Motivation:

When a mover is killed/cancelled, the mover
may already have finished and been removed
from the pool's mover service.

In this case, a CacheException is thrown
and an error reply returned.

For the purposes of the frontend, this
error should simply be regarded as equivalent
to having cancelled the mover, particularly
in terms of the snapshot of transfers
cached there.  Hence, it should be
temporarily invalidated, until the full
set of transfers is refreshed.

Modification:

If the thrown CacheException is because of
mover not found, invalidate the cache
entry and return OK.

A new error code, MOVER_NOT_FOUND, is introduced to CacheException.
The code is used in the message reply and checked by
the frontend.

Result:

The front end sees (temporarily) "CANCELED"
as with a successful kill, instead of
getting an error.

NOTE:  This introduces an incompatibility with the preceding pool version.

Target: master
Request: 4.2
Request: 4.1
Acked-by: Paul

modules/dcache-frontend/src/main/java/org/dcache/restful/resources/pool/PoolInfoResources.java

@@ -488,11 +488,17 @@ public Response killMovers(@ApiParam(value = "The pool with the mover to be kill
         try {
             poolStub.sendAndWait(new CellPath(pool),
                                  new PoolMoverKillMessage(pool, id,
-                                    "Killed by user."));
+                                                          "Killed by user."));
             transferInfoService.setCancelled(pool, id);
         } catch (IllegalArgumentException e) {
             throw new BadRequestException(e);
-        } catch (InterruptedException | NoRouteToCellException | CacheException e) {
+        } catch (CacheException e) {
+            if (e.getRc() == CacheException.MOVER_NOT_FOUND) {
+                transferInfoService.setCancelled(pool, id);
+            } else {
+                throw new InternalServerErrorException(e);
+            }
+        } catch (InterruptedException | NoRouteToCellException e) {
             throw new InternalServerErrorException(e);
         }
 

modules/dcache-vehicles/src/main/java/diskCacheV111/util/CacheException.java

@@ -121,6 +121,9 @@ public class CacheException extends Exception
     /** Transfer between pool and remote site failed. */
     public static final int THIRD_PARTY_TRANSFER_FAILED = 10027;
 
+    /** Mover was not found **/
+    public static final int MOVER_NOT_FOUND = 10028;
+
     /**
      * default error code. <b>It's recommended to use more specific error
      * codes</b>

modules/dcache/src/main/java/org/dcache/pool/classic/PoolV4.java

@@ -3,7 +3,6 @@
 package org.dcache.pool.classic;
 
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
 import com.google.common.net.InetAddresses;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
@@ -24,7 +23,6 @@
 import java.net.UnknownHostException;
 import java.nio.channels.CompletionHandler;
 import java.nio.file.OpenOption;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -82,6 +80,7 @@
 import diskCacheV111.vehicles.ProtocolInfo;
 import diskCacheV111.vehicles.RemoveFileInfoMessage;
 import diskCacheV111.vehicles.StorageInfo;
+
 import dmg.cells.nucleus.AbstractCellComponent;
 import dmg.cells.nucleus.CellCommandListener;
 import dmg.cells.nucleus.CellInfo;
@@ -99,6 +98,7 @@
 import dmg.util.command.Argument;
 import dmg.util.command.Command;
 import dmg.util.command.Option;
+
 import org.dcache.alarms.AlarmMarkerFactory;
 import org.dcache.alarms.PredefinedAlarm;
 import org.dcache.cells.CellStub;
@@ -126,7 +126,6 @@
 import org.dcache.pool.repository.SpaceRecord;
 import org.dcache.pool.repository.StateChangeEvent;
 import org.dcache.pool.repository.StickyRecord;
-import org.dcache.pool.repository.v5.ReplicaRepository;
 import org.dcache.util.IoPriority;
 import org.dcache.util.NetworkUtils;
 import org.dcache.util.Version;
@@ -1068,7 +1067,7 @@ public PoolMoverKillMessage messageArrived(PoolMoverKillMessage kill)
             kill.setSucceeded();
         } catch (NoSuchElementException e) {
             LOGGER.info(e.toString());
-            kill.setReply(1, e);
+            kill.setReply(CacheException.MOVER_NOT_FOUND, e);
         }
         return kill;
     }

modules/dcache/src/main/java/org/dcache/util/CacheExceptionFactory.java

@@ -94,6 +94,7 @@ public static CacheException exceptionOf(int errorCode, String message, Throwabl
             case POOL_DISABLED:
             case NO_POOL_CONFIGURED:
             case NO_POOL_ONLINE:
+            case MOVER_NOT_FOUND:
             default:
                 return new CacheException(errorCode, message, cause);
         }