Merge pull request #6938 from lemora/pr/7.2/cleaner-stale-pool-handling

dcache-chimera: cleaner better handles stale delete reply
dCache · Jan 11, 2023 · d93496c · d93496c
2 parents 0da9c6f + 26d65f2
commit d93496c
Showing 1 changed file with 18 additions and 7 deletions.
diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/HsmCleaner.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/HsmCleaner.java
@@ -309,24 +309,33 @@ public synchronized void messageArrived(PoolRemoveFilesFromHSMMessage msg) {
         }
 
         String hsm = msg.getHsm();
+        String poolName = msg.getPoolName();
         Collection<URI> locations = _locationsToDelete.get(hsm);
         Collection<URI> success = msg.getSucceeded();
         Collection<URI> failures = msg.getFailed();
 
+        boolean isStaleReply = false;
+        HsmCleaner.Timeout currHsmTimeout = _requestTimeoutPerHsm.get(hsm);
+        if (currHsmTimeout == null || !poolName.equals(currHsmTimeout.getPool())) {
+            LOGGER.warn(
+                  "Received a remove reply from pool {}, which is no longer waited for. "
+                        + "The cleaner pool timeout might be too small.",
+                  poolName);
+            isStaleReply = true;
+        }
+
         if (locations == null) {
             /* Seems we got a reply for something this instance did
              * not request. We log this as a warning, but otherwise
              * ignore it.
              */
             LOGGER.warn(
-                  "Received confirmation from a pool, for an action this cleaner did not request.");
+                  "Received confirmation from a pool for an action this cleaner did not request.");
             return;
         }
 
-        if (!failures.isEmpty()) {
-            LOGGER.warn("Failed to delete {} files from HSM {}. Will try again later.",
-                  failures.size(), hsm);
-        }
+        LOGGER.info("Pool delete responses for HSM {}: {} success, {} failures", hsm,
+              success.size(), failures.size());
 
         for (URI location : success) {
             assert location.getAuthority().equals(hsm);
@@ -342,8 +351,10 @@ public synchronized void messageArrived(PoolRemoveFilesFromHSMMessage msg) {
             }
         }
 
-        removeHsmRequestTimeout(hsm);
-        flush(hsm);
+        if (!isStaleReply) {
+            removeHsmRequestTimeout(hsm);
+            flush(hsm);
+        }
     }
 
     /**