dcache-qos: improve exception reported on aborted verification

Motivation:

When the verifier aborts an operation, the
error code is always the same (10011, unexpected
system error), and the cause may not reflect
the root cause.

Modification:

Fix this so that more specific error info
is registered.

Result:

Better understanding of what went wrong.

Target: master
Request: 9.1
Request: 9.0
Request: 8.2
Patch: https://rb.dcache.org/r/14020/
Requires-notes: yes
Acked-by: Lea

modules/dcache-qos/src/main/java/org/dcache/qos/services/verifier/handlers/VerifyOperationHandler.java

@@ -65,8 +65,10 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import static org.dcache.qos.services.verifier.data.VerifyOperationState.CANCELED;
 import static org.dcache.qos.services.verifier.handlers.FileStatusVerifier.VERIFY_FAILURE_MESSAGE;
 
+import com.google.common.base.Throwables;
 import diskCacheV111.util.CacheException;
 import diskCacheV111.util.PnfsId;
+import java.io.IOException;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -358,17 +360,24 @@ public void handleVerification(PnfsId pnfsId) {
 
             action = statusVerifier.verify(requirements, operation);
         } catch (QoSException e) {
+            Throwable c = e.getCause();
             String message = CacheExceptionUtils.getCacheExceptionErrorMessage(
                   VERIFY_FAILURE_MESSAGE,
                   pnfsId,
                   VOID,
-                  null, e.getCause());
+                  null, Throwables.getRootCause(e));
             /*
              *  FATAL error, should abort operation.
              */
-            CacheException exception = new CacheException(
-                  CacheException.UNEXPECTED_SYSTEM_EXCEPTION,
-                  message, e.getCause());
+            CacheException exception;
+            if (c instanceof CacheException) {
+                exception = new CacheException(((CacheException)c).getRc(), message);
+            } else if (c instanceof IOException) {
+                exception = new CacheException(CacheException.ERROR_IO_DISK, message);
+            } else {
+                exception = new CacheException(
+                      CacheException.UNEXPECTED_SYSTEM_EXCEPTION, message, c);
+            }
             fileOpMap.updateOperation(pnfsId, exception);
             return;
         } catch (InterruptedException e) {

modules/dcache/src/main/java/org/dcache/qos/util/CacheExceptionUtils.java

@@ -64,6 +64,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import static org.dcache.qos.util.CacheExceptionUtils.FailureType.NEWTARGET;
 import static org.dcache.qos.util.CacheExceptionUtils.FailureType.RETRIABLE;
 
+import com.google.common.base.Throwables;
 import diskCacheV111.util.CacheException;
 import diskCacheV111.util.PnfsId;
 import java.io.Serializable;
@@ -108,7 +109,7 @@ public static CacheException getCacheExceptionFrom(Serializable errorObject) {
 
         if (errorObject instanceof Throwable) {
             Throwable t = (Throwable) errorObject;
-            return new CacheException(t.getMessage(), t.getCause());
+            return new CacheException(t.getMessage(), Throwables.getRootCause(t));
         }
 
         return new CacheException(String.valueOf(errorObject));