Merge pull request #2250 from gbehrmann/fix/2.15/rb9084

pnfsmanager: Fix race leading to transaction failures in Chimera
dCache · Mar 7, 2016 · 542d7ad · 542d7ad
2 parents 12764b7 + 1b8e541
commit 542d7ad
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/...s/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraNameSpaceProvider.java b/...s/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraNameSpaceProvider.java
@@ -40,6 +40,7 @@
 import diskCacheV111.util.FileNotFoundCacheException;
 import diskCacheV111.util.FsPath;
 import diskCacheV111.util.InvalidMessageCacheException;
+import diskCacheV111.util.LockedCacheException;
 import diskCacheV111.util.NotDirCacheException;
 import diskCacheV111.util.NotFileCacheException;
 import diskCacheV111.util.PermissionDeniedCacheException;
@@ -1109,9 +1110,9 @@ private ExtendedInode installSystemDirectory(FsPath path, int mode, List<ACE> ac
             try {
                 inode = parentOfPath.mkdir(path.getName(), 0, 0, mode, acl, tags);
             } catch (FileExistsChimeraFsException e1) {
-                /* Concurrent directory creation. Do another lookup.
+                /* Concurrent directory creation. Current transaction is invalid.
                  */
-                inode = lookupDirectory(Subjects.ROOT, path);
+                throw new LockedCacheException("Concurrent access prevented this operation from completing. Please retry.");
             }
         }
         return inode;
@@ -1127,9 +1128,9 @@ private ExtendedInode installDirectory(Subject subject, FsPath path, int mode) t
             try {
                 inode = mkdir(subject, parentOfPath, path.getName(), DEFAULT, DEFAULT, mode);
             } catch (FileExistsChimeraFsException e1) {
-                /* Concurrent directory creation. Do another lookup.
+                /* Concurrent directory creation. Current transaction is invalid.
                  */
-                inode = lookupDirectory(subject, path);
+                throw new LockedCacheException("Concurrent access prevented this operation from completing. Please retry.");
             }
         }
         return inode;

diff --git a/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java b/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java
@@ -1114,6 +1114,8 @@ private static Map<String,List<String>> resolve(String name, String[] attrIds)
             CellStub.addCallback(_pnfsStub.send(msg),
                                  new AbstractMessageCallback<PnfsCreateUploadPath>()
                                  {
+                                     int failures = 0;
+
                                      @Override
                                      public void success(PnfsCreateUploadPath message)
                                      {
@@ -1123,6 +1125,7 @@ public void success(PnfsCreateUploadPath message)
                                      @Override
                                      public void failure(int rc, Object error)
                                      {
+                                         failures++;
                                          String msg = Objects.toString(error, "");
                                          switch (rc) {
                                          case CacheException.PERMISSION_DENIED:
@@ -1134,6 +1137,20 @@ public void failure(int rc, Object error)
                                          case CacheException.FILE_NOT_FOUND:
                                              future.setException(new SRMInvalidPathException(msg));
                                              break;
+                                         case CacheException.LOCKED:
+                                             if (failures < 3) {
+                                                 /* Usually due to concurrent uploads to the same non-existing target
+                                                  * directory. Retry a few times.
+                                                  */
+                                                 PnfsCreateUploadPath retry =
+                                                         new PnfsCreateUploadPath(subject, restriction, fullPath,
+                                                                                  user.getRoot(),
+                                                                                  size, al, rp, spaceToken, options);
+                                                 CellStub.addCallback(_pnfsStub.send(retry), this, _executor);
+                                             } else {
+                                                 future.setException(new SRMInternalErrorException(msg));
+                                             }
+                                             break;
                                          case CacheException.TIMEOUT:
                                          default:
                                              future.setException(new SRMInternalErrorException(msg));