spacemanager: Allow unreserved transfers even when space reservation …

…for non-srm transfers is enabled

Space manager can be configured to create space reservations for non-srm
transfers. The problem is that as soon as this feature is enabled, all
authenticated non-srm transfers that support space manager must go to a link
group. It isn't possible to mix reserved and unreserved transfers.

This patch addresses this problem by letting space manager fall back to an
unreserved transfer. This fallback already exists if the transfer is
unauthenticated or when a protocol without space- manager support is used.

The patch thus changes the semantics of the property
spacemanager.enable.reserve-space-for-non-srm-transfers. Whereas before
enabling this property would cause some transfers that succeeded before to
fail, the property now only adds additional ways in which a transfer can
succeed. This is easily demonstrated using the system-test package.

Without this patch and with reserve-space-for-non-srm-transfers disabled,
uploading to the reserved directory with gridftp would fail (e.g. 'test
--gridftp -d reserved'), while upload to the disk directory would succeed (e.g.
'test --gridftp -d disk'). If reserve-space-for-non-srm-transfers is enabled,
the success status of these two transfers would be reversed (the former would
now succeed, the latter would now fail). With this patch, enabling
reserve-space-for-non-srm-transfers will cause both uploads to succeed. The
patch thus also changes the default system-test setup to enabel space
reservation for non-srm transfers.

Target: trunk
Require-notes: yes
Require-book: yes
Acked-by: Dmitry Litvintsev <litvinse@fnal.gov>
Patch: http://rb.dcache.org/r/6356

modules/dcache/src/main/java/diskCacheV111/services/space/Manager.java

@@ -4402,13 +4402,6 @@ private LinkGroup selectLinkGroupForWrite(Subject subject, ProtocolInfo protocol
         {
             Set<LinkGroup> linkGroups =
                     findLinkGroupIds(size, fileAttributes.getAccessLatency(), fileAttributes.getRetentionPolicy());
-            if(linkGroups.isEmpty()) {
-                logger.warn("failed to find matching linkgroup");
-                throw new NoFreeSpaceException(" no space available");
-            }
-            //
-            // filter out groups we are not authorized to use
-            //
             List<String> linkGroupNames = new ArrayList<>();
             for (LinkGroup linkGroup : linkGroups) {
                 try {
@@ -4418,36 +4411,16 @@ private LinkGroup selectLinkGroupForWrite(Subject subject, ProtocolInfo protocol
                 catch (SpaceAuthorizationException e) {
                 }
             }
-            if(linkGroupNames.isEmpty()) {
-                logger.warn("failed to find linkgroup where user is " +
-                        "authorized to reserve space.");
-                throw new SpaceAuthorizationException("Failed to find LinkGroup where user is authorized to reserve space.");
-            }
+            linkGroupNames = selectLinkGroupForWrite(protocolInfo, fileAttributes, linkGroupNames);
             logger.trace("Found {} linkgroups protocolInfo={}, fileAttributes={}",
                     linkGroups.size(), protocolInfo, fileAttributes);
-            if (linkGroupNames.size()>1 &&
-                    protocolInfo != null &&
-                    fileAttributes != null) {
-                try {
-                    linkGroupNames = selectLinkGroupForWrite(protocolInfo, fileAttributes, linkGroupNames);
-                    if(linkGroupNames.isEmpty()) {
-                        throw new SpaceAuthorizationException("PoolManagerSelectLinkGroupForWriteMessage: Failed to find LinkGroup where user is authorized to reserve space.");
-                    }
-                }
-                catch (SpaceAuthorizationException e)  {
-                    logger.warn("authorization problem: {}",
-                            e.getMessage());
-                    throw e;
-                }
-                catch(Exception e) {
-                    throw new SpaceException("Internal error : Failed to get list of link group ids from Pool Manager "+e.getMessage());
-                }
 
-            }
-            String linkGroupName = linkGroupNames.get(0);
-            for (LinkGroup lg : linkGroups) {
-                if (lg.getName().equals(linkGroupName) ) {
-                    return lg;
+            if (!linkGroupNames.isEmpty()) {
+                String linkGroupName = linkGroupNames.get(0);
+                for (LinkGroup lg : linkGroups) {
+                    if (lg.getName().equals(linkGroupName) ) {
+                        return lg;
+                    }
                 }
             }
             return null;
@@ -4692,10 +4665,15 @@ public void selectPoolOnRequest(CellMessage envelope, PoolMgrSelectWritePoolMsg
                 LinkGroup linkGroup =
                         selectLinkGroupForWrite(subject, selectWritePool
                                 .getProtocolInfo(), fileAttributes, selectWritePool.getPreallocated());
-                String linkGroupName = linkGroup.getName();
-                selectWritePool.setLinkGroup(linkGroupName);
-                logger.trace("selectPoolOnRequest: found linkGroup = {}, " +
-                        "forwarding message", linkGroupName);
+                if (linkGroup != null) {
+                    String linkGroupName = linkGroup.getName();
+                    selectWritePool.setLinkGroup(linkGroupName);
+                    logger.trace("selectPoolOnRequest: found linkGroup = {}, " +
+                            "forwarding message", linkGroupName);
+                } else {
+                    logger.trace("selectPoolOnRequest: did not find linkGroup that can " +
+                            "hold this file, processing file without space reservation.");
+                }
             } else {
                 logger.trace("selectPoolOnRequest: file is " +
                         "not found, no prior " +

modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrGetPoolMsg.java

@@ -2,6 +2,8 @@
 
 package diskCacheV111.vehicles;
 
+import javax.annotation.Nonnull;
+
 import java.util.Collection;
 import java.util.EnumSet;
 
@@ -32,16 +34,19 @@ public PoolMgrGetPoolMsg(FileAttributes fileAttributes)
 	setReplyRequired(true);
     }
 
+    @Nonnull
     public FileAttributes getFileAttributes()
     {
 	return _fileAttributes;
     }
 
+    @Nonnull
     public StorageInfo getStorageInfo()
     {
 	return _fileAttributes.getStorageInfo();
     }
 
+    @Nonnull
     public PnfsId getPnfsId()
     {
 	return _fileAttributes.getPnfsId();

modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectPoolMsg.java

@@ -2,12 +2,16 @@
 
 package diskCacheV111.vehicles ;
 
+import javax.annotation.Nonnull;
+
 import java.util.EnumSet;
 
 import diskCacheV111.poolManager.RequestContainerV5;
 
 import org.dcache.vehicles.FileAttributes;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 public class PoolMgrSelectPoolMsg extends PoolMgrGetPoolMsg {
 
     private static final long serialVersionUID = -5874326080375390208L;
@@ -31,8 +35,8 @@ public PoolMgrSelectPoolMsg(FileAttributes fileAttributes,
                                 EnumSet<RequestContainerV5.RequestState> allowedStates)
     {
         super(fileAttributes);
-        _protocolInfo = protocolInfo;
-        _allowedStates = allowedStates;
+        _protocolInfo = checkNotNull(protocolInfo);
+        _allowedStates = checkNotNull(allowedStates);
     }
 
     public void setSkipCostUpdate(boolean value)
@@ -45,8 +49,9 @@ public boolean getSkipCostUpdate()
         return _skipCostUpdate;
     }
 
+    @Nonnull
     public ProtocolInfo getProtocolInfo(){ return _protocolInfo; }
-    public void setProtocolInfo( ProtocolInfo protocolInfo ){ _protocolInfo = protocolInfo ; }
+    public void setProtocolInfo( ProtocolInfo protocolInfo ){ _protocolInfo = checkNotNull(protocolInfo); }
     public void setIoQueueName( String ioQueueName ){ _ioQueueName = ioQueueName ; }
     public String getIoQueueName(){ return _ioQueueName ; }
 
@@ -66,6 +71,7 @@ public String getLinkGroup() {
         return _linkGroup;
     }
 
+    @Nonnull
     public EnumSet<RequestContainerV5.RequestState> getAllowedStates() {
         return _allowedStates;
     }

modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectWritePoolMsg.java

@@ -2,8 +2,6 @@
 
 import org.dcache.vehicles.FileAttributes;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 /**
  * Requests pool manager to provide a pool to which a file with the
  * given properties can be written.
@@ -33,8 +31,6 @@ public PoolMgrSelectWritePoolMsg(FileAttributes fileAttributes,
                                      long preallocated)
     {
 	super(fileAttributes, protocolInfo);
-        checkNotNull(fileAttributes);
-        checkNotNull(protocolInfo);
         _preallocated = preallocated;
     }
 

packages/system-test/src/main/skel/etc/layouts/system-test.conf

@@ -81,6 +81,7 @@ pool.lfs=precious
 
 [dCacheDomain/spacemanager]
 spacemanager.authz.link-group-file-name=${dcache.paths.etc}/linkgroup.conf
+spacemanager.enable.reserve-space-for-non-srm-transfers=true
 
 [dCacheDomain/pinmanager]
 [dCacheDomain/cleaner]