poolmanager: Fix missing access latency and retention policy on pool …

…to pool copy

Motivation:

Pool manager may as part of a read pool selection trigger pool to pool
transfers and staging from tape. A pool persists certain meta data. Pool
manager fails to ensure that in particular access latency, retention policy and
checksums are included in the request message sent to the pool. The consequence
is that files created on behalf of pool manager as part of a stage or pool to
pool will lack these attributes.

Modification:

Adds the missing attributes to the list of required attributes. Further, these
attributes are also made required for pool selection. This avoids that the p2p
companion has to refetch the missing attributes, but more importantly this
ensures that these attributes are available for staging too (staging has no
ability to fetch the missing attributes itself).

Result:

Replicas created by a pool to pool copy will have the correct attributes. The
price is that for the common case of not needing to stage or replicate a file,
opening a file for read has become more expensive. Future patches may try to
optimize this, but correctness is more important.

Target: trunk
Require-notes: yes
Require-book: no
Request: 2.14
Request: 2.13
Request: 2.12
Request: 2.11
Request: 2.10
Acked-by: Paul Millar <paul.millar@desy.de>
Acked-by: Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>
Patch: https://rb.dcache.org/r/8881/
(cherry picked from commit f8416b5)

modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java

@@ -14,13 +14,14 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static org.dcache.namespace.FileAttribute.PNFSID;
+import static org.dcache.namespace.FileAttribute.*;
 
 
 public class Pool2PoolTransferMsg extends PoolMessage {
 
     public static final ImmutableSet<FileAttribute> NEEDED_ATTRIBUTES =
-            Sets.immutableEnumSet(FileAttribute.PNFSID, FileAttribute.STORAGEINFO, FileAttribute.CHECKSUM, FileAttribute.SIZE);
+            Sets.immutableEnumSet(PNFSID, STORAGEINFO, CHECKSUM, SIZE, ACCESS_LATENCY, RETENTION_POLICY,
+                                  STORAGECLASS, CACHECLASS, HSM, FLAGS);
 
     public final static int   UNDETERMINED = 0 ;
     public final static int   PRECIOUS     = 1 ;

modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectReadPoolMsg.java

@@ -47,6 +47,15 @@
  */
 public class PoolMgrSelectReadPoolMsg extends PoolMgrSelectPoolMsg
 {
+    private static final EnumSet<FileAttribute> NEEDED_ATTRIBUTES;
+
+    static {
+        EnumSet<FileAttribute> attributes =
+                EnumSet.of(PNFSID, STORAGEINFO, STORAGECLASS, CACHECLASS, HSM, LOCATIONS, SIZE, ACCESS_LATENCY, RETENTION_POLICY);
+        attributes.addAll(Pool2PoolTransferMsg.NEEDED_ATTRIBUTES);
+        NEEDED_ATTRIBUTES = attributes;
+    }
+
     private static final long serialVersionUID = -2126253028981131441L;
 
     private Context _context;
@@ -76,7 +85,7 @@ public PoolMgrSelectReadPoolMsg(FileAttributes fileAttributes,
 
     public static EnumSet<FileAttribute> getRequiredAttributes()
     {
-        return EnumSet.of(PNFSID, STORAGEINFO, STORAGECLASS, CACHECLASS, HSM, LOCATIONS, SIZE, ACCESS_LATENCY, RETENTION_POLICY);
+        return NEEDED_ATTRIBUTES.clone();
     }
 
     public Context getContext()

modules/dcache/src/test/java/org/dcache/pinmanager/PinManagerTests.java

@@ -104,6 +104,10 @@ private FileAttributes getAttributes(PnfsId pnfsId)
         attributes.setSize(0L);
         attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY);
         attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY);
+        attributes.setChecksums(Collections.emptySet());
+        attributes.setCacheClass(null);
+        attributes.setHsm("osm");
+        attributes.setFlags(Collections.emptyMap());
         return attributes;
     }
 

modules/dcache/src/test/java/org/dcache/tests/poolmanager/HsmRestoreTest.java

@@ -138,6 +138,7 @@ public void testRestoreNoLocations() throws Exception {
         attributes.setSize(5);
         attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY);
         attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY);
+        attributes.setChecksums(Collections.emptySet());
         fileAttributesMessage.setFileAttributes(attributes);
         _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true);
 
@@ -215,6 +216,7 @@ public void testRestoreNoLocationsOnePoolCantStage() throws Exception {
         attributes.setSize(5);
         attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY);
         attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY);
+        attributes.setChecksums(Collections.emptySet());
         fileAttributesMessage.setFileAttributes(attributes);
         _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true);
 
@@ -315,6 +317,7 @@ public void testRestoreNoLocationsSinglePool() throws Exception {
         attributes.setSize(5);
         attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY);
         attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY);
+        attributes.setChecksums(Collections.emptySet());
         fileAttributesMessage.setFileAttributes(attributes);
         _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true);
 
@@ -414,6 +417,7 @@ public void testRestoreNoLocationsAllPoolsCantStage() throws Exception {
         attributes.setSize(5);
         attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY);
         attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY);
+        attributes.setChecksums(Collections.emptySet());
         fileAttributesMessage.setFileAttributes(attributes);
         _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true);