From c74b297fe745b2bd317a0e5b164b509038fdfeaf Mon Sep 17 00:00:00 2001 From: Gerd Behrmann Date: Wed, 16 Dec 2015 00:32:14 +0100 Subject: [PATCH] poolmanager: Fix missing access latency and retention policy on pool to pool copy Motivation: Pool manager may as part of a read pool selection trigger pool to pool transfers and staging from tape. A pool persists certain meta data. Pool manager fails to ensure that in particular access latency, retention policy and checksums are included in the request message sent to the pool. The consequence is that files created on behalf of pool manager as part of a stage or pool to pool will lack these attributes. Modification: Adds the missing attributes to the list of required attributes. Further, these attributes are also made required for pool selection. This avoids that the p2p companion has to refetch the missing attributes, but more importantly this ensures that these attributes are available for staging too (staging has no ability to fetch the missing attributes itself). Result: Replicas created by a pool to pool copy will have the correct attributes. The price is that for the common case of not needing to stage or replicate a file, opening a file for read has become more expensive. Future patches may try to optimize this, but correctness is more important. Target: trunk Require-notes: yes Require-book: no Request: 2.14 Request: 2.13 Request: 2.12 Request: 2.11 Request: 2.10 Acked-by: Paul Millar Acked-by: Tigran Mkrtchyan Patch: https://rb.dcache.org/r/8881/ (cherry picked from commit f8416b5fdb7db54fc4c37903bf3c695255cd3b2b) --- .../diskCacheV111/vehicles/Pool2PoolTransferMsg.java | 5 +++-- .../vehicles/PoolMgrSelectReadPoolMsg.java | 11 ++++++++++- .../java/org/dcache/pinmanager/PinManagerTests.java | 4 ++++ .../org/dcache/tests/poolmanager/HsmRestoreTest.java | 4 ++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java index 5488c41cf3a..0245336a336 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java @@ -14,13 +14,14 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static org.dcache.namespace.FileAttribute.PNFSID; +import static org.dcache.namespace.FileAttribute.*; public class Pool2PoolTransferMsg extends PoolMessage { public static final ImmutableSet NEEDED_ATTRIBUTES = - Sets.immutableEnumSet(FileAttribute.PNFSID, FileAttribute.STORAGEINFO, FileAttribute.CHECKSUM, FileAttribute.SIZE); + Sets.immutableEnumSet(PNFSID, STORAGEINFO, CHECKSUM, SIZE, ACCESS_LATENCY, RETENTION_POLICY, + STORAGECLASS, CACHECLASS, HSM, FLAGS); public final static int UNDETERMINED = 0 ; public final static int PRECIOUS = 1 ; diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectReadPoolMsg.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectReadPoolMsg.java index d766df2c012..66ddb6d8de8 100644 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectReadPoolMsg.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolMgrSelectReadPoolMsg.java @@ -47,6 +47,15 @@ */ public class PoolMgrSelectReadPoolMsg extends PoolMgrSelectPoolMsg { + private static final EnumSet NEEDED_ATTRIBUTES; + + static { + EnumSet attributes = + EnumSet.of(PNFSID, STORAGEINFO, STORAGECLASS, CACHECLASS, HSM, LOCATIONS, SIZE, ACCESS_LATENCY, RETENTION_POLICY); + attributes.addAll(Pool2PoolTransferMsg.NEEDED_ATTRIBUTES); + NEEDED_ATTRIBUTES = attributes; + } + private static final long serialVersionUID = -2126253028981131441L; private Context _context; @@ -76,7 +85,7 @@ public PoolMgrSelectReadPoolMsg(FileAttributes fileAttributes, public static EnumSet getRequiredAttributes() { - return EnumSet.of(PNFSID, STORAGEINFO, STORAGECLASS, CACHECLASS, HSM, LOCATIONS, SIZE, ACCESS_LATENCY, RETENTION_POLICY); + return NEEDED_ATTRIBUTES.clone(); } public Context getContext() diff --git a/modules/dcache/src/test/java/org/dcache/pinmanager/PinManagerTests.java b/modules/dcache/src/test/java/org/dcache/pinmanager/PinManagerTests.java index 3220842f531..9c5afafccfb 100644 --- a/modules/dcache/src/test/java/org/dcache/pinmanager/PinManagerTests.java +++ b/modules/dcache/src/test/java/org/dcache/pinmanager/PinManagerTests.java @@ -104,6 +104,10 @@ private FileAttributes getAttributes(PnfsId pnfsId) attributes.setSize(0L); attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY); attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY); + attributes.setChecksums(Collections.emptySet()); + attributes.setCacheClass(null); + attributes.setHsm("osm"); + attributes.setFlags(Collections.emptyMap()); return attributes; } diff --git a/modules/dcache/src/test/java/org/dcache/tests/poolmanager/HsmRestoreTest.java b/modules/dcache/src/test/java/org/dcache/tests/poolmanager/HsmRestoreTest.java index 9c7dcd8513a..a7e174b2909 100644 --- a/modules/dcache/src/test/java/org/dcache/tests/poolmanager/HsmRestoreTest.java +++ b/modules/dcache/src/test/java/org/dcache/tests/poolmanager/HsmRestoreTest.java @@ -138,6 +138,7 @@ public void testRestoreNoLocations() throws Exception { attributes.setSize(5); attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY); attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY); + attributes.setChecksums(Collections.emptySet()); fileAttributesMessage.setFileAttributes(attributes); _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true); @@ -215,6 +216,7 @@ public void testRestoreNoLocationsOnePoolCantStage() throws Exception { attributes.setSize(5); attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY); attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY); + attributes.setChecksums(Collections.emptySet()); fileAttributesMessage.setFileAttributes(attributes); _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true); @@ -315,6 +317,7 @@ public void testRestoreNoLocationsSinglePool() throws Exception { attributes.setSize(5); attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY); attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY); + attributes.setChecksums(Collections.emptySet()); fileAttributesMessage.setFileAttributes(attributes); _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true); @@ -414,6 +417,7 @@ public void testRestoreNoLocationsAllPoolsCantStage() throws Exception { attributes.setSize(5); attributes.setAccessLatency(StorageInfo.DEFAULT_ACCESS_LATENCY); attributes.setRetentionPolicy(StorageInfo.DEFAULT_RETENTION_POLICY); + attributes.setChecksums(Collections.emptySet()); fileAttributesMessage.setFileAttributes(attributes); _cell.prepareMessage(new CellPath("PnfsManager"), fileAttributesMessage, true);