diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraHsmStorageInfoExtractor.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraHsmStorageInfoExtractor.java index 0c8e04fcac9..08f08483399 100644 --- a/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraHsmStorageInfoExtractor.java +++ b/modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraHsmStorageInfoExtractor.java @@ -58,7 +58,7 @@ public final RetentionPolicy getDefaultRetentionPolicy() { /* * (non-Javadoc) * - * @see diskCacheV111.util.StorageInfoExtractable#getStorageInfo(java.lang.String, + * @see diskCacheV111.util.StorageInfoExtractable#getFileAttributes(java.lang.String, * diskCacheV111.util.PnfsId) */ diff --git a/modules/dcache-dcap/src/main/java/diskCacheV111/doors/DCapDoorInterpreterV3.java b/modules/dcache-dcap/src/main/java/diskCacheV111/doors/DCapDoorInterpreterV3.java index 19b058507fd..66958ce51ff 100755 --- a/modules/dcache-dcap/src/main/java/diskCacheV111/doors/DCapDoorInterpreterV3.java +++ b/modules/dcache-dcap/src/main/java/diskCacheV111/doors/DCapDoorInterpreterV3.java @@ -2221,17 +2221,15 @@ private void storeChecksumInPnfs( PnfsId pnfsId , String checksumString){ poolMessage = new PoolDeliverFileMessage( pool, - _fileAttributes.getPnfsId(), _protocolInfo , - _fileAttributes.getStorageInfo()); + _fileAttributes); }else if( reply instanceof PoolMgrSelectWritePoolMsg ){ poolMessage = new PoolAcceptFileMessage( pool, - _fileAttributes.getPnfsId(), _protocolInfo , - _fileAttributes.getStorageInfo()); + _fileAttributes); }else{ sendReply( "poolMgrGetPoolArrived" , 7 , "Illegal Message arrived : "+reply.getClass().getName() ) ; diff --git a/modules/dcache-dcap/src/main/java/org/dcache/pool/movers/DCapProtocol_3_nio.java b/modules/dcache-dcap/src/main/java/org/dcache/pool/movers/DCapProtocol_3_nio.java index 5026175395e..f1d71e7cf85 100755 --- a/modules/dcache-dcap/src/main/java/org/dcache/pool/movers/DCapProtocol_3_nio.java +++ b/modules/dcache-dcap/src/main/java/org/dcache/pool/movers/DCapProtocol_3_nio.java @@ -10,6 +10,7 @@ import java.util.Map; import java.security.MessageDigest; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -278,15 +279,13 @@ protected String getCellDomainName() } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access ) - - throws Exception { - + throws Exception + { Exception ioException = null; if(! (protocol instanceof DCapProtocolInfo)) { @@ -295,7 +294,8 @@ public void runIO(RepositoryChannel fileChannel, } DCapProtocolInfo dcapProtocolInfo = (DCapProtocolInfo)protocol; - _pnfsId = pnfsId; + StorageInfo storage = fileAttributes.getStorageInfo(); + _pnfsId = fileAttributes.getPnfsId(); _spaceMonitorHandler = new SpaceMonitorHandler(allocator); //////////////////////////////////////////////////////////////////////// diff --git a/modules/dcache-ftp/src/main/java/org/dcache/pool/movers/GFtpProtocol_2_nio.java b/modules/dcache-ftp/src/main/java/org/dcache/pool/movers/GFtpProtocol_2_nio.java index 59b0f35e3f5..4b06c98033b 100644 --- a/modules/dcache-ftp/src/main/java/org/dcache/pool/movers/GFtpProtocol_2_nio.java +++ b/modules/dcache-ftp/src/main/java/org/dcache/pool/movers/GFtpProtocol_2_nio.java @@ -1,8 +1,6 @@ package org.dcache.pool.movers; import java.text.MessageFormat; -import java.nio.channels.FileChannel; -import java.io.RandomAccessFile; import java.io.IOException; import java.net.InetSocketAddress; import java.net.InetAddress; @@ -20,17 +18,16 @@ import diskCacheV111.vehicles.ProtocolInfo; import diskCacheV111.vehicles.GFtpProtocolInfo; import diskCacheV111.vehicles.GFtpTransferStartedMessage; -import diskCacheV111.vehicles.StorageInfo; import org.dcache.util.Checksum; import org.dcache.util.ChecksumType; import java.security.NoSuchAlgorithmException; import diskCacheV111.util.ChecksumFactory; -import diskCacheV111.util.PnfsId; import diskCacheV111.util.CacheException; import org.dcache.pool.repository.Allocator; import org.dcache.util.PortRange; import org.dcache.util.NetworkUtils; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.dcache.ftp.*; @@ -401,10 +398,9 @@ public void transfer(RepositoryChannel fileChannel, Role role, /** Part of the MoverProtocol interface. */ @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access) throws Exception @@ -480,7 +476,7 @@ public void runIO(RepositoryChannel fileChannel, int localPort = channel.socket().getLocalPort(); message = - new GFtpTransferStartedMessage(pnfsId.getId(), + new GFtpTransferStartedMessage(fileAttributes.getPnfsId().getId(), localHostName, localPort); mode.setPassive(channel); @@ -489,7 +485,7 @@ public void runIO(RepositoryChannel fileChannel, * active mode. When notified about this, the door * will fall back to proxy mode. */ - message = new GFtpTransferStartedMessage(pnfsId.getId()); + message = new GFtpTransferStartedMessage(fileAttributes.getPnfsId().getId()); } CellPath path = new CellPath(gftpProtocolInfo.getDoorCellName(), gftpProtocolInfo.getDoorCellDomainName()); diff --git a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdProtocol_3.java b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdProtocol_3.java index 5e31336b78c..6143aeef691 100644 --- a/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdProtocol_3.java +++ b/modules/dcache-xrootd/src/main/java/org/dcache/xrootd/pool/XrootdProtocol_3.java @@ -1,18 +1,15 @@ package org.dcache.xrootd.pool; -import java.io.IOException; import java.net.InetSocketAddress; import java.net.InetAddress; import java.net.Inet4Address; import java.net.NetworkInterface; -import java.net.UnknownHostException; import java.net.SocketException; import java.util.Collection; import java.util.ArrayList; import java.util.Enumeration; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import dmg.cells.nucleus.CellPath; import dmg.cells.nucleus.CellEndpoint; @@ -23,19 +20,15 @@ import org.dcache.pool.movers.MoverProtocol; import org.dcache.pool.movers.MoverChannel; import org.dcache.pool.movers.IoMode; -import org.dcache.pool.movers.MoverChannel; import org.dcache.pool.repository.RepositoryChannel; import org.dcache.pool.repository.Allocator; -import org.dcache.pool.repository.RepositoryChannel; import org.dcache.util.NetworkUtils; +import org.dcache.vehicles.FileAttributes; import org.dcache.vehicles.XrootdProtocolInfo; import org.dcache.vehicles.XrootdDoorAdressInfoMessage; import diskCacheV111.util.CacheException; -import diskCacheV111.util.PnfsId; -import diskCacheV111.util.TimeoutCacheException; import diskCacheV111.movers.NetIFContainer; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,10 +128,9 @@ public XrootdProtocol_3(CellEndpoint endpoint) } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access) throws Exception diff --git a/modules/dcache/src/main/java/diskCacheV111/admin/UserAdminShell.java b/modules/dcache/src/main/java/diskCacheV111/admin/UserAdminShell.java index c8f658650ef..8b087a0b16a 100755 --- a/modules/dcache/src/main/java/diskCacheV111/admin/UserAdminShell.java +++ b/modules/dcache/src/main/java/diskCacheV111/admin/UserAdminShell.java @@ -968,8 +968,10 @@ private PnfsFlagReply setPnfsFlag( String dest = args.argv(2) ; PnfsId pnfsId = new PnfsId( args.argv(0) ) ; + FileAttributes fileAttributes = new FileAttributes(); + fileAttributes.setPnfsId(pnfsId); Pool2PoolTransferMsg p2p = - new Pool2PoolTransferMsg( source , dest , pnfsId , null ) ; + new Pool2PoolTransferMsg( source , dest , fileAttributes ) ; cellEndPoint.sendMessage( diff --git a/modules/dcache/src/main/java/diskCacheV111/cells/DebugCommander.java b/modules/dcache/src/main/java/diskCacheV111/cells/DebugCommander.java index 86be0aee6ef..554ccbc0ff0 100755 --- a/modules/dcache/src/main/java/diskCacheV111/cells/DebugCommander.java +++ b/modules/dcache/src/main/java/diskCacheV111/cells/DebugCommander.java @@ -13,6 +13,7 @@ import dmg.util.Args; import dmg.util.CommandSyntaxException; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,14 +52,12 @@ public void messageArrived( CellMessage msg ){ } String pnfsid = PnfsId.toCompleteId( args.argv(1) ) ; - + FileAttributes fileAttributes = new FileAttributes(); + fileAttributes.setPnfsId(new PnfsId(pnfsid)); + fileAttributes.setStorageInfo(new GenericStorageInfo(hsmName, "any")); _nucleus.sendMessage( new CellMessage( - path , - new PoolFetchFileMessage( poolName , - new GenericStorageInfo(hsmName,"any") , - pnfsid ) - ) ) ; + path, new PoolFetchFileMessage( poolName , fileAttributes))); return "Stay tuned" ; } public static final String hh_send_getpool = diff --git a/modules/dcache/src/main/java/diskCacheV111/namespace/PnfsManagerV3.java b/modules/dcache/src/main/java/diskCacheV111/namespace/PnfsManagerV3.java index 0c66203bb34..571d35c0338 100644 --- a/modules/dcache/src/main/java/diskCacheV111/namespace/PnfsManagerV3.java +++ b/modules/dcache/src/main/java/diskCacheV111/namespace/PnfsManagerV3.java @@ -1963,10 +1963,10 @@ public void getFileAttributes(PnfsGetFileAttributes message) Set requested = message.getRequestedAttributes(); if(requested.contains(FileAttribute.STORAGEINFO)) { /* - * TODO: The 'classic' result of getStorageInfo was a + * TODO: The 'classic' result of getFileAttributes was a * cobination of fileMetadata + storageInfo. This was - * used add the owner and group information into - * sorageInfo's internal Map. Uid and Gid used by the + * used to add the owner and group information into + * storageInfo's internal Map. Uid and Gid are used by the * HSM flush scripts. * * This atavism will have to be cut out when HSM diff --git a/modules/dcache/src/main/java/diskCacheV111/poolManager/CostModuleV1.java b/modules/dcache/src/main/java/diskCacheV111/poolManager/CostModuleV1.java index 5291fe4c151..3076c78aa6a 100755 --- a/modules/dcache/src/main/java/diskCacheV111/poolManager/CostModuleV1.java +++ b/modules/dcache/src/main/java/diskCacheV111/poolManager/CostModuleV1.java @@ -14,6 +14,8 @@ import java.util.ArrayList; import java.util.regex.Pattern; +import org.dcache.namespace.FileAttribute; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.dcache.cells.CellCommandListener; @@ -262,15 +264,16 @@ public synchronized void messageToForward(PoolIoFileMessage msg) int diff = 0; long pinned = 0; + FileAttributes attributes = msg.getFileAttributes(); if (msg.isReply() && msg.getReturnCode() != 0) { diff = -1; - if (msg instanceof PoolAcceptFileMessage) { - pinned = -msg.getStorageInfo().getFileSize(); + if (msg instanceof PoolAcceptFileMessage && attributes.isDefined(FileAttribute.SIZE)) { + pinned = -msg.getFileAttributes().getSize(); } } else if (!msg.isReply() && !_magic) { diff = 1; - if (msg instanceof PoolAcceptFileMessage) { - pinned = msg.getStorageInfo().getFileSize(); + if (msg instanceof PoolAcceptFileMessage && attributes.isDefined(FileAttribute.SIZE)) { + pinned = msg.getFileAttributes().getSize(); } } @@ -333,14 +336,19 @@ public synchronized void messageToForward(PoolFetchFileMessage msg) int diff; long pinned; - if (!msg.isReply()) { - diff = 1; - pinned = msg.getStorageInfo().getFileSize(); - } else { - diff = -1; - pinned = 0; - } - queue.modifyQueue(diff); + if (msg.isReply()) { + diff = -1; + pinned = 0; + } else { + diff = 1; + FileAttributes attributes = msg.getFileAttributes(); + if (attributes.isDefined(FileAttribute.SIZE)) { + pinned = attributes.getSize(); + } else { + pinned = 0; + } + } + queue.modifyQueue(diff); spaceInfo.modifyPinnedSpace(pinned); considerInvalidatingCache(currentPerformanceCost, costInfo); xsay("Restore", poolName, diff, pinned, msg); @@ -421,7 +429,7 @@ public synchronized void messageToForward(Pool2PoolTransferMsg msg) destinationCostInfo.getSpaceInfo(); int diff = msg.isReply() ? -1 : 1; - long pinned = msg.getStorageInfo().getFileSize(); + long pinned = msg.getFileAttributes().getSize(); sourceQueue.modifyQueue(diff); destinationQueue.modifyQueue(diff); diff --git a/modules/dcache/src/main/java/diskCacheV111/poolManager/RequestContainerV5.java b/modules/dcache/src/main/java/diskCacheV111/poolManager/RequestContainerV5.java index bf3415e771d..8b74e808af3 100755 --- a/modules/dcache/src/main/java/diskCacheV111/poolManager/RequestContainerV5.java +++ b/modules/dcache/src/main/java/diskCacheV111/poolManager/RequestContainerV5.java @@ -1051,16 +1051,15 @@ private void setError( int errorCode , String errorMessage ){ _currentRc = errorCode ; _currentRm = errorMessage ; } - private boolean sendFetchRequest( String poolName , StorageInfo storageInfo ) + + private boolean sendFetchRequest(String poolName) throws NoRouteToCellException { - CellMessage cellMessage = new CellMessage( new CellPath( poolName ), new PoolFetchFileMessage( poolName, - storageInfo, - _pnfsId ) + _fileAttributes) ); synchronized( _messageHash ){ if( ( _maxRestore >=0 ) && @@ -1078,7 +1077,7 @@ private void sendPool2PoolRequest( String sourcePool , String destPool ) throws NoRouteToCellException { Pool2PoolTransferMsg pool2pool = - new Pool2PoolTransferMsg(sourcePool,destPool,_pnfsId,_storageInfo) ; + new Pool2PoolTransferMsg(sourcePool, destPool, _fileAttributes); pool2pool.setDestinationFileStatus( _destinationFileStatus ) ; _log.info("[p2p] Sending transfer request: "+pool2pool); CellMessage cellMessage = @@ -2117,7 +2116,7 @@ private int askForStaging() _stageCandidateHost = pool.getHostName(); _log.info("[staging] poolCandidate -> {}", _poolCandidate); - if (!sendFetchRequest(_poolCandidate, _storageInfo)) { + if (!sendFetchRequest(_poolCandidate)) { return RT_OUT_OF_RESOURCES; } diff --git a/modules/dcache/src/main/java/diskCacheV111/replicaManager/CopyManager.java b/modules/dcache/src/main/java/diskCacheV111/replicaManager/CopyManager.java index 848583dbb1a..80b80348509 100644 --- a/modules/dcache/src/main/java/diskCacheV111/replicaManager/CopyManager.java +++ b/modules/dcache/src/main/java/diskCacheV111/replicaManager/CopyManager.java @@ -15,6 +15,7 @@ import java.io.* ; import java.util.*; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -511,8 +512,10 @@ private void startTransfer( PoolFileEntry entry ){ entry.timestamp = System.currentTimeMillis() ; + FileAttributes fileAttributes = new FileAttributes(); + fileAttributes.setPnfsId(entry.getPnfsId()); Pool2PoolTransferMsg pool2pool = - new Pool2PoolTransferMsg( _source , entry._destination , entry.getPnfsId() , null ) ; + new Pool2PoolTransferMsg(_source, entry._destination, fileAttributes); CellMessage msg = new CellMessage( new CellPath(entry._destination) , pool2pool ) ; diff --git a/modules/dcache/src/main/java/diskCacheV111/replicaManager/DCacheCoreControllerV2.java b/modules/dcache/src/main/java/diskCacheV111/replicaManager/DCacheCoreControllerV2.java index 0bfa7a6719f..95f9c51a54c 100644 --- a/modules/dcache/src/main/java/diskCacheV111/replicaManager/DCacheCoreControllerV2.java +++ b/modules/dcache/src/main/java/diskCacheV111/replicaManager/DCacheCoreControllerV2.java @@ -18,6 +18,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -687,7 +688,7 @@ public String toString(){ protected TaskObserver movePnfsId( PnfsId pnfsId , String source , String destination ) throws Exception { - StorageInfo storageInfo = getStorageInfo( pnfsId ) ; + FileAttributes fileAttributes = getFileAttributes(pnfsId); Collection hash = new HashSet<>(getCacheLocationList( pnfsId , false )) ; @@ -706,8 +707,7 @@ protected TaskObserver movePnfsId( PnfsId pnfsId , String source , String destin Pool2PoolTransferMsg req = - new Pool2PoolTransferMsg( source , destination , - pnfsId , storageInfo ) ; + new Pool2PoolTransferMsg(source, destination, fileAttributes); req.setDestinationFileStatus( Pool2PoolTransferMsg.PRECIOUS ) ; CellMessage msg = new CellMessage( new CellPath(destination) , req ) ; @@ -1014,8 +1014,8 @@ protected MoverTask replicatePnfsId( PnfsId pnfsId, Set readablePools, S + " no pools found in online state and not having listed pnfsId=" + pnfsId); } - StorageInfo storageInfo = getStorageInfo( pnfsId ) ; - long fileSize = storageInfo.getFileSize(); + FileAttributes fileAttributes = getFileAttributes(pnfsId) ; + long fileSize = fileAttributes.getSize(); // do not use pools on the same host Set sourceHosts = new HashSet<>(); @@ -1028,7 +1028,7 @@ protected MoverTask replicatePnfsId( PnfsId pnfsId, Set readablePools, S String destination = bestDestPool(destPools, fileSize, sourceHosts ); - return replicatePnfsId( storageInfo, pnfsId, source, destination); + return replicatePnfsId( fileAttributes, source, destination); } /** @@ -1039,15 +1039,14 @@ protected MoverTask replicatePnfsId( PnfsId pnfsId, Set readablePools, S * to facilitate implementation of external loop over destination pools. * */ - private MoverTask replicatePnfsId(StorageInfo storageInfo, - PnfsId pnfsId, String source, + private MoverTask replicatePnfsId(FileAttributes attributes, String source, String destination) throws Exception { + PnfsId pnfsId = attributes.getPnfsId(); _log.info("Sending p2p for " + pnfsId + " " + source + " -> " + destination); Pool2PoolTransferMsg req = - new Pool2PoolTransferMsg(source, destination, - pnfsId, storageInfo); + new Pool2PoolTransferMsg(source, destination, attributes); req.setDestinationFileStatus( Pool2PoolTransferMsg.PRECIOUS ) ; CellMessage msg = new CellMessage(new CellPath(destination), req); @@ -1475,7 +1474,7 @@ void cacheLocationModified( void taskFinished( TaskObserver task ); /** - * Returns the storage info of the specified pnfsId. Mainly used by + * Returns the file attributes of the specified pnfsId. Mainly used by * other DCacheCoreController methods. * * @param pnfsId pnfsId for which the method should return the cache locations. @@ -1486,7 +1485,7 @@ void cacheLocationModified( * @throws InterruptedException if the method was interrupted. * */ - protected StorageInfo getStorageInfo( PnfsId pnfsId ) + protected FileAttributes getFileAttributes(PnfsId pnfsId) throws MissingResourceException, NoRouteToCellException, InterruptedException @@ -1497,7 +1496,7 @@ protected StorageInfo getStorageInfo( PnfsId pnfsId ) CellMessage cellMessage = new CellMessage( new CellPath( "PnfsManager" ) , msg ) ; CellMessage answer; -// _log.debug("getStorageInfo: sendAndWait, pnfsId=" +pnfsId ); +// _log.debug("getFileAttributes: sendAndWait, pnfsId=" +pnfsId ); answer = sendAndWait( cellMessage , _TO_GetStorageInfo ) ; if( answer == null ) { @@ -1511,7 +1510,7 @@ protected StorageInfo getStorageInfo( PnfsId pnfsId ) msg = (PnfsGetStorageInfoMessage) answer.getMessageObject() ; if( msg.getReturnCode() != 0 ) { - _log.debug("getStorageInfo() PnfsGetStorageInfoMessage answer error: err=" + _log.debug("getFileAttributes() PnfsGetStorageInfoMessage answer error: err=" +msg.getReturnCode() + ", message='" + msg + "'" ); @@ -1528,7 +1527,7 @@ protected StorageInfo getStorageInfo( PnfsId pnfsId ) "PnfsManager", "PnfsGetStorageInfoMessage"); } - return msg.getStorageInfo(); + return msg.getFileAttributes(); } protected void removeCopy( PnfsId pnfsId , String poolName , boolean force ) diff --git a/modules/dcache/src/main/java/diskCacheV111/repository/CacheRepositoryEntryInfo.java b/modules/dcache/src/main/java/diskCacheV111/repository/CacheRepositoryEntryInfo.java index 6effef44ee6..b4637c3752f 100644 --- a/modules/dcache/src/main/java/diskCacheV111/repository/CacheRepositoryEntryInfo.java +++ b/modules/dcache/src/main/java/diskCacheV111/repository/CacheRepositoryEntryInfo.java @@ -49,7 +49,7 @@ public CacheRepositoryEntryInfo(CacheEntry entry) pnfsId = entry.getPnfsId(); lastAccessTime = entry.getLastAccessTime(); creationTime = entry.getCreationTime(); - storageInfo = entry.getStorageInfo(); + storageInfo = entry.getFileAttributes().getStorageInfo(); size = entry.getReplicaSize(); switch (entry.getState()) { case PRECIOUS: diff --git a/modules/dcache/src/main/java/diskCacheV111/services/TransferManagerHandler.java b/modules/dcache/src/main/java/diskCacheV111/services/TransferManagerHandler.java index fea9117f3b8..73325b6fbfa 100644 --- a/modules/dcache/src/main/java/diskCacheV111/services/TransferManagerHandler.java +++ b/modules/dcache/src/main/java/diskCacheV111/services/TransferManagerHandler.java @@ -21,7 +21,6 @@ import diskCacheV111.vehicles.PnfsGetStorageInfoMessage; import diskCacheV111.vehicles.PnfsGetFileMetaDataMessage; import diskCacheV111.vehicles.PnfsMessage; -import diskCacheV111.vehicles.StorageInfo; import diskCacheV111.util.CacheException; import diskCacheV111.vehicles.DoorRequestInfoMessage; import diskCacheV111.vehicles.PnfsCreateEntryMessage; @@ -43,8 +42,6 @@ import java.io.Serializable; import java.net.URI; import java.net.InetAddress; -import java.io.IOException; -import java.util.Iterator; import java.util.EnumSet; import javax.security.auth.Subject; import diskCacheV111.doors.FTPTransactionLog; @@ -57,7 +54,6 @@ import org.dcache.namespace.FileAttribute; import org.dcache.vehicles.FileAttributes; import org.dcache.acl.enums.AccessMask; -import org.dcache.acl.enums.AccessType; import org.dcache.auth.Subjects; public class TransferManagerHandler implements CellMessageAnswerable @@ -74,7 +70,6 @@ public class TransferManagerHandler implements CellMessageAnswerable private PnfsId pnfsId; private String pnfsIdString; private String remoteUrl; - private StorageInfo storageInfo; transient boolean locked; private String pool; private FTPTransactionLog tlog; @@ -330,7 +325,6 @@ public void createEntryResponseArrived(PnfsCreateEntryMessage create) created = true; manager.persist(this); - storageInfo = create.getStorageInfo(); fileAttributes = create.getFileAttributes(); pnfsId = create.getPnfsId(); pnfsIdString = pnfsId.toString(); @@ -370,17 +364,14 @@ public void storageInfoArrived(PnfsGetStorageInfoMessage storage_info_msg) manager.justRequestedIDs.add(pnfsId); } } - if(storageInfo == null) { - storageInfo = storage_info_msg.getStorageInfo(); - } if(fileAttributes == null) { fileAttributes = storage_info_msg.getFileAttributes(); } - log.debug("storageInfoArrived(uid={} gid={} pnfsid={} storageInfo={} fileAttributes={}", info.getUid(), info.getGid(), - pnfsId, storageInfo, fileAttributes); + log.debug("storageInfoArrived(uid={} gid={} pnfsid={} fileAttributes={}", info.getUid(), info.getGid(), + pnfsId, fileAttributes); selectPool(); } @@ -448,14 +439,12 @@ public void startMoverOnThePool() { PoolIoFileMessage poolMessage = store ? new PoolAcceptFileMessage( pool, - pnfsId, protocol_info , - storageInfo ) : + fileAttributes) : new PoolDeliverFileMessage( pool, - pnfsId, protocol_info , - storageInfo ); + fileAttributes ); if( manager.getIoQueueName() != null ) { poolMessage.setIoQueueName(manager.getIoQueueName()); @@ -795,8 +784,8 @@ public synchronized String toString(boolean long_format) { if(pnfsId != null) { sb.append("\n pnfsId=").append(pnfsId); } - if(storageInfo != null) { - sb.append("\n storageInfo=").append(storageInfo); + if(fileAttributes != null) { + sb.append("\n fileAttributes=").append(fileAttributes); } if(pool != null) { sb.append("\n pool=").append(pool); diff --git a/modules/dcache/src/main/java/diskCacheV111/services/space/Manager.java b/modules/dcache/src/main/java/diskCacheV111/services/space/Manager.java index 32ec01014f8..3fa04f27344 100644 --- a/modules/dcache/src/main/java/diskCacheV111/services/space/Manager.java +++ b/modules/dcache/src/main/java/diskCacheV111/services/space/Manager.java @@ -4217,7 +4217,7 @@ private void transferToBeStarted(PoolAcceptFileMessage poolRequest){ try { File f = getFile(pnfsId); Space s = getSpace(f.getSpaceId()); - StorageInfo info = poolRequest.getStorageInfo(); + StorageInfo info = poolRequest.getFileAttributes().getStorageInfo(); info.setAccessLatency(s.getAccessLatency()); info.isSetAccessLatency(true); info.setRetentionPolicy(s.getRetentionPolicy()); diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java index 5c35b987eb2..92471591841 100755 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/Pool2PoolTransferMsg.java @@ -2,6 +2,15 @@ package diskCacheV111.vehicles; import diskCacheV111.util.PnfsId ; +import org.dcache.namespace.FileAttribute; +import org.dcache.vehicles.FileAttributes; + +import java.io.IOException; +import java.util.EnumSet; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.dcache.namespace.FileAttribute.*; public class Pool2PoolTransferMsg extends PoolMessage { @@ -10,8 +19,14 @@ public class Pool2PoolTransferMsg extends PoolMessage { public final static int PRECIOUS = 1 ; public final static int CACHED = 2 ; + private FileAttributes _fileAttributes; + + @Deprecated // Remove in 2.7 private PnfsId _pnfsId; + + @Deprecated // Remove in 2.7 private StorageInfo _storageInfo; + private String _destinationPoolName; private int _destinationFileStatus = UNDETERMINED ; @@ -19,19 +34,26 @@ public class Pool2PoolTransferMsg extends PoolMessage { public Pool2PoolTransferMsg( String sourcePoolName , String destinationPoolName , - PnfsId pnfsId , - StorageInfo storageInfo ){ - super( sourcePoolName ) ; - _pnfsId = pnfsId ; - _storageInfo = storageInfo ; - _destinationPoolName = destinationPoolName ; - setReplyRequired(true); + FileAttributes fileAttributes){ + super( sourcePoolName ) ; + + checkNotNull(fileAttributes); + checkArgument(fileAttributes.isDefined(EnumSet.of(PNFSID, SIZE))); + + _fileAttributes = fileAttributes; + _pnfsId = fileAttributes.getPnfsId(); + if (fileAttributes.isDefined(STORAGEINFO)) { + _storageInfo = fileAttributes.getStorageInfo(); + } + _destinationPoolName = destinationPoolName ; + setReplyRequired(true); } - public PnfsId getPnfsId(){ - return _pnfsId ; + public PnfsId getPnfsId() + { + return _fileAttributes.getPnfsId(); } - public StorageInfo getStorageInfo(){ return _storageInfo ; } + public String getSourcePoolName(){ return getPoolName() ; } public String getDestinationPoolName(){ return _destinationPoolName ; } @@ -41,10 +63,27 @@ public void setDestinationFileStatus( int status ){ public int getDestinationFileStatus(){ return _destinationFileStatus ; } + + public FileAttributes getFileAttributes() + { + return _fileAttributes; + } + public String toString(){ - return getPoolName()+";pnfsid="+_pnfsId + ";mode="+ + return getPoolName()+";pnfsid=" + _fileAttributes.getPnfsId() + ";mode="+ ( _destinationFileStatus==UNDETERMINED? "Undetermined": ( _destinationFileStatus==PRECIOUS?"Precious":"Cached" )); } + + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException + { + stream.defaultReadObject(); + if (_fileAttributes == null) { + _fileAttributes = new FileAttributes(); + _fileAttributes.setStorageInfo(_storageInfo); + _fileAttributes.setPnfsId(_pnfsId); + } + } } diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolAcceptFileMessage.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolAcceptFileMessage.java index 7ce49934cd4..d67ef002320 100755 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolAcceptFileMessage.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolAcceptFileMessage.java @@ -2,17 +2,15 @@ package diskCacheV111.vehicles; -import diskCacheV111.util.*; +import org.dcache.vehicles.FileAttributes; public class PoolAcceptFileMessage extends PoolIoFileMessage { private static final long serialVersionUID = 7898737438685700742L; public PoolAcceptFileMessage( String pool , - PnfsId pnfsId , ProtocolInfo protocolInfo , - StorageInfo storageInfo ){ - super( pool , pnfsId , protocolInfo , storageInfo ) ; + FileAttributes fileAttributes){ + super( pool , protocolInfo , fileAttributes ) ; } - } diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolDeliverFileMessage.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolDeliverFileMessage.java index 6610e3661a0..274efe18265 100755 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolDeliverFileMessage.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolDeliverFileMessage.java @@ -3,15 +3,15 @@ package diskCacheV111.vehicles; import diskCacheV111.util.* ; +import org.dcache.vehicles.FileAttributes; public class PoolDeliverFileMessage extends PoolIoFileMessage { private static final long serialVersionUID = 1168612224820572770L; public PoolDeliverFileMessage( String pool , - PnfsId pnfsId , ProtocolInfo protocolInfo , - StorageInfo storageInfo ){ - super( pool , pnfsId , protocolInfo , storageInfo ) ; + FileAttributes fileAttributes ){ + super( pool , protocolInfo , fileAttributes ) ; } } diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolFetchFileMessage.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolFetchFileMessage.java index 3feda104487..3d480f9429e 100755 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolFetchFileMessage.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolFetchFileMessage.java @@ -3,6 +3,9 @@ package diskCacheV111.vehicles; import diskCacheV111.util.PnfsId; +import org.dcache.vehicles.FileAttributes; + +import java.io.IOException; /** * restore file from HSM @@ -12,39 +15,39 @@ public class PoolFetchFileMessage extends PoolMessage { private PnfsId _pnfsId; private StorageInfo _storageInfo; + private FileAttributes _fileAttributes; private static final long serialVersionUID = 1856537534158868883L; - public PoolFetchFileMessage(String poolName, StorageInfo storageInfo, - PnfsId pnfsId) { - super(poolName); - _pnfsId = pnfsId; - _storageInfo = storageInfo; - setReplyRequired(true); - } - - public PoolFetchFileMessage(String poolName, StorageInfo storageInfo, - String pnfsId) { + public PoolFetchFileMessage(String poolName, FileAttributes fileAttributes) + { super(poolName); - _pnfsId = new PnfsId(pnfsId); - _storageInfo = storageInfo; + _fileAttributes = fileAttributes; + _pnfsId = fileAttributes.getPnfsId(); + _storageInfo = fileAttributes.getStorageInfo(); setReplyRequired(true); } - public void setPnfsId(PnfsId pnfsId) { - _pnfsId = pnfsId; - } - - public PnfsId getPnfsId() { - return _pnfsId; + public FileAttributes getFileAttributes() + { + return _fileAttributes; } - public void setStorageInfo(StorageInfo storageInfo) { - _storageInfo = storageInfo; + public PnfsId getPnfsId() + { + return _fileAttributes.getPnfsId(); } - public StorageInfo getStorageInfo() { - return _storageInfo; + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException + { + stream.defaultReadObject(); + if (_fileAttributes == null) { + _fileAttributes = new FileAttributes(); + if (_storageInfo != null) { + _fileAttributes.setStorageInfo(_storageInfo); + } + _fileAttributes.setPnfsId(_pnfsId); + } } - } diff --git a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolIoFileMessage.java b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolIoFileMessage.java index 1fae0da6b1c..8efa985dccc 100755 --- a/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolIoFileMessage.java +++ b/modules/dcache/src/main/java/diskCacheV111/vehicles/PoolIoFileMessage.java @@ -1,12 +1,27 @@ package diskCacheV111.vehicles; import diskCacheV111.util.PnfsId; +import org.dcache.vehicles.FileAttributes; + +import java.io.IOException; +import java.util.EnumSet; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.dcache.namespace.FileAttribute.*; public class PoolIoFileMessage extends PoolMessage { + private FileAttributes _fileAttributes; + + @Deprecated // Remove in 2.7 private StorageInfo _storageInfo; + private ProtocolInfo _protocolInfo; + + @Deprecated // Remove in 2.7 private PnfsId _pnfsId; + private boolean _isPool2Pool; private String _ioQueueName; private int _moverId; @@ -16,13 +31,18 @@ public class PoolIoFileMessage extends PoolMessage { private static final long serialVersionUID = -6549886547049510754L; public PoolIoFileMessage( String pool , - PnfsId pnfsId , ProtocolInfo protocolInfo , - StorageInfo storageInfo ){ + FileAttributes fileAttributes ){ super( pool ) ; - _storageInfo = storageInfo ; + + checkNotNull(fileAttributes); + checkArgument(fileAttributes.isDefined( + EnumSet.of(STORAGEINFO, PNFSID))); + + _fileAttributes = fileAttributes; + _storageInfo = fileAttributes.getStorageInfo(); _protocolInfo = protocolInfo ; - _pnfsId = pnfsId ; + _pnfsId = fileAttributes.getPnfsId(); } public PoolIoFileMessage( String pool , @@ -31,9 +51,10 @@ public PoolIoFileMessage( String pool , super( pool ) ; _protocolInfo = protocolInfo ; _pnfsId = pnfsId ; + _fileAttributes = new FileAttributes(); + _fileAttributes.setPnfsId(pnfsId); } - public PnfsId getPnfsId(){ return _pnfsId ; } - public StorageInfo getStorageInfo(){ return _storageInfo ; } + public PnfsId getPnfsId(){ return _fileAttributes.getPnfsId(); } public ProtocolInfo getProtocolInfo(){ return _protocolInfo ; } public boolean isPool2Pool(){ return _isPool2Pool ; } @@ -70,6 +91,24 @@ public String getInitiator() { return _initiator; } + public FileAttributes getFileAttributes() + { + return _fileAttributes; + } + + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException + { + stream.defaultReadObject(); + if (_fileAttributes == null) { + _fileAttributes = new FileAttributes(); + if (_storageInfo != null) { + _fileAttributes.setStorageInfo(_storageInfo); + } + _fileAttributes.setPnfsId(_pnfsId); + } + } + public void setForceSourceMode(boolean forceSourceMode) { _forceSourceMode = forceSourceMode; diff --git a/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/NFSv41ProtocolMover.java b/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/NFSv41ProtocolMover.java index 94c3175fd03..d428693ee88 100644 --- a/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/NFSv41ProtocolMover.java +++ b/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/NFSv41ProtocolMover.java @@ -1,13 +1,12 @@ package org.dcache.chimera.nfsv41.mover; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.dcache.pool.movers.ManualMover; import org.dcache.pool.repository.Allocator; -import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; import dmg.cells.nucleus.CellEndpoint; import org.dcache.pool.movers.IoMode; import org.dcache.pool.repository.RepositoryChannel; @@ -65,8 +64,7 @@ public long getTransferTime() { * SpaceMonitor spaceMonitor, int access) */ @Override - public void runIO(RepositoryChannel raf, ProtocolInfo protocol, - StorageInfo storage, PnfsId pnfsId, + public void runIO(FileAttributes fileAttributes, RepositoryChannel raf, ProtocolInfo protocol, Allocator allocator, IoMode access) throws Exception { diff --git a/modules/dcache/src/main/java/org/dcache/http/HttpProtocol_2.java b/modules/dcache/src/main/java/org/dcache/http/HttpProtocol_2.java index b704ced0d36..004c53b24f5 100644 --- a/modules/dcache/src/main/java/org/dcache/http/HttpProtocol_2.java +++ b/modules/dcache/src/main/java/org/dcache/http/HttpProtocol_2.java @@ -6,7 +6,6 @@ import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; -import java.net.UnknownHostException; import java.util.Enumeration; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -20,6 +19,7 @@ import org.dcache.pool.repository.Allocator; import org.dcache.util.Checksum; import org.dcache.util.NetworkUtils; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +28,6 @@ import diskCacheV111.vehicles.HttpDoorUrlInfoMessage; import diskCacheV111.vehicles.HttpProtocolInfo; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; import dmg.cells.nucleus.CellEndpoint; import dmg.cells.nucleus.CellMessage; import dmg.cells.nucleus.CellPath; @@ -140,10 +139,9 @@ public HttpProtocol_2(CellEndpoint endpoint) { } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access) throws Exception @@ -164,7 +162,7 @@ public void runIO(RepositoryChannel fileChannel, try { UUID uuid = _server.register(_wrappedChannel); InetSocketAddress address = _server.getServerAddress(); - sendAddressToDoor(address.getPort(), uuid, pnfsId); + sendAddressToDoor(address.getPort(), uuid, fileAttributes.getPnfsId()); _server.await(_wrappedChannel, _connectTimeout); } finally { _logger.debug("Shutting down mover"); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/ChecksumScanner.java b/modules/dcache/src/main/java/org/dcache/pool/classic/ChecksumScanner.java index 21abeab3110..9debee116c1 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/ChecksumScanner.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/ChecksumScanner.java @@ -9,6 +9,7 @@ import java.util.Date; import java.util.EnumSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -24,7 +25,6 @@ import diskCacheV111.util.FileNotInCacheException; import diskCacheV111.util.NotInTrashCacheException; import diskCacheV111.util.TimeoutCacheException; -import diskCacheV111.vehicles.StorageInfo; import org.dcache.cells.CellCommandListener; import org.dcache.cells.CellLifeCycleAware; @@ -38,6 +38,7 @@ import org.dcache.pool.repository.Repository.OpenFlags; import org.dcache.pool.repository.ReplicaDescriptor; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,14 +102,14 @@ private Checksum checkFile(File file, double throughputLimit) private Checksum readChecksum(CacheEntry entry) throws CacheException { - StorageInfo info = entry.getStorageInfo(); - String flags = (info == null ? null : info.getKey("flag-c")); - - if (flags == null) { - return _csm.getDefaultChecksumFactory().find(_pnfs.getFileAttributes(entry.getPnfsId(), EnumSet.of(FileAttribute.CHECKSUM)).getChecksums()); + Set checksums; + FileAttributes attributes = entry.getFileAttributes(); + if (attributes.isDefined(FileAttribute.CHECKSUM)) { + checksums = attributes.getChecksums(); } else { - return Checksum.parseChecksum(flags); + checksums = _pnfs.getFileAttributes(entry.getPnfsId(), EnumSet.of(FileAttribute.CHECKSUM)).getChecksums(); } + return _csm.getDefaultChecksumFactory().find(checksums); } private class FullScan extends Singleton diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageHandler2.java b/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageHandler2.java index 1ca6203cd37..6e903cbe0a9 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageHandler2.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageHandler2.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.io.RandomAccessFile; import java.io.BufferedReader; import java.io.StringReader; import java.io.FileNotFoundException; @@ -15,7 +14,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -28,6 +26,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +34,6 @@ import dmg.cells.nucleus.CellMessage; import dmg.cells.nucleus.CellPath; import dmg.cells.nucleus.NoRouteToCellException; -import dmg.cells.nucleus.CellInfo; import diskCacheV111.util.Batchable; import diskCacheV111.util.CacheException; @@ -43,7 +41,6 @@ import diskCacheV111.util.FileNotInCacheException; import diskCacheV111.util.FileInCacheException; import diskCacheV111.util.HsmSet; -import diskCacheV111.util.InconsistentCacheException; import diskCacheV111.util.JobScheduler; import diskCacheV111.util.PnfsHandler; import diskCacheV111.util.PnfsId; @@ -269,9 +266,12 @@ public synchronized void getInfo(PrintWriter pw) } private synchronized String - getSystemCommand(File file, PnfsId pnfsId, StorageInfo storageInfo, + getSystemCommand(File file, FileAttributes fileAttributes, HsmSet.HsmInfo hsm, String direction) { + PnfsId pnfsId = fileAttributes.getPnfsId(); + StorageInfo storageInfo = fileAttributes.getStorageInfo(); + String hsmCommand = hsm.getAttribute("command"); if (hsmCommand == null) { throw new @@ -324,14 +324,13 @@ public JobScheduler getFetchScheduler() return _fetchQueue; } - public synchronized void fetch(PnfsId pnfsId, - StorageInfo storageInfo, + public synchronized void fetch(FileAttributes fileAttributes, CacheFileAvailable callback) throws FileInCacheException, CacheException { assertInitialized(); - FetchThread info = _restorePnfsidList.get(pnfsId); + FetchThread info = _restorePnfsidList.get(fileAttributes.getPnfsId()); if (info != null) { if (callback != null) { @@ -340,14 +339,14 @@ public synchronized void fetch(PnfsId pnfsId, return; } - info = new FetchThread(pnfsId, storageInfo); + info = new FetchThread(fileAttributes); if (callback != null) { info.addCallback(callback); } try { _fetchQueue.add(info); - _restorePnfsidList.put(pnfsId, info); + _restorePnfsidList.put(fileAttributes.getPnfsId(), info); } catch (InvocationTargetException e) { /* This happens when the queued method of the FetchThread * throws an exception. They have been designed not to @@ -395,8 +394,9 @@ public synchronized Info getRestoreInfoByPnfsId(PnfsId pnfsId) * Returns the name of an HSM accessible for this pool and which * contains the given file. Returns null if no such HSM exists. */ - private String findAccessibleLocation(StorageInfo file) + private String findAccessibleLocation(FileAttributes fileAttributes) { + StorageInfo file = fileAttributes.getStorageInfo(); if (file.locations().isEmpty() && _hsmSet.getHsmInstances().contains(file.getHsm())) { // This is for backwards compatibility until all info @@ -413,20 +413,19 @@ private String findAccessibleLocation(StorageInfo file) } private synchronized String - getFetchCommand(File file, PnfsId pnfsId, StorageInfo storageInfo) + getFetchCommand(File file, FileAttributes fileAttributes) { - String instance = findAccessibleLocation(storageInfo); + String instance = findAccessibleLocation(fileAttributes); if (instance == null) { throw new IllegalArgumentException("HSM not defined on this pool: " + - storageInfo.locations()); + fileAttributes.getStorageInfo().locations()); } HsmSet.HsmInfo hsm = _hsmSet.getHsmInfoByName(instance); - _log.debug("getFetchCommand for pnfsid=" + pnfsId + - ";hsm=" + instance + ";si=" + storageInfo); + _log.debug("getFetchCommand for {} on HSM {}", fileAttributes, instance); - return getSystemCommand(file, pnfsId, storageInfo, hsm, "get"); + return getSystemCommand(file, fileAttributes, hsm, "get"); } private class FetchThread extends Info implements Batchable @@ -437,15 +436,15 @@ private class FetchThread extends Info implements Batchable private int _id; private Thread _thread; - public FetchThread(PnfsId pnfsId, StorageInfo storageInfo) + public FetchThread(FileAttributes fileAttributes) throws CacheException, FileInCacheException { - super(pnfsId); + super(fileAttributes.getPnfsId()); String address = _cellName + "@" + _domainName; - _infoMsg = new StorageInfoMessage(address, pnfsId, true); - _infoMsg.setStorageInfo(storageInfo); + _infoMsg = new StorageInfoMessage(address, fileAttributes.getPnfsId(), true); + _infoMsg.setStorageInfo(fileAttributes.getStorageInfo()); - long fileSize = storageInfo.getFileSize(); + long fileSize = fileAttributes.getSize(); _infoMsg.setFileSize(fileSize); @@ -457,8 +456,7 @@ public FetchThread(PnfsId pnfsId, StorageInfo storageInfo) // } List stickyRecords = Collections.emptyList(); - _handle = _repository.createEntry(pnfsId, - storageInfo, + _handle = _repository.createEntry(fileAttributes, EntryState.FROM_STORE, EntryState.CACHED, stickyRecords); @@ -549,7 +547,7 @@ public void run() Exception excep = null; PnfsId pnfsId = getPnfsId(); CacheEntry entry = _handle.getEntry(); - StorageInfo storageInfo = entry.getStorageInfo(); + FileAttributes attributes = entry.getFileAttributes(); try { setThread(Thread.currentThread()); @@ -561,8 +559,8 @@ public void run() _timestamp = now; String fetchCommand = - getFetchCommand(_handle.getFile(), pnfsId, storageInfo); - long fileSize = storageInfo.getFileSize(); + getFetchCommand(_handle.getFile(), attributes); + long fileSize = attributes.getSize(); _log.debug("Waiting for space (" + fileSize + " bytes)"); _handle.allocate(fileSize); @@ -764,10 +762,11 @@ public synchronized void remove(CellMessage message) // the store part // private synchronized String - getStoreCommand(File file, PnfsId pnfsId, StorageInfo storageInfo) + getStoreCommand(File file, FileAttributes fileAttributes) { + StorageInfo storageInfo = fileAttributes.getStorageInfo(); String hsmType = storageInfo.getHsm(); - _log.debug("getStoreCommand for pnfsid=" + pnfsId + + _log.debug("getStoreCommand for pnfsid=" + fileAttributes.getPnfsId() + ";hsm=" + hsmType + ";si=" + storageInfo); List hsms = _hsmSet.getHsmInfoByType(hsmType); if (hsms.isEmpty()) { @@ -780,7 +779,7 @@ public synchronized void remove(CellMessage message) // choice. HsmSet.HsmInfo hsm = hsms.get(0); - return getSystemCommand(file, pnfsId, storageInfo, hsm, "put"); + return getSystemCommand(file, fileAttributes, hsm, "put"); } public synchronized Info getStoreInfoByPnfsId(PnfsId pnfsId) @@ -858,11 +857,6 @@ public String toString() return getPnfsId().toString(); } - public double getTransferRate() - { - return 10.0; - } - @Override public String getClient() { @@ -961,21 +955,22 @@ public void run() throw e; } - StorageInfo storageInfo; Set flags = Collections.emptySet(); ReplicaDescriptor handle = _repository.openEntry(pnfsId, flags); + StorageInfo storageInfo; try { doChecksum(handle); - storageInfo = handle.getEntry().getStorageInfo().clone(); + FileAttributes fileAttributes = handle.getEntry().getFileAttributes(); + storageInfo = fileAttributes.getStorageInfo().clone(); _infoMsg.setStorageInfo(storageInfo); - _infoMsg.setFileSize(storageInfo.getFileSize()); + _infoMsg.setFileSize(fileAttributes.getSize()); long now = System.currentTimeMillis(); _infoMsg.setTimeQueued(now - _timestamp); _timestamp = now; String storeCommand = - getStoreCommand(handle.getFile(), pnfsId, storageInfo); + getStoreCommand(handle.getFile(), fileAttributes); RunSystem run = new RunSystem(storeCommand, _maxLines, _maxStoreRun); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageInterpreter.java b/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageInterpreter.java index dd933c35e23..193658854e7 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageInterpreter.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/HsmStorageInterpreter.java @@ -9,9 +9,9 @@ import dmg.util.Args; import diskCacheV111.util.* ; -import diskCacheV111.vehicles.StorageInfo; import org.dcache.cells.CellCommandListener; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,9 +210,8 @@ public void cacheFileAvailable(PnfsId pnfsId, Throwable ee) { @Override public void run() { try { - StorageInfo si = _pnfs.getStorageInfoByPnfsId(pnfsId).getStorageInfo(); - _storageHandler.fetch(pnfsId, si, - block ? cfa : null); + FileAttributes attributes = _pnfs.getStorageInfoByPnfsId(pnfsId).getFileAttributes(); + _storageHandler.fetch(attributes, block ? cfa : null); } catch (CacheException e) { cfa.cacheFileAvailable(pnfsId, e); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOReadTransfer.java b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOReadTransfer.java index 167d2184e71..fc9a1065539 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOReadTransfer.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOReadTransfer.java @@ -16,6 +16,7 @@ import org.dcache.pool.repository.FileRepositoryChannel; import org.dcache.pool.repository.ReplicaDescriptor; import org.dcache.pool.repository.Repository.OpenFlags; +import org.dcache.vehicles.FileAttributes; /** * Encapsulates a read transfer, that is, sending a file. It acts as a @@ -27,17 +28,16 @@ public class PoolIOReadTransfer private final ReplicaDescriptor _handle; private final long _size; - public PoolIOReadTransfer(PnfsId pnfsId, + public PoolIOReadTransfer(FileAttributes fileAttributes, ProtocolInfo protocolInfo, Subject subject, - StorageInfo storageInfo, MoverProtocol mover, Set openFlags, Repository repository) throws CacheException, InterruptedException { - super(pnfsId, protocolInfo, subject, storageInfo, mover); - _handle = repository.openEntry(pnfsId, openFlags); + super(fileAttributes, protocolInfo, subject, mover); + _handle = repository.openEntry(fileAttributes.getPnfsId(), openFlags); _size = _handle.getFile().length(); } @@ -51,10 +51,9 @@ public void transfer() // say("Trying to open " + file); try (RepositoryChannel fileIoChannel = new FileRepositoryChannel(file, "r")) { - _mover.runIO(fileIoChannel, + _mover.runIO(_fileAttributes, + fileIoChannel, _protocolInfo, - _storageInfo, - _pnfsId, null, IoMode.READ); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIORequest.java b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIORequest.java index 4edfbd46dca..5df08a70f07 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIORequest.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIORequest.java @@ -1,11 +1,10 @@ package org.dcache.pool.classic; -import diskCacheV111.util.PnfsId; import diskCacheV111.util.CacheException; +import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.DoorTransferFinishedMessage; import diskCacheV111.vehicles.MoverInfoMessage; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; import dmg.cells.nucleus.CellEndpoint; import dmg.cells.nucleus.CellMessage; import dmg.cells.nucleus.CellPath; @@ -13,6 +12,7 @@ import java.io.IOException; import java.nio.channels.CompletionHandler; import org.dcache.pool.FaultListener; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.dcache.pool.classic.IoRequestState.*; @@ -69,8 +69,6 @@ public class PoolIORequest implements IoProcessable { * @param door the cell path to the cell requesting the * transfer * @param poolName the name of the pool - * @param pool the name of the pool - * @param cellEndpoint the cellEndpoint of the pool * @param queue the name of the queue used for the request * @param cellEndpoint the cellEndpoint of the pool * @param billingCell the CellPath of the billing cell @@ -91,12 +89,12 @@ public PoolIORequest(PoolIOTransfer transfer, long id, String initiator, void sendBillingMessage() { MoverInfoMessage info = new MoverInfoMessage(_cellEndpoint.getCellInfo().getCellName() + "@" + _cellEndpoint.getCellInfo().getDomainName(), - getPnfsId()); + getFileAttributes().getPnfsId()); info.setSubject(_transfer.getSubject()); info.setInitiator(_initiator); info.setFileCreated(_transfer instanceof PoolIOWriteTransfer); - info.setStorageInfo(getStorageInfo()); + info.setStorageInfo(getFileAttributes().getStorageInfo()); info.setFileSize(_transfer.getFileSize()); info.setResult(_errorCode, _errorMessage); info.setTransferAttributes(getBytesTransferred(), @@ -113,9 +111,9 @@ void sendBillingMessage() { void sendFinished() { DoorTransferFinishedMessage finished = new DoorTransferFinishedMessage(getClientId(), - getPnfsId(), + getFileAttributes().getPnfsId(), getProtocolInfo(), - getStorageInfo(), + getFileAttributes().getStorageInfo(), _poolName); finished.setIoQueueName(_queue); if (_errorCode == 0) { @@ -135,8 +133,8 @@ protected ProtocolInfo getProtocolInfo() { return _transfer.getProtocolInfo(); } - protected StorageInfo getStorageInfo() { - return _transfer.getStorageInfo(); + protected FileAttributes getFileAttributes() { + return _transfer.getFileAttributes(); } @Override @@ -160,7 +158,7 @@ public long getLastTransferred() { } public PnfsId getPnfsId() { - return _transfer.getPnfsId(); + return getFileAttributes().getPnfsId(); } @Override diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOTransfer.java b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOTransfer.java index 8fdae0b848d..3301953f5e3 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOTransfer.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOTransfer.java @@ -1,17 +1,14 @@ package org.dcache.pool.classic; -import diskCacheV111.util.PnfsId; import diskCacheV111.util.CacheException; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; import org.dcache.pool.movers.MoverProtocol; -import dmg.cells.nucleus.NoRouteToCellException; - import java.io.IOException; import javax.security.auth.Subject; import org.dcache.pool.movers.IoMode; import org.dcache.pool.repository.ReplicaDescriptor; +import org.dcache.vehicles.FileAttributes; /** * Abstract bridge between repository and movers. PoolIOTransfer @@ -31,27 +28,24 @@ public abstract class PoolIOTransfer { protected final MoverProtocol _mover; - protected final PnfsId _pnfsId; + protected final FileAttributes _fileAttributes; protected final ProtocolInfo _protocolInfo; - protected final StorageInfo _storageInfo; protected final Subject _subject; - public PoolIOTransfer(PnfsId pnfsId, + public PoolIOTransfer(FileAttributes fileAttributes, ProtocolInfo protocolInfo, Subject subject, - StorageInfo storageInfo, MoverProtocol mover) { - _pnfsId = pnfsId; + _fileAttributes = fileAttributes; _protocolInfo = protocolInfo; _subject = subject; - _storageInfo = storageInfo; _mover = mover; } - public StorageInfo getStorageInfo() + public FileAttributes getFileAttributes() { - return _storageInfo; + return _fileAttributes; } public ProtocolInfo getProtocolInfo() @@ -59,11 +53,6 @@ public ProtocolInfo getProtocolInfo() return _protocolInfo; } - public PnfsId getPnfsId() - { - return _pnfsId; - } - public long getTransferTime() { return _mover.getTransferTime(); @@ -94,7 +83,7 @@ public MoverProtocol getMover() { public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(_pnfsId); + sb.append(_fileAttributes.getPnfsId()); sb.append(" h={") .append(_mover.toString()) .append("} bytes=").append(getBytesTransferred()) diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOWriteTransfer.java b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOWriteTransfer.java index 222dd670699..30d2806303c 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOWriteTransfer.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolIOWriteTransfer.java @@ -5,7 +5,6 @@ import org.dcache.pool.repository.EntryState; import org.dcache.pool.repository.Repository; import diskCacheV111.util.PnfsId; -import diskCacheV111.util.FileNotInCacheException; import org.dcache.util.Checksum; import diskCacheV111.util.ChecksumFactory; import diskCacheV111.util.AccessLatency; @@ -17,8 +16,6 @@ import org.dcache.pool.movers.MoverProtocol; import org.dcache.pool.movers.ChecksumMover; -import dmg.cells.nucleus.NoRouteToCellException; - import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,6 +27,7 @@ import org.dcache.pool.repository.RepositoryChannel; import org.dcache.pool.repository.FileRepositoryChannel; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,10 +72,9 @@ public static EntryState getTargetState(StorageInfo info) } } - public PoolIOWriteTransfer(PnfsId pnfsId, + public PoolIOWriteTransfer(FileAttributes fileAttributes, ProtocolInfo protocolInfo, Subject subject, - StorageInfo storageInfo, MoverProtocol mover, Repository repository, ChecksumModuleV1 checksumModule, @@ -85,11 +82,10 @@ public PoolIOWriteTransfer(PnfsId pnfsId, List stickyRecords) throws FileInCacheException, IOException { - super(pnfsId, protocolInfo, subject, storageInfo, mover); + super(fileAttributes, protocolInfo, subject, mover); _checksumModule = checksumModule; - _handle = repository.createEntry(pnfsId, - _storageInfo, + _handle = repository.createEntry(fileAttributes, EntryState.FROM_CLIENT, targetState, stickyRecords); @@ -100,10 +96,9 @@ public PoolIOWriteTransfer(PnfsId pnfsId, private void runMover(RepositoryChannel fileIoChannel) throws Exception { - _mover.runIO(fileIoChannel, + _mover.runIO(_fileAttributes, + fileIoChannel, _protocolInfo, - _storageInfo, - _pnfsId, _handle, IoMode.WRITE); } @@ -173,7 +168,7 @@ public void close() _checksumFactory = _checksumModule.getDefaultChecksumFactory(); } - _checksumModule.setMoverChecksums(_pnfsId, + _checksumModule.setMoverChecksums(_fileAttributes.getPnfsId(), _file, _checksumFactory, clientChecksum, @@ -185,7 +180,7 @@ public void close() /* Temporary workaround to ensure that the correct size is * logged in billing and send back to the door. */ - _storageInfo.setFileSize(getFileSize()); + _fileAttributes.getStorageInfo().setFileSize(getFileSize()); } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolV4.java b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolV4.java index b3da1fd7889..fcd25bb886c 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/PoolV4.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/PoolV4.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collections; @@ -55,7 +54,6 @@ import diskCacheV111.util.PnfsId; import diskCacheV111.util.UnitInteger; import diskCacheV111.vehicles.DCapProtocolInfo; -import diskCacheV111.vehicles.InfoMessage; import diskCacheV111.vehicles.IoJobInfo; import diskCacheV111.vehicles.JobInfo; import diskCacheV111.vehicles.Message; @@ -577,7 +575,7 @@ public void stateChanged(StateChangeEvent event) RemoveFileInfoMessage msg = new RemoveFileInfoMessage(source, entry.getPnfsId()); msg.setFileSize(entry.getReplicaSize()); - msg.setStorageInfo(entry.getStorageInfo()); + msg.setStorageInfo(entry.getFileAttributes().getStorageInfo()); sendMessage(new CellMessage(_billingCell, msg)); } catch (NoRouteToCellException e) { _log.error("Failed to send message to " + _billingCell + ": " @@ -687,12 +685,13 @@ private int queueIoRequest(PoolIoFileMessage message, private void ioFile(CellMessage envelope, PoolIoFileMessage message) { - PnfsId pnfsId = message.getPnfsId(); + FileAttributes attributes = message.getFileAttributes(); + PnfsId pnfsId = attributes.getPnfsId(); try { long id = message.getId(); ProtocolInfo pi = message.getProtocolInfo(); Subject subject = message.getSubject(); - StorageInfo si = message.getStorageInfo(); + StorageInfo si = attributes.getStorageInfo(); String initiator = message.getInitiator(); String pool = message.getPoolName(); String queueName = message.getIoQueueName(); @@ -749,12 +748,12 @@ private void ioFile(CellMessage envelope, PoolIoFileMessage message) EntryState targetState = _replicaStatePolicy.getTargetState(si); transfer = - new PoolIOWriteTransfer(pnfsId, pi, subject, si, mover, _repository, + new PoolIOWriteTransfer(attributes, pi, subject, mover, _repository, _checksumModule, targetState, stickyRecords); } else { transfer = - new PoolIOReadTransfer(pnfsId, pi, subject, si, mover, openFlags, _repository); + new PoolIOReadTransfer(attributes, pi, subject, mover, openFlags, _repository); } try { source.revert(); @@ -917,7 +916,7 @@ private void _initiateReplication(CacheEntry entry, String source) throws NoRouteToCellException { PnfsId pnfsId = entry.getPnfsId(); - StorageInfo storageInfo = entry.getStorageInfo().clone(); + StorageInfo storageInfo = entry.getFileAttributes().getStorageInfo().clone(); storageInfo.setKey("replication.source", source); @@ -1159,8 +1158,7 @@ public DelayedReply messageArrived(Pool2PoolTransferMsg msg) } String poolName = msg.getPoolName(); - PnfsId pnfsId = msg.getPnfsId(); - StorageInfo storageInfo = msg.getStorageInfo(); + FileAttributes fileAttributes = msg.getFileAttributes(); CompanionFileAvailableCallback callback = new CompanionFileAvailableCallback(msg); @@ -1176,7 +1174,7 @@ public DelayedReply messageArrived(Pool2PoolTransferMsg msg) } List stickyRecords = Collections.emptyList(); - _p2pClient.newCompanion(pnfsId, poolName, storageInfo, + _p2pClient.newCompanion(poolName, fileAttributes, targetState, stickyRecords, callback, false); return callback; } @@ -1194,14 +1192,15 @@ public Object messageArrived(PoolFetchFileMessage msg) throw new CacheException(CacheException.POOL_DISABLED, "Pool has no tape backend"); } - PnfsId pnfsId = msg.getPnfsId(); - StorageInfo storageInfo = msg.getStorageInfo(); + FileAttributes fileAttributes = msg.getFileAttributes(); + PnfsId pnfsId = fileAttributes.getPnfsId(); + StorageInfo storageInfo = fileAttributes.getStorageInfo(); _log.info("Pool " + _poolName + " asked to fetch file " + pnfsId + " (hsm=" + storageInfo.getHsm() + ")"); try { ReplyToPoolFetch reply = new ReplyToPoolFetch(msg); - _storageHandler.fetch(pnfsId, storageInfo, reply); + _storageHandler.fetch(fileAttributes, reply); return reply; } catch (FileInCacheException e) { _log.warn("Pool already contains replica"); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/SpaceSweeper2.java b/modules/dcache/src/main/java/org/dcache/pool/classic/SpaceSweeper2.java index 1373a35db57..4200d9e0285 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/SpaceSweeper2.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/SpaceSweeper2.java @@ -8,6 +8,7 @@ import diskCacheV111.vehicles.StorageInfo; import org.dcache.cells.CellCommandListener; import org.dcache.cells.CellSetupProvider; +import org.dcache.namespace.FileAttribute; import org.dcache.pool.repository.Account; import org.dcache.pool.repository.Repository; import org.dcache.pool.repository.StateChangeListener; @@ -31,6 +32,7 @@ import java.text.SimpleDateFormat; import java.io.PrintWriter; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -255,9 +257,9 @@ public String ac_sweeper_ls(Args args) sb.append(__format.format(new Date(entry.getCreationTime()))).append(" "); sb.append(__format.format(new Date(entry.getLastAccessTime()))).append(" "); if (s) { - StorageInfo info = entry.getStorageInfo(); - if (info != null) { - sb.append("\n ").append(info); + FileAttributes attributes = entry.getFileAttributes(); + if (attributes.isDefined(FileAttribute.STORAGEINFO)) { + sb.append("\n ").append(attributes.getStorageInfo()); } } sb.append("\n"); diff --git a/modules/dcache/src/main/java/org/dcache/pool/classic/StorageClassContainer.java b/modules/dcache/src/main/java/org/dcache/pool/classic/StorageClassContainer.java index 4a2617dee7f..51dff17f358 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/classic/StorageClassContainer.java +++ b/modules/dcache/src/main/java/org/dcache/pool/classic/StorageClassContainer.java @@ -166,8 +166,9 @@ public synchronized boolean addCacheEntry(PnfsId id) throws CacheException, InterruptedException { CacheEntry entry = _repository.getEntry(id); - String storageClass = entry.getStorageInfo().getStorageClass(); - String hsmName = entry.getStorageInfo().getHsm().toLowerCase(); + StorageInfo storageInfo = entry.getFileAttributes().getStorageInfo(); + String storageClass = storageInfo.getStorageClass(); + String hsmName = storageInfo.getHsm().toLowerCase(); String composedName = storageClass+"@"+hsmName; synchronized (_storageClassLock) { @@ -338,7 +339,7 @@ public String ac_queue_ls_queue(Args args) try { CacheEntry info = _repository.getEntry(id); long time = info.getLastAccessTime(); - StorageInfo sinfo = info.getStorageInfo(); + StorageInfo sinfo = info.getFileAttributes().getStorageInfo(); String sclass = sinfo.getStorageClass(); String hsm = sinfo.getHsm(); String cclass = sinfo.getCacheClass(); @@ -363,7 +364,7 @@ public String ac_queue_ls_queue(Args args) } CacheEntry info = _repository.getEntry(id); long time = info.getLastAccessTime(); - StorageInfo sinfo = info.getStorageInfo(); + StorageInfo sinfo = info.getFileAttributes().getStorageInfo(); String sclass = sinfo.getStorageClass(); String hsm = sinfo.getHsm(); String cclass = sinfo.getCacheClass(); diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/AccessLatencyFilter.java b/modules/dcache/src/main/java/org/dcache/pool/migration/AccessLatencyFilter.java index 886188d071e..f802f1ee231 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/AccessLatencyFilter.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/AccessLatencyFilter.java @@ -1,9 +1,11 @@ package org.dcache.pool.migration; +import org.dcache.namespace.FileAttribute; import org.dcache.pool.repository.CacheEntry; import diskCacheV111.util.AccessLatency; import diskCacheV111.vehicles.StorageInfo; +import org.dcache.vehicles.FileAttributes; /** * Repository entry filter which only accepts files with a certain @@ -21,7 +23,7 @@ public AccessLatencyFilter(AccessLatency accessLatency) @Override public boolean accept(CacheEntry entry) { - StorageInfo info = entry.getStorageInfo(); - return info != null && _accessLatency.equals(info.getAccessLatency()); + FileAttributes attributes = entry.getFileAttributes(); + return attributes.isDefined(FileAttribute.ACCESS_LATENCY) && _accessLatency.equals(attributes.getAccessLatency()); } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/MigrationModuleServer.java b/modules/dcache/src/main/java/org/dcache/pool/migration/MigrationModuleServer.java index 39c0bc91825..1ffaca8e3a1 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/MigrationModuleServer.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/MigrationModuleServer.java @@ -36,6 +36,7 @@ import dmg.cells.nucleus.CellPath; import dmg.cells.nucleus.NoRouteToCellException; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -176,7 +177,7 @@ private class Request implements CacheFileAvailable, Runnable private final CellPath _requestor; private final UUID _uuid; private final PnfsId _pnfsId; - private final StorageInfo _storageInfo; + private final FileAttributes _fileAttributes; private final List _stickyRecords; private final EntryState _targetState; private final String _pool; @@ -189,7 +190,7 @@ public Request(CellPath requestor, PoolMigrationCopyReplicaMessage message) { _requestor = requestor; _pnfsId = message.getPnfsId(); - _storageInfo = message.getStorageInfo(); + _fileAttributes = message.getFileAttributes(); _stickyRecords = message.getStickyRecords(); _targetState = message.getState(); _pool = message.getPool(); @@ -222,7 +223,7 @@ public synchronized void start() { EntryState state = _repository.getState(_pnfsId); if (state == EntryState.NEW) { - _companion = _p2p.newCompanion(_pnfsId, _pool, _storageInfo, + _companion = _p2p.newCompanion(_pool, _fileAttributes, _targetState, _stickyRecords, this, _forceSourceMode); } else { diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationCopyReplicaMessage.java b/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationCopyReplicaMessage.java index 63bcf4535ea..ca099b06bcb 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationCopyReplicaMessage.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationCopyReplicaMessage.java @@ -1,13 +1,18 @@ package org.dcache.pool.migration; +import java.io.IOException; import java.util.List; import java.util.UUID; +import com.google.common.base.Preconditions; import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.StorageInfo; import org.dcache.pool.repository.EntryState; import org.dcache.pool.repository.StickyRecord; +import org.dcache.vehicles.FileAttributes; + +import static com.google.common.base.Preconditions.*; /** * MigrationModuleServer message to request that a replica is @@ -17,33 +22,36 @@ public class PoolMigrationCopyReplicaMessage extends PoolMigrationMessage { private static final long serialVersionUID = 6328444770149191656L; - private final StorageInfo _storageInfo; + private FileAttributes _fileAttributes; + + @Deprecated // Remove in 2.7 + private StorageInfo _storageInfo; + private final EntryState _state; private final List _stickyRecords; private final boolean _computeChecksumOnUpdate; private final boolean _forceSourceMode; public PoolMigrationCopyReplicaMessage(UUID uuid, String pool, - PnfsId pnfsId, - StorageInfo storageInfo, + FileAttributes fileAttributes, EntryState state, List stickyRecords, boolean computeChecksumOnUpdate, boolean forceSourceMode) { - super(uuid, pool, pnfsId); - _storageInfo = storageInfo; + super(uuid, pool, fileAttributes.getPnfsId()); + + checkNotNull(state); + checkNotNull(stickyRecords); + + _fileAttributes = fileAttributes; + _storageInfo = fileAttributes.getStorageInfo(); _state = state; _stickyRecords = stickyRecords; _computeChecksumOnUpdate = computeChecksumOnUpdate; _forceSourceMode = forceSourceMode; } - public StorageInfo getStorageInfo() - { - return _storageInfo; - } - public EntryState getState() { return _state; @@ -59,8 +67,24 @@ public boolean getComputeChecksumOnUpdate() return _computeChecksumOnUpdate; } + public FileAttributes getFileAttributes() + { + return _fileAttributes; + } + public boolean isForceSourceMode() { return _forceSourceMode; } + + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException + { + stream.defaultReadObject(); + if (_fileAttributes == null) { + _fileAttributes = new FileAttributes(); + _fileAttributes.setStorageInfo(_storageInfo); + _fileAttributes.setPnfsId(getPnfsId()); + } + } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationMessage.java b/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationMessage.java index 721d6a68ad5..6032d35a63a 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationMessage.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/PoolMigrationMessage.java @@ -4,6 +4,8 @@ import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.Message; +import static com.google.common.base.Preconditions.checkNotNull; + /** * Base class for MigrationModuleServer messages. */ @@ -17,6 +19,10 @@ public class PoolMigrationMessage extends Message public PoolMigrationMessage(UUID uuid, String pool, PnfsId pnfsId) { + checkNotNull(uuid); + checkNotNull(pool); + checkNotNull(pnfsId); + _uuid = uuid; _pool = pool; _pnfsId = pnfsId; diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/RetentionPolicyFilter.java b/modules/dcache/src/main/java/org/dcache/pool/migration/RetentionPolicyFilter.java index b12bfb71d75..a58c6c42495 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/RetentionPolicyFilter.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/RetentionPolicyFilter.java @@ -1,9 +1,11 @@ package org.dcache.pool.migration; +import org.dcache.namespace.FileAttribute; import org.dcache.pool.repository.CacheEntry; import diskCacheV111.util.RetentionPolicy; import diskCacheV111.vehicles.StorageInfo; +import org.dcache.vehicles.FileAttributes; /** * Repository entry filter which only accepts files with a certain @@ -21,7 +23,8 @@ public RetentionPolicyFilter(RetentionPolicy retentionPolicy) @Override public boolean accept(CacheEntry entry) { - StorageInfo info = entry.getStorageInfo(); - return info != null && _retentionPolicy.equals(info.getRetentionPolicy()); + FileAttributes fileAttributes = entry.getFileAttributes(); + return fileAttributes.isDefined(FileAttribute.RETENTION_POLICY) && + _retentionPolicy.equals(fileAttributes.getRetentionPolicy()); } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/StorageClassFilter.java b/modules/dcache/src/main/java/org/dcache/pool/migration/StorageClassFilter.java index 39739e1e845..3defc24f9a6 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/StorageClassFilter.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/StorageClassFilter.java @@ -19,7 +19,7 @@ public StorageClassFilter(String sc) @Override public boolean accept(CacheEntry entry) { - StorageInfo info = entry.getStorageInfo(); - return info != null && _sc.equals(info.getStorageClass()); + StorageInfo info = entry.getFileAttributes().getStorageInfo(); + return _sc.equals(info.getStorageClass()); } } diff --git a/modules/dcache/src/main/java/org/dcache/pool/migration/Task.java b/modules/dcache/src/main/java/org/dcache/pool/migration/Task.java index abd9d8dd76c..1c146c00ff0 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/migration/Task.java +++ b/modules/dcache/src/main/java/org/dcache/pool/migration/Task.java @@ -36,6 +36,7 @@ import dmg.cells.nucleus.CellMessage; import dmg.cells.nucleus.CellMessageAnswerable; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -317,14 +318,12 @@ public void run() { private synchronized void initiateCopy(CellPath target) { - PnfsId pnfsId = _entry.getPnfsId(); - StorageInfo storageInfo = _entry.getStorageInfo(); + FileAttributes fileAttributes = _entry.getFileAttributes(); _target = target; _pool.send(_target, new PoolMigrationCopyReplicaMessage(_uuid, _source, - pnfsId, - storageInfo, + fileAttributes, getTargetState(), getTargetStickyRecords(), _definition.computeChecksumOnUpdate, diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/DCapClientProtocol_1.java b/modules/dcache/src/main/java/org/dcache/pool/movers/DCapClientProtocol_1.java index 9ce9b704a4d..ab70bfc6de2 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/DCapClientProtocol_1.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/DCapClientProtocol_1.java @@ -14,6 +14,7 @@ import diskCacheV111.util.CacheException; import org.dcache.pool.repository.Allocator; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,14 +61,15 @@ private void esay(Throwable t) } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol , - StorageInfo storage , - PnfsId pnfsId , Allocator allocator , IoMode access) throws Exception { + PnfsId pnfsId = fileAttributes.getPnfsId(); + StorageInfo storage = fileAttributes.getStorageInfo(); say("runIO()\n\tprotocol="+ protocol+",\n\tStorageInfo="+storage+",\n\tPnfsId="+pnfsId+ ",\n\taccess ="+access); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java index 4101ab35150..d9c290e24e6 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/MoverProtocol.java @@ -1,10 +1,9 @@ package org.dcache.pool.movers; import diskCacheV111.vehicles.ProtocolInfo; -import diskCacheV111.vehicles.StorageInfo; -import diskCacheV111.util.PnfsId; import org.dcache.pool.repository.Allocator; import org.dcache.pool.repository.RepositoryChannel; +import org.dcache.vehicles.FileAttributes; public interface MoverProtocol { @@ -13,10 +12,9 @@ public interface MoverProtocol * @param allocator Space allocator. May be null for a read-only * transfer. */ - public void runIO(RepositoryChannel diskFile, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel diskFile, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access) throws Exception; diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteGsiftpTransferProtocol_1.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteGsiftpTransferProtocol_1.java index 1e799e2d184..ef3639f0c27 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteGsiftpTransferProtocol_1.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteGsiftpTransferProtocol_1.java @@ -81,15 +81,10 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import diskCacheV111.vehicles.StorageInfo; import diskCacheV111.vehicles.transferManager.RemoteGsiftpTransferProtocolInfo; import dmg.cells.nucleus.CellEndpoint; -import dmg.cells.nucleus.CellMessage; import dmg.cells.nucleus.CellPath; import dmg.cells.nucleus.NoRouteToCellException; import java.io.IOException; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.EnumSet; import java.util.Set; import java.security.MessageDigest; @@ -97,18 +92,11 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.pool.repository.RepositoryChannel; import org.dcache.srm.util.GridftpClient.IDiskDataSourceSink; import org.dcache.srm.util.GridftpClient; -import org.dcache.srm.security.SslGsiSocketFactory; -import org.dcache.util.NetworkUtils; import org.globus.ftp.Buffer; import org.globus.ftp.exception.ClientException; import org.globus.ftp.exception.ServerException; -import org.globus.gsi.gssapi.auth.Authorization; -import org.globus.gsi.gssapi.net.GssSocket; -import org.globus.gsi.gssapi.net.impl.GSIGssSocket; import org.globus.gsi.CredentialException; import org.globus.util.GlobusURL; -import org.ietf.jgss.GSSContext; -import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; public class RemoteGsiftpTransferProtocol_1 @@ -169,10 +157,9 @@ private void createFtpClient(RemoteGsiftpTransferProtocolInfo protocolInfo) } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId, Allocator allocator, IoMode access) throws CacheException, IOException, @@ -180,10 +167,11 @@ public void runIO(RepositoryChannel fileChannel, ServerException, ClientException, CredentialException, GSSException { - _pnfsId = pnfsId; + _pnfsId = fileAttributes.getPnfsId(); + StorageInfo storage = fileAttributes.getStorageInfo(); _log.debug("runIO()\n\tprotocol=" + protocol + ",\n\tStorageInfo=" + storage + ",\n\tPnfsId=" - + pnfsId + ",\n\taccess =" + + _pnfsId + ",\n\taccess =" + access ); if (!(protocol instanceof RemoteGsiftpTransferProtocolInfo)) { throw new CacheException("protocol info is not RemoteGsiftpransferProtocolInfo"); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol_1.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol_1.java index 01e6ecce71f..74458a829e6 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol_1.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol_1.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import org.dcache.pool.repository.RepositoryChannel; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,14 +55,15 @@ private void say(String str){ } @Override - public void runIO(RepositoryChannel fileChannel, + public void runIO(FileAttributes fileAttributes, + RepositoryChannel fileChannel, ProtocolInfo protocol, - StorageInfo storage, - PnfsId pnfsId , Allocator allocator, IoMode access) throws Exception { + PnfsId pnfsId = fileAttributes.getPnfsId(); + StorageInfo storage = fileAttributes.getStorageInfo(); say("runIO()\n\tprotocol="+ protocol+",\n\tStorageInfo="+storage+",\n\tPnfsId="+pnfsId+ ",\n\taccess ="+ access); diff --git a/modules/dcache/src/main/java/org/dcache/pool/p2p/Companion.java b/modules/dcache/src/main/java/org/dcache/pool/p2p/Companion.java index 58f88fad7a8..0bc5b657666 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/p2p/Companion.java +++ b/modules/dcache/src/main/java/org/dcache/pool/p2p/Companion.java @@ -1,5 +1,6 @@ package org.dcache.pool.p2p; +import java.util.EnumSet; import java.util.List; import java.util.ArrayList; import java.util.concurrent.ScheduledExecutorService; @@ -15,6 +16,7 @@ import org.dcache.cells.CellStub; import org.dcache.cells.AbstractMessageCallback; +import org.dcache.namespace.FileAttribute; import org.dcache.pool.repository.EntryState; import org.dcache.pool.repository.StickyRecord; import org.dcache.pool.repository.ReplicaDescriptor; @@ -29,8 +31,6 @@ import org.dcache.util.ChecksumType; import org.dcache.util.FireAndForgetTask; import diskCacheV111.util.Adler32; -import diskCacheV111.vehicles.StorageInfo; -import diskCacheV111.vehicles.PnfsGetStorageInfoMessage; import diskCacheV111.vehicles.PoolDeliverFileMessage; import diskCacheV111.vehicles.DoorTransferFinishedMessage; import diskCacheV111.vehicles.HttpDoorUrlInfoMessage; @@ -38,17 +38,18 @@ import diskCacheV111.vehicles.IoJobInfo; import dmg.cells.nucleus.CellPath; - import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.net.MalformedURLException; import java.util.concurrent.atomic.AtomicInteger; +import org.dcache.vehicles.FileAttributes; +import org.dcache.vehicles.PnfsGetFileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; /** * Encapsulates the tasks to be performed on the destination of a pool @@ -75,7 +76,6 @@ class Companion private final InetAddress _address; private final Repository _repository; private final ChecksumModuleV1 _checksumModule; - private final PnfsId _pnfsId; private final String _sourcePoolName; private final String _destinationPoolCellname; private final String _destinationPoolCellDomainName; @@ -94,7 +94,7 @@ class Companion private final int _id; /** Storage info for the file. */ - private StorageInfo _storageInfo; + private FileAttributes _fileAttributes; /** Description of error condition, or null. */ private Object _error; @@ -118,9 +118,8 @@ class Companion * compute checksums * @param pnfs Cell stub for PNFS communication * @param pool Cell stub for pool communication - * @param pnfsId PNFS ID of replica to copy - * @param storageInfo Storage info of the file. May be null. - * @param poolName Name of source pool + * @param fileAttributes File attributes of the file. May be null. + * @param sourcePoolName Name of source pool * @param destinationPoolCellname Cell name of the destination pool * @param destinationPoolCellDomainName Domain name of the destination pool * @param targetState The repository state used for the new replica @@ -134,8 +133,7 @@ class Companion ChecksumModuleV1 checksumModule, CellStub pnfs, CellStub pool, - PnfsId pnfsId, - StorageInfo storageInfo, + FileAttributes fileAttributes, String sourcePoolName, String destinationPoolCellname, String destinationPoolCellDomainName, @@ -152,11 +150,11 @@ class Companion _checksumModule = checksumModule; _pnfs = pnfs; _pool = pool; - _pnfsId = pnfsId; _sourcePoolName = sourcePoolName; - checkArgument(destinationPoolCellname != null, "Destination pool name is unknown. Aborting the request."); - checkArgument(destinationPoolCellDomainName != null, "Destination domain name is unknown. Aborting the request"); + checkNotNull(fileAttributes); + checkNotNull(destinationPoolCellname, "Destination pool name is unknown. Aborting the request."); + checkNotNull(destinationPoolCellDomainName, "Destination domain name is unknown. Aborting the request"); _destinationPoolCellname = destinationPoolCellname; _destinationPoolCellDomainName = destinationPoolCellDomainName; @@ -165,8 +163,8 @@ class Companion _forceSourceMode = forceSourceMode; _targetState = targetState; _stickyRecords = new ArrayList<>(stickyRecords); - if (storageInfo != null) { - setStorageInfo(storageInfo); + if (fileAttributes != null) { + setFileAttributes(fileAttributes); } _id = _nextId.getAndIncrement(); @@ -189,7 +187,7 @@ synchronized public int getId() */ synchronized public PnfsId getPnfsId() { - return _pnfsId; + return _fileAttributes.getPnfsId(); } synchronized public long getPingPeriod() @@ -212,7 +210,7 @@ synchronized public String toString() return "" + _id + " " - + _pnfsId + + getPnfsId() + " " + _fsm.getState(); } @@ -260,7 +258,7 @@ private void transfer(String uri) try { File file = handle.getFile(); CacheEntry entry = handle.getEntry(); - long size = entry.getStorageInfo().getFileSize(); + long size = entry.getFileAttributes().getSize(); handle.allocate(size); @@ -296,8 +294,7 @@ private void transfer(String uri) private ReplicaDescriptor createReplicaEntry() throws FileInCacheException { - return _repository.createEntry(_pnfsId, - _storageInfo, + return _repository.createEntry(_fileAttributes, EntryState.FROM_POOL, _targetState, _stickyRecords); @@ -327,7 +324,7 @@ private void setChecksum(File file, MessageDigest digest) checksum = new Checksum(ChecksumType.ADLER32, digest.digest()); } - _checksumModule.setMoverChecksums(_pnfsId, + _checksumModule.setMoverChecksums(getPnfsId(), file, _checksumModule.getDefaultChecksumFactory(), null, @@ -384,28 +381,29 @@ synchronized void clearError() /** Returns true iff storage info is known. */ synchronized boolean hasStorageInfo() { - return _storageInfo != null; + return _fileAttributes.isDefined(FileAttribute.STORAGEINFO); } - /** Asynchronously retrieves the storage info. */ - synchronized void getStorageInfo() + /** Asynchronously retrieves the file attributes. */ + synchronized void fetchFileAttributes() { - _pnfs.send(new PnfsGetStorageInfoMessage(_pnfsId), - PnfsGetStorageInfoMessage.class, - new Callback() + _pnfs.send(new PnfsGetFileAttributes(getPnfsId(), + EnumSet.of(FileAttribute.PNFSID, FileAttribute.STORAGEINFO)), + PnfsGetFileAttributes.class, + new Callback() { @Override - public void success(PnfsGetStorageInfoMessage message) + public void success(PnfsGetFileAttributes message) { - setStorageInfo(message.getStorageInfo()); + setFileAttributes(message.getFileAttributes()); super.success(message); } }); } - synchronized void setStorageInfo(StorageInfo info) + synchronized void setFileAttributes(FileAttributes fileAttributes) { - _storageInfo = info; + _fileAttributes = fileAttributes; } /** FSM Action */ @@ -449,15 +447,14 @@ synchronized void sendDeliveryRequest() new InetSocketAddress(_address, 0), _destinationPoolCellname, _destinationPoolCellDomainName, - "/" + _pnfsId.toIdString(), + "/" + getPnfsId(), null); protocolInfo.setSessionId(_id); PoolDeliverFileMessage request = new PoolDeliverFileMessage(_sourcePoolName, - _pnfsId, protocolInfo, - _storageInfo); + _fileAttributes); request.setPool2Pool(); request.setInitiator(getInitiator()); request.setId(_id); @@ -498,7 +495,7 @@ synchronized void ping() */ synchronized void beginTransfer(final String uri) { - new Thread("P2P Transfer - " + _pnfsId) { + new Thread("P2P Transfer - " + getPnfsId()) { @Override public void run() { @@ -519,13 +516,13 @@ synchronized void done() if (_error != null) { if (_error instanceof RuntimeException) { - _log.error(String.format("P2P for %s failed: %s", _pnfsId, _error), + _log.error(String.format("P2P for %s failed: %s", getPnfsId(), _error), (Exception) _error); } else { - _log.error(String.format("P2P for %s failed: %s", _pnfsId, _error)); + _log.error(String.format("P2P for %s failed: %s", getPnfsId(), _error)); } } else { - _log.info(String.format("P2P for %s completed", _pnfsId)); + _log.info(String.format("P2P for %s completed", getPnfsId())); } if (_callback != null) { @@ -544,7 +541,7 @@ public void run() { } else { t = new CacheException(error.toString()); } - _callback.cacheFileAvailable(_pnfsId, t); + _callback.cacheFileAvailable(getPnfsId(), t); } })); } diff --git a/modules/dcache/src/main/java/org/dcache/pool/p2p/P2PClient.java b/modules/dcache/src/main/java/org/dcache/pool/p2p/P2PClient.java index b24180403d0..3e91081def8 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/p2p/P2PClient.java +++ b/modules/dcache/src/main/java/org/dcache/pool/p2p/P2PClient.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.concurrent.ScheduledExecutorService; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.dcache.pool.classic.ChecksumModuleV1; @@ -213,9 +214,8 @@ public void cacheFileAvailable(PnfsId pnfsId, Throwable t) } } - public synchronized int newCompanion(PnfsId pnfsId, - String sourcePoolName, - StorageInfo storageInfo, + public synchronized int newCompanion(String sourcePoolName, + FileAttributes fileAttributes, EntryState targetState, List stickyRecords, CacheFileAvailable callback, @@ -240,7 +240,7 @@ public synchronized int newCompanion(PnfsId pnfsId, if (_pnfs == null) { throw new IllegalStateException("PNFS stub not initialized"); } - if (_repository.getState(pnfsId) != EntryState.NEW) { + if (_repository.getState(fileAttributes.getPnfsId()) != EntryState.NEW) { throw new IllegalStateException("Replica already exists"); } @@ -250,7 +250,7 @@ public synchronized int newCompanion(PnfsId pnfsId, new Companion(_executor, getInterface(), _repository, _checksumModule, _pnfs, _pool, - pnfsId, storageInfo, + fileAttributes, sourcePoolName, _destinationPoolCellname, _destinationPoolCellDomainName, @@ -367,10 +367,11 @@ public synchronized void printSetup(PrintWriter pw) public synchronized String ac_pp_get_file_$_2(Args args) throws CacheException, IOException, InterruptedException { - PnfsId pnfsId = new PnfsId(args.argv(0)); + FileAttributes fileAttributes = new FileAttributes(); + fileAttributes.setPnfsId(new PnfsId(args.argv(0))); String pool = args.argv(1); List stickyRecords = Collections.emptyList(); - newCompanion(pnfsId, pool, null, EntryState.CACHED, stickyRecords, null, + newCompanion(pool, fileAttributes, EntryState.CACHED, stickyRecords, null, false); return "Transfer Initiated"; } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/CacheEntry.java b/modules/dcache/src/main/java/org/dcache/pool/repository/CacheEntry.java index bca7dff3d81..2778b63dc4f 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/CacheEntry.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/CacheEntry.java @@ -1,8 +1,7 @@ package org.dcache.pool.repository; -import diskCacheV111.util.CacheException; import diskCacheV111.util.PnfsId; -import diskCacheV111.vehicles.StorageInfo; +import org.dcache.vehicles.FileAttributes; import java.util.Collection; @@ -19,10 +18,9 @@ public interface CacheEntry public long getReplicaSize(); /** - * Get the storage info of the related entry. - * @return storage info of the entry or null if storage info is not available yet. + * @return file attributes of this entry */ - public StorageInfo getStorageInfo(); + public FileAttributes getFileAttributes(); /** * diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/Repository.java b/modules/dcache/src/main/java/org/dcache/pool/repository/Repository.java index 60f48920fa0..7bea816781a 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/Repository.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/Repository.java @@ -4,14 +4,11 @@ import diskCacheV111.util.FileInCacheException; import diskCacheV111.util.FileNotInCacheException; import diskCacheV111.util.CacheException; -import diskCacheV111.vehicles.StorageInfo; -import org.dcache.pool.repository.v3.RepositoryException; import org.dcache.pool.FaultListener; +import org.dcache.vehicles.FileAttributes; import java.io.IOException; -import java.io.FileNotFoundException; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -70,8 +67,7 @@ void load() * @throws FileInCacheException if an entry with the same ID * already exists. */ - ReplicaDescriptor createEntry(PnfsId id, - StorageInfo info, + ReplicaDescriptor createEntry(FileAttributes fileAttributes, EntryState transferState, EntryState targetState, List sticky) diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/RepositoryInterpreter.java b/modules/dcache/src/main/java/org/dcache/pool/repository/RepositoryInterpreter.java index 1001a8c26ec..d3d661f2e76 100755 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/RepositoryInterpreter.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/RepositoryInterpreter.java @@ -182,7 +182,7 @@ private Serializable list() for (PnfsId id: _repository) { try { CacheEntry entry = _repository.getEntry(id); - StorageInfo info = entry.getStorageInfo(); + StorageInfo info = entry.getFileAttributes().getStorageInfo(); if (info == null) { continue; } @@ -313,7 +313,7 @@ public void run() try { CacheEntry entry = _repository.getEntry(id); - StorageInfo info = entry.getStorageInfo(); + StorageInfo info = entry.getFileAttributes().getStorageInfo(); if (info == null) { continue; } diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheEntryImpl.java b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheEntryImpl.java index 4dcf6bf6556..9b57cae7dc6 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheEntryImpl.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheEntryImpl.java @@ -1,6 +1,6 @@ package org.dcache.pool.repository.v5; -import diskCacheV111.util.CacheException; +import com.google.common.collect.Sets; import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.StorageInfo; @@ -8,6 +8,8 @@ import org.dcache.pool.repository.CacheEntry; import org.dcache.pool.repository.EntryState; import org.dcache.pool.repository.StickyRecord; +import org.dcache.util.Checksum; +import org.dcache.vehicles.FileAttributes; import java.util.Collection; @@ -56,13 +58,34 @@ public long getReplicaSize() return _size; } - /** - * @see org.dcache.pool.repository.CacheEntry#getStorageInfo() - */ @Override - public StorageInfo getStorageInfo() + public FileAttributes getFileAttributes() { - return _info; + FileAttributes attributes = new FileAttributes(); + attributes.setStorageInfo(_info); + attributes.setPnfsId(_id); + attributes.setSize(_info.getFileSize()); + if (_info.isSetAccessLatency()) { + attributes.setAccessLatency(_info.getAccessLatency()); + } + if (_info.isSetRetentionPolicy()) { + attributes.setRetentionPolicy(_info.getRetentionPolicy()); + } + String cFlag = _info.getKey("flag-c"); + if (cFlag != null) { + attributes.setChecksums(Sets.newHashSet(Checksum + .parseChecksum(cFlag))); + } + String uid = _info.getKey("uid"); + if (uid != null) { + attributes.setOwner(Integer.parseInt(uid)); + } + String gid = _info.getKey("gid"); + if (gid != null) { + attributes.setOwner(Integer.parseInt(gid)); + } + attributes.setFlags(_info.getMap()); + return attributes; } /** diff --git a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheRepositoryV5.java b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheRepositoryV5.java index b02105e6694..783d6d13e97 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheRepositoryV5.java +++ b/modules/dcache/src/main/java/org/dcache/pool/repository/v5/CacheRepositoryV5.java @@ -9,6 +9,7 @@ import diskCacheV111.util.UnitInteger; import diskCacheV111.vehicles.StorageInfo; +import org.dcache.namespace.FileAttribute; import org.dcache.pool.repository.StickyRecord; import org.dcache.pool.repository.StateChangeEvent; import org.dcache.pool.repository.EntryChangeEvent; @@ -34,9 +35,13 @@ import org.dcache.cells.AbstractCellComponent; import org.dcache.cells.CellCommandListener; import org.dcache.util.CacheExceptionFactory; + +import static org.dcache.namespace.FileAttribute.PNFSID; +import static org.dcache.namespace.FileAttribute.STORAGEINFO; import static org.dcache.pool.repository.EntryState.*; import java.io.PrintWriter; +import java.util.EnumSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -51,6 +56,7 @@ import dmg.util.Args; +import org.dcache.vehicles.FileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -424,20 +430,19 @@ public Iterator iterator() } @Override - public ReplicaDescriptor createEntry(PnfsId id, - StorageInfo info, + public ReplicaDescriptor createEntry(FileAttributes fileAttributes, EntryState transferState, EntryState targetState, List stickyRecords) throws FileInCacheException { try { - if (stickyRecords == null) { - throw new IllegalArgumentException("List of sticky records must not be null"); + if (!fileAttributes.isDefined(EnumSet.of(PNFSID, STORAGEINFO))) { + throw new IllegalArgumentException("StorageInfo must not be null"); } - if (info == null) { - throw new IllegalArgumentException("StorageInfo must not be null"); + if (stickyRecords == null) { + throw new IllegalArgumentException("List of sticky records must not be null"); } assertOpen(); @@ -459,9 +464,9 @@ public ReplicaDescriptor createEntry(PnfsId id, throw new IllegalArgumentException("Invalid target state"); } - MetaDataRecord entry = _store.create(id); + MetaDataRecord entry = _store.create(fileAttributes.getPnfsId()); synchronized (entry) { - entry.setStorageInfo(info); + entry.setStorageInfo(fileAttributes.getStorageInfo()); setState(entry, transferState); return new WriteHandleImpl(this, @@ -472,7 +477,7 @@ public ReplicaDescriptor createEntry(PnfsId id, stickyRecords); } } catch (DuplicateEntryException e) { - throw new FileInCacheException("Entry already exists: " + id); + throw new FileInCacheException("Entry already exists: " + fileAttributes.getPnfsId()); } catch (CacheException e) { fail(FaultAction.READONLY, "Internal repository error", e); throw new RuntimeException("Internal repository error", e); diff --git a/modules/dcache/src/main/java/org/dcache/util/Transfer.java b/modules/dcache/src/main/java/org/dcache/util/Transfer.java index 913191805c8..0eb18487686 100644 --- a/modules/dcache/src/main/java/org/dcache/util/Transfer.java +++ b/modules/dcache/src/main/java/org/dcache/util/Transfer.java @@ -758,11 +758,11 @@ public void startMover(String queue, long timeout) throws CacheException, InterruptedException { PnfsId pnfsId = getPnfsId(); - StorageInfo storageInfo = getStorageInfo(); + FileAttributes fileAttributes = getFileAttributes(); String pool = getPool(); - if (pnfsId == null || storageInfo == null || pool == null) { - throw new IllegalStateException("Need PNFS ID, storage info and pool before a mover can be started"); + if (fileAttributes == null|| pool == null) { + throw new IllegalStateException("Need PNFS ID, file attributes and pool before a mover can be started"); } setStatus("Pool " + pool + ": Creating mover"); @@ -771,12 +771,10 @@ public void startMover(String queue, long timeout) PoolIoFileMessage message; if (isWrite()) { message = - new PoolAcceptFileMessage(pool, pnfsId, - protocolInfo, storageInfo); + new PoolAcceptFileMessage(pool, protocolInfo, fileAttributes); } else { message = - new PoolDeliverFileMessage(pool, pnfsId, - protocolInfo, storageInfo); + new PoolDeliverFileMessage(pool, protocolInfo, fileAttributes); } message.setIoQueueName(queue); message.setInitiator(getTransaction()); diff --git a/modules/dcache/src/main/java/org/dcache/vehicles/FileAttributes.java b/modules/dcache/src/main/java/org/dcache/vehicles/FileAttributes.java index 3096230bcf0..b2644c78f30 100644 --- a/modules/dcache/src/main/java/org/dcache/vehicles/FileAttributes.java +++ b/modules/dcache/src/main/java/org/dcache/vehicles/FileAttributes.java @@ -18,12 +18,26 @@ import org.dcache.util.Checksum; /** + * FileAttributes encapsulates attributes about a logical file. * - * FileAttributes is a set off all file attributes - * used by dcache including StorageInfo and Location. - * The only part of attributes may be defined. If client asks for an attribute - * which is not in returned by attributes() then return value is - * unpredictable. + * The attributes represented by an instance of this class belong to + * a logical file as seen by a client or user of dCache. That is, + * FileAttributes represent the information about a file stored, or that + * should be stored, in the name space or other central components. + * + * Besides their location, the class does not represent any properties + * of physical replicas on a pool. Eg the size or checksum stored in a + * FileAttributes instance represents the expected file size and expected + * checksum of the file. An broken replica may have a different size or a + * different checksum. + * + * The distinction between the logical and physical instance is relevant when + * considering response types to pool query messages: These should NOT return + * attributes of a replica using FileAttributes, except when those represent + * cached information from the name space or other central components. + * + * Not all attributes may be defined. Attempts to read undefined attributes + * will throw IllegalStateException. * * @since 1.9.5 */ diff --git a/modules/dcache/src/main/smc/org/dcache/pool/p2p/Companion.sm b/modules/dcache/src/main/smc/org/dcache/pool/p2p/Companion.sm index 59987205e31..fb139f58cff 100644 --- a/modules/dcache/src/main/smc/org/dcache/pool/p2p/Companion.sm +++ b/modules/dcache/src/main/smc/org/dcache/pool/p2p/Companion.sm @@ -16,7 +16,7 @@ Init { start [ !ctxt.hasStorageInfo() ] - GettingStorageInfo + GettingFileAttributes { } start @@ -26,26 +26,26 @@ Init } } -GettingStorageInfo +GettingFileAttributes Entry { - getStorageInfo(); + fetchFileAttributes(); } { timeout Failed { - setError("Failed to get storage info (timeout)"); + setError("Failed to get file attributes (timeout)"); } noroute Failed { - setError("Failed to get storage info (no route to cell)"); + setError("Failed to get file attributes (no route to cell)"); } failure(rc: Integer, cause: Object) Failed { - setError("Failed to get storage info (" + cause + ")"); + setError("Failed to get file attributes (" + cause + ")"); } success() CreatingMover diff --git a/modules/dcache/src/test/java/org/dcache/tests/repository/RepositorySubsystemTest.java b/modules/dcache/src/test/java/org/dcache/tests/repository/RepositorySubsystemTest.java index b516949bc61..143538d9364 100644 --- a/modules/dcache/src/test/java/org/dcache/tests/repository/RepositorySubsystemTest.java +++ b/modules/dcache/src/test/java/org/dcache/tests/repository/RepositorySubsystemTest.java @@ -9,7 +9,6 @@ import org.junit.*; import com.sleepycat.je.DatabaseException; -import diskCacheV111.repository.*; import diskCacheV111.util.PnfsId; import diskCacheV111.util.PnfsHandler; import diskCacheV111.util.CacheException; @@ -18,7 +17,6 @@ import diskCacheV111.vehicles.*; import org.dcache.vehicles.*; import dmg.cells.nucleus.*; -import dmg.util.*; import org.dcache.namespace.FileAttribute; import org.dcache.tests.cells.CellAdapterHelper; import org.dcache.tests.cells.CellStubHelper; @@ -26,7 +24,6 @@ import org.dcache.pool.repository.Account; import org.dcache.pool.repository.v5.CacheRepositoryV5; -import org.dcache.pool.repository.Repository; import org.dcache.pool.repository.Repository.OpenFlags; import org.dcache.pool.repository.IllegalTransitionException; @@ -44,7 +41,6 @@ import org.dcache.pool.repository.FileStore; import org.dcache.pool.repository.FlatFileStore; import org.dcache.pool.repository.meta.file.FileMetaDataRepository; -import org.dcache.pool.repository.v3.RepositoryException; public class RepositorySubsystemTest extends AbstractStateChangeListener @@ -67,6 +63,12 @@ public class RepositorySubsystemTest private StorageInfo info4; private StorageInfo info5; + private FileAttributes attributes1; + private FileAttributes attributes2; + private FileAttributes attributes3; + private FileAttributes attributes4; + private FileAttributes attributes5; + private PnfsHandler pnfs; private Account account; @@ -94,8 +96,7 @@ private void createFile(File file, long size) } - private void createEntry(final PnfsId id, - final StorageInfo info, + private void createEntry(final FileAttributes attributes, final EntryState state, final List sticky) throws Throwable @@ -113,14 +114,13 @@ protected void run() throws CacheException, IOException, InterruptedException { ReplicaDescriptor handle = - repository.createEntry(id, - info, + repository.createEntry(attributes, EntryState.FROM_CLIENT, state, sticky); try { - handle.allocate(info.getFileSize()); - createFile(handle.getFile(), info.getFileSize()); + handle.allocate(attributes.getSize()); + createFile(handle.getFile(), attributes.getSize()); handle.commit(null); } finally { handle.close(); @@ -136,6 +136,15 @@ private StorageInfo createStorageInfo(long size) return info; } + private FileAttributes createFileAttributes(PnfsId pnfsId, StorageInfo info) + { + FileAttributes attributes = new FileAttributes(); + attributes.setPnfsId(pnfsId); + attributes.setStorageInfo(info); + attributes.setSize(info.getFileSize()); + return attributes; + } + private void deleteDirectory(File dir) { File[] fileArray = dir.listFiles(); @@ -197,6 +206,12 @@ public void setUp() info4 = createStorageInfo(0); info5 = createStorageInfo(size5); + attributes1 = createFileAttributes(id1, info1); + attributes2 = createFileAttributes(id2, info2); + attributes3 = createFileAttributes(id3, info3); + attributes4 = createFileAttributes(id4, info4); + attributes5 = createFileAttributes(id5, info5); + root = File.createTempFile("dtest", null); if (!root.delete()) { throw new IOException("Could not delete temp file"); @@ -228,11 +243,11 @@ public void setUp() initRepository(); repository.init(); repository.load(); - createEntry(id1, info1, EntryState.PRECIOUS, + createEntry(attributes1, EntryState.PRECIOUS, Arrays.asList(new StickyRecord("system", 0))); - createEntry(id2, info2, EntryState.CACHED, + createEntry(attributes2, EntryState.CACHED, Arrays.asList(new StickyRecord("system", 0))); - createEntry(id3, info3, EntryState.CACHED, + createEntry(attributes3, EntryState.CACHED, Arrays.asList(new StickyRecord("system", -1))); repository.shutdown(); metaDataStore.close(); @@ -341,7 +356,10 @@ public void testCreateEntryFailsBeforeLoad() throws Exception { repository.init(); List stickyRecords = Collections.emptyList(); - repository.createEntry(id1, info1, FROM_CLIENT, PRECIOUS, stickyRecords); + FileAttributes attributes = new FileAttributes(); + attributes.setPnfsId(id1); + attributes.setStorageInfo(info1); + repository.createEntry(attributes, FROM_CLIENT, PRECIOUS, stickyRecords); } @Test(expected=IllegalStateException.class) @@ -457,7 +475,7 @@ public Object message(PnfsSetFileAttributes msg) { protected void run() throws CacheException, InterruptedException { List stickyRecords = Collections.emptyList(); - ReplicaDescriptor handle = repository.createEntry(id5, info5, FROM_STORE, CACHED, stickyRecords); + ReplicaDescriptor handle = repository.createEntry(attributes5, FROM_STORE, CACHED, stickyRecords); try { handle.allocate(info5.getFileSize()); createFile(handle.getFile(), info5.getFileSize()); @@ -705,7 +723,7 @@ public void testCreateEntryFileExists() stateChangeEvents.clear(); List stickyRecords = Collections.emptyList(); - repository.createEntry(id1, info1, FROM_CLIENT, PRECIOUS, stickyRecords); + repository.createEntry(attributes1, FROM_CLIENT, PRECIOUS, stickyRecords); } /* Helper method for creating a fourth entry in the repository. @@ -766,7 +784,7 @@ protected void run() { List stickyRecords = Collections.emptyList(); ReplicaDescriptor handle = - repository.createEntry(id4, info4, transferState, + repository.createEntry(attributes4, transferState, finalState, stickyRecords); try { handle.allocate(size4 + overallocation); @@ -869,7 +887,7 @@ public void testCreateEntryNegativeAllocation() stateChangeEvents.clear(); ReplicaDescriptor handle = - repository.createEntry(id4, info4, FROM_CLIENT, PRECIOUS, null); + repository.createEntry(attributes4, FROM_CLIENT, PRECIOUS, null); handle.allocate(-1); }