Skip to content

Commit

Permalink
[#46] NFSRODS can now recover from restarts of the iRODS server.
Browse files Browse the repository at this point in the history 
- Uses the object ID of collections and data objects as the inode number
- Provides a lookup for cache misses on previously encountered object IDs to
  generate logical paths
  • Loading branch information
@ganning127 @trel
ganning127 authored and trel committed Sep 29, 2023
1 parent 4e7c98e commit aab87f6
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 39 deletions.
150 changes: 137 additions & 13 deletions irods-vfs-impl/src/main/java/org/irods/nfsrods/vfs/IRODSVirtualFileSystem.java
View file
Expand Up @@ -64,6 +64,7 @@
import org.dcache.nfs.vfs.Stat;
import org.dcache.nfs.vfs.Stat.Type;
import org.dcache.nfs.vfs.VirtualFileSystem;

import org.irods.jargon.core.connection.IRODSAccount;
import org.irods.jargon.core.exception.DataNotFoundException;
import org.irods.jargon.core.exception.JargonException;
Expand All @@ -76,6 +77,7 @@
import org.irods.jargon.core.pub.DataObjectAO;
import org.irods.jargon.core.pub.IRODSAccessObjectFactory;
import org.irods.jargon.core.pub.IRODSFileSystemAO;
import org.irods.jargon.core.pub.IRODSGenQueryExecutor;
import org.irods.jargon.core.pub.UserAO;
import org.irods.jargon.core.pub.UserGroupAO;
import org.irods.jargon.core.pub.domain.ObjStat;
Expand All @@ -86,12 +88,21 @@
import org.irods.jargon.core.pub.io.IRODSFile;
import org.irods.jargon.core.pub.io.IRODSFileFactory;
import org.irods.jargon.core.pub.io.IRODSRandomAccessFile;
import org.irods.jargon.core.query.CollectionAndDataObjectListingEntry;
import org.irods.jargon.core.query.CollectionAndDataObjectListingEntry.ObjectType;
import org.irods.jargon.core.query.CollectionAndDataObjectListingEntry;
import org.irods.jargon.core.query.GenQueryBuilderException;
import org.irods.jargon.core.query.IRODSGenQueryBuilder;
import org.irods.jargon.core.query.IRODSGenQueryFromBuilder;
import org.irods.jargon.core.query.IRODSQueryResultRow;
import org.irods.jargon.core.query.IRODSQueryResultSet;
import org.irods.jargon.core.query.JargonQueryException;
import org.irods.jargon.core.query.QueryConditionOperators;
import org.irods.jargon.core.query.RodsGenQueryEnum;
import org.irods.nfsrods.config.IRODSClientConfig;
import org.irods.nfsrods.config.IRODSProxyAdminAccountConfig;
import org.irods.nfsrods.config.NFSServerConfig;
import org.irods.nfsrods.config.ServerConfig;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -279,7 +290,10 @@ public Inode create(Inode _parent, Type _type, String _name, Subject _subject, i
}
}

long newInodeNumber = inodeToPathMapper_.getAndIncrementFileID();
CollectionAndDataObjectListAndSearchAO lao = factory_.getCollectionAndDataObjectListAndSearchAO(adminAcct_);
ObjStat objStat = lao.retrieveObjectStatForPath(newFile.getAbsolutePath());

long newInodeNumber = objStat.getDataId();
inodeToPathMapper_.map(newInodeNumber, path);

return toFh(newInodeNumber);
Expand Down Expand Up @@ -684,7 +698,8 @@ public Access checkAcl(Subject _subject, Inode _inode, int _accessMask) throws C
return Access.DENY;
}

String path = getPath(toInodeNumber(_inode)).toString();
long inodeNum = toInodeNumber(_inode);
String path = getPath(inodeNum).toString();

// Key (String) => <user_id>#<access_mask>#<path>
// Value (Access) => ALLOW/DENY
Expand Down Expand Up @@ -853,7 +868,7 @@ public NfsIdMapping getIdMapper()
@Override
public Inode getRootInode() throws IOException
{
return toFh(1);
return toFh(getInodeNumber(ZONE_COLLECTION)); // ZONE_COLLECTION is set to rodsSvrConfig.getZone()
}

@Override
Expand Down Expand Up @@ -955,7 +970,7 @@ private List<CollectionAndDataObjectListingEntry> listDataObjectsAndCollectionsU

return entries;
}

@Override
public DirectoryStream list(Inode _inode, byte[] _verifier, long _cookie) throws IOException
{
Expand Down Expand Up @@ -1006,7 +1021,9 @@ public DirectoryStream list(Inode _inode, byte[] _verifier, long _cookie) throws

if (null == inodeNumber)
{
inodeNumber = inodeToPathMapper_.getAndIncrementFileID();
CollectionAndDataObjectListAndSearchAO lao = factory_.getCollectionAndDataObjectListAndSearchAO(adminAcct_);
ObjStat objStat = lao.retrieveObjectStatForPath(path.toAbsolutePath().toString());
inodeNumber = (long) objStat.getDataId();
inodeToPathMapper_.map(inodeNumber, path);
}

Expand All @@ -1018,7 +1035,6 @@ public DirectoryStream list(Inode _inode, byte[] _verifier, long _cookie) throws
}
catch (JargonException e)
{
log_.error(e.getMessage());
throw new IOException(e);
}
finally
Expand Down Expand Up @@ -1049,10 +1065,11 @@ public Inode lookup(Inode _parent, String _path) throws IOException
CollectionAndDataObjectListAndSearchAO lao = null;
lao = factory_.getCollectionAndDataObjectListAndSearchAO(adminAcct_);
boolean isTargetValid = false;

ObjStat objStat = null;
try
{
isTargetValid = (lao.retrieveObjectStatForPath(targetPath.toString()) != null);
objStat = lao.retrieveObjectStatForPath(targetPath.toAbsolutePath().toString());
isTargetValid = (null != objStat);
}
catch (Exception e)
{
Expand All @@ -1070,7 +1087,8 @@ public Inode lookup(Inode _parent, String _path) throws IOException
return toFh(inodeNumber);
}

inodeNumber = inodeToPathMapper_.getAndIncrementFileID();
inodeNumber = (long) objStat.getDataId();

inodeToPathMapper_.map(inodeNumber, targetPath);

return toFh(inodeNumber);
Expand Down Expand Up @@ -1123,7 +1141,10 @@ public Inode mkdir(Inode _inode, String _path, Subject _subject, int _mode) thro
// so that the user sees the updates.
listOpCache_.remove(acct.getUserName() + "#" + parentPath.toString());

long inodeNumber = inodeToPathMapper_.getAndIncrementFileID();
CollectionAndDataObjectListAndSearchAO lao = factory_.getCollectionAndDataObjectListAndSearchAO(adminAcct_);
ObjStat objStat = lao.retrieveObjectStatForPath(file.getAbsolutePath());
long inodeNumber = objStat.getDataId();

inodeToPathMapper_.map(inodeNumber, file.getAbsolutePath());

return toFh(inodeNumber);
Expand Down Expand Up @@ -1645,15 +1666,118 @@ private static Inode toFh(long _inodeNumber)
return Inode.forFile(Longs.toByteArray(_inodeNumber));
}

private Optional<String> findLogicalPathByInodeNumber(long _inodeNumber, ObjectType _option)
{
// if getPath() returns null, we'll use GenQuery to find the path
log_.debug("""
findLogicalPathByInodeNumber - _inodeNumber = {}
findLogicalPathByInodeNumber - _option = {}""",
_inodeNumber, _option);

String strPath = null;
IRODSGenQueryBuilder builder = null;
IRODSGenQueryExecutor gqe = null;
IRODSGenQueryFromBuilder query = null;
IRODSQueryResultSet resultSet = null;
List<IRODSQueryResultRow> results = null;

if (ObjectType.COLLECTION == _option)
{
try
{
builder = new IRODSGenQueryBuilder(true, null);

builder.addSelectAsGenQueryValue(RodsGenQueryEnum.COL_COLL_ID)
.addSelectAsGenQueryValue(RodsGenQueryEnum.COL_COLL_NAME)
.addConditionAsGenQueryField(RodsGenQueryEnum.COL_COLL_ID, QueryConditionOperators.EQUAL, String.valueOf(_inodeNumber));

// the constructed GenQuery is: SELECT COLL_ID, COLL_NAME WHERE COLL_ID = '<_inodeNumber>'
gqe = factory_.getIRODSGenQueryExecutor(adminAcct_);
query = builder.exportIRODSQueryFromBuilder(50);
resultSet = gqe.executeIRODSQueryAndCloseResultInZone(query, 0, adminAcct_.getZone());
results = resultSet.getResults();

log_.debug("found row is (coll id) [{}]", results);
if (!results.isEmpty())
{
// column 1 = COLL_NAME; COLL_NAME value example: /tempZone/home/<username>/<coll>
strPath = results.get(0).getColumn(1);
log_.debug("findLogicalPathByInodeNumber - Path from coll_id [{}]", strPath);
}
}
catch (GenQueryBuilderException | JargonException | JargonQueryException e)
{
log_.error(e.getMessage());
}
}
else if (ObjectType.DATA_OBJECT == _option)
{
try
{
builder = new IRODSGenQueryBuilder(true, null);

builder.addSelectAsGenQueryValue(RodsGenQueryEnum.COL_D_DATA_ID)
.addSelectAsGenQueryValue(RodsGenQueryEnum.COL_COLL_NAME)
.addSelectAsGenQueryValue(RodsGenQueryEnum.COL_DATA_NAME)
.addConditionAsGenQueryField(RodsGenQueryEnum.COL_D_DATA_ID, QueryConditionOperators.EQUAL, String.valueOf(_inodeNumber));

// the constructed GenQuery is: SELECT DATA_ID, COLL_NAME, DATA_NAME WHERE DATA_ID = '<_inodeNumber>'

gqe = factory_.getIRODSGenQueryExecutor(adminAcct_);
query = builder.exportIRODSQueryFromBuilder(50);
resultSet = gqe.executeIRODSQueryAndCloseResultInZone(query, 0, adminAcct_.getZone());
results = resultSet.getResults();
log_.debug("findLogicalPathByInodeNumber - Found row is (data id) [{}]", results);
if (!results.isEmpty())
{
// since DATA_NAME is only the filename (no path), we'll need to combine it with COLL_NAME to get the full path
// column 1 = COLL_NAME, column 2 = DATA_NAME
strPath = results.get(0).getColumn(1) + '/' + results.get(0).getColumn(2);

log_.debug("findLogicalPathByInodeNumber - Combined path is [{}]", strPath);
}
}
catch (GenQueryBuilderException | JargonException | JargonQueryException e)
{
log_.error(e.getMessage());
}
}
else
{
log_.warn("findLogicalPathByInodeNumber - Option [{}] is not supported. choose from `ObjectType.COLLECTION` or `ObjectType.DATA_OBJECT`", _option);
}

return Optional.ofNullable(strPath);
}

private Path getPath(long _inodeNumber) throws IOException
{
Path path = inodeToPathMapper_.getPathByInodeNumber(_inodeNumber);

// If no path matches _inodeNumber, we'll need to find where we via GenQuery.
// Since a path may lead to a data object or a collection, we'll need to check both when using GenQuery.
if (path == null)
{
throw new NoEntException("Path does not exist for [" + _inodeNumber + "]");
}
log_.debug("getPath - Path was null (maybe because NFSRODS was restarted without remounting), looking up in GenQuery");

Optional<String> genQueryPath = findLogicalPathByInodeNumber(_inodeNumber, ObjectType.COLLECTION);

if (genQueryPath.isEmpty())
{
genQueryPath = findLogicalPathByInodeNumber(_inodeNumber, ObjectType.DATA_OBJECT);

// if GenQuery didn't find a valid path, throw an exception
if (genQueryPath.isEmpty())
{
throw new NoEntException("getPath - Path does not exist for inode number [" + _inodeNumber + "]");
}
}

path = ROOT_COLLECTION.resolve(genQueryPath.get()); // genQueryPath example: /tempZone/home/<username>/<file>
log_.debug("getPath - Path found from GenQuery [{}]", path);
inodeToPathMapper_.map(_inodeNumber, path);
return path;
}
return path;
}

Expand Down
37 changes: 11 additions & 26 deletions irods-vfs-impl/src/main/java/org/irods/nfsrods/vfs/InodeToPathMapper.java
View file
Expand Up @@ -4,7 +4,6 @@
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.Lock;
Expand All @@ -14,7 +13,9 @@
import org.irods.jargon.core.connection.IRODSAccount;
import org.irods.jargon.core.exception.DataNotFoundException;
import org.irods.jargon.core.exception.JargonException;
import org.irods.jargon.core.pub.CollectionAndDataObjectListAndSearchAO;
import org.irods.jargon.core.pub.IRODSAccessObjectFactory;
import org.irods.jargon.core.pub.domain.ObjStat;
import org.irods.jargon.core.pub.io.IRODSFile;
import org.irods.nfsrods.config.IRODSClientConfig;
import org.irods.nfsrods.config.IRODSProxyAdminAccountConfig;
Expand All @@ -30,15 +31,13 @@ class InodeToPathMapper
private Map<Long, Path> inodeToPath_;
private Map<Path, Long> pathToInode_;
private Set<Long> availableInodeNumbers_;
private long fileID_;
private ReadWriteLock lock_;

public InodeToPathMapper(ServerConfig _config, IRODSAccessObjectFactory _factory) throws JargonException
{
inodeToPath_ = new HashMap<>();
pathToInode_ = new HashMap<>();
availableInodeNumbers_ = new HashSet<>();
fileID_ = 1; // Inode numbers start at 1
lock_ = new ReentrantReadWriteLock();

NFSServerConfig nfsSvrConfig = _config.getNfsServerConfig();
Expand All @@ -58,44 +57,30 @@ public InodeToPathMapper(ServerConfig _config, IRODSAccessObjectFactory _factory

try
{
establishRoot(_factory.getIRODSFileFactory(acct).instanceIRODSFile(rootPath));
establishRoot(_factory.getIRODSFileFactory(acct).instanceIRODSFile(rootPath), _factory, acct);
}
finally
{
_factory.closeSessionAndEatExceptions();
}
}

public Long getAndIncrementFileID()
{
final Wrapper<Long> ref = new Wrapper<>();

new AtomicRead(() -> {
if (!availableInodeNumbers_.isEmpty())
{
Iterator<Long> it = availableInodeNumbers_.iterator();
ref.value = it.next();
it.remove();

return;
}

ref.value = fileID_++;
});

return ref.value;
}

private void establishRoot(IRODSFile _irodsMountPoint) throws DataNotFoundException
private void establishRoot(IRODSFile _irodsMountPoint, IRODSAccessObjectFactory _factory, IRODSAccount _adminAcct) throws DataNotFoundException, JargonException
{
if (!_irodsMountPoint.exists())
{
throw new DataNotFoundException("Cannot establish root at [" + _irodsMountPoint + "]");
}


CollectionAndDataObjectListAndSearchAO lao = _factory.getCollectionAndDataObjectListAndSearchAO(_adminAcct);
ObjStat objStat = lao.retrieveObjectStatForPath(_irodsMountPoint.getAbsolutePath());
long newInodeNumber = objStat.getDataId();

log_.debug("establishRoot - Mapping root to [{}] ...", _irodsMountPoint);

map(getAndIncrementFileID(), _irodsMountPoint.getAbsolutePath());
map((long) newInodeNumber, _irodsMountPoint.getAbsolutePath());


log_.debug("establishRoot - Mapping successful.");
}
Expand Down

0 comments on commit aab87f6

Please sign in to comment.