Skip to content

Commit

Permalink
pool: mover grizzly IO buffer initialization into NfsTransferService
Browse files Browse the repository at this point in the history
Motivation:
the grizzly IO buffers are pre-initialized on startup. However, the
initialization is bound to EDSOperationREAD class, thus, effectively,
during the first NFS read request.

Modification:
mover grizzly IO buffer initialization into NfsTransferService, which is
initialized at pool stat.

Result:
More predicable buffer initialization

Acked-by: Lea Morschel
Target: master
Require-book: no
Require-notes: no
(cherry picked from commit 426d7c4)
Signed-off-by: Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>
  • Loading branch information
kofemann committed Jan 30, 2024
1 parent 82cd451 commit 7647790
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 18 deletions.
Expand Up @@ -10,34 +10,18 @@
import org.dcache.nfs.v4.xdr.nfs_argop4;
import org.dcache.nfs.v4.xdr.nfs_opnum4;
import org.dcache.nfs.v4.xdr.nfs_resop4;
import org.dcache.oncrpc4j.grizzly.GrizzlyUtils;
import org.dcache.oncrpc4j.rpc.OncRpcException;
import org.dcache.oncrpc4j.xdr.Xdr;
import org.dcache.oncrpc4j.xdr.XdrEncodingStream;
import org.dcache.pool.repository.RepositoryChannel;
import org.dcache.util.ByteUnit;
import org.glassfish.grizzly.Buffer;
import org.glassfish.grizzly.memory.MemoryManager;
import org.glassfish.grizzly.memory.PooledMemoryManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class EDSOperationREAD extends AbstractNFSv4Operation {

private static final Logger _log = LoggerFactory.getLogger(EDSOperationREAD.class.getName());

// one pool with 1MB chunks (max NFS rsize)
private final static MemoryManager<? extends Buffer> POOLED_BUFFER_ALLOCATOR =
new PooledMemoryManager(
ByteUnit.MiB.toBytes(1), // base chunk size
1, // number of pools
2, // grow facter per pool, ignored, see above
GrizzlyUtils.getDefaultWorkerPoolSize(), // expected concurrency
PooledMemoryManager.DEFAULT_HEAP_USAGE_PERCENTAGE,
PooledMemoryManager.DEFAULT_PREALLOCATED_BUFFERS_PERCENTAGE,
true // direct buffers
);

private final NfsTransferService nfsTransferService;

public EDSOperationREAD(nfs_argop4 args, NfsTransferService nfsTransferService) {
Expand All @@ -57,12 +41,12 @@ public void process(CompoundContext context, nfs_resop4 result) {
NfsMover mover = nfsTransferService.getMoverByStateId(context, _args.opread.stateid);
if (mover == null) {
res.status = nfsstat.NFSERR_BAD_STATEID;
_log.debug("No mover associated with given stateid: ", _args.opread.stateid);
_log.debug("No mover associated with given stateid: {}", _args.opread.stateid);
return;
}

RepositoryChannel fc = mover.getMoverChannel();
var gBuffer = POOLED_BUFFER_ALLOCATOR.allocate(count);
var gBuffer = nfsTransferService.getIOBufferAllocator().allocate(count);
int bytesToRead = count;

int rc = -1;
Expand Down
Expand Up @@ -69,6 +69,7 @@
import org.dcache.nfs.v4.xdr.nfs_argop4;
import org.dcache.nfs.v4.xdr.nfs_opnum4;
import org.dcache.nfs.v4.xdr.stateid4;
import org.dcache.oncrpc4j.grizzly.GrizzlyUtils;
import org.dcache.oncrpc4j.rpc.IoStrategy;
import org.dcache.oncrpc4j.rpc.OncRpcException;
import org.dcache.oncrpc4j.rpc.OncRpcProgram;
Expand All @@ -84,9 +85,13 @@
import org.dcache.pool.movers.MoverFactory;
import org.dcache.pool.repository.ReplicaDescriptor;
import org.dcache.pool.repository.Repository;
import org.dcache.util.ByteUnit;
import org.dcache.util.NetworkUtils;
import org.dcache.util.PortRange;
import org.dcache.vehicles.DoorValidateMoverMessage;
import org.glassfish.grizzly.Buffer;
import org.glassfish.grizzly.memory.MemoryManager;
import org.glassfish.grizzly.memory.PooledMemoryManager;
import org.ietf.jgss.GSSException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -156,6 +161,21 @@ public class NfsTransferService

private CellAddressCore _cellAddress;

/**
* Buffer pool for IO operations.
* One pool with 1MB chunks (max NFS rsize).
*/
private final MemoryManager<? extends Buffer> pooledBufferAllocator =
new PooledMemoryManager(// one pool with 1MB chunks (max NFS rsize)
ByteUnit.MiB.toBytes(1), // base chunk size
1, // number of pools
2, // grow facter per pool, ignored, see above
GrizzlyUtils.getDefaultWorkerPoolSize(), // expected concurrency
PooledMemoryManager.DEFAULT_HEAP_USAGE_PERCENTAGE,
PooledMemoryManager.DEFAULT_PREALLOCATED_BUFFERS_PERCENTAGE,
true // direct buffers
);

@Override
public void setCellAddress(CellAddressCore address) {
_cellAddress = address;
Expand Down Expand Up @@ -563,4 +583,12 @@ public void getInfo(PrintWriter pw) {
var endpoint = _rpcService.getInetSocketAddress(IpProtocolType.TCP);
pw.printf(" Listening on: %s:%d\n", InetAddresses.toUriString(endpoint.getAddress()), endpoint.getPort());
}

/**
* Get IO buffer allocator.
* @return IO buffer allocator.
*/
public MemoryManager<? extends Buffer> getIOBufferAllocator() {
return pooledBufferAllocator;
}
}

0 comments on commit 7647790

Please sign in to comment.