From 58cc1f30f5b39bfe45c47af467186695997d6935 Mon Sep 17 00:00:00 2001 From: Tigran Mkrtchyan Date: Tue, 5 Mar 2024 22:41:38 +0100 Subject: [PATCH] nfs: calculate desired memory fraction for correct memory allocation Motivation: Grizzly memory management uses heap memory size fraction even if direct memory is used. https://github.com/eclipse-ee4j/grizzly/issues/2201 Moreover, for each memory slice (a memory segment per thread) allocates at least x16 chunks, which ends up at 16MB per slice (with 1MB chunk) Modification: as dCache needs `2 * #Cores * maxIObuf` memory, pre-calculate the required amount of direct memory and the corresponding fraction in relation to heap. Initialise the memory pool with 1/16 of the desired slice size to compensate memory allocator internal x16 increase. Result: dCache starts with 256m of direct memory (we still have xroot mover) Fixes: #7522 Acked-by: Svenja Meyer Acked-by: Lea Morschel Target: master, 9.2 Require-book: no Require-notes: yes (cherry picked from commit 8f3b984fab80beba8e42a61a3988367e4cd5ee87) Signed-off-by: Tigran Mkrtchyan --- .../nfsv41/mover/EDSOperationREAD.java | 2 +- .../nfsv41/mover/NfsTransferService.java | 20 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/EDSOperationREAD.java b/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/EDSOperationREAD.java index b8c492f8c09..3577ccfa1ec 100644 --- a/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/EDSOperationREAD.java +++ b/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/EDSOperationREAD.java @@ -57,7 +57,7 @@ public void process(CompoundContext context, nfs_resop4 result) { Buffer[] bufs = bArray.getArray(); int size = bArray.size(); - for(int i = 0; i < size; i++) { + for(int i = 0; bytesToRead > 0 && i < size; i++) { ByteBuffer directChunk = bufs[i].toByteBuffer(); directChunk.clear().limit(Math.min(directChunk.capacity(), bytesToRead)); diff --git a/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/NfsTransferService.java b/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/NfsTransferService.java index 9cb07b21b8f..a54e82203fa 100644 --- a/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/NfsTransferService.java +++ b/modules/dcache-nfs/src/main/java/org/dcache/chimera/nfsv41/mover/NfsTransferService.java @@ -161,17 +161,31 @@ public class NfsTransferService private CellAddressCore _cellAddress; + // This is a workaround for the issue with the grizzly allocator. + // (which uses a fraction of heap memory for direct buffers, instead of configured direct memory limit + // See: https://github.com/eclipse-ee4j/grizzly/issues/2201 + + // as we know in advance how much memory is going to be used, we can pre-calculate the desired fraction. + // The expected direct buffer allocation is ` * ` (with an assumption, + // that we use only one memory pool, i.g. no grow). + + private final int expectedConcurrency = GrizzlyUtils.getDefaultWorkerPoolSize(); + private final int allocationChunkSize = ByteUnit.MiB.toBytes(1); // one pool with 1MB chunks (max NFS rsize) + private final float heapFraction = (allocationChunkSize * expectedConcurrency) / (float) Runtime.getRuntime().maxMemory(); + /** * Buffer pool for IO operations. * One pool with 1MB chunks (max NFS rsize). */ private final MemoryManager pooledBufferAllocator = new PooledMemoryManager(// one pool with 1MB chunks (max NFS rsize) - ByteUnit.MiB.toBytes(1), // base chunk size + allocationChunkSize / 16, // Grizzly allocates at least 16 chunks per slice, + // for 1MB buffers 16MB in total. + // Pass 1/16 of the desired buffer size to compensate the over commitment. 1, // number of pools 2, // grow facter per pool, ignored, see above - GrizzlyUtils.getDefaultWorkerPoolSize(), // expected concurrency - PooledMemoryManager.DEFAULT_HEAP_USAGE_PERCENTAGE, + expectedConcurrency, // expected concurrency + heapFraction, // fraction of heap memory to use for direct buffers PooledMemoryManager.DEFAULT_PREALLOCATED_BUFFERS_PERCENTAGE, true // direct buffers );