From 4f9bc4b648d2ad84eb0641125452ae8002fa9a9d Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Mon, 29 May 2017 09:27:27 +0200 Subject: [PATCH] Hide the native memory allocation scheme behind an interface --- .../impl/muninn/MuninnPageCache.java | 6 +-- .../io/pagecache/impl/muninn/PageList.java | 14 +++--- .../neo4j/io/pagecache/PageSwapperTest.java | 5 +-- .../pagecache/impl/muninn/PageListTest.java | 6 +-- ...{MemoryManager.java => GrabAllocator.java} | 26 +++++------ .../internal/dragons/MemoryAllocator.java | 44 +++++++++++++++++++ ...agerTest.java => MemoryAllocatorTest.java} | 17 ++++--- 7 files changed, 80 insertions(+), 38 deletions(-) rename community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/{MemoryManager.java => GrabAllocator.java} (87%) create mode 100644 community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocator.java rename community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/{MemoryManagerTest.java => MemoryAllocatorTest.java} (82%) diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java index cf89ff9347a7e..03406292ce8da 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java @@ -46,7 +46,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; -import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; +import org.neo4j.unsafe.impl.internal.dragons.MemoryAllocator; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static org.neo4j.unsafe.impl.internal.dragons.FeatureToggles.flag; @@ -220,10 +220,10 @@ public MuninnPageCache( long alignment = swapperFactory.getRequiredBufferAlignment(); long expectedMaxMemory = ((long) maxPages) * cachePageSize; // cast to long prevents overflow - MemoryManager memoryManager = new MemoryManager( expectedMaxMemory, alignment ); + MemoryAllocator memoryAllocator = MemoryAllocator.createAllocator( expectedMaxMemory, alignment ); this.victimPage = VictimPageReference.getVictimPage( cachePageSize ); - this.pages = new PageList( maxPages, cachePageSize, memoryManager, new SwapperSet(), victimPage ); + this.pages = new PageList( maxPages, cachePageSize, memoryAllocator, new SwapperSet(), victimPage ); setFreelistHead( new AtomicInteger() ); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java index 05de7fee2c8ab..eef4f7b03b6fa 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java @@ -27,7 +27,7 @@ import org.neo4j.io.pagecache.tracing.EvictionEventOpportunity; import org.neo4j.io.pagecache.tracing.FlushEvent; import org.neo4j.io.pagecache.tracing.PageFaultEvent; -import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; +import org.neo4j.unsafe.impl.internal.dragons.MemoryAllocator; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static java.lang.String.format; @@ -91,20 +91,20 @@ class PageList private final int pageCount; private final int cachePageSize; - private final MemoryManager memoryManager; + private final MemoryAllocator memoryAllocator; private final SwapperSet swappers; private final long victimPageAddress; private final long baseAddress; - PageList( int pageCount, int cachePageSize, MemoryManager memoryManager, SwapperSet swappers, long victimPageAddress ) + PageList( int pageCount, int cachePageSize, MemoryAllocator memoryAllocator, SwapperSet swappers, long victimPageAddress ) { this.pageCount = pageCount; this.cachePageSize = cachePageSize; - this.memoryManager = memoryManager; + this.memoryAllocator = memoryAllocator; this.swappers = swappers; this.victimPageAddress = victimPageAddress; long bytes = ((long) pageCount) * META_DATA_BYTES_PER_PAGE; - this.baseAddress = memoryManager.allocateAligned( bytes ); + this.baseAddress = memoryAllocator.allocateAligned( bytes ); clearMemory( baseAddress, pageCount ); } @@ -119,7 +119,7 @@ class PageList { this.pageCount = pageList.pageCount; this.cachePageSize = pageList.cachePageSize; - this.memoryManager = pageList.memoryManager; + this.memoryAllocator = pageList.memoryAllocator; this.swappers = pageList.swappers; this.victimPageAddress = pageList.victimPageAddress; this.baseAddress = pageList.baseAddress; @@ -306,7 +306,7 @@ public void initBuffer( long pageRef ) { if ( getAddress( pageRef ) == 0L ) { - long addr = memoryManager.allocateAligned( getCachePageSize() ); + long addr = memoryAllocator.allocateAligned( getCachePageSize() ); UnsafeUtil.putLong( offAddress( pageRef ), addr ); } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java index cfa16b03399b4..d73ab04136ea3 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageSwapperTest.java @@ -41,11 +41,10 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import org.neo4j.io.ByteUnit; import org.neo4j.test.rule.TestDirectory; -import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; +import org.neo4j.unsafe.impl.internal.dragons.MemoryAllocator; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static org.hamcrest.Matchers.is; @@ -72,7 +71,7 @@ public abstract class PageSwapperTest private final ConcurrentLinkedQueue openedFactories = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue openedSwappers = new ConcurrentLinkedQueue<>(); - private final MemoryManager mman = new MemoryManager( ByteUnit.kibiBytes( 32 ), 1 ); + private final MemoryAllocator mman = MemoryAllocator.createAllocator( ByteUnit.kibiBytes( 32 ), 1 ); protected abstract PageSwapperFactory swapperFactory() throws Exception; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java index 9c1f4dc6522d5..c51a4dda047dd 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java @@ -50,7 +50,7 @@ import org.neo4j.io.pagecache.tracing.FlushEvent; import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.PageFaultEvent; -import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; +import org.neo4j.unsafe.impl.internal.dragons.MemoryAllocator; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static org.hamcrest.Matchers.equalTo; @@ -82,13 +82,13 @@ public static Iterable parameters() } private static ExecutorService executor; - private static MemoryManager mman; + private static MemoryAllocator mman; @BeforeClass public static void setUpStatics() { executor = Executors.newCachedThreadPool( new DaemonThreadFactory() ); - mman = new MemoryManager( ByteUnit.mebiBytes( 1 ), ALIGNMENT ); + mman = MemoryAllocator.createAllocator( ByteUnit.mebiBytes( 1 ), ALIGNMENT ); } @AfterClass diff --git a/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManager.java b/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/GrabAllocator.java similarity index 87% rename from community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManager.java rename to community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/GrabAllocator.java index 2e60a93f6c8e8..3cce423d5b536 100644 --- a/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManager.java +++ b/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/GrabAllocator.java @@ -19,20 +19,18 @@ */ package org.neo4j.unsafe.impl.internal.dragons; +import static org.neo4j.unsafe.impl.internal.dragons.FeatureToggles.getInteger; + /** - * The memory manager is simple: it only allocates memory, until it itself is finalizable and frees it all in one go. - * - * The memory is allocated in large segments, called "grabs", and the memory returned by the memory manager is page - * aligned, and plays well with transparent huge pages and other operating system optimisations. - * - * The memory manager assumes that the memory claimed from it is evenly divisible in units of pages. + * This memory allocator is allocating memory in large segments, called "grabs", and the memory returned by the memory + * manager is page aligned, and plays well with transparent huge pages and other operating system optimisations. */ -public final class MemoryManager +public final class GrabAllocator implements MemoryAllocator { /** * The amount of memory, in bytes, to grab in each Grab. */ - private static final long GRAB_SIZE = FeatureToggles.getInteger( MemoryManager.class, "GRAB_SIZE", 512 * 1024 ); // 512 KiB + private static final long GRAB_SIZE = getInteger( GrabAllocator.class, "GRAB_SIZE", 512 * 1024 ); // 512 KiB /** * The amount of memory that this memory manager can still allocate. @@ -43,13 +41,13 @@ public final class MemoryManager private Grab grabs; /** - * Create a new MemoryManager that will allocate the given amount of memory, to pointers that are aligned to the + * Create a new GrabAllocator that will allocate the given amount of memory, to pointers that are aligned to the * given alignment size. * @param expectedMaxMemory The maximum amount of memory that this memory manager is expected to allocate. The * actual amount of memory used can end up greater than this value, if some of it gets wasted on alignment padding. * @param alignment The byte multiple that the allocated pointers have to be aligned at. */ - public MemoryManager( long expectedMaxMemory, long alignment ) + GrabAllocator( long expectedMaxMemory, long alignment ) { if ( alignment == 0 ) { @@ -59,6 +57,7 @@ public MemoryManager( long expectedMaxMemory, long alignment ) this.alignment = alignment; } + @Override public synchronized long sumUsedMemory() { long sum = 0; @@ -71,12 +70,7 @@ public synchronized long sumUsedMemory() return sum; } - /** - * Allocate a contiguous, aligned region of memory of the given size in bytes. - * @param bytes the number of bytes to allocate. - * @return A pointer to the allocated memory. - * @throws OutOfMemoryError if the requested memory could not be allocated. - */ + @Override public synchronized long allocateAligned( long bytes ) { if ( bytes > GRAB_SIZE ) diff --git a/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocator.java b/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocator.java new file mode 100644 index 0000000000000..b5a5ed1efe2ac --- /dev/null +++ b/community/unsafe/src/main/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocator.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.unsafe.impl.internal.dragons; + +/** + * A MemoryAllocator is simple: it only allocates memory, until it itself is finalizable and frees it all in one go. + */ +public interface MemoryAllocator +{ + static MemoryAllocator createAllocator( long expectedMaxMemory, long alignment ) + { + return new GrabAllocator( expectedMaxMemory, alignment ); + } + + /** + * @return The sum, in bytes, of all the memory currently allocating through this allocator. + */ + long sumUsedMemory(); + + /** + * Allocate a contiguous, aligned region of memory of the given size in bytes. + * @param bytes the number of bytes to allocate. + * @return A pointer to the allocated memory. + * @throws OutOfMemoryError if the requested memory could not be allocated. + */ + long allocateAligned( long bytes ); +} diff --git a/community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManagerTest.java b/community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocatorTest.java similarity index 82% rename from community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManagerTest.java rename to community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocatorTest.java index ed85a98f79b91..2cd423d53ec1c 100644 --- a/community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryManagerTest.java +++ b/community/unsafe/src/test/java/org/neo4j/unsafe/impl/internal/dragons/MemoryAllocatorTest.java @@ -25,12 +25,17 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertThat; -public class MemoryManagerTest +public class MemoryAllocatorTest { + protected MemoryAllocator createAllocator( long expectedMaxMemory, long alignment ) + { + return MemoryAllocator.createAllocator( expectedMaxMemory, alignment ); + } + @Test public void allocatedPointerMustNotBeNull() throws Exception { - MemoryManager mman = new MemoryManager( 16 * 4096, 8 ); + MemoryAllocator mman = createAllocator( 16 * 4096, 8 ); long address = mman.allocateAligned( 8192 ); assertThat( address, is( not( 0L ) ) ); } @@ -38,7 +43,7 @@ public void allocatedPointerMustNotBeNull() throws Exception @Test public void allocatedPointerMustBePageAligned() throws Exception { - MemoryManager mman = new MemoryManager( 16 * 4096, UnsafeUtil.pageSize() ); + MemoryAllocator mman = createAllocator( 16 * 4096, UnsafeUtil.pageSize() ); long address = mman.allocateAligned( 8192 ); assertThat( address % UnsafeUtil.pageSize(), is( 0L ) ); } @@ -46,7 +51,7 @@ public void allocatedPointerMustBePageAligned() throws Exception @Test public void mustBeAbleToAllocatePastMemoryLimit() throws Exception { - MemoryManager mman = new MemoryManager( 8192, 2 ); + MemoryAllocator mman = createAllocator( 8192, 2 ); for ( int i = 0; i < 4100; i++ ) { assertThat( mman.allocateAligned( 1 ) % 2, is( 0L ) ); @@ -57,13 +62,13 @@ public void mustBeAbleToAllocatePastMemoryLimit() throws Exception @Test( expected = IllegalArgumentException.class ) public void alignmentCannotBeZero() throws Exception { - new MemoryManager( 8192, 0 ); + createAllocator( 8192, 0 ); } @Test public void mustBeAbleToAllocateSlabsLargerThanGrabSize() throws Exception { - MemoryManager mman = new MemoryManager( 32 * 1024 * 1024, 1 ); + MemoryAllocator mman = createAllocator( 32 * 1024 * 1024, 1 ); long page1 = mman.allocateAligned( UnsafeUtil.pageSize() ); long largeBlock = mman.allocateAligned( 1024 * 1024 ); // 1 MiB long page2 = mman.allocateAligned( UnsafeUtil.pageSize() );