From b468000418e224cbac8631a9c324da5eb21d0c24 Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Fri, 11 Mar 2016 12:25:04 +0100 Subject: [PATCH] Better page cache memory calculation in importer in that Configuration can specify total mapped memory (lower than normal in import than normal online db) and BatchingNeoStores will calculate optimal (as big as possible) page size from that, while at the same time have room for 60 pages. Default is page cache memory of 240 MiB which would mean a page size of 4 MiB. Also removes unnecessarily overridden default values from two types of Configuration. --- .../java/org/neo4j/tooling/ImportTool.java | 6 +- .../impl/batchimport/Configuration.java | 34 ++++++------ .../batchimport/staging/Configuration.java | 4 +- .../batchimport/store/BatchingNeoStores.java | 40 +++++--------- .../store/BatchingNeoStoresTest.java | 55 +++++++++++++++++++ 5 files changed, 91 insertions(+), 48 deletions(-) diff --git a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java index b50c5059b776a..00af6dbc61f2f 100644 --- a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java +++ b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java @@ -75,7 +75,7 @@ import static org.neo4j.helpers.Exceptions.launderedException; import static org.neo4j.helpers.Format.bytes; import static org.neo4j.helpers.Strings.TAB; -import static org.neo4j.io.ByteUnit.kibiBytes; +import static org.neo4j.io.ByteUnit.mebiBytes; import static org.neo4j.kernel.impl.util.Converters.withDefault; import static org.neo4j.unsafe.impl.batchimport.Configuration.BAD_FILE_NAME; import static org.neo4j.unsafe.impl.batchimport.input.Collectors.badCollector; @@ -498,9 +498,9 @@ private static org.neo4j.unsafe.impl.batchimport.Configuration importConfigurati return new org.neo4j.unsafe.impl.batchimport.Configuration.Default() { @Override - public long pageSize() + public long pageCacheMemory() { - return defaultSettingsSuitableForTests ? kibiBytes( 32 ) : super.pageSize(); + return defaultSettingsSuitableForTests ? mebiBytes( 8 ) : super.pageCacheMemory(); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/Configuration.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/Configuration.java index 6ef6a64cbeec5..81b1d1d4d55e8 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/Configuration.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/Configuration.java @@ -22,6 +22,9 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.kernel.configuration.Config; +import static java.lang.Long.parseLong; +import static java.lang.Math.min; + import static org.neo4j.io.ByteUnit.mebiBytes; /** @@ -41,24 +44,25 @@ public interface Configuration extends org.neo4j.unsafe.impl.batchimport.staging int denseNodeThreshold(); /** - * @return page size for the page cache managing the store. + * @return amount of memory to reserve for the page cache. This should just be "enough" for it to be able + * to sequentially read and write a couple of stores at a time. If configured too high then there will + * be less memory available for other caches which are critical during the import. Optimal size is + * estimated to be 100-200 MiB. The importer will figure out an optimal page size from this value, + * with slightly bigger page size than "normal" random access use cases. */ - long pageSize(); + long pageCacheMemory(); class Default extends org.neo4j.unsafe.impl.batchimport.staging.Configuration.Default implements Configuration { @Override - public int batchSize() - { - return 10_000; - } - - @Override - public long pageSize() + public long pageCacheMemory() { - return mebiBytes( 8 ); + long upperBound = parseLong( GraphDatabaseSettings.pagecache_memory.getDefaultValue() ); + // Get the upper bound of what we can get from the default config calculation + // We even want to limit amount of memory a bit more since we don't need very much during import + return min( mebiBytes( 240 ), upperBound ); } @Override @@ -66,12 +70,6 @@ public int denseNodeThreshold() { return Integer.parseInt( GraphDatabaseSettings.dense_node_threshold.getDefaultValue() ); } - - @Override - public int movingAverageSize() - { - return 100; - } } Configuration DEFAULT = new Default(); @@ -96,9 +94,9 @@ public Overridden( Config config ) } @Override - public long pageSize() + public long pageCacheMemory() { - return defaults.pageSize(); + return defaults.pageCacheMemory(); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Configuration.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Configuration.java index 3ad9e2f7718c4..0dde97be98692 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Configuration.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Configuration.java @@ -55,13 +55,13 @@ class Default implements Configuration @Override public int batchSize() { - return 20_000; + return 10_000; } @Override public int movingAverageSize() { - return 1000; + return 100; } @Override diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index 18bb1139c93dd..10e49c34297e2 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -47,7 +47,6 @@ import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.kernel.impl.transaction.state.NeoStoresSupplier; import org.neo4j.kernel.impl.util.Dependencies; -import org.neo4j.kernel.impl.util.OsBeanUtil; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.logging.LogProvider; import org.neo4j.logging.NullLogProvider; @@ -59,14 +58,14 @@ import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingRelationshipTypeTokenRepository; import org.neo4j.unsafe.impl.batchimport.store.io.IoTracer; -import static java.lang.Math.max; -import static java.lang.Math.min; import static java.lang.String.valueOf; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.mapped_memory_page_size; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.io.ByteUnit.kibiBytes; +import static org.neo4j.io.ByteUnit.mebiBytes; /** * Creator and accessor of {@link NeoStores} with some logic to provide very batch friendly services to the @@ -94,17 +93,14 @@ public BatchingNeoStores( FileSystemAbstraction fileSystem, File storeDir, Confi this.logProvider = logService.getInternalLogProvider(); this.storeDir = storeDir; - long pageSize = config.pageSize(); + long mappedMemory = config.pageCacheMemory(); // 30 is the minimum number of pages the page cache wants to keep free at all times. // Having less than that might result in an evicted page will reading, which would mean // unnecessary re-reading. Having slightly more leaves some leg room. - long optimalMappedMemorySize = pageSize * 40; - long limitedMemorySize = max( - 2 * pageSize, // page cache requires at the very least memory enough for two pages - applyEnvironmentLimitationsTo( optimalMappedMemorySize ) ); + int pageSize = calculateOptimalPageSize( mappedMemory, 60 /*pages*/ ); this.neo4jConfig = new Config( stringMap( dbConfig.getParams(), dense_node_threshold.name(), valueOf( config.denseNodeThreshold() ), - pagecache_memory.name(), valueOf( limitedMemorySize ), + pagecache_memory.name(), valueOf( mappedMemory ), mapped_memory_page_size.name(), valueOf( pageSize ) ), GraphDatabaseSettings.class ); final PageCacheTracer tracer = new DefaultPageCacheTracer(); @@ -170,25 +166,19 @@ public File storeDir() LabelScanStoreProvider.HIGHEST_PRIORITIZED ).getLabelScanStore() ); } - /** - * An attempt to limit amount of memory used by the page cache in a severely limited environment. - * This shouldn't be a problem in most scenarios since the optimal mapped memory size is in the range - * of 100-200 MiB and so shouldn't impose a noticeable dent in memory usage. - * - * @param optimalMappedMemorySize amount of mapped memory that would be considered optimal for the import. - * @return in most cases the optimal size, although in some very limited environments a smaller size. - */ - private long applyEnvironmentLimitationsTo( long optimalMappedMemorySize ) + static int calculateOptimalPageSize( long memorySize, int numberOfPages ) { - long freePhysicalMemory = OsBeanUtil.getFreePhysicalMemory(); - if ( freePhysicalMemory == OsBeanUtil.VALUE_UNAVAILABLE ) + int pageSize = (int) mebiBytes( 8 ); + int lowest = (int) kibiBytes( 8 ); + while ( pageSize > lowest ) { - // We have no idea how much free memory there is, let's simply go with what we'd like to have - return optimalMappedMemorySize; + if ( memorySize / pageSize >= numberOfPages ) + { + return pageSize; + } + pageSize >>>= 1; } - // We got a hint about amount of free memory. Let's acquire tops a fifth of the free memory - // since other parts of the importer also needs memory to function. - return min( optimalMappedMemorySize, freePhysicalMemory / 5 ); + return lowest; } private static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogProvider log, diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStoresTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStoresTest.java index 580e71c041468..1a6cf2f57d025 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStoresTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStoresTest.java @@ -40,9 +40,12 @@ import static org.junit.Assert.fail; import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.io.ByteUnit.kibiBytes; +import static org.neo4j.io.ByteUnit.mebiBytes; import static org.neo4j.kernel.impl.store.AbstractDynamicStore.BLOCK_HEADER_SIZE; import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY; import static org.neo4j.unsafe.impl.batchimport.Configuration.DEFAULT; +import static org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores.calculateOptimalPageSize; public class BatchingNeoStoresTest { @@ -84,6 +87,58 @@ public void shouldRespectDbConfig() throws Exception } } + @Test + public void shouldCalculateBigPageSizeForBiggerMemory() throws Exception + { + // GIVEN + long memorySize = mebiBytes( 240 ); + + // WHEN + int pageSize = calculateOptimalPageSize( memorySize, 60 ); + + // THEN + assertEquals( mebiBytes( 4 ), pageSize ); + } + + @Test + public void shouldCalculateSmallPageSizeForSmallerMemory() throws Exception + { + // GIVEN + long memorySize = mebiBytes( 100 ); + + // WHEN + int pageSize = calculateOptimalPageSize( memorySize, 60 ); + + // THEN + assertEquals( mebiBytes( 1 ), pageSize ); + } + + @Test + public void shouldNotGoLowerThan8kPageSizeForSmallMemory() throws Exception + { + // GIVEN + long memorySize = kibiBytes( 8*30 ); + + // WHEN + int pageSize = calculateOptimalPageSize( memorySize, 60 ); + + // THEN + assertEquals( kibiBytes( 8 ), pageSize ); + } + + @Test + public void shouldNotGoHigherThan8mPageSizeForBigMemory() throws Exception + { + // GIVEN + long memorySize = mebiBytes( 700 ); + + // WHEN + int pageSize = calculateOptimalPageSize( memorySize, 60 ); + + // THEN + assertEquals( mebiBytes( 8 ), pageSize ); + } + private void someDataInTheDatabase() { GraphDatabaseService db = new TestGraphDatabaseFactory().setFileSystem( fsr.get() )