Better page cache memory calculation in importer

in that Configuration can specify total mapped memory
(lower than normal in import than normal online db) and BatchingNeoStores
will calculate optimal (as big as possible) page size from that,
while at the same time have room for 60 pages. Default is page cache memory
of 240 MiB which would mean a page size of 4 MiB.

Also removes unnecessarily overridden default values from two types
of Configuration.

community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java

@@ -75,7 +75,7 @@
 import static org.neo4j.helpers.Exceptions.launderedException;
 import static org.neo4j.helpers.Format.bytes;
 import static org.neo4j.helpers.Strings.TAB;
-import static org.neo4j.io.ByteUnit.kibiBytes;
+import static org.neo4j.io.ByteUnit.mebiBytes;
 import static org.neo4j.kernel.impl.util.Converters.withDefault;
 import static org.neo4j.unsafe.impl.batchimport.Configuration.BAD_FILE_NAME;
 import static org.neo4j.unsafe.impl.batchimport.input.Collectors.badCollector;
@@ -498,9 +498,9 @@ private static org.neo4j.unsafe.impl.batchimport.Configuration importConfigurati
         return new org.neo4j.unsafe.impl.batchimport.Configuration.Default()
         {
             @Override
-            public long pageSize()
+            public long pageCacheMemory()
             {
-                return defaultSettingsSuitableForTests ? kibiBytes( 32 ) : super.pageSize();
+                return defaultSettingsSuitableForTests ? mebiBytes( 8 ) : super.pageCacheMemory();
             }
 
             @Override

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/Configuration.java

@@ -22,6 +22,9 @@
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.kernel.configuration.Config;
 
+import static java.lang.Long.parseLong;
+import static java.lang.Math.min;
+
 import static org.neo4j.io.ByteUnit.mebiBytes;
 
 /**
@@ -41,37 +44,32 @@ public interface Configuration extends org.neo4j.unsafe.impl.batchimport.staging
     int denseNodeThreshold();
 
     /**
-     * @return page size for the page cache managing the store.
+     * @return amount of memory to reserve for the page cache. This should just be "enough" for it to be able
+     * to sequentially read and write a couple of stores at a time. If configured too high then there will
+     * be less memory available for other caches which are critical during the import. Optimal size is
+     * estimated to be 100-200 MiB. The importer will figure out an optimal page size from this value,
+     * with slightly bigger page size than "normal" random access use cases.
      */
-    long pageSize();
+    long pageCacheMemory();
 
     class Default
             extends org.neo4j.unsafe.impl.batchimport.staging.Configuration.Default
             implements Configuration
     {
         @Override
-        public int batchSize()
-        {
-            return 10_000;
-        }
-
-        @Override
-        public long pageSize()
+        public long pageCacheMemory()
         {
-            return mebiBytes( 8 );
+            long upperBound = parseLong( GraphDatabaseSettings.pagecache_memory.getDefaultValue() );
+            // Get the upper bound of what we can get from the default config calculation
+            // We even want to limit amount of memory a bit more since we don't need very much during import
+            return min( mebiBytes( 240 ), upperBound );
         }
 
         @Override
         public int denseNodeThreshold()
         {
             return Integer.parseInt( GraphDatabaseSettings.dense_node_threshold.getDefaultValue() );
         }
-
-        @Override
-        public int movingAverageSize()
-        {
-            return 100;
-        }
     }
 
     Configuration DEFAULT = new Default();
@@ -96,9 +94,9 @@ public Overridden( Config config )
         }
 
         @Override
-        public long pageSize()
+        public long pageCacheMemory()
         {
-            return defaults.pageSize();
+            return defaults.pageCacheMemory();
         }
 
         @Override

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Configuration.java

@@ -55,13 +55,13 @@ class Default implements Configuration
         @Override
         public int batchSize()
         {
-            return 20_000;
+            return 10_000;
         }
 
         @Override
         public int movingAverageSize()
         {
-            return 1000;
+            return 100;
         }
 
         @Override

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java

@@ -47,7 +47,6 @@
 import org.neo4j.kernel.impl.store.counts.CountsTracker;
 import org.neo4j.kernel.impl.transaction.state.NeoStoresSupplier;
 import org.neo4j.kernel.impl.util.Dependencies;
-import org.neo4j.kernel.impl.util.OsBeanUtil;
 import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.logging.LogProvider;
 import org.neo4j.logging.NullLogProvider;
@@ -59,14 +58,14 @@
 import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingRelationshipTypeTokenRepository;
 import org.neo4j.unsafe.impl.batchimport.store.io.IoTracer;
 
-import static java.lang.Math.max;
-import static java.lang.Math.min;
 import static java.lang.String.valueOf;
 
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold;
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.mapped_memory_page_size;
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory;
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
+import static org.neo4j.io.ByteUnit.kibiBytes;
+import static org.neo4j.io.ByteUnit.mebiBytes;
 
 /**
  * Creator and accessor of {@link NeoStores} with some logic to provide very batch friendly services to the
@@ -94,17 +93,14 @@ public BatchingNeoStores( FileSystemAbstraction fileSystem, File storeDir, Confi
         this.logProvider = logService.getInternalLogProvider();
         this.storeDir = storeDir;
 
-        long pageSize = config.pageSize();
+        long mappedMemory = config.pageCacheMemory();
         // 30 is the minimum number of pages the page cache wants to keep free at all times.
         // Having less than that might result in an evicted page will reading, which would mean
         // unnecessary re-reading. Having slightly more leaves some leg room.
-        long optimalMappedMemorySize = pageSize * 40;
-        long limitedMemorySize = max(
-                2 * pageSize, // page cache requires at the very least memory enough for two pages
-                applyEnvironmentLimitationsTo( optimalMappedMemorySize ) );
+        int pageSize = calculateOptimalPageSize( mappedMemory, 60 /*pages*/ );
         this.neo4jConfig = new Config( stringMap( dbConfig.getParams(),
                 dense_node_threshold.name(), valueOf( config.denseNodeThreshold() ),
-                pagecache_memory.name(), valueOf( limitedMemorySize ),
+                pagecache_memory.name(), valueOf( mappedMemory ),
                 mapped_memory_page_size.name(), valueOf( pageSize ) ),
                 GraphDatabaseSettings.class );
         final PageCacheTracer tracer = new DefaultPageCacheTracer();
@@ -170,25 +166,19 @@ public File storeDir()
                 LabelScanStoreProvider.HIGHEST_PRIORITIZED ).getLabelScanStore() );
     }
 
-    /**
-     * An attempt to limit amount of memory used by the page cache in a severely limited environment.
-     * This shouldn't be a problem in most scenarios since the optimal mapped memory size is in the range
-     * of 100-200 MiB and so shouldn't impose a noticeable dent in memory usage.
-     *
-     * @param optimalMappedMemorySize amount of mapped memory that would be considered optimal for the import.
-     * @return in most cases the optimal size, although in some very limited environments a smaller size.
-     */
-    private long applyEnvironmentLimitationsTo( long optimalMappedMemorySize )
+    static int calculateOptimalPageSize( long memorySize, int numberOfPages )
     {
-        long freePhysicalMemory = OsBeanUtil.getFreePhysicalMemory();
-        if ( freePhysicalMemory == OsBeanUtil.VALUE_UNAVAILABLE )
+        int pageSize = (int) mebiBytes( 8 );
+        int lowest = (int) kibiBytes( 8 );
+        while ( pageSize > lowest )
         {
-            // We have no idea how much free memory there is, let's simply go with what we'd like to have
-            return optimalMappedMemorySize;
+            if ( memorySize / pageSize >= numberOfPages )
+            {
+                return pageSize;
+            }
+            pageSize >>>= 1;
         }
-        // We got a hint about amount of free memory. Let's acquire tops a fifth of the free memory
-        // since other parts of the importer also needs memory to function.
-        return min( optimalMappedMemorySize, freePhysicalMemory / 5 );
+        return lowest;
     }
 
     private static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogProvider log,

community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStoresTest.java

@@ -40,9 +40,12 @@
 import static org.junit.Assert.fail;
 
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
+import static org.neo4j.io.ByteUnit.kibiBytes;
+import static org.neo4j.io.ByteUnit.mebiBytes;
 import static org.neo4j.kernel.impl.store.AbstractDynamicStore.BLOCK_HEADER_SIZE;
 import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY;
 import static org.neo4j.unsafe.impl.batchimport.Configuration.DEFAULT;
+import static org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores.calculateOptimalPageSize;
 
 public class BatchingNeoStoresTest
 {
@@ -84,6 +87,58 @@ public void shouldRespectDbConfig() throws Exception
         }
     }
 
+    @Test
+    public void shouldCalculateBigPageSizeForBiggerMemory() throws Exception
+    {
+        // GIVEN
+        long memorySize = mebiBytes( 240 );
+
+        // WHEN
+        int pageSize = calculateOptimalPageSize( memorySize, 60 );
+
+        // THEN
+        assertEquals( mebiBytes( 4 ), pageSize );
+    }
+
+    @Test
+    public void shouldCalculateSmallPageSizeForSmallerMemory() throws Exception
+    {
+        // GIVEN
+        long memorySize = mebiBytes( 100 );
+
+        // WHEN
+        int pageSize = calculateOptimalPageSize( memorySize, 60 );
+
+        // THEN
+        assertEquals( mebiBytes( 1 ), pageSize );
+    }
+
+    @Test
+    public void shouldNotGoLowerThan8kPageSizeForSmallMemory() throws Exception
+    {
+        // GIVEN
+        long memorySize = kibiBytes( 8*30 );
+
+        // WHEN
+        int pageSize = calculateOptimalPageSize( memorySize, 60 );
+
+        // THEN
+        assertEquals( kibiBytes( 8 ), pageSize );
+    }
+
+    @Test
+    public void shouldNotGoHigherThan8mPageSizeForBigMemory() throws Exception
+    {
+        // GIVEN
+        long memorySize = mebiBytes( 700 );
+
+        // WHEN
+        int pageSize = calculateOptimalPageSize( memorySize, 60 );
+
+        // THEN
+        assertEquals( mebiBytes( 8 ), pageSize );
+    }
+
     private void someDataInTheDatabase()
     {
         GraphDatabaseService db = new TestGraphDatabaseFactory().setFileSystem( fsr.get() )