Address review comments for PageCacheNumberArray

community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/PackedMultiFieldCache.java

@@ -24,7 +24,7 @@
 import org.neo4j.unsafe.impl.batchimport.cache.LongBitsManipulator;
 
 import static org.neo4j.consistency.checking.cache.CacheSlots.ID_SLOT_SIZE;
-import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO;
+import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE;
 
 /**
  * Simply combining a {@link LongArray} with {@link LongBitsManipulator}, so that each long can be split up into
@@ -38,7 +38,7 @@ public class PackedMultiFieldCache
 
     public PackedMultiFieldCache( int... slotSizes )
     {
-        this( AUTO.newDynamicLongArray( 1_000_000, 0 ), slotSizes );
+        this( AUTO_WITHOUT_PAGECACHE.newDynamicLongArray( 1_000_000, 0 ), slotSizes );
     }
 
     public PackedMultiFieldCache( LongArray array, int... slotSizes )

community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java

@@ -85,7 +85,7 @@
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.io.ByteUnit.mebiBytes;
 import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY;
-import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO;
+import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE;
 import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators.fromInput;
 import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators.startingFromTheBeginning;
 import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers.longs;
@@ -153,13 +153,13 @@ public static Collection<Object[]> data()
         return Arrays.<Object[]>asList(
 
                 // synchronous I/O, actual node id input
-                new Object[]{new LongInputIdGenerator(), longs( AUTO ), fromInput(), true},
+                new Object[]{new LongInputIdGenerator(), longs( AUTO_WITHOUT_PAGECACHE ), fromInput(), true},
                 // synchronous I/O, string id input
-                new Object[]{new StringInputIdGenerator(), strings( AUTO ), startingFromTheBeginning(), true},
+                new Object[]{new StringInputIdGenerator(), strings( AUTO_WITHOUT_PAGECACHE ), startingFromTheBeginning(), true},
                 // synchronous I/O, string id input
-                new Object[]{new StringInputIdGenerator(), strings( AUTO ), startingFromTheBeginning(), false},
+                new Object[]{new StringInputIdGenerator(), strings( AUTO_WITHOUT_PAGECACHE ), startingFromTheBeginning(), false},
                 // extra slow parallel I/O, actual node id input
-                new Object[]{new LongInputIdGenerator(), longs( AUTO ), fromInput(), false}
+                new Object[]{new LongInputIdGenerator(), longs( AUTO_WITHOUT_PAGECACHE ), fromInput(), false}
         );
     }
 

community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java

@@ -19,6 +19,7 @@
  */
 package org.neo4j.kernel.impl.store;
 
+import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.impl.api.CountsAccessor;
 import org.neo4j.kernel.impl.store.counts.CountsTracker;
 import org.neo4j.kernel.impl.store.kvstore.DataInitializer;
@@ -35,13 +36,13 @@ public class CountsComputer implements DataInitializer<CountsAccessor.Updater>
 
     private final NumberArrayFactory numberArrayFactory;
 
-    public static void recomputeCounts( NeoStores stores )
+    public static void recomputeCounts( NeoStores stores, PageCache pageCache )
     {
         MetaDataStore metaDataStore = stores.getMetaDataStore();
         CountsTracker counts = stores.getCounts();
         try ( CountsAccessor.Updater updater = counts.reset( metaDataStore.getLastCommittedTransactionId() ) )
         {
-            new CountsComputer( stores ).initialize( updater );
+            new CountsComputer( stores, pageCache ).initialize( updater );
         }
     }
 
@@ -51,13 +52,13 @@ public static void recomputeCounts( NeoStores stores )
     private final int highRelationshipTypeId;
     private final long lastCommittedTransactionId;
 
-    public CountsComputer( NeoStores stores )
+    public CountsComputer( NeoStores stores, PageCache pageCache )
     {
         this( stores.getMetaDataStore().getLastCommittedTransactionId(),
                 stores.getNodeStore(), stores.getRelationshipStore(),
                 (int) stores.getLabelTokenStore().getHighId(),
                 (int) stores.getRelationshipTypeTokenStore().getHighId(),
-                NumberArrayFactory.autoWithPageCacheFallback( stores.getPageCache(), stores.getStoreDir() ) );
+                NumberArrayFactory.auto( pageCache, stores.getStoreDir() ) );
     }
 
     public CountsComputer( long lastCommittedTransactionId, NodeStore nodes, RelationshipStore relationships,

community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java

@@ -151,11 +151,6 @@ public File getStoreDir()
         return storeDir;
     }
 
-    public PageCache getPageCache()
-    {
-        return pageCache;
-    }
-
     private File getStoreFile( String substoreName )
     {
         return new File( neoStoreFileName.getPath() + substoreName );
@@ -637,7 +632,7 @@ CountsTracker createCountStore( String storeName )
             public void initialize( CountsAccessor.Updater updater )
             {
                 log.warn( "Missing counts store, rebuilding it." );
-                new CountsComputer( neoStores ).initialize( updater );
+                new CountsComputer( neoStores, pageCache ).initialize( updater );
                 log.warn( "Counts store rebuild completed." );
             }
 

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java

@@ -423,7 +423,7 @@ private void rebuildCountsFromScratch( File storeDir, long lastTxId, String vers
                 int highRelationshipTypeId = (int) neoStores.getRelationshipTypeTokenStore().getHighId();
                 CountsComputer initializer = new CountsComputer(
                         lastTxId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId,
-                        NumberArrayFactory.autoWithPageCacheFallback( pageCache, storeDir ) );
+                        NumberArrayFactory.auto( pageCache, storeDir ) );
                 life.add( new CountsTracker(
                         logService.getInternalLogProvider(), fileSystem, pageCache, config, storeFileBase )
                         .setInitializer( initializer ) );

community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java

@@ -187,6 +187,7 @@ public class BatchInserterImpl implements BatchInserter, IndexConfigStoreProvide
     private final Config config;
     private final BatchInserterImpl.BatchSchemaActions actions;
     private final StoreLocker storeLocker;
+    private final PageCache pageCache;
     private boolean labelsTouched;
     private boolean isShutdown;
 
@@ -260,6 +261,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS
         life.start();
         neoStores = sf.openAllNeoStores( true );
         neoStores.verifyStoreOk();
+        this.pageCache = pageCache;
 
         nodeStore = neoStores.getNodeStore();
         relationshipStore = neoStores.getRelationshipStore();
@@ -526,7 +528,7 @@ private void rebuildCounts()
             throw new UnderlyingStorageException( e );
         }
 
-        CountsComputer.recomputeCounts( neoStores );
+        CountsComputer.recomputeCounts( neoStores, pageCache );
     }
 
     private class InitialNodeLabelCreationVisitor implements Visitor<NodeLabelUpdate, IOException>, Closeable

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java

@@ -169,7 +169,7 @@ public void doImport( Input input ) throws IOException
               InputCache inputCache = new InputCache( fileSystem, storeDir, recordFormats, config ) )
         {
             NumberArrayFactory numberArrayFactory =
-                    NumberArrayFactory.autoWithPageCacheFallback( neoStore.getPageCache(), storeDir );
+                    NumberArrayFactory.auto( pageCache, storeDir );
             Collector badCollector = input.badCollector();
             // Some temporary caches and indexes in the import
             IoMonitor writeMonitor = new IoMonitor( neoStore.getIoTracer() );

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java

@@ -44,6 +44,9 @@
 public class RelationshipGroupDefragmenter
 {
 
+    private final Configuration config;
+    private final ExecutionMonitor executionMonitor;
+    private final Monitor monitor;
     private final NumberArrayFactory numberArrayFactory;
 
     public interface Monitor
@@ -58,15 +61,11 @@ public interface Monitor
         default void defragmentingNodeRange( long fromNodeId, long toNodeId )
         {   // empty
         }
-
         Monitor EMPTY = new Monitor()
         {   // empty
         };
-    }
 
-    private final Configuration config;
-    private final ExecutionMonitor executionMonitor;
-    private final Monitor monitor;
+    }
 
     public RelationshipGroupDefragmenter( Configuration config, ExecutionMonitor executionMonitor,
                                           NumberArrayFactory numberArrayFactory )

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/ChunkedNumberArrayFactory.java

@@ -28,60 +28,53 @@
  */
 public class ChunkedNumberArrayFactory extends NumberArrayFactory.Adapter
 {
+    static final int MAGIC_CHUNK_COUNT = 10;
+    // This is a safe bet on the maximum number of items the JVM can store in an array. It is commonly slightly less
+    // than Integer.MAX_VALUE
+    private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - Short.MAX_VALUE;
     private final NumberArrayFactory delegate;
 
-    public ChunkedNumberArrayFactory()
+    ChunkedNumberArrayFactory()
     {
         this( OFF_HEAP, HEAP );
     }
 
-    public ChunkedNumberArrayFactory( NumberArrayFactory... delegateList )
+    ChunkedNumberArrayFactory( NumberArrayFactory... delegateList )
     {
         delegate = new Auto( delegateList );
     }
 
     @Override
     public LongArray newLongArray( long length, long defaultValue, long base )
     {
-        // Here we want to have the property of a dynamic array which makes some parts of the array
-        // live on heap, some off. At the same time we want a fixed size array. Therefore first create
-        // the array as a dynamic array and make it grow to the requested length.
-        LongArray array = newDynamicLongArray( fractionOf( length ), defaultValue );
-        array.at( length - 1 );
-        return array;
+        // Here we want to have the property of a dynamic array so that some parts of the array
+        // can live on heap, some off.
+        return newDynamicLongArray( fractionOf( length ), defaultValue );
     }
 
     @Override
     public IntArray newIntArray( long length, int defaultValue, long base )
     {
-        // Here we want to have the property of a dynamic array which makes some parts of the array
-        // live on heap, some off. At the same time we want a fixed size array. Therefore first create
-        // the array as a dynamic array and make it grow to the requested length.
-        IntArray array = newDynamicIntArray( fractionOf( length ), defaultValue );
-        array.at( length - 1 );
-        return array;
+        // Here we want to have the property of a dynamic array so that some parts of the array
+        // can live on heap, some off.
+        return newDynamicIntArray( fractionOf( length ), defaultValue );
     }
 
     @Override
     public ByteArray newByteArray( long length, byte[] defaultValue, long base )
     {
-        // Here we want to have the property of a dynamic array which makes some parts of the array
-        // live on heap, some off. At the same time we want a fixed size array. Therefore first create
-        // the array as a dynamic array and make it grow to the requested length.
-        ByteArray array = newDynamicByteArray( fractionOf( length ), defaultValue );
-        array.at( length - 1 );
-        return array;
+        // Here we want to have the property of a dynamic array so that some parts of the array
+        // can live on heap, some off.
+        return newDynamicByteArray( fractionOf( length ), defaultValue );
     }
 
     private long fractionOf( long length )
     {
-        int idealChunkCount = 10;
-        if ( length < idealChunkCount )
+        if ( length < MAGIC_CHUNK_COUNT )
         {
             return length;
         }
-        int maxArraySize = Integer.MAX_VALUE - Short.MAX_VALUE;
-        return min( length / idealChunkCount, maxArraySize );
+        return min( length / MAGIC_CHUNK_COUNT, MAX_ARRAY_SIZE );
     }
 
     @Override
@@ -105,6 +98,6 @@ public ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue )
     @Override
     public String toString()
     {
-        return "CHUNKED_FIXED_SIZE";
+        return "ChunkedNumberArrayFactory with delegate " + delegate;
     }
 }

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArray.java

@@ -32,8 +32,9 @@ public interface NumberArray<N extends NumberArray<N>> extends MemoryStatsVisito
     long length();
 
     /**
-     * Swaps {@code numberOfEntries} items from {@code fromIndex} to {@code toIndex}, such that
+     * Swaps items from {@code fromIndex} to {@code toIndex}, such that
      * {@code fromIndex} and {@code toIndex}, {@code fromIndex+1} and {@code toIndex} a.s.o swaps places.
+     * The number of items swapped is equal to the length of the default value of the array.
      *  @param fromIndex where to start swapping from.
      * @param toIndex where to start swapping to.
      */