diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/PackedMultiFieldCache.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/PackedMultiFieldCache.java index ab2d2309be6dc..f15c7b0ca18bd 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/PackedMultiFieldCache.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/PackedMultiFieldCache.java @@ -24,7 +24,7 @@ import org.neo4j.unsafe.impl.batchimport.cache.LongBitsManipulator; import static org.neo4j.consistency.checking.cache.CacheSlots.ID_SLOT_SIZE; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE; /** * Simply combining a {@link LongArray} with {@link LongBitsManipulator}, so that each long can be split up into @@ -38,7 +38,7 @@ public class PackedMultiFieldCache public PackedMultiFieldCache( int... slotSizes ) { - this( AUTO.newDynamicLongArray( 1_000_000, 0 ), slotSizes ); + this( AUTO_WITHOUT_PAGECACHE.newDynamicLongArray( 1_000_000, 0 ), slotSizes ); } public PackedMultiFieldCache( LongArray array, int... slotSizes ) diff --git a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java index b71b939b83766..becb4d5b18752 100644 --- a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java @@ -85,7 +85,7 @@ import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.io.ByteUnit.mebiBytes; import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE; import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators.fromInput; import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators.startingFromTheBeginning; import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers.longs; @@ -153,13 +153,13 @@ public static Collection data() return Arrays.asList( // synchronous I/O, actual node id input - new Object[]{new LongInputIdGenerator(), longs( AUTO ), fromInput(), true}, + new Object[]{new LongInputIdGenerator(), longs( AUTO_WITHOUT_PAGECACHE ), fromInput(), true}, // synchronous I/O, string id input - new Object[]{new StringInputIdGenerator(), strings( AUTO ), startingFromTheBeginning(), true}, + new Object[]{new StringInputIdGenerator(), strings( AUTO_WITHOUT_PAGECACHE ), startingFromTheBeginning(), true}, // synchronous I/O, string id input - new Object[]{new StringInputIdGenerator(), strings( AUTO ), startingFromTheBeginning(), false}, + new Object[]{new StringInputIdGenerator(), strings( AUTO_WITHOUT_PAGECACHE ), startingFromTheBeginning(), false}, // extra slow parallel I/O, actual node id input - new Object[]{new LongInputIdGenerator(), longs( AUTO ), fromInput(), false} + new Object[]{new LongInputIdGenerator(), longs( AUTO_WITHOUT_PAGECACHE ), fromInput(), false} ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java index f76ddf1c85ba7..4f78c831c4a5c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CountsComputer.java @@ -19,6 +19,7 @@ */ package org.neo4j.kernel.impl.store; +import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.kernel.impl.store.kvstore.DataInitializer; @@ -35,13 +36,13 @@ public class CountsComputer implements DataInitializer private final NumberArrayFactory numberArrayFactory; - public static void recomputeCounts( NeoStores stores ) + public static void recomputeCounts( NeoStores stores, PageCache pageCache ) { MetaDataStore metaDataStore = stores.getMetaDataStore(); CountsTracker counts = stores.getCounts(); try ( CountsAccessor.Updater updater = counts.reset( metaDataStore.getLastCommittedTransactionId() ) ) { - new CountsComputer( stores ).initialize( updater ); + new CountsComputer( stores, pageCache ).initialize( updater ); } } @@ -51,13 +52,13 @@ public static void recomputeCounts( NeoStores stores ) private final int highRelationshipTypeId; private final long lastCommittedTransactionId; - public CountsComputer( NeoStores stores ) + public CountsComputer( NeoStores stores, PageCache pageCache ) { this( stores.getMetaDataStore().getLastCommittedTransactionId(), stores.getNodeStore(), stores.getRelationshipStore(), (int) stores.getLabelTokenStore().getHighId(), (int) stores.getRelationshipTypeTokenStore().getHighId(), - NumberArrayFactory.autoWithPageCacheFallback( stores.getPageCache(), stores.getStoreDir() ) ); + NumberArrayFactory.auto( pageCache, stores.getStoreDir() ) ); } public CountsComputer( long lastCommittedTransactionId, NodeStore nodes, RelationshipStore relationships, diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java index d884afbee703e..523a331ab9a0d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java @@ -151,11 +151,6 @@ public File getStoreDir() return storeDir; } - public PageCache getPageCache() - { - return pageCache; - } - private File getStoreFile( String substoreName ) { return new File( neoStoreFileName.getPath() + substoreName ); @@ -637,7 +632,7 @@ CountsTracker createCountStore( String storeName ) public void initialize( CountsAccessor.Updater updater ) { log.warn( "Missing counts store, rebuilding it." ); - new CountsComputer( neoStores ).initialize( updater ); + new CountsComputer( neoStores, pageCache ).initialize( updater ); log.warn( "Counts store rebuild completed." ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index 29c1e468b3cdc..a649fa2050d05 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -423,7 +423,7 @@ private void rebuildCountsFromScratch( File storeDir, long lastTxId, String vers int highRelationshipTypeId = (int) neoStores.getRelationshipTypeTokenStore().getHighId(); CountsComputer initializer = new CountsComputer( lastTxId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId, - NumberArrayFactory.autoWithPageCacheFallback( pageCache, storeDir ) ); + NumberArrayFactory.auto( pageCache, storeDir ) ); life.add( new CountsTracker( logService.getInternalLogProvider(), fileSystem, pageCache, config, storeFileBase ) .setInitializer( initializer ) ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java index 2583b36f0cea5..997fb75b976ce 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java @@ -187,6 +187,7 @@ public class BatchInserterImpl implements BatchInserter, IndexConfigStoreProvide private final Config config; private final BatchInserterImpl.BatchSchemaActions actions; private final StoreLocker storeLocker; + private final PageCache pageCache; private boolean labelsTouched; private boolean isShutdown; @@ -260,6 +261,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS life.start(); neoStores = sf.openAllNeoStores( true ); neoStores.verifyStoreOk(); + this.pageCache = pageCache; nodeStore = neoStores.getNodeStore(); relationshipStore = neoStores.getRelationshipStore(); @@ -526,7 +528,7 @@ private void rebuildCounts() throw new UnderlyingStorageException( e ); } - CountsComputer.recomputeCounts( neoStores ); + CountsComputer.recomputeCounts( neoStores, pageCache ); } private class InitialNodeLabelCreationVisitor implements Visitor, Closeable diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java index a5e336a16019a..2c393e35ec3e0 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java @@ -169,7 +169,7 @@ public void doImport( Input input ) throws IOException InputCache inputCache = new InputCache( fileSystem, storeDir, recordFormats, config ) ) { NumberArrayFactory numberArrayFactory = - NumberArrayFactory.autoWithPageCacheFallback( neoStore.getPageCache(), storeDir ); + NumberArrayFactory.auto( pageCache, storeDir ); Collector badCollector = input.badCollector(); // Some temporary caches and indexes in the import IoMonitor writeMonitor = new IoMonitor( neoStore.getIoTracer() ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java index 4459c9c752e63..6e4f26a915159 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java @@ -44,6 +44,9 @@ public class RelationshipGroupDefragmenter { + private final Configuration config; + private final ExecutionMonitor executionMonitor; + private final Monitor monitor; private final NumberArrayFactory numberArrayFactory; public interface Monitor @@ -58,15 +61,11 @@ public interface Monitor default void defragmentingNodeRange( long fromNodeId, long toNodeId ) { // empty } - Monitor EMPTY = new Monitor() { // empty }; - } - private final Configuration config; - private final ExecutionMonitor executionMonitor; - private final Monitor monitor; + } public RelationshipGroupDefragmenter( Configuration config, ExecutionMonitor executionMonitor, NumberArrayFactory numberArrayFactory ) diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/ChunkedNumberArrayFactory.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/ChunkedNumberArrayFactory.java index 045c05587f5f0..bbd6ee93543ae 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/ChunkedNumberArrayFactory.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/ChunkedNumberArrayFactory.java @@ -28,14 +28,18 @@ */ public class ChunkedNumberArrayFactory extends NumberArrayFactory.Adapter { + static final int MAGIC_CHUNK_COUNT = 10; + // This is a safe bet on the maximum number of items the JVM can store in an array. It is commonly slightly less + // than Integer.MAX_VALUE + private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - Short.MAX_VALUE; private final NumberArrayFactory delegate; - public ChunkedNumberArrayFactory() + ChunkedNumberArrayFactory() { this( OFF_HEAP, HEAP ); } - public ChunkedNumberArrayFactory( NumberArrayFactory... delegateList ) + ChunkedNumberArrayFactory( NumberArrayFactory... delegateList ) { delegate = new Auto( delegateList ); } @@ -43,45 +47,34 @@ public ChunkedNumberArrayFactory( NumberArrayFactory... delegateList ) @Override public LongArray newLongArray( long length, long defaultValue, long base ) { - // Here we want to have the property of a dynamic array which makes some parts of the array - // live on heap, some off. At the same time we want a fixed size array. Therefore first create - // the array as a dynamic array and make it grow to the requested length. - LongArray array = newDynamicLongArray( fractionOf( length ), defaultValue ); - array.at( length - 1 ); - return array; + // Here we want to have the property of a dynamic array so that some parts of the array + // can live on heap, some off. + return newDynamicLongArray( fractionOf( length ), defaultValue ); } @Override public IntArray newIntArray( long length, int defaultValue, long base ) { - // Here we want to have the property of a dynamic array which makes some parts of the array - // live on heap, some off. At the same time we want a fixed size array. Therefore first create - // the array as a dynamic array and make it grow to the requested length. - IntArray array = newDynamicIntArray( fractionOf( length ), defaultValue ); - array.at( length - 1 ); - return array; + // Here we want to have the property of a dynamic array so that some parts of the array + // can live on heap, some off. + return newDynamicIntArray( fractionOf( length ), defaultValue ); } @Override public ByteArray newByteArray( long length, byte[] defaultValue, long base ) { - // Here we want to have the property of a dynamic array which makes some parts of the array - // live on heap, some off. At the same time we want a fixed size array. Therefore first create - // the array as a dynamic array and make it grow to the requested length. - ByteArray array = newDynamicByteArray( fractionOf( length ), defaultValue ); - array.at( length - 1 ); - return array; + // Here we want to have the property of a dynamic array so that some parts of the array + // can live on heap, some off. + return newDynamicByteArray( fractionOf( length ), defaultValue ); } private long fractionOf( long length ) { - int idealChunkCount = 10; - if ( length < idealChunkCount ) + if ( length < MAGIC_CHUNK_COUNT ) { return length; } - int maxArraySize = Integer.MAX_VALUE - Short.MAX_VALUE; - return min( length / idealChunkCount, maxArraySize ); + return min( length / MAGIC_CHUNK_COUNT, MAX_ARRAY_SIZE ); } @Override @@ -105,6 +98,6 @@ public ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue ) @Override public String toString() { - return "CHUNKED_FIXED_SIZE"; + return "ChunkedNumberArrayFactory with delegate " + delegate; } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArray.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArray.java index 75bd9eeec91fe..ebb93f7bbaadd 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArray.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArray.java @@ -32,8 +32,9 @@ public interface NumberArray> extends MemoryStatsVisito long length(); /** - * Swaps {@code numberOfEntries} items from {@code fromIndex} to {@code toIndex}, such that + * Swaps items from {@code fromIndex} to {@code toIndex}, such that * {@code fromIndex} and {@code toIndex}, {@code fromIndex+1} and {@code toIndex} a.s.o swaps places. + * The number of items swapped is equal to the length of the default value of the array. * @param fromIndex where to start swapping from. * @param toIndex where to start swapping to. */ diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayFactory.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayFactory.java index 423b94ff21b86..bffef6f142839 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayFactory.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayFactory.java @@ -31,111 +31,12 @@ import static org.neo4j.unsafe.impl.batchimport.Utils.safeCastLongToInt; /** - * Factory of {@link LongArray} and {@link IntArray} instances. Users can select in which type of memory - * the arrays will be placed, either in {@link #HEAP} or {@link #OFF_HEAP}, or even {@link #AUTO} which - * will have each instance placed where it fits best, favoring off-heap. + * Factory of {@link LongArray}, {@link IntArray} and {@link ByteArray} instances. Users can select in which type of + * memory the arrays will be placed, either in {@link #HEAP}, {@link #OFF_HEAP}, or use an auto allocator which + * will have each instance placed where it fits best, favoring the primary candidates. */ public interface NumberArrayFactory { - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @return a fixed size {@link IntArray}. - */ - default IntArray newIntArray( long length, int defaultValue ) - { - return newIntArray( length, defaultValue, 0 ); - } - - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @param base base index to rebase all requested indexes with. - * @return a fixed size {@link IntArray}. - */ - IntArray newIntArray( long length, int defaultValue, long base ); - - /** - * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. - * @param defaultValue value which will represent unset values. - * @return dynamically growing {@link IntArray}. - */ - IntArray newDynamicIntArray( long chunkSize, int defaultValue ); - - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @return a fixed size {@link LongArray}. - */ - default LongArray newLongArray( long length, long defaultValue ) - { - return newLongArray( length, defaultValue, 0 ); - } - - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @param base base index to rebase all requested indexes with. - * @return a fixed size {@link LongArray}. - */ - LongArray newLongArray( long length, long defaultValue, long base ); - - /** - * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. - * @param defaultValue value which will represent unset values. - * @return dynamically growing {@link LongArray}. - */ - LongArray newDynamicLongArray( long chunkSize, long defaultValue ); - - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @return a fixed size {@link ByteArray}. - */ - default ByteArray newByteArray( long length, byte[] defaultValue ) - { - return newByteArray( length, defaultValue, 0 ); - } - - /** - * @param length size of the array. - * @param defaultValue value which will represent unset values. - * @param base base index to rebase all requested indexes with. - * @return a fixed size {@link ByteArray}. - */ - ByteArray newByteArray( long length, byte[] defaultValue, long base ); - - /** - * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. - * @param defaultValue value which will represent unset values. - * @return dynamically growing {@link ByteArray}. - */ - ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue ); - - /** - * Implements the dynamic array methods, because they are the same in most implementations. - */ - abstract class Adapter implements NumberArrayFactory - { - @Override - public IntArray newDynamicIntArray( long chunkSize, int defaultValue ) - { - return new DynamicIntArray( this, chunkSize, defaultValue ); - } - - @Override - public LongArray newDynamicLongArray( long chunkSize, long defaultValue ) - { - return new DynamicLongArray( this, chunkSize, defaultValue ); - } - - @Override - public ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue ) - { - return new DynamicByteArray( this, chunkSize, defaultValue ); - } - } - /** * Puts arrays inside the heap. */ @@ -196,13 +97,36 @@ public String toString() } }; + /** + * Used as part of the fallback strategy for {@link Auto}. Tries to split up fixed-size arrays + * ({@link #newLongArray(long, long)} and {@link #newIntArray(long, int)} into smaller chunks where + * some can live on heap and some off heap. + */ + NumberArrayFactory CHUNKED_FIXED_SIZE = new ChunkedNumberArrayFactory(); + + /** + * {@link Auto} factory which uses JVM stats for gathering information about available memory. + */ + NumberArrayFactory AUTO_WITHOUT_PAGECACHE = new Auto( OFF_HEAP, HEAP, CHUNKED_FIXED_SIZE ); + + /** + * {@link Auto} factory which has a page cache backed number array as final fallback, in order to prevent OOM + * errors. + */ + static NumberArrayFactory auto( PageCache pageCache, File dir ) + { + PageCachedNumberArrayFactory pagedArrayFactory = new PageCachedNumberArrayFactory( pageCache, dir ); + ChunkedNumberArrayFactory chunkedArrayFactory = new ChunkedNumberArrayFactory( OFF_HEAP, HEAP, + pagedArrayFactory ); + return new Auto( OFF_HEAP, HEAP, chunkedArrayFactory ); + } /** * Looks at available memory and decides where the requested array fits best. Tries to allocate the whole - * array off-heap, then inside heap. If all else fails a dynamic array is returned with a smaller chunk size - * so that collectively the whole array will fit in memory available on the machine. + * array with the first candidate, falling back to others as needed. */ class Auto extends Adapter { + private final NumberArrayFactory[] candidates; public Auto( NumberArrayFactory... candidates ) @@ -221,8 +145,16 @@ public LongArray newLongArray( long length, long defaultValue, long base ) return candidate.newLongArray( length, defaultValue, base ); } catch ( Throwable e ) - { // Allright let's try the next one - error = e; + { // Alright let's try the next one + if ( error == null ) + { + error = e; + } + else + { + e.addSuppressed( error ); + error = e; + } } } throw launderedException( error( length, 8, error ) ); @@ -263,32 +195,113 @@ public ByteArray newByteArray( long length, byte[] defaultValue, long base ) } throw launderedException( error( length, defaultValue.length, error ) ); } - private Throwable error( long length, int itemSize, Throwable error ) { return Exceptions.withMessage( error, format( "%s: Not enough memory available for allocating %s, tried %s", error.getMessage(), bytes( length * itemSize ), Arrays.toString( candidates ) ) ); } + } /** - * Used as part of the fallback strategy for {@link Auto}. Tries to split up fixed-size arrays - * ({@link #newLongArray(long, long)} and {@link #newIntArray(long, int)} into smaller chunks where - * some can live on heap and some off heap. + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @return a fixed size {@link IntArray}. */ - NumberArrayFactory CHUNKED_FIXED_SIZE = new ChunkedNumberArrayFactory(); + default IntArray newIntArray( long length, int defaultValue ) + { + return newIntArray( length, defaultValue, 0 ); + } /** - * {@link Auto} factory which uses JVM stats for gathering information about available memory. + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @param base base index to rebase all requested indexes with. + * @return a fixed size {@link IntArray}. */ - NumberArrayFactory AUTO = new Auto( OFF_HEAP, HEAP, CHUNKED_FIXED_SIZE ); + IntArray newIntArray( long length, int defaultValue, long base ); - static NumberArrayFactory autoWithPageCacheFallback( PageCache pageCache, File dir ) + /** + * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. + * @param defaultValue value which will represent unset values. + * @return dynamically growing {@link IntArray}. + */ + IntArray newDynamicIntArray( long chunkSize, int defaultValue ); + + /** + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @return a fixed size {@link LongArray}. + */ + default LongArray newLongArray( long length, long defaultValue ) { - PageCachedNumberArrayFactory pagedArrayFactory = - new PageCachedNumberArrayFactory( pageCache, dir ); - ChunkedNumberArrayFactory chunkedArrayFactory = - new ChunkedNumberArrayFactory( OFF_HEAP, HEAP, pagedArrayFactory ); - return new Auto( OFF_HEAP, HEAP, chunkedArrayFactory ); + return newLongArray( length, defaultValue, 0 ); + } + + /** + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @param base base index to rebase all requested indexes with. + * @return a fixed size {@link LongArray}. + */ + LongArray newLongArray( long length, long defaultValue, long base ); + + /** + * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. + * @param defaultValue value which will represent unset values. + * @return dynamically growing {@link LongArray}. + */ + LongArray newDynamicLongArray( long chunkSize, long defaultValue ); + + /** + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @return a fixed size {@link ByteArray}. + */ + default ByteArray newByteArray( long length, byte[] defaultValue ) + { + return newByteArray( length, defaultValue, 0 ); + } + + /** + * @param length size of the array. + * @param defaultValue value which will represent unset values. + * @param base base index to rebase all requested indexes with. + * @return a fixed size {@link ByteArray}. + */ + ByteArray newByteArray( long length, byte[] defaultValue, long base ); + + /** + * @param chunkSize the size of each array (number of items). Where new chunks are added when needed. + * @param defaultValue value which will represent unset values. + * @return dynamically growing {@link ByteArray}. + */ + ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue ); + + /** + * Implements the dynamic array methods, because they are the same in most implementations. + */ + + abstract class Adapter implements NumberArrayFactory + { + + @Override + public IntArray newDynamicIntArray( long chunkSize, int defaultValue ) + { + return new DynamicIntArray( this, chunkSize, defaultValue ); + } + + @Override + public LongArray newDynamicLongArray( long chunkSize, long defaultValue ) + { + return new DynamicLongArray( this, chunkSize, defaultValue ); + } + + @Override + public ByteArray newDynamicByteArray( long chunkSize, byte[] defaultValue ) + { + return new DynamicByteArray( this, chunkSize, defaultValue ); + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheByteArray.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheByteArray.java index 667dc3df87ac5..2cc9f78c376de 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheByteArray.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheByteArray.java @@ -33,10 +33,11 @@ public class PageCacheByteArray extends PageCacheNumberArray implemen { private final byte[] defaultValue; - public PageCacheByteArray( PagedFile pagedFile, long length, byte[] defaultValue, long base ) throws IOException + PageCacheByteArray( PagedFile pagedFile, long length, byte[] defaultValue, long base ) throws IOException { - // '0' default value means we skip filling it out in super - super( pagedFile, defaultValue.length, length, 0, base ); + // Default value is handled locally in this class, in contrast to its siblings, which lets the superclass + // handle it. + super( pagedFile, defaultValue.length, length, base ); this.defaultValue = defaultValue; setDefaultValue( -1 ); } @@ -167,7 +168,7 @@ public long get6ByteLong( long index, int offset ) { long low4b = cursor.getInt( offset ) & 0xFFFFFFFFL; long high2b = cursor.getShort( offset + Integer.BYTES ); - result = low4b | (high2b << 32); + result = low4b | (high2b << Integer.SIZE); } while ( cursor.shouldRetry() ); checkBounds( cursor ); @@ -208,7 +209,7 @@ public void set( long index, byte[] value ) assert value.length == entrySize; long pageId = pageId( index ); int offset = offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); for ( int i = 0; i < value.length; i++ ) @@ -228,7 +229,7 @@ public void setByte( long index, int offset, byte value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putByte( offset, value ); @@ -245,7 +246,7 @@ public void setShort( long index, int offset, short value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putShort( offset, value ); @@ -262,7 +263,7 @@ public void setInt( long index, int offset, int value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putInt( offset, value ); @@ -279,11 +280,11 @@ public void set6ByteLong( long index, int offset, long value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putInt( offset, (int) value ); - cursor.putShort( offset + Integer.BYTES, (short) (value >>> 32) ); + cursor.putShort( offset + Integer.BYTES, (short) (value >>> Integer.SIZE) ); checkBounds( cursor ); } catch ( IOException e ) @@ -297,7 +298,7 @@ public void setLong( long index, int offset, long value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putLong( offset, value ); @@ -340,7 +341,7 @@ public void set3ByteInt( long index, int offset, int value ) { long pageId = pageId( index ); offset += offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putShort( offset, (short) value ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheIntArray.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheIntArray.java index 23282bc5340c7..eaf6545868316 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheIntArray.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheIntArray.java @@ -31,9 +31,9 @@ public class PageCacheIntArray extends PageCacheNumberArray implements IntArray { - public PageCacheIntArray( PagedFile pagedFile, long length, long defaultValue, long base ) throws IOException + PageCacheIntArray( PagedFile pagedFile, long length, long defaultValue, long base ) throws IOException { - super( pagedFile, Integer.BYTES, length, defaultValue, base ); + super( pagedFile, Integer.BYTES, length, defaultValue | defaultValue << Integer.SIZE, base ); } @Override @@ -64,7 +64,7 @@ public void set( long index, int value ) { long pageId = pageId( index ); int offset = offset( index ); - try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ); ) + try ( PageCursor cursor = pagedFile.io( pageId, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { cursor.next(); cursor.putInt( offset, value ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArray.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArray.java index b038a482fb087..c516f7eef0fc7 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArray.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArray.java @@ -31,7 +31,7 @@ public class PageCacheLongArray extends PageCacheNumberArray implements LongArray { - public PageCacheLongArray( PagedFile pagedFile, long length, long defaultValue, long base ) throws IOException + PageCacheLongArray( PagedFile pagedFile, long length, long defaultValue, long base ) throws IOException { super( pagedFile, Long.BYTES, length, defaultValue, base ); } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheNumberArray.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheNumberArray.java index 79eecac439e5a..0c589976c2e81 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheNumberArray.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheNumberArray.java @@ -29,6 +29,11 @@ import static org.neo4j.io.pagecache.PagedFile.PF_NO_GROW; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; +/** + * Abstraction over page cache backed number arrays. + * + * @see PageCachedNumberArrayFactory + */ public abstract class PageCacheNumberArray> implements NumberArray { protected final PagedFile pagedFile; @@ -39,8 +44,13 @@ public abstract class PageCacheNumberArray> implements private final long base; private boolean closed; - protected PageCacheNumberArray( PagedFile pagedFile, int entrySize, long length, - long defaultValue, long base ) throws IOException + PageCacheNumberArray( PagedFile pagedFile, int entrySize, long length, long base ) throws IOException + { + this( pagedFile, entrySize, length, 0, base ); + } + + PageCacheNumberArray( PagedFile pagedFile, int entrySize, long length, long defaultValue, long base ) + throws IOException { this.pagedFile = pagedFile; this.entrySize = entrySize; @@ -64,7 +74,8 @@ private void setLength( PageCursor cursor, long length ) throws IOException { if ( !cursor.next( (length - 1) / entriesPerPage ) ) { - throw new IllegalStateException(); + throw new IllegalStateException( + String.format( "Unable to extend the backing file %s to desired size %d.", pagedFile, length ) ); } } @@ -75,7 +86,7 @@ protected long pageId( long index ) protected int offset( long index ) { - return toIntExact( rebase( index ) % entriesPerPage ) * entrySize; + return toIntExact( rebase( index ) % entriesPerPage * entrySize ); } private long rebase( long index ) @@ -85,10 +96,6 @@ private long rebase( long index ) protected void setDefaultValue( long defaultValue ) throws IOException { - if ( entrySize == Integer.BYTES ) - { - defaultValue |= defaultValue << 32; - } try ( PageCursor writeCursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) { writeCursor.next(); @@ -101,6 +108,7 @@ protected void setDefaultValue( long defaultValue ) throws IOException while ( cursor.next() ) { writeCursor.copyTo( 0, cursor, 0, pageSize ); + checkBounds( writeCursor ); } } } @@ -114,6 +122,7 @@ protected void fillPageWithDefaultValue( PageCursor writeCursor, long defaultVal { writeCursor.putLong( defaultValue ); } + checkBounds( writeCursor ); } @Override @@ -172,7 +181,9 @@ protected void checkBounds( PageCursor cursor ) { if ( cursor.checkAndClearBoundsFlag() ) { - throw new IllegalStateException(); + throw new IllegalStateException( + String.format( "Cursor %s access out of bounds, page id %d, offset %d", cursor.toString(), + cursor.getCurrentPageId(), cursor.getOffset() ) ); } } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCachedNumberArrayFactory.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCachedNumberArrayFactory.java index 60724a77cd564..734e650676ad3 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCachedNumberArrayFactory.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/PageCachedNumberArrayFactory.java @@ -29,12 +29,16 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE; +/** + * Factory of page cache backed number arrays. + * @see NumberArrayFactory + */ public class PageCachedNumberArrayFactory extends NumberArrayFactory.Adapter { private final PageCache pageCache; private final File storeDir; - public PageCachedNumberArrayFactory( PageCache pageCache, File storeDir ) + PageCachedNumberArrayFactory( PageCache pageCache, File storeDir ) { this.pageCache = pageCache; this.storeDir = storeDir; diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/Tracker.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/Tracker.java index 8efdc83f8c7a8..581f71e7f2507 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/Tracker.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/cache/idmapping/string/Tracker.java @@ -48,7 +48,7 @@ public interface Tracker extends MemoryStatsVisitor.Visitable, AutoCloseable long get( long index ); /** - * Swaps values from {@code fromIndex} to {@code toIndex}, as many items as {@code count} specifies. + * Swaps values from {@code fromIndex} to {@code toIndex}. * * @param fromIndex index to swap from. * @param toIndex index to swap to. diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index 4255469d327e2..bc87890b5d602 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -283,11 +283,6 @@ public CountsTracker getCountsStore() return neoStores.getCounts(); } - public PageCache getPageCache() - { - return pageCache; - } - @Override public void close() throws IOException { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java index 3bc3dbaf47e36..b534d173958ea 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java @@ -329,7 +329,7 @@ private void rebuildCounts( long lastCommittedTransactionId ) throws IOException int highRelationshipTypeId = (int) neoStores.getRelationshipTypeTokenStore().getHighId(); CountsComputer countsComputer = new CountsComputer( lastCommittedTransactionId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId, - NumberArrayFactory.AUTO ); + NumberArrayFactory.AUTO_WITHOUT_PAGECACHE ); CountsTracker countsTracker = createCountsTracker(); life.add( countsTracker.setInitializer( countsComputer ) ); } diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipCountsProcessorTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipCountsProcessorTest.java index c1c317374122d..2a0fa675bdadc 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipCountsProcessorTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipCountsProcessorTest.java @@ -87,7 +87,7 @@ public void testRelationshipCountersUpdates() when( nodeLabelCache.get( eq( client ), eq( 4L ), any( int[].class ) ) ).thenReturn( new int[]{} ); RelationshipCountsProcessor countsProcessor = new RelationshipCountsProcessor( nodeLabelCache, labels, - relationTypes, countsUpdater, NumberArrayFactory.AUTO ); + relationTypes, countsUpdater, NumberArrayFactory.AUTO_WITHOUT_PAGECACHE ); countsProcessor.process( 1, 0, 3 ); countsProcessor.process( 2, 1, 4 ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java index 90c5f58512fe9..ee0dfc59d6ff9 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java @@ -59,7 +59,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE; @RunWith( Parameterized.class ) public class RelationshipGroupDefragmenterTest @@ -193,7 +193,7 @@ private void defrag( int nodeCount, RecordStore groupSt { Monitor monitor = mock( Monitor.class ); RelationshipGroupDefragmenter defragmenter = new RelationshipGroupDefragmenter( CONFIG, - ExecutionMonitors.invisible(), monitor, AUTO ); + ExecutionMonitors.invisible(), monitor, AUTO_WITHOUT_PAGECACHE ); // Calculation below correlates somewhat to calculation in RelationshipGroupDefragmenter. // Anyway we verify below that we exercise the multi-pass bit, which is what we want diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/ByteArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/ByteArrayTest.java index 59583a161d7f8..3cc26349fc2b2 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/ByteArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/ByteArrayTest.java @@ -39,10 +39,10 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.HEAP; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.OFF_HEAP; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.autoWithPageCacheFallback; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.auto; @RunWith( Parameterized.class ) public class ByteArrayTest extends NumberArrayPageCacheTestSupport @@ -57,16 +57,16 @@ public static Collection> data() throws IOException fixture = prepareDirectoryAndPageCache( ByteArrayTest.class ); PageCache pageCache = fixture.pageCache; File dir = fixture.directory; - NumberArrayFactory autoWithPageCacheFallback = autoWithPageCacheFallback( pageCache, dir ); + NumberArrayFactory autoWithPageCacheFallback = auto( pageCache, dir ); NumberArrayFactory pageCacheArrayFactory = new PageCachedNumberArrayFactory( pageCache, dir ); - int chunkSize = LENGTH / 10; + int chunkSize = LENGTH / ChunkedNumberArrayFactory.MAGIC_CHUNK_COUNT; return Arrays.asList( () -> HEAP.newByteArray( LENGTH, DEFAULT ), () -> HEAP.newDynamicByteArray( chunkSize, DEFAULT ), () -> OFF_HEAP.newByteArray( LENGTH, DEFAULT ), () -> OFF_HEAP.newDynamicByteArray( chunkSize, DEFAULT ), - () -> AUTO.newByteArray( LENGTH, DEFAULT ), - () -> AUTO.newDynamicByteArray( chunkSize, DEFAULT ), + () -> AUTO_WITHOUT_PAGECACHE.newByteArray( LENGTH, DEFAULT ), + () -> AUTO_WITHOUT_PAGECACHE.newDynamicByteArray( chunkSize, DEFAULT ), () -> autoWithPageCacheFallback.newByteArray( LENGTH, DEFAULT ), () -> autoWithPageCacheFallback.newDynamicByteArray( chunkSize, DEFAULT ), () -> pageCacheArrayFactory.newByteArray( LENGTH, DEFAULT ), diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicIntArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicIntArrayTest.java index ee7e37141b47c..f30e7807e9d3d 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicIntArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicIntArrayTest.java @@ -30,7 +30,7 @@ public void shouldWorkOnSingleChunk() throws Exception { // GIVEN int defaultValue = 0; - IntArray array = NumberArrayFactory.AUTO.newDynamicIntArray( 10, defaultValue ); + IntArray array = NumberArrayFactory.AUTO_WITHOUT_PAGECACHE.newDynamicIntArray( 10, defaultValue ); array.set( 4, 5 ); // WHEN @@ -44,7 +44,7 @@ public void shouldWorkOnSingleChunk() throws Exception public void shouldChunksAsNeeded() throws Exception { // GIVEN - IntArray array = NumberArrayFactory.AUTO.newDynamicIntArray( 10, 0 ); + IntArray array = NumberArrayFactory.AUTO_WITHOUT_PAGECACHE.newDynamicIntArray( 10, 0 ); // WHEN long index = 243; diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicLongArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicLongArrayTest.java index a186850d346f6..beaaab3748469 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicLongArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/DynamicLongArrayTest.java @@ -30,7 +30,7 @@ public void shouldWorkOnSingleChunk() throws Exception { // GIVEN long defaultValue = 0; - LongArray array = NumberArrayFactory.AUTO.newDynamicLongArray( 10, defaultValue ); + LongArray array = NumberArrayFactory.AUTO_WITHOUT_PAGECACHE.newDynamicLongArray( 10, defaultValue ); array.set( 4, 5 ); // WHEN @@ -44,7 +44,7 @@ public void shouldWorkOnSingleChunk() throws Exception public void shouldChunksAsNeeded() throws Exception { // GIVEN - LongArray array = NumberArrayFactory.AUTO.newDynamicLongArray( 10, 0 ); + LongArray array = NumberArrayFactory.AUTO_WITHOUT_PAGECACHE.newDynamicLongArray( 10, 0 ); // WHEN long index = 243; diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/IntArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/IntArrayTest.java index 670edcf232d36..ffddf7ef4f300 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/IntArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/IntArrayTest.java @@ -38,7 +38,7 @@ import static org.junit.Assert.assertEquals; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.HEAP; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.OFF_HEAP; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.autoWithPageCacheFallback; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.auto; @RunWith( Parameterized.class ) public class IntArrayTest extends NumberArrayPageCacheTestSupport @@ -106,7 +106,7 @@ public static Collection data() throws IOException fixture = prepareDirectoryAndPageCache( IntArrayTest.class ); PageCache pageCache = fixture.pageCache; File dir = fixture.directory; - NumberArrayFactory autoWithPageCacheFallback = autoWithPageCacheFallback( pageCache, dir ); + NumberArrayFactory autoWithPageCacheFallback = auto( pageCache, dir ); NumberArrayFactory pageCacheArrayFactory = new PageCachedNumberArrayFactory( pageCache, dir ); return Arrays.asList( HEAP, OFF_HEAP, autoWithPageCacheFallback, pageCacheArrayFactory ); } diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/LongArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/LongArrayTest.java index beab2f1929d90..010ff894f2ca4 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/LongArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/LongArrayTest.java @@ -38,7 +38,7 @@ import static org.junit.Assert.assertEquals; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.HEAP; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.OFF_HEAP; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.autoWithPageCacheFallback; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.auto; @RunWith( Parameterized.class ) public class LongArrayTest extends NumberArrayPageCacheTestSupport @@ -106,7 +106,7 @@ public static Collection data() throws IOException fixture = prepareDirectoryAndPageCache( LongArrayTest.class ); PageCache pageCache = fixture.pageCache; File dir = fixture.directory; - NumberArrayFactory autoWithPageCacheFallback = autoWithPageCacheFallback( pageCache, dir ); + NumberArrayFactory autoWithPageCacheFallback = auto( pageCache, dir ); NumberArrayFactory pageCacheArrayFactory = new PageCachedNumberArrayFactory( pageCache, dir ); return Arrays.asList( HEAP, OFF_HEAP, autoWithPageCacheFallback, pageCacheArrayFactory ); } diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeLabelsCacheTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeLabelsCacheTest.java index 10864badbb409..59aeb84e631e0 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeLabelsCacheTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeLabelsCacheTest.java @@ -36,7 +36,7 @@ public class NodeLabelsCacheTest public void shouldCacheSmallSetOfLabelsPerNode() throws Exception { // GIVEN - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, 5, CHUNK_SIZE ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 5, CHUNK_SIZE ); NodeLabelsCache.Client client = cache.newClient(); long nodeId = 0; @@ -54,7 +54,7 @@ public void shouldHandleLargeAmountOfLabelsPerNode() throws Exception { // GIVEN int highLabelId = 1000; - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, highLabelId, CHUNK_SIZE ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, highLabelId, CHUNK_SIZE ); NodeLabelsCache.Client client = cache.newClient(); long nodeId = 0; @@ -73,7 +73,7 @@ public void shouldHandleLabelsForManyNodes() throws Exception { // GIVEN a really weird scenario where we have 5000 different labels int highLabelId = 1_000; - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, highLabelId, 1_000_000 ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, highLabelId, 1_000_000 ); NodeLabelsCache.Client client = cache.newClient(); int numberOfNodes = 100_000; int[][] expectedLabels = new int[numberOfNodes][]; @@ -97,7 +97,7 @@ public void shouldHandleLabelsForManyNodes() throws Exception public void shouldEndTargetArrayWithMinusOne() throws Exception { // GIVEN - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, 10 ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 10 ); NodeLabelsCache.Client client = cache.newClient(); cache.put( 10, new long[] { 5, 6, 7, 8 } ); @@ -117,7 +117,7 @@ public void shouldEndTargetArrayWithMinusOne() throws Exception public void shouldReturnEmptyArrayForNodeWithNoLabelsAndNoLabelsWhatsoever() throws Exception { // GIVEN - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, 0 ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 0 ); NodeLabelsCache.Client client = cache.newClient(); // WHEN @@ -134,7 +134,7 @@ public void shouldSupportConcurrentGet() throws Throwable // GIVEN int highLabelId = 10, numberOfNodes = 100; int[][] expectedLabels = new int[numberOfNodes][]; - NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO, highLabelId ); + NodeLabelsCache cache = new NodeLabelsCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, highLabelId ); for ( int i = 0; i < numberOfNodes; i++ ) { cache.put( i, asLongArray( expectedLabels[i] = randomLabels( random.nextInt( 5 ), highLabelId ) ) ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeRelationshipCacheTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeRelationshipCacheTest.java index dad7c367d4a70..eefa78b9a6454 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeRelationshipCacheTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NodeRelationshipCacheTest.java @@ -87,7 +87,7 @@ public static Collection data() public void shouldReportCorrectNumberOfDenseNodes() throws Exception { // GIVEN - cache = new NodeRelationshipCache( NumberArrayFactory.AUTO, 5, 100, base ); + cache = new NodeRelationshipCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 5, 100, base ); cache.setHighNodeId( 26 ); increment( cache, 2, 10 ); increment( cache, 5, 2 ); @@ -133,7 +133,7 @@ public void shouldGoThroughThePhases() throws Exception public void shouldObserveFirstRelationshipAsEmptyInEachDirection() throws Exception { // GIVEN - cache = new NodeRelationshipCache( NumberArrayFactory.AUTO, 1, 100, base ); + cache = new NodeRelationshipCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 1, 100, base ); int nodes = 100; int typeId = 5; Direction[] directions = Direction.values(); @@ -170,7 +170,7 @@ public void shouldObserveFirstRelationshipAsEmptyInEachDirection() throws Except public void shouldResetCountAfterGetOnDenseNodes() throws Exception { // GIVEN - cache = new NodeRelationshipCache( NumberArrayFactory.AUTO, 1, 100, base ); + cache = new NodeRelationshipCache( NumberArrayFactory.AUTO_WITHOUT_PAGECACHE, 1, 100, base ); long nodeId = 0; int typeId = 3; cache.setHighNodeId( 1 ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayPageCacheTestSupport.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayPageCacheTestSupport.java index 6cf0ef8934aee..dcebc08254c48 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayPageCacheTestSupport.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayPageCacheTestSupport.java @@ -35,7 +35,6 @@ public static Fixture prepareDirectoryAndPageCache( Class testClass ) throws DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); TestDirectory testDirectory = TestDirectory.testDirectory( testClass, fileSystem ); File dir = testDirectory.prepareDirectoryForTest( "test" ); - fileSystem.mkdirs( dir ); PageCache pageCache = StandalonePageCacheFactory.createPageCache( fileSystem ); return new Fixture( pageCache, fileSystem, dir ); } diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayTest.java index 2b9cc3f9d21f0..6a015fd643470 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/NumberArrayTest.java @@ -40,11 +40,11 @@ import org.neo4j.test.rule.RandomRule; import static org.junit.Assert.assertEquals; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.AUTO_WITHOUT_PAGECACHE; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.CHUNKED_FIXED_SIZE; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.HEAP; import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.OFF_HEAP; -import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.autoWithPageCacheFallback; +import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.auto; @RunWith( Parameterized.class ) public class NumberArrayTest extends NumberArrayPageCacheTestSupport @@ -75,9 +75,9 @@ public static Collection arrays() throws IOException Map factories = new HashMap<>(); factories.put( "HEAP", HEAP ); factories.put( "OFF_HEAP", OFF_HEAP ); - factories.put( "AUTO", AUTO ); + factories.put( "AUTO_WITHOUT_PAGECACHE", AUTO_WITHOUT_PAGECACHE ); factories.put( "CHUNKED_FIXED_SIZE", CHUNKED_FIXED_SIZE ); - factories.put( "autoWithPageCacheFallback", autoWithPageCacheFallback( pageCache, dir ) ); + factories.put( "autoWithPageCacheFallback", auto( pageCache, dir ) ); factories.put( "PageCachedNumberArrayFactory", new PageCachedNumberArrayFactory( pageCache, dir ) ); for ( Map.Entry entry : factories.entrySet() ) { diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArrayTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArrayTest.java index de9b2ddd7449b..8356f42faf553 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArrayTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/cache/PageCacheLongArrayTest.java @@ -53,6 +53,7 @@ public void verifyPageCacheLongArray() throws Exception { PageCache pageCache = pageCacheRule.getPageCache( fs ); PagedFile file = pageCache.map( dir.file( "file" ), pageCache.pageSize(), CREATE, DELETE_ON_CLOSE ); + try ( LongArray array = new PageCacheLongArray( file, COUNT, 0, 0 ) ) { verifyBehaviour( array ); @@ -64,7 +65,7 @@ public void verifyChunkingArrayWithPageCacheLongArray() throws Exception { PageCache pageCache = pageCacheRule.getPageCache( fs ); File directory = dir.directory(); - NumberArrayFactory numberArrayFactory = NumberArrayFactory.autoWithPageCacheFallback( pageCache, directory ); + NumberArrayFactory numberArrayFactory = NumberArrayFactory.auto( pageCache, directory ); try ( LongArray array = numberArrayFactory.newDynamicLongArray( COUNT / 1_000, 0 ) ) { verifyBehaviour( array );