diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java index c0fb5555d209e..2dc0ce485d06b 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java @@ -36,7 +36,6 @@ import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.cursor.RawCursor; import org.neo4j.helpers.Exceptions; -import org.neo4j.io.pagecache.CursorException; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; @@ -124,19 +123,6 @@ */ public class GBPTree implements Closeable { - /** - * Version of the format that makes up the tree. This includes: - * - * If any of the above changes the on-page format then this version should be bumped, so that opening - * an index on wrong format version fails and user will need to rebuild. - */ - static final int FORMAT_VERSION = 2; - /** * For monitoring {@link GBPTree}. */ @@ -424,11 +410,22 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i boolean success = false; try { - this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize, layout ); + this.pagedFile = openOrCreate( pageCache, indexFile, tentativePageSize ); this.pageSize = pagedFile.pageSize(); closed = false; - this.bTreeNode = layout.fixedSize() ? new TreeNodeFixedSize<>( pageSize, layout ) - : new TreeNodeDynamicSize<>( pageSize, layout ); + TreeNodeSelector.Factory format; + if ( created ) + { + format = TreeNodeSelector.selectByLayout( layout ); + writeMeta( layout, format, pagedFile ); + } + else + { + Meta meta = readMeta( layout, pagedFile ); + meta.verify( layout ); + format = TreeNodeSelector.selectByFormat( meta.getFormatIdentifier(), meta.getFormatVersion() ); + } + this.bTreeNode = format.create( pageSize, layout ); this.freeList = new FreeListIdProvider( pagedFile, pageSize, rootId, FreeListIdProvider.NO_MONITOR ); this.writer = new SingleWriter( new InternalTreeLogic<>( freeList, bTreeNode, layout ) ); @@ -468,9 +465,6 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i private void initializeAfterCreation( Layout layout, Consumer headerWriter ) throws IOException { - // Initialize meta - writeMeta( layout, pagedFile ); - // Initialize state try ( PageCursor cursor = pagedFile.io( 0 /*ignored*/, PagedFile.PF_SHARED_WRITE_LOCK ) ) { @@ -496,11 +490,11 @@ private void initializeAfterCreation( Layout layout, Consumer layout ) throws IOException + int pageSizeForCreation ) throws IOException { try { - return openExistingIndexFile( pageCache, indexFile, layout ); + return openExistingIndexFile( pageCache, indexFile ); } catch ( NoSuchFileException e ) { @@ -508,8 +502,7 @@ private PagedFile openOrCreate( PageCache pageCache, File indexFile, } } - private static PagedFile openExistingIndexFile( PageCache pageCache, File indexFile, Layout layout ) - throws IOException + private static PagedFile openExistingIndexFile( PageCache pageCache, File indexFile ) throws IOException { PagedFile pagedFile = pageCache.map( indexFile, pageCache.pageSize() ); // This index already exists, verify meta data aligns with expectations @@ -517,8 +510,9 @@ private static PagedFile openExistingIndexFile( PageCache pageCache boolean success = false; try { - int pageSize = readMeta( layout, pagedFile ); - pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile, pageSize ); + // We're only interested in the page size really, so don't involve layout at this point + Meta meta = readMeta( null, pagedFile ); + pagedFile = mapWithCorrectPageSize( pageCache, indexFile, pagedFile, meta.getPageSize() ); success = true; return pagedFile; } @@ -588,8 +582,9 @@ private void loadState( PagedFile pagedFile, Header.Reader headerReader ) throws */ public static void readHeader( PageCache pageCache, File indexFile, Layout layout, Header.Reader headerReader ) throws IOException { - try ( PagedFile pagedFile = openExistingIndexFile( pageCache, indexFile, layout ) ) + try ( PagedFile pagedFile = openExistingIndexFile( pageCache, indexFile ) ) { + readMeta( layout, pagedFile ).verify( layout ); Pair states = loadStatePages( pagedFile ); TreeState state = TreeStatePair.selectNewestValidState( states ); try ( PageCursor cursor = pagedFile.io( state.pageId(), PagedFile.PF_SHARED_READ_LOCK ) ) @@ -731,66 +726,21 @@ private static PageCursor openMetaPageCursor( PagedFile pagedFile, int pfFlags ) return metaCursor; } - private static int readMeta( Layout layout, PagedFile pagedFile ) + private static Meta readMeta( Layout layout, PagedFile pagedFile ) throws IOException { - // Read meta - int formatVersion; - int pageSize; - long layoutIdentifier; - int majorVersion; - int minorVersion; try ( PageCursor metaCursor = openMetaPageCursor( pagedFile, PagedFile.PF_SHARED_READ_LOCK ) ) { - do - { - formatVersion = metaCursor.getInt(); - pageSize = metaCursor.getInt(); - layoutIdentifier = metaCursor.getLong(); - majorVersion = metaCursor.getInt(); - minorVersion = metaCursor.getInt(); - layout.readMetaData( metaCursor ); - } - while ( metaCursor.shouldRetry() ); - checkOutOfBounds( metaCursor ); - metaCursor.checkAndClearCursorException(); - } - catch ( CursorException e ) - { - throw new MetadataMismatchException( e, - "Tried to open, but caught an error while reading meta data. " + - "File is expected to be corrupt, try to rebuild." ); - } - - if ( formatVersion != FORMAT_VERSION ) - { - throw new MetadataMismatchException( - "Tried to open with a different format version than " + - "what it was created with. Created with %d, opened with %d", - formatVersion, FORMAT_VERSION ); - } - if ( !layout.compatibleWith( layoutIdentifier, majorVersion, minorVersion ) ) - { - throw new MetadataMismatchException( - "Tried to open using layout not compatible with " + - "what the index was created with. Created with: layoutIdentifier=%d,majorVersion=%d,minorVersion=%d. " + - "Opened with layoutIdentifier=%d,majorVersion=%d,minorVersion=%d", - layoutIdentifier, majorVersion, minorVersion, layout.identifier(), layout.majorVersion(), layout.minorVersion() ); + return Meta.read( metaCursor, layout ); } - return pageSize; } - private void writeMeta( Layout layout, PagedFile pagedFile ) throws IOException + private void writeMeta( Layout layout, TreeNodeSelector.Factory format, PagedFile pagedFile ) throws IOException { + Meta meta = new Meta( format.formatIdentifier(), format.formatVersion(), pageSize, layout ); try ( PageCursor metaCursor = openMetaPageCursor( pagedFile, PagedFile.PF_SHARED_WRITE_LOCK ) ) { - metaCursor.putInt( FORMAT_VERSION ); - metaCursor.putInt( pageSize ); - metaCursor.putLong( layout.identifier() ); - metaCursor.putInt( layout.majorVersion() ); - metaCursor.putInt( layout.minorVersion() ); - layout.writeMetaData( metaCursor ); - checkOutOfBounds( metaCursor ); + meta.write( metaCursor, layout ); } } diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/Meta.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/Meta.java new file mode 100644 index 0000000000000..2dd5c40913ea4 --- /dev/null +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/Meta.java @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.index.internal.gbptree; + +import java.io.IOException; + +import org.neo4j.index.internal.gbptree.TreeNodeSelector.Factory; +import org.neo4j.io.pagecache.CursorException; +import org.neo4j.io.pagecache.PageCursor; +import static org.neo4j.index.internal.gbptree.PageCursorUtil.checkOutOfBounds; + +/** + * About versioning (i.e. the format version {@code int}): + * The format version started out as one int controlling the entire version of the tree and its different types of formats. + * For compatibility reasons this int has been kept but used differently, i.e. split up into four individual versions, + * one {@code byte} each. These are: + * + * 
+ *     <------- int ------>
+ * msb [ 3 ][ 2 ][ 1 ][ 0 ] lsb
+ *       ▲    ▲    ▲    ▲
+ *       │    │    │    │
+ *       │    │    │    └──────────── {@link TreeNode#formatIdentifier()}
+ *       │    │    └───────────────── {@link TreeNode#formatVersion()}
+ *       │    └────────────────────── {@link #CURRENT_STATE_VERSION}
+ *       └─────────────────────────── {@link #CURRENT_GBPTREE_VERSION}
+ *
+ * + * {@link #CURRENT_STATE_VERSION} and {@link #CURRENT_GBPTREE_VERSION} aren't used yet because they have + * never needed to be versioned yet, but remain reserved for future use. The are fixed at 0 a.t.m. + */ +class Meta +{ + static final byte CURRENT_STATE_VERSION = 0; + static final byte CURRENT_GBPTREE_VERSION = 0; + + private static final int MASK_BYTE = 0xFF; + + private static final int SHIFT_FORMAT_IDENTIFIER = Byte.SIZE * 0; + private static final int SHIFT_FORMAT_VERSION = Byte.SIZE * 1; + private static final int SHIFT_UNUSED_VERSION_SLOT_3 = Byte.SIZE * 2; + private static final int SHIFT_UNUSED_VERSION_SLOT_4 = Byte.SIZE * 3; + static final byte UNUSED_VERSION = 0; + + private final byte formatIdentifier; + private final byte formatVersion; + private final byte unusedVersionSlot3; + private final byte unusedVersionSlot4; + private final int pageSize; + private final long layoutIdentifier; + private final int layoutMajorVersion; + private final int layoutMinorVersion; + + private Meta( byte formatIdentifier, byte formatVersion, byte unusedVersionSlot3, byte unusedVersionSlot4, + int pageSize, long layoutIdentifier, int layoutMajorVersion, int layoutMinorVersion ) + { + this.formatIdentifier = formatIdentifier; + this.formatVersion = formatVersion; + this.unusedVersionSlot3 = unusedVersionSlot3; + this.unusedVersionSlot4 = unusedVersionSlot4; + this.pageSize = pageSize; + this.layoutIdentifier = layoutIdentifier; + this.layoutMajorVersion = layoutMajorVersion; + this.layoutMinorVersion = layoutMinorVersion; + } + + Meta( byte formatIdentifier, byte formatVersion, int pageSize, Layout layout ) + { + this( formatIdentifier, formatVersion, UNUSED_VERSION, UNUSED_VERSION, + pageSize, layout.identifier(), layout.majorVersion(), layout.minorVersion() ); + } + + private static Meta parseMeta( int format, int pageSize, long layoutIdentifier, int majorVersion, int minorVersion ) + { + return new Meta( extractIndividualVersion( format, SHIFT_FORMAT_IDENTIFIER ), + extractIndividualVersion( format, SHIFT_FORMAT_VERSION ), + extractIndividualVersion( format, SHIFT_UNUSED_VERSION_SLOT_3 ), + extractIndividualVersion( format, SHIFT_UNUSED_VERSION_SLOT_4 ), + pageSize, layoutIdentifier, majorVersion, minorVersion ); + } + + /** + * Reads meta information from the meta page. Reading meta information also involves {@link Layout} in that + * it can have written layout-specific information to this page too. The layout identifier and its version + * that the returned {@link Meta} instance will have are the ones read from the page, not retrieved from {@link Layout}. + * + * @param cursor {@link PageCursor} to read meta information from. + * @param layout {@link Layout} instance that will get the opportunity to read layout-specific data from the meta page. + * {@code layout} is allowed to be {@code null} where it won't be told to read layout-specific data from the meta page. + * @return {@link Meta} instance with all meta information. + * @throws IOException on {@link PageCursor} I/O error. + */ + static Meta read( PageCursor cursor, Layout layout ) throws IOException + { + int format; + int pageSize; + long layoutIdentifier; + int layoutMajorVersion; + int layoutMinorVersion; + try + { + do + { + format = cursor.getInt(); + pageSize = cursor.getInt(); + layoutIdentifier = cursor.getLong(); + layoutMajorVersion = cursor.getInt(); + layoutMinorVersion = cursor.getInt(); + if ( layout != null ) + { + layout.readMetaData( cursor ); + } + } + while ( cursor.shouldRetry() ); + checkOutOfBounds( cursor ); + cursor.checkAndClearCursorException(); + } + catch ( CursorException e ) + { + throw new MetadataMismatchException( e, + "Tried to open, but caught an error while reading meta data. " + + "File is expected to be corrupt, try to rebuild." ); + } + + return parseMeta( format, pageSize, layoutIdentifier, layoutMajorVersion, layoutMinorVersion ); + } + + void verify( Layout layout ) + { + if ( unusedVersionSlot3 != Meta.UNUSED_VERSION ) + { + throw new MetadataMismatchException( "Unexpected version " + unusedVersionSlot3 + " for unused version slot 3" ); + } + if ( unusedVersionSlot4 != Meta.UNUSED_VERSION ) + { + throw new MetadataMismatchException( "Unexpected version " + unusedVersionSlot4 + " for unused version slot 4" ); + } + + if ( !layout.compatibleWith( layoutIdentifier, layoutMajorVersion, layoutMinorVersion ) ) + { + throw new MetadataMismatchException( + "Tried to open using layout not compatible with " + + "what the index was created with. Created with: layoutIdentifier=%d,majorVersion=%d,minorVersion=%d. " + + "Opened with layoutIdentifier=%d,majorVersion=%d,minorVersion=%d", + layoutIdentifier, layoutMajorVersion, layoutMinorVersion, + layout.identifier(), layout.majorVersion(), layout.minorVersion() ); + } + + Factory formatByLayout = TreeNodeSelector.selectByLayout( layout ); + if ( formatByLayout.formatIdentifier() != formatIdentifier || + formatByLayout.formatVersion() != formatVersion ) + { + throw new MetadataMismatchException( "Tried to open using layout not compatible with what index was created with. " + + "Created with formatIdentifier:%d,formatVersion:%d. Opened with formatIdentifier:%d,formatVersion%d", + formatIdentifier, formatVersion, formatByLayout.formatIdentifier(), formatByLayout.formatVersion() ); + } + } + + /** + * Writes meta information to the meta page. Writing meta information also involves {@link Layout} in that + * it can write layout-specific information to this page too. + * + * @param cursor {@link PageCursor} to read meta information from. + * @param layout {@link Layout} instance that will get the opportunity to write layout-specific data to the meta page. + */ + void write( PageCursor cursor, Layout layout ) + { + cursor.putInt( allVersionsCombined() ); + cursor.putInt( getPageSize() ); + cursor.putLong( getLayoutIdentifier() ); + cursor.putInt( getLayoutMajorVersion() ); + cursor.putInt( getLayoutMinorVersion() ); + layout.writeMetaData( cursor ); + checkOutOfBounds( cursor ); + } + + private static byte extractIndividualVersion( int format, int shift ) + { + return (byte) ((format >>> shift) & MASK_BYTE); + } + + int allVersionsCombined() + { + return formatIdentifier >>> SHIFT_FORMAT_IDENTIFIER | formatVersion >>> SHIFT_FORMAT_VERSION; + } + + public int getPageSize() + { + return pageSize; + } + + byte getFormatIdentifier() + { + return formatIdentifier; + } + + public byte getFormatVersion() + { + return formatVersion; + } + + byte getUnusedVersionSlot3() + { + return unusedVersionSlot3; + } + + byte getUnusedVersionSlot4() + { + return unusedVersionSlot4; + } + + long getLayoutIdentifier() + { + return layoutIdentifier; + } + + int getLayoutMajorVersion() + { + return layoutMajorVersion; + } + + int getLayoutMinorVersion() + { + return layoutMinorVersion; + } +} diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java index bbd13e1f39e52..edae65a97210f 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java @@ -61,6 +61,9 @@ */ public class TreeNodeDynamicSize extends TreeNode { + static final byte FORMAT_IDENTIFIER = 3; + static final byte FORMAT_VERSION = 0; + /** * Concepts * Total space - The space available for data (pageSize - headerSize) @@ -88,8 +91,8 @@ public class TreeNodeDynamicSize extends TreeNode private final int maxKeyCount = pageSize / bytesKeyOffset() + bytesKeySize() + bytesValueSize(); private final int[] oldOffset = new int[maxKeyCount]; private final int[] newOffset = new int[maxKeyCount]; - private int totalSpace; - private int halfSpace; + private final int totalSpace; + private final int halfSpace; TreeNodeDynamicSize( int pageSize, Layout layout ) { diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSize.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSize.java index f0ce94948f234..bb2696b9a715c 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSize.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSize.java @@ -64,6 +64,9 @@ */ class TreeNodeFixedSize extends TreeNode { + static final byte FORMAT_IDENTIFIER = 2; + static final byte FORMAT_VERSION = 0; + private final int internalMaxKeyCount; private final int leafMaxKeyCount; private final int keySize; diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeSelector.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeSelector.java new file mode 100644 index 0000000000000..035d09f408e5f --- /dev/null +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeSelector.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.index.internal.gbptree; + +import static java.lang.String.format; + +/** + * Able to select implementation of {@link TreeNode} to use in different scenarios, should be used in favor of directly + * instantiating {@link TreeNode} instances. + */ +class TreeNodeSelector +{ + /** + * Creates {@link TreeNodeFixedSize} instances. + */ + static Factory FIXED = new Factory() + { + @Override + public TreeNode create( int pageSize, Layout layout ) + { + return new TreeNodeFixedSize<>( pageSize, layout ); + } + + @Override + public byte formatIdentifier() + { + return TreeNodeFixedSize.FORMAT_IDENTIFIER; + } + + @Override + public byte formatVersion() + { + return TreeNodeFixedSize.FORMAT_VERSION; + } + }; + + /** + * Creates {@link TreeNodeDynamicSize} instances. + */ + static Factory DYNAMIC = new Factory() + { + @Override + public TreeNode create( int pageSize, Layout layout ) + { + return new TreeNodeDynamicSize<>( pageSize, layout ); + } + + @Override + public byte formatIdentifier() + { + return TreeNodeDynamicSize.FORMAT_IDENTIFIER; + } + + @Override + public byte formatVersion() + { + return TreeNodeDynamicSize.FORMAT_VERSION; + } + }; + + /** + * Selects a format based on the given {@link Layout}. + * + * @param layout {@link Layout} dictating which {@link TreeNode} to instantiate. + * @return a {@link Factory} capable of instantiating the selected format. + */ + static Factory selectByLayout( Layout layout ) + { + // For now the selection is done in a simple fashion, by looking at layout.fixedSize(). + return layout.fixedSize() ? FIXED : DYNAMIC; + } + + /** + * Selects a format based on the given format specification. + * + * @param formatIdentifier format identifier, see {@link Meta#getFormatIdentifier()} + * @param formatVersion format version, see {@link Meta#getFormatVersion()}. + * @return a {@link Factory} capable of instantiating the selected format. + */ + static Factory selectByFormat( byte formatIdentifier, byte formatVersion ) + { + // For now do a simple selection of the two formats we know. Moving forward this can contain + // many more identifiers and different versions of each. + if ( formatIdentifier == TreeNodeFixedSize.FORMAT_IDENTIFIER && formatVersion == TreeNodeFixedSize.FORMAT_VERSION ) + { + return FIXED; + } + else if ( formatIdentifier == TreeNodeDynamicSize.FORMAT_IDENTIFIER && formatVersion == TreeNodeDynamicSize.FORMAT_VERSION ) + { + return DYNAMIC; + } + throw new IllegalArgumentException( + format( "Unknown format identifier:%d and version:%d combination", formatIdentifier, formatVersion ) ); + } + + /** + * Able to instantiate {@link TreeNode} of a specific format and version. + */ + interface Factory + { + /** + * Instantiates a {@link TreeNode} of a specific format and version that this factory represents. + * + * @param pageSize page size, i.e. size of tree nodes. + * @param layout {@link Layout} that will be used in this format. + * @return the instantiated {@link TreeNode}. + */ + TreeNode create( int pageSize, Layout layout ); + + /** + * Specifies the format identifier of the physical layout of tree nodes. + * A format identifier must be unique among all possible existing format identifiers. + * It's used to differentiate between different types of formats. + * On top of this a specific format identifier may be {@link #formatVersion()}. + * + * @return format identifier for the specific {@link TreeNode} that this factory represents. + * Can return this w/o instantiating the {@link TreeNode}. + */ + byte formatIdentifier(); + + /** + * Specifies the version of this particular {@link #formatIdentifier() format}. It must be unique + * among all other versions of this {@link #formatIdentifier() format}. + * + * @return format version for the specific {@link TreeNode} that this factory represents. + * Can return this w/o instantiating the {@link TreeNode}. + */ + byte formatVersion(); + } +} diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/ConsistencyCheckerTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/ConsistencyCheckerTest.java index 8cb80169e2f76..97d72fe65634e 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/ConsistencyCheckerTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/ConsistencyCheckerTest.java @@ -32,6 +32,7 @@ import static org.neo4j.index.internal.gbptree.ConsistencyChecker.assertNoCrashOrBrokenPointerInGSPP; import static org.neo4j.index.internal.gbptree.GenerationSafePointer.MIN_GENERATION; import static org.neo4j.index.internal.gbptree.PageCursorUtil.goTo; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; public class ConsistencyCheckerTest { @@ -48,7 +49,7 @@ public void shouldThrowDescriptiveExceptionOnBrokenGSPP() throws Exception long pointer = 123; cursor.next( 0 ); - new TreeNodeFixedSize<>( pageSize, new SimpleLongLayout() ).initializeInternal( cursor, stableGeneration, crashGeneration ); + new TreeNodeFixedSize<>( pageSize, layout().build() ).initializeInternal( cursor, stableGeneration, crashGeneration ); TreeNode.setSuccessor( cursor, pointer, stableGeneration, crashGeneration ); // WHEN @@ -75,7 +76,7 @@ public void shouldDetectUnusedPages() throws Exception { // GIVEN int pageSize = 256; - Layout layout = new SimpleLongLayout(); + Layout layout = layout().build(); TreeNode node = new TreeNodeFixedSize<>( pageSize, layout ); long stableGeneration = GenerationSafePointer.MIN_GENERATION; long unstableGeneration = stableGeneration + 1; diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/CrashGenerationCleanerTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/CrashGenerationCleanerTest.java index 0b51da2639023..11354fedd8369 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/CrashGenerationCleanerTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/CrashGenerationCleanerTest.java @@ -44,6 +44,8 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.DELETE_ON_CLOSE; import static org.junit.Assert.assertEquals; + +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; import static org.neo4j.index.internal.gbptree.TreeNode.BYTE_POS_LEFTSIBLING; import static org.neo4j.index.internal.gbptree.TreeNode.BYTE_POS_RIGHTSIBLING; import static org.neo4j.index.internal.gbptree.TreeNode.BYTE_POS_SUCCESSOR; @@ -54,10 +56,10 @@ public class CrashGenerationCleanerTest { - private FileSystemRule fileSystemRule = new DefaultFileSystemRule(); - private PageCacheRule pageCacheRule = new PageCacheRule(); - private TestDirectory testDirectory = TestDirectory.testDirectory( this.getClass(), fileSystemRule.get() ); - private RandomRule randomRule = new RandomRule(); + private final FileSystemRule fileSystemRule = new DefaultFileSystemRule(); + private final PageCacheRule pageCacheRule = new PageCacheRule(); + private final TestDirectory testDirectory = TestDirectory.testDirectory( this.getClass(), fileSystemRule.get() ); + private final RandomRule randomRule = new RandomRule(); @Rule public RuleChain ruleChain = RuleChain .outerRule( fileSystemRule ).around( testDirectory ).around( pageCacheRule ).around( randomRule ); @@ -66,7 +68,7 @@ public class CrashGenerationCleanerTest private static final int PAGE_SIZE = 256; private PagedFile pagedFile; - private final Layout layout = new SimpleLongLayout(); + private final Layout layout = layout().build(); private final CorruptibleTreeNode corruptibleTreeNode = new CorruptibleTreeNode( PAGE_SIZE, layout ); private final int oldStableGeneration = 9; private final int stableGeneration = 10; diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java index 37d2d0a564d60..e733ee3b77ad7 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java @@ -49,6 +49,8 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; import static org.neo4j.test.rule.PageCacheRule.config; /** @@ -95,7 +97,7 @@ public void shouldDetectFormatChange() throws Throwable // THEN everything should work, otherwise there has likely been a format change PageCache pageCache = pageCacheRule.getPageCache( fsRule.get() ); try ( GBPTree tree = - new GBPTreeBuilder<>( pageCache, storeFile, new SimpleLongLayout() ).build() ) + new GBPTreeBuilder<>( pageCache, storeFile, layout().build() ).build() ) { try { @@ -171,7 +173,7 @@ private void createAndZipTree( File storeFile ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fsRule.get() ); try ( GBPTree tree = - new GBPTreeBuilder<>( pageCache, storeFile, new SimpleLongLayout() ).build() ) + new GBPTreeBuilder<>( pageCache, storeFile, layout().build() ).build() ) { MutableLong insertKey = new MutableLong(); MutableLong insertValue = new MutableLong(); diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyFIxedSizeIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyFIxedSizeIT.java index 5606c70082e1c..8d4eb8bca2e38 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyFIxedSizeIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyFIxedSizeIT.java @@ -23,11 +23,13 @@ import org.neo4j.test.rule.RandomRule; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class GBPTreeConcurrencyFIxedSizeIT extends GBPTreeConcurrencyITBase { @Override protected TestLayout getLayout( RandomRule random ) { - return new SimpleLongLayout( random.intBetween( 0, 10 ) ); + return layout().withKeyPadding( random.intBetween( 0, 10 ) ).build(); } } diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeFixedSizeIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeFixedSizeIT.java index a60e1c8dd3b2c..27aa16b3871f8 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeFixedSizeIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeFixedSizeIT.java @@ -23,12 +23,14 @@ import org.neo4j.test.rule.RandomRule; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class GBPTreeFixedSizeIT extends GBPTreeITBase { @Override protected TestLayout getLayout( RandomRule random ) { - return new SimpleLongLayout( random.intBetween( 0, 10 ) ); + return layout().withKeyPadding( random.intBetween( 0, 10 ) ).build(); } @Override diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java index 29a62163e5123..d2005d4d5b0bb 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java @@ -42,6 +42,7 @@ import static java.util.Arrays.asList; import static org.neo4j.graphdb.config.Configuration.EMPTY; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_READER; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; /** * Tests functionality around process crashing, or similar, when having started, but not completed creation of an index file, @@ -69,7 +70,7 @@ public void shouldDetectAndThrowIOExceptionOnPartiallyCreatedFile() throws Excep // then reading it should either work or throw IOException try ( PageCache pageCache = storage.pageCache() ) { - SimpleLongLayout layout = new SimpleLongLayout(); + SimpleLongLayout layout = layout().build(); // check readHeader try @@ -106,7 +107,7 @@ public static void main( String[] args ) throws IOException try ( PageCache pageCache = new MuninnPageCache( swapper, 10, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ) ) { fs.deleteFile( file ); - new GBPTreeBuilder<>( pageCache, file, new SimpleLongLayout() ).build().close(); + new GBPTreeBuilder<>( pageCache, file, layout().build() ).build().close(); } } } diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeReadWriteFixedSizeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeReadWriteFixedSizeTest.java index 30705d1b981e3..872c9d1d03b61 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeReadWriteFixedSizeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeReadWriteFixedSizeTest.java @@ -21,11 +21,13 @@ import org.apache.commons.lang3.mutable.MutableLong; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class GBPTreeReadWriteFixedSizeTest extends GBPTreeReadWriteTestBase { @Override TestLayout getLayout() { - return new SimpleLongLayout(); + return layout().build(); } } diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryFixedSizeIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryFixedSizeIT.java index 1150a06b4c920..d11040f4f45a3 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryFixedSizeIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryFixedSizeIT.java @@ -23,11 +23,13 @@ import org.neo4j.test.rule.RandomRule; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class GBPTreeRecoveryFixedSizeIT extends GBPTreeRecoveryITBase { @Override protected TestLayout getLayout( RandomRule random ) { - return new SimpleLongLayout( random.intBetween( 0, 10 ) ); + return layout().withKeyPadding( random.intBetween( 0, 10 ) ).build(); } } diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java index f6aa170a75943..2fb2300937cf8 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java @@ -80,6 +80,7 @@ import static org.junit.Assert.fail; import static org.junit.rules.RuleChain.outerRule; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_READER; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; import static org.neo4j.index.internal.gbptree.ThrowingRunnable.throwing; import static org.neo4j.io.pagecache.IOLimiter.unlimited; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; @@ -90,7 +91,7 @@ public class GBPTreeTest { private static final int DEFAULT_PAGE_SIZE = 256; - private static final Layout layout = new SimpleLongLayout(); + private static final Layout layout = layout().build(); private final DefaultFileSystemRule fs = new DefaultFileSystemRule(); private final TestDirectory directory = TestDirectory.testDirectory( getClass(), fs.get() ); @@ -143,7 +144,7 @@ public void shouldFailToOpenOnDifferentMetaData() throws Exception } // WHEN - SimpleLongLayout otherLayout = new SimpleLongLayout( 0, "Something else" ); + SimpleLongLayout otherLayout = layout().withCustomerNameAsMetaData( "Something else" ).build(); try ( GBPTree ignored = index().with( otherLayout ).build() ) { fail( "Should not load" ); @@ -166,14 +167,7 @@ public void shouldFailToOpenOnDifferentLayout() throws Exception } // WHEN - SimpleLongLayout otherLayout = new SimpleLongLayout() - { - @Override - public long identifier() - { - return 123456; - } - }; + SimpleLongLayout otherLayout = layout().withIdentifier( 123456 ).build(); try ( GBPTree ignored = index().with( otherLayout ).build() ) { fail( "Should not load" ); @@ -193,14 +187,7 @@ public void shouldFailToOpenOnDifferentMajorVersion() throws Exception } // WHEN - SimpleLongLayout otherLayout = new SimpleLongLayout() - { - @Override - public int majorVersion() - { - return super.majorVersion() + 1; - } - }; + SimpleLongLayout otherLayout = layout().withMajorVersion( 123 ).build(); try ( GBPTree ignored = index().with( otherLayout ).build() ) { fail( "Should not load" ); @@ -220,14 +207,7 @@ public void shouldFailToOpenOnDifferentMinorVersion() throws Exception } // WHEN - SimpleLongLayout otherLayout = new SimpleLongLayout() - { - @Override - public int minorVersion() - { - return super.minorVersion() + 1; - } - }; + SimpleLongLayout otherLayout = layout().withMinorVersion( 123 ).build(); try ( GBPTree ignored = index().with( otherLayout ).build() ) { fail( "Should not load" ); @@ -363,7 +343,7 @@ public void shouldRemapFileIfMappedWithPageSizeLargerThanCreationSize() throws E } @Test - public void shouldFailWhenTryingToOpenWithDifferentFormatVersion() throws Exception + public void shouldFailWhenTryingToOpenWithDifferentFormatIdentifier() throws Exception { // GIVEN int pageSize = DEFAULT_PAGE_SIZE; @@ -372,12 +352,11 @@ public void shouldFailWhenTryingToOpenWithDifferentFormatVersion() throws Except try ( GBPTree ignored = builder.build() ) { // Open/close is enough } - setFormatVersion( pageCache, pageSize, GBPTree.FORMAT_VERSION - 1 ); try { // WHEN - builder.build(); + builder.with( layout().withFixedSize( false ).build() ).build(); fail( "Should have failed" ); } catch ( MetadataMismatchException e ) @@ -1806,16 +1785,6 @@ public int count() } } - private void setFormatVersion( PageCache pageCache, int pageSize, int formatVersion ) throws IOException - { - try ( PagedFile pagedFile = pageCache.map( indexFile, pageSize ); - PageCursor cursor = pagedFile.io( IdSpace.META_PAGE_ID, PF_SHARED_WRITE_LOCK ) ) - { - assertTrue( cursor.next() ); - cursor.putInt( formatVersion ); - } - } - private static class MonitorDirty extends Monitor.Adaptor { private boolean called; diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/InternalTreeLogicFixedSizeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/InternalTreeLogicFixedSizeTest.java index 730c30404be6c..4e5a5ca7c4e39 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/InternalTreeLogicFixedSizeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/InternalTreeLogicFixedSizeTest.java @@ -21,9 +21,11 @@ import org.apache.commons.lang3.mutable.MutableLong; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class InternalTreeLogicFixedSizeTest extends InternalTreeLogicTestBase { - SimpleLongLayout layout = new SimpleLongLayout(); + SimpleLongLayout layout = layout().build(); @Override protected ValueMerger getAdder() diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/KeySearchTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/KeySearchTest.java index aace9f8c062fb..24a288b16ea4d 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/KeySearchTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/KeySearchTest.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertTrue; import static org.neo4j.index.internal.gbptree.GBPTreeTestUtil.contains; import static org.neo4j.index.internal.gbptree.KeySearch.search; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; import static org.neo4j.index.internal.gbptree.TreeNode.Overflow.NO; import static org.neo4j.index.internal.gbptree.TreeNode.Type.INTERNAL; import static org.neo4j.index.internal.gbptree.TreeNode.Type.LEAF; @@ -46,7 +47,7 @@ public class KeySearchTest private static final int KEY_COUNT = 10; private static final int PAGE_SIZE = 512; private final PageCursor cursor = wrap( new byte[PAGE_SIZE], 0, PAGE_SIZE ); - private final Layout layout = new SimpleLongLayout(); + private final Layout layout = layout().build(); private final TreeNode node = new TreeNodeFixedSize<>( PAGE_SIZE, layout ); private final MutableLong readKey = layout.newKey(); private final MutableLong searchKey = layout.newKey(); diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/SeekCursorFixedSizeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/SeekCursorFixedSizeTest.java index 257ec90310014..3464dd8d36a76 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/SeekCursorFixedSizeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/SeekCursorFixedSizeTest.java @@ -21,12 +21,14 @@ import org.apache.commons.lang3.mutable.MutableLong; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class SeekCursorFixedSizeTest extends SeekCursorTestBase { @Override TestLayout getLayout() { - return new SimpleLongLayout(); + return layout().build(); } @Override diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/SimpleLongLayout.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/SimpleLongLayout.java index 8605d4007825f..dd526d2bac84b 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/SimpleLongLayout.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/SimpleLongLayout.java @@ -29,21 +29,75 @@ class SimpleLongLayout extends TestLayout { private final int keyPadding; private String customNameAsMetaData; + private final boolean fixedSize; + private final int identifier; + private final int majorVersion; + private final int minorVersion; - SimpleLongLayout( int keyPadding, String customNameAsMetaData ) + static class Builder { - this.keyPadding = keyPadding; - this.customNameAsMetaData = customNameAsMetaData; + private int keyPadding; + private int identifier = 999; + private int majorVersion; + private int minorVersion; + private String customNameAsMetaData = "test"; + private boolean fixedSize = true; + + Builder withKeyPadding( int keyPadding ) + { + this.keyPadding = keyPadding; + return this; + } + + Builder withIdentifier( int identifier ) + { + this.identifier = identifier; + return this; + } + + Builder withMajorVersion( int majorVersion ) + { + this.majorVersion = majorVersion; + return this; + } + + Builder withMinorVersion( int minorVersion ) + { + this.minorVersion = minorVersion; + return this; + } + + Builder withCustomerNameAsMetaData( String customNameAsMetaData ) + { + this.customNameAsMetaData = customNameAsMetaData; + return this; + } + + Builder withFixedSize( boolean fixedSize ) + { + this.fixedSize = fixedSize; + return this; + } + + SimpleLongLayout build() + { + return new SimpleLongLayout( keyPadding, customNameAsMetaData, fixedSize, identifier, majorVersion, minorVersion ); + } } - SimpleLongLayout( int keyPadding ) + static Builder layout() { - this( keyPadding, "test" ); + return new Builder(); } - SimpleLongLayout() + SimpleLongLayout( int keyPadding, String customNameAsMetaData, boolean fixedSize, int identifier, int majorVersion, int minorVersion ) { - this( 0 ); + this.keyPadding = keyPadding; + this.customNameAsMetaData = customNameAsMetaData; + this.fixedSize = fixedSize; + this.identifier = identifier; + this.majorVersion = majorVersion; + this.minorVersion = minorVersion; } @Override @@ -117,25 +171,25 @@ public void readValue( PageCursor cursor, MutableLong into, int valueSize ) @Override public boolean fixedSize() { - return true; + return fixedSize; } @Override public long identifier() { - return 999; + return identifier; } @Override public int majorVersion() { - return 0; + return majorVersion; } @Override public int minorVersion() { - return 0; + return minorVersion; } @Override diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSizeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSizeTest.java index f110195c063ff..7e0466afa368e 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSizeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/TreeNodeFixedSizeTest.java @@ -23,9 +23,11 @@ import org.neo4j.io.pagecache.PageCursor; +import static org.neo4j.index.internal.gbptree.SimpleLongLayout.layout; + public class TreeNodeFixedSizeTest extends TreeNodeTestBase { - private SimpleLongLayout layout = new SimpleLongLayout(); + private final SimpleLongLayout layout = layout().build(); @Override protected TestLayout getLayout()