From 8ebde0269647384b65e5de9bc960c955ac453517 Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Tue, 13 Jun 2017 16:50:25 +0200 Subject: [PATCH] Support for writing header on newly instantiated GBPTree --- .../neo4j/index/internal/gbptree/GBPTree.java | 20 +++--- .../gbptree/FormatCompatibilityTest.java | 9 +-- .../gbptree/GBPTreeConcurrencyIT.java | 3 +- .../index/internal/gbptree/GBPTreeIT.java | 3 +- .../internal/gbptree/GBPTreeRecoveryIT.java | 3 +- .../index/internal/gbptree/GBPTreeTest.java | 61 ++++++++++++++++++- 6 files changed, 84 insertions(+), 15 deletions(-) diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java index 51d16818225d7..9e6bddd452726 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java @@ -201,6 +201,11 @@ public void startupState( boolean clean ) */ static final Header.Reader NO_HEADER = (cursor,length) -> {}; + /** + * No-op header writer. + */ + static final Consumer NO_HEADER_WRITER = pc -> {}; + /** * Paged file in a {@link PageCache} providing the means of storage. */ @@ -340,12 +345,13 @@ public void startupState( boolean clean ) * @param tentativePageSize page size, i.e. tree node size. Must be less than or equal to that of the page cache. * A pageSize of {@code 0} means to use whatever the page cache has (at creation) * @param monitor {@link Monitor} for monitoring {@link GBPTree}. - * @param headerReader reads header data, previously written using {@link #checkpoint(IOLimiter, Consumer)} - * or {@link #close()} + * @param headerReader reads header data if indexFile already exists, + * previously written using {@link #checkpoint(IOLimiter, Consumer)} or {@link #close()} + * @param headerWriter writes header data if indexFile is created as a result of this call. * @throws IOException on page cache error */ public GBPTree( PageCache pageCache, File indexFile, Layout layout, int tentativePageSize, - Monitor monitor, Header.Reader headerReader ) throws IOException + Monitor monitor, Header.Reader headerReader, Consumer headerWriter ) throws IOException { this.indexFile = indexFile; this.monitor = monitor; @@ -363,7 +369,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i // Create or load state if ( created ) { - initializeAfterCreation( layout ); + initializeAfterCreation( layout, headerWriter ); } else { @@ -396,7 +402,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i } } - private void initializeAfterCreation( Layout layout ) throws IOException + private void initializeAfterCreation( Layout layout, Consumer headerWriter ) throws IOException { // Initialize meta writeMeta( layout, pagedFile ); @@ -419,7 +425,7 @@ private void initializeAfterCreation( Layout layout ) throws IOExcept // Initialize free-list freeList.initializeAfterCreation(); changesSinceLastCheckpoint = true; - checkpoint( IOLimiter.unlimited() ); + checkpoint( IOLimiter.unlimited(), headerWriter ); clean = true; } @@ -523,7 +529,7 @@ private static void writerHeader( PagedFile pagedFile, Header.Writer headerWrite // Write/carry over header int headerOffset = cursor.getOffset(); int headerDataOffset = headerOffset + Integer.BYTES; // will contain length of written header data (below) - if ( otherState.isValid() ) + if ( otherState.isValid() || headerWriter != CARRY_OVER_PREVIOUS_HEADER ) { PageCursor previousCursor = pagedFile.io( otherState.pageId(), PagedFile.PF_SHARED_READ_LOCK ); PageCursorUtil.goTo( previousCursor, "previous state page", otherState.pageId() ); diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java index 43b14dd75d398..39d7a324563aa 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/FormatCompatibilityTest.java @@ -50,6 +50,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER; import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; import static org.neo4j.test.rule.PageCacheRule.config; @@ -96,8 +97,8 @@ public void shouldDetectFormatChange() throws Throwable // WHEN reading from the tree // THEN everything should work, otherwise there has likely been a format change PageCache pageCache = pageCacheRule.getPageCache( fsRule.get() ); - try ( GBPTree tree = - new GBPTree<>( pageCache, storeFile, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER ) ) + try ( GBPTree tree = new GBPTree<>( pageCache, storeFile, new SimpleLongLayout(), 0, + NO_MONITOR, NO_HEADER, NO_HEADER_WRITER ) ) { try { @@ -172,8 +173,8 @@ private void unzipTo( File storeFile ) throws IOException private void createAndZipTree( File storeFile ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fsRule.get() ); - try ( GBPTree tree = - new GBPTree<>( pageCache, storeFile, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER ) ) + try ( GBPTree tree = new GBPTree<>( pageCache, storeFile, new SimpleLongLayout(), 0, + NO_MONITOR, NO_HEADER, NO_HEADER_WRITER ) ) { MutableLong insertKey = new MutableLong(); MutableLong insertValue = new MutableLong(); diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyIT.java index dee79c74e2891..983e735df2d25 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeConcurrencyIT.java @@ -61,6 +61,7 @@ import static org.junit.Assert.fail; import static org.junit.rules.RuleChain.outerRule; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER; import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; import static org.neo4j.test.rule.PageCacheRule.config; @@ -106,7 +107,7 @@ private GBPTree createIndex( GBPTree.Monitor monitor ) PageCache pageCache = pageCacheRule.getPageCache( fs.get(), config().withPageSize( pageSize ).withAccessChecks( true ) ); return index = new GBPTree<>( pageCache, directory.file( "index" ), - layout, 0/*use whatever page cache says*/, monitor, NO_HEADER ); + layout, 0/*use whatever page cache says*/, monitor, NO_HEADER, NO_HEADER_WRITER ); } @After diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeIT.java index 57bc906228546..2f279762a20e9 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeIT.java @@ -46,6 +46,7 @@ import static org.junit.Assert.fail; import static org.junit.rules.RuleChain.outerRule; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER; import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; import static org.neo4j.test.rule.PageCacheRule.config; @@ -75,7 +76,7 @@ private GBPTree createIndex( int pageSize, GBPTree.Moni { pageCache = pageCacheRule.getPageCache( fs.get(), config().withPageSize( pageSize ).withAccessChecks( true ) ); return index = new GBPTree<>( pageCache, directory.file( "index" ), - layout, 0/*use whatever page cache says*/, monitor, NO_HEADER ); + layout, 0/*use whatever page cache says*/, monitor, NO_HEADER, NO_HEADER_WRITER ); } @After diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryIT.java index 451d0ed5b2969..e4b28a27baa50 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeRecoveryIT.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.rules.RuleChain.outerRule; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER; import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; import static org.neo4j.index.internal.gbptree.ThrowingRunnable.throwing; import static org.neo4j.io.pagecache.IOLimiter.unlimited; @@ -381,7 +382,7 @@ private long[] modificationData( int min, int max ) private static GBPTree createIndex( PageCache pageCache, File file ) throws IOException { - return new GBPTree<>( pageCache, file, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER ); + return new GBPTree<>( pageCache, file, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER, NO_HEADER_WRITER ); } private PageCache createPageCache() diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java index 876b7369d069a..f5ad60399c9a4 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeTest.java @@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.neo4j.collection.primitive.Primitive; import org.neo4j.collection.primitive.PrimitiveLongCollections; @@ -73,6 +74,7 @@ import static org.junit.Assert.fail; import static org.junit.rules.RuleChain.outerRule; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER; import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; import static org.neo4j.index.internal.gbptree.ThrowingRunnable.throwing; import static org.neo4j.io.pagecache.IOLimiter.unlimited; @@ -557,6 +559,50 @@ public void shouldReplaceHeaderDataInNextCheckPoint() throws Exception verifyHeaderDataAfterClose( beforeClose ); } + @Test + public void mustWriteHeaderOnInitialization() throws Exception + { + // GIVEN + byte[] headerBytes = new byte[12]; + ThreadLocalRandom.current().nextBytes( headerBytes ); + Consumer headerWriter = pc -> pc.putBytes( headerBytes ); + + // WHEN + try ( GBPTree ignore = index().with( headerWriter ).build() ) + { + } + + // THEN + verifyHeader( headerBytes ); + } + + @Test + public void mustNotOverwriteHeaderOnExistingTree() throws Exception + { + // GIVEN + byte[] expectedBytes = new byte[12]; + ThreadLocalRandom.current().nextBytes( expectedBytes ); + Consumer headerWriter = pc -> pc.putBytes( expectedBytes ); + try ( GBPTree ignore = index().with( headerWriter ).build() ) + { + } + + // WHEN + byte[] fraudulentBytes = new byte[12]; + do + { + ThreadLocalRandom.current().nextBytes( fraudulentBytes ); + } + while ( Arrays.equals( expectedBytes, fraudulentBytes ) ); + + try ( GBPTree ignore = index().with( headerWriter ).build() ) + { + } + + // THEN + verifyHeader( expectedBytes ); + } + private void verifyHeaderDataAfterClose( BiConsumer,byte[]> beforeClose ) throws IOException { byte[] expectedHeader = new byte[12]; @@ -568,6 +614,11 @@ private void verifyHeaderDataAfterClose( BiConsumer layout = GBPTreeTest.layout; private PageCache specificPageCache; + private Consumer headerWriter = NO_HEADER_WRITER; private GBPTreeBuilder withPageCachePageSize( int pageSize ) { @@ -1250,6 +1302,12 @@ private GBPTreeBuilder with( PageCache pageCache ) return this; } + public GBPTreeBuilder with( Consumer headerWriter ) + { + this.headerWriter = headerWriter; + return this; + } + private GBPTree build() throws IOException { PageCache pageCacheToUse; @@ -1267,7 +1325,8 @@ private GBPTree build() throws IOException pageCacheToUse = specificPageCache; } - return new GBPTree<>( pageCacheToUse, indexFile, layout, tentativePageSize, monitor, headerReader ); + return new GBPTree<>( pageCacheToUse, indexFile, layout, tentativePageSize, monitor, headerReader, + headerWriter ); } }