From 4e779997ffc0167d7b8f22894aedf149f2f7181f Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Mon, 2 Jan 2017 17:00:36 +0100 Subject: [PATCH] Complete the first draft implementation of PageList.tryEvict --- .../io/pagecache/impl/muninn/PageList.java | 55 ++- .../tracing/DefaultPageCacheTracer.java | 2 +- .../io/pagecache/tracing/EvictionEvent.java | 4 +- .../tracing/EvictionEventOpportunity.java | 31 ++ .../pagecache/tracing/EvictionRunEvent.java | 8 +- .../tracing/FlushEventOpportunity.java | 2 +- .../io/pagecache/tracing/PageFaultEvent.java | 11 +- .../neo4j/io/pagecache/tracing/PinEvent.java | 4 +- .../cursor/DefaultPageCursorTracer.java | 8 +- .../pagecache/impl/muninn/PageListTest.java | 342 ++++++++++++++++-- .../impl/muninn/StubPageFaultEvent.java | 4 +- .../pagecache/tracing/DummyPageSwapper.java | 4 +- .../io/pagecache/tracing/linear/HEvents.java | 20 +- .../recording/RecordingPageCacheTracer.java | 2 +- .../recording/RecordingPageCursorTracer.java | 4 +- 15 files changed, 421 insertions(+), 80 deletions(-) create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEventOpportunity.java diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java index 0f91aae99a80e..cebc91287cd97 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java @@ -23,6 +23,9 @@ import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PageSwapper; +import org.neo4j.io.pagecache.tracing.EvictionEvent; +import org.neo4j.io.pagecache.tracing.EvictionEventOpportunity; +import org.neo4j.io.pagecache.tracing.FlushEvent; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; @@ -117,7 +120,7 @@ private void clearMemory( long baseAddress, long pageCount ) */ public long deref( long pageId ) { - return baseAddress + pageId * META_DATA_BYTES_PER_PAGE; + return baseAddress + (pageId * META_DATA_BYTES_PER_PAGE); } private long offLock( long pageRef ) @@ -337,32 +340,54 @@ private static IllegalStateException cannotFaultException( long pageRef, PageSwa return new IllegalStateException( msg ); } - public boolean tryEvict( long pageRef ) throws IOException + public boolean tryEvict( long pageRef, EvictionEventOpportunity evictionOpportunity ) throws IOException { if ( tryExclusiveLock( pageRef ) ) { if ( isLoaded( pageRef ) ) { - int swapperId = getSwapperId( pageRef ); - SwapperSet.Allocation allocation = swappers.getAllocation( swapperId ); - PageSwapper swapper = allocation.swapper; - long filePageId = getFilePageId( pageRef ); - if ( isModified( pageRef ) ) + try ( EvictionEvent evictionEvent = evictionOpportunity.beginEviction() ) { - long address = getAddress( pageRef ); - swapper.write( filePageId, address, allocation.filePageSize ); - explicitlyMarkPageUnmodifiedUnderExclusiveLock( pageRef ); + evict( pageRef, evictionEvent ); + return true; } - swapper.evicted( filePageId, null ); - clearBinding( pageRef ); - return true; } - else + unlockExclusive( pageRef ); + } + return false; + } + + private void evict( long pageRef, EvictionEvent evictionEvent ) throws IOException + { + int swapperId = getSwapperId( pageRef ); + SwapperSet.Allocation allocation = swappers.getAllocation( swapperId ); + PageSwapper swapper = allocation.swapper; + long filePageId = getFilePageId( pageRef ); + evictionEvent.setFilePageId( filePageId ); + evictionEvent.setCachePageId( pageRef ); + evictionEvent.setSwapper( swapper ); + if ( isModified( pageRef ) ) + { + FlushEvent flushEvent = evictionEvent.flushEventOpportunity().beginFlush( filePageId, pageRef, swapper ); + try + { + long address = getAddress( pageRef ); + long bytesWritten = swapper.write( filePageId, address, allocation.filePageSize ); + explicitlyMarkPageUnmodifiedUnderExclusiveLock( pageRef ); + flushEvent.addBytesWritten( bytesWritten ); + flushEvent.addPagesFlushed( 1 ); + flushEvent.done(); + } + catch ( IOException e ) { unlockExclusive( pageRef ); + flushEvent.done( e ); + evictionEvent.threwException( e ); + throw e; } } - return false; + swapper.evicted( filePageId, null ); + clearBinding( pageRef ); } private void clearBinding( long pageRef ) diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java index 9e9688698a0fa..a7d982624a517 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java @@ -95,7 +95,7 @@ public void threwException( IOException exception ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEvent.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEvent.java index 890ef0425a1f9..2e79d9b2b25f7 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEvent.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEvent.java @@ -55,7 +55,7 @@ public void threwException( IOException exception ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } @@ -89,5 +89,5 @@ public void close() /** * The cache page id of the evicted page. */ - void setCachePageId( int cachePageId ); + void setCachePageId( long cachePageId ); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEventOpportunity.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEventOpportunity.java new file mode 100644 index 0000000000000..f260cfc203c4a --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionEventOpportunity.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing; + +/** + * Interface for any event that in turn presents the opportunity to evict a page. + */ +public interface EvictionEventOpportunity +{ + /** + * Begin an eviction event. + */ + EvictionEvent beginEviction(); +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionRunEvent.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionRunEvent.java index 3c51e2d805c76..0b3342cede121 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionRunEvent.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/EvictionRunEvent.java @@ -24,14 +24,13 @@ * needs to evict a batch of pages. The dedicated eviction thread is * mostly sleeping when it is not performing an eviction run. */ -public interface EvictionRunEvent extends AutoCloseablePageCacheTracerEvent +public interface EvictionRunEvent extends AutoCloseablePageCacheTracerEvent, EvictionEventOpportunity { /** * An EvictionRunEvent that does nothing other than return the EvictionEvent.NULL. */ EvictionRunEvent NULL = new EvictionRunEvent() { - @Override public EvictionEvent beginEviction() { @@ -43,9 +42,4 @@ public void close() { } }; - - /** - * An eviction is started as part of this eviction run. - */ - EvictionEvent beginEviction(); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/FlushEventOpportunity.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/FlushEventOpportunity.java index 95f269a302757..024df8d2c5a7b 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/FlushEventOpportunity.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/FlushEventOpportunity.java @@ -36,5 +36,5 @@ public interface FlushEventOpportunity /** * Begin flushing the given page. */ - FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ); + FlushEvent beginFlush( long filePageId, long cachePageId, PageSwapper swapper ); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageFaultEvent.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageFaultEvent.java index 834a26727c159..35d802f212df9 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageFaultEvent.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageFaultEvent.java @@ -22,7 +22,7 @@ /** * Begin a page fault as part of a pin event. */ -public interface PageFaultEvent +public interface PageFaultEvent extends EvictionEventOpportunity { /** * A PageFaultEvent that does nothing. @@ -51,7 +51,7 @@ public EvictionEvent beginEviction() } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } }; @@ -64,7 +64,7 @@ public void setCachePageId( int cachePageId ) /** * The id of the cache page that is being faulted into. */ - void setCachePageId( int cachePageId ); + void setCachePageId( long cachePageId ); /** * The page fault completed successfully. @@ -75,9 +75,4 @@ public void setCachePageId( int cachePageId ) * The page fault did not complete successfully, but instead caused the given Throwable to be thrown. */ void done( Throwable throwable ); - - /** - * Begin an eviction event caused by this page fault event. - */ - EvictionEvent beginEviction(); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PinEvent.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PinEvent.java index 08abd82e10f37..1e08f2a4db099 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PinEvent.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PinEvent.java @@ -30,7 +30,7 @@ public interface PinEvent PinEvent NULL = new PinEvent() { @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } @@ -54,7 +54,7 @@ public void done() /** * The id of the cache page that holds the file page we pinned. */ - void setCachePageId( int cachePageId ); + void setCachePageId( long cachePageId ); /** * The page we want to pin is not in memory, so being a page fault to load it in. diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java index 2671ec01ac7cb..4c839ccd3d03f 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java @@ -191,7 +191,7 @@ public void threwException( IOException exception ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } @@ -229,7 +229,7 @@ public EvictionEvent beginEviction() } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } }; @@ -237,7 +237,7 @@ public void setCachePageId( int cachePageId ) private final FlushEventOpportunity flushEventOpportunity = new FlushEventOpportunity() { @Override - public FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ) + public FlushEvent beginFlush( long filePageId, long cachePageId, PageSwapper swapper ) { return flushEvent; } @@ -274,7 +274,7 @@ private class DefaultPinEvent implements PinEvent int eventHits = 1; @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java index 8b1dec01e33c6..689d6dd102de2 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/PageListTest.java @@ -46,6 +46,10 @@ import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.DummyPageSwapper; +import org.neo4j.io.pagecache.tracing.EvictionEvent; +import org.neo4j.io.pagecache.tracing.EvictionRunEvent; +import org.neo4j.io.pagecache.tracing.FlushEvent; +import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; @@ -55,6 +59,8 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -1233,7 +1239,7 @@ public void tryEvictMustFailIfPageIsAlreadyExclusivelyLocked() throws Exception int swapperId = swappers.allocate( DUMMY_SWAPPER, pageSize ); doFault( swapperId, 42 ); // page is now loaded // pages are delivered from the fault routine with the exclusive lock already held! - assertFalse( pageList.tryEvict( pageRef ) ); + assertFalse( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); } @Test @@ -1242,20 +1248,20 @@ public void tryEvictThatFailsOnExclusiveLockMustNotUndoSaidLock() throws Excepti int swapperId = swappers.allocate( DUMMY_SWAPPER, pageSize ); doFault( swapperId, 42 ); // page is now loaded // pages are delivered from the fault routine with the exclusive lock already held! - pageList.tryEvict( pageRef ); // This attempt will fail + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); // This attempt will fail assertTrue( pageList.isExclusivelyLocked( pageRef ) ); // page should still have its lock } @Test public void tryEvictMustFailIfPageIsNotLoaded() throws Exception { - assertFalse( pageList.tryEvict( pageRef ) ); + assertFalse( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); } @Test public void tryEvictMustWhenPageIsNotLoadedMustNotLeavePageLocked() throws Exception { - pageList.tryEvict( pageRef ); // This attempt fails + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); // This attempt fails assertFalse( pageList.isExclusivelyLocked( pageRef ) ); // Page should not be left in locked state } @@ -1265,7 +1271,7 @@ public void tryEvictMustLeavePageExclusivelyLockedOnSuccess() throws Exception int swapperId = swappers.allocate( DUMMY_SWAPPER, pageSize ); doFault( swapperId, 42 ); // page now bound & exclusively locked pageList.unlockExclusive( pageRef ); // no longer exclusively locked; can now be evicted - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); pageList.unlockExclusive( pageRef ); // will throw if lock is not held } @@ -1276,7 +1282,7 @@ public void pageMustNotBeLoadedAfterSuccessfulEviction() throws Exception doFault( swapperId, 42 ); // page now bound & exclusively locked pageList.unlockExclusive( pageRef ); // no longer exclusively locked; can now be evicted assertTrue( pageList.isLoaded( pageRef ) ); - pageList.tryEvict( pageRef ); + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); assertFalse( pageList.isLoaded( pageRef ) ); } @@ -1289,7 +1295,7 @@ public void pageMustNotBeBoundAfterSuccessfulEviction() throws Exception assertTrue( pageList.isBoundTo( pageRef, 1, 42 ) ); assertTrue( pageList.isLoaded( pageRef ) ); assertThat( pageList.getSwapperId( pageRef ), is( 1 ) ); - pageList.tryEvict( pageRef ); + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); assertFalse( pageList.isBoundTo( pageRef, 1, 42 ) ); assertFalse( pageList.isLoaded( pageRef ) ); assertThat( pageList.getSwapperId( pageRef ), is( 0 ) ); @@ -1303,7 +1309,7 @@ public void pageMustNotBeModifiedAfterSuccessfulEviction() throws Exception pageList.unlockExclusiveAndTakeWriteLock( pageRef ); pageList.unlockWrite( pageRef ); // page is now modified assertTrue( pageList.isModified( pageRef ) ); - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); assertFalse( pageList.isModified( pageRef ) ); } @@ -1329,7 +1335,7 @@ public long write( long filePageId, long bufferAddress, int bufferSize ) throws pageList.unlockExclusiveAndTakeWriteLock( pageRef ); pageList.unlockWrite( pageRef ); // page is now modified assertTrue( pageList.isModified( pageRef ) ); - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); assertThat( writtenFilePageId.get(), is( 42L ) ); assertThat( writtenBufferAddress.get(), is( pageList.getAddress( pageRef ) ) ); assertThat( writtenBufferSize.get(), is( 313 ) ); @@ -1352,7 +1358,7 @@ public long write( long filePageId, long bufferAddress, int bufferSize ) throws doFault( swapperId, 42 ); pageList.unlockExclusive( pageRef ); // we take no write lock, so page is not modified assertFalse( pageList.isModified( pageRef ) ); - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); assertThat( writes.get(), is( 0 ) ); } @@ -1372,7 +1378,7 @@ public void evicted( long pageId, Page page ) int swapperId = swappers.allocate( swapper, 313 ); doFault( swapperId, 42 ); pageList.unlockExclusive( pageRef ); - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); assertTrue( evictionNotified.get() ); } @@ -1394,21 +1400,311 @@ public void evicted( long pageId, Page page ) pageList.unlockExclusiveAndTakeWriteLock( pageRef ); pageList.unlockWrite( pageRef ); // page is now modified assertTrue( pageList.isModified( pageRef ) ); - assertTrue( pageList.tryEvict( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); assertTrue( evictionNotified.get() ); assertFalse( pageList.isModified( pageRef ) ); } - // todo try evict must leave page unlocked if flush throws - // todo try evict must leave page loaded if flush throws - // todo try evict must leave page bound if flush throws - // todo try evict must leave page modified if flush throws - // todo try evict must not notify swapper of eviction if flush throws - // todo try evict must report to eviction event - // todo try evict that flushes must report to flush event - // todo try evict that succeeds must not interfere with adjacent pages - // todo try evict that fails must not interfere with adjacent pages - - // todo evict + + @Test + public void tryEvictMustLeavePageUnlockedAndLoadedAndBoundAndModifiedIfFlushThrows() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ) + { + @Override + public long write( long filePageId, long bufferAddress, int bufferSize ) throws IOException + { + throw new IOException(); + } + }; + int swapperId = swappers.allocate( swapper, 313 ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modified + assertTrue( pageList.isModified( pageRef ) ); + try + { + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); + fail( "tryEvict should have thrown" ); + } + catch ( IOException e ) + { + // good + } + // there should be no lock preventing us from taking an exclusive lock + assertTrue( pageList.tryExclusiveLock( pageRef ) ); + // page should still be loaded... + assertTrue( pageList.isLoaded( pageRef ) ); + // ... and bound + assertTrue( pageList.isBoundTo( pageRef, swapperId, 42 ) ); + // ... and modified + assertTrue( pageList.isModified( pageRef ) ); + } + + @Test + public void tryEvictMustNotNotifySwapperOfEvictionIfFlushThrows() throws Exception + { + AtomicBoolean evictionNotified = new AtomicBoolean(); + PageSwapper swapper = new DummyPageSwapper( "a" ) + { + @Override + public long write( long filePageId, long bufferAddress, int bufferSize ) throws IOException + { + throw new IOException(); + } + + @Override + public void evicted( long pageId, Page page ) + { + evictionNotified.set( true ); + } + }; + int swapperId = swappers.allocate( swapper, 313 ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modified + assertTrue( pageList.isModified( pageRef ) ); + try + { + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); + fail( "tryEvict should have thrown" ); + } + catch ( IOException e ) + { + // good + } + // we should not have gotten any notification about eviction + assertFalse( evictionNotified.get() ); + } + + private static class EvictionAndFlushRecorder implements EvictionEvent, FlushEventOpportunity, FlushEvent + { + private long filePageId; + private PageSwapper swapper; + private IOException evictionException; + private long cachePageId; + private boolean evictionClosed; + private long bytesWritten; + private boolean flushDone; + private IOException flushException; + private int pagesFlushed; + + // --- EvictionEvent: + + @Override + public void close() + { + this.evictionClosed = true; + } + + @Override + public void setFilePageId( long filePageId ) + { + this.filePageId = filePageId; + } + + @Override + public void setSwapper( PageSwapper swapper ) + { + this.swapper = swapper; + } + + @Override + public FlushEventOpportunity flushEventOpportunity() + { + return this; + } + + @Override + public void threwException( IOException exception ) + { + this.evictionException = exception; + } + + @Override + public void setCachePageId( long cachePageId ) + { + this.cachePageId = cachePageId; + } + + // --- FlushEventOpportunity: + + @Override + public FlushEvent beginFlush( long filePageId, long cachePageId, PageSwapper swapper ) + { + return this; + } + + // --- FlushEvent: + + @Override + public void addBytesWritten( long bytes ) + { + this.bytesWritten += bytes; + } + + @Override + public void done() + { + this.flushDone = true; + } + + @Override + public void done( IOException exception ) + { + this.flushDone = true; + this.flushException = exception; + + } + + @Override + public void addPagesFlushed( int pageCount ) + { + this.pagesFlushed += pageCount; + } + } + + @Test + public void tryEvictMustReportToEvictionEvent() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ); + int swapperId = swappers.allocate( swapper, 313 ); + doFault( swapperId, 42 ); + pageList.unlockExclusive( pageRef ); + EvictionAndFlushRecorder recorder = new EvictionAndFlushRecorder(); + assertTrue( pageList.tryEvict( pageRef, () -> recorder ) ); + assertThat( recorder.evictionClosed, is( true ) ); + assertThat( recorder.filePageId, is( 42L ) ) ; + assertThat( recorder.swapper, sameInstance( swapper ) ); + assertThat( recorder.evictionException, is( nullValue() ) ); + assertThat( recorder.cachePageId, is( pageRef ) ); + assertThat( recorder.bytesWritten, is( 0L ) ); + assertThat( recorder.flushDone, is( false ) ); + assertThat( recorder.flushException, is( nullValue() ) ); + assertThat( recorder.pagesFlushed, is( 0 ) ); + } + + @Test + public void tryEvictThatFlushesMustReportToEvictionAndFlushEvents() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ); + int filePageSize = 313; + int swapperId = swappers.allocate( swapper, filePageSize ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modified + assertTrue( pageList.isModified( pageRef ) ); + EvictionAndFlushRecorder recorder = new EvictionAndFlushRecorder(); + assertTrue( pageList.tryEvict( pageRef, () -> recorder ) ); + assertThat( recorder.evictionClosed, is( true ) ); + assertThat( recorder.filePageId, is( 42L ) ) ; + assertThat( recorder.swapper, sameInstance( swapper ) ); + assertThat( recorder.evictionException, is( nullValue() ) ); + assertThat( recorder.cachePageId, is( pageRef ) ); + assertThat( recorder.bytesWritten, is( (long) filePageSize ) ); + assertThat( recorder.flushDone, is( true ) ); + assertThat( recorder.flushException, is( nullValue() ) ); + assertThat( recorder.pagesFlushed, is( 1 ) ); + } + + @Test + public void tryEvictThatFailsMustReportExceptionsToEvictionAndFlushEvents() throws Exception + { + IOException ioException = new IOException(); + PageSwapper swapper = new DummyPageSwapper( "a" ) + { + @Override + public long write( long filePageId, long bufferAddress, int bufferSize ) throws IOException + { + throw ioException; + } + }; + int swapperId = swappers.allocate( swapper, 313 ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modified + assertTrue( pageList.isModified( pageRef ) ); + EvictionAndFlushRecorder recorder = new EvictionAndFlushRecorder(); + try + { + pageList.tryEvict( pageRef, () -> recorder ); + fail( "tryEvict should have thrown" ); + } + catch ( IOException e ) + { + // Ok + } + assertThat( recorder.evictionClosed, is( true ) ); + assertThat( recorder.filePageId, is( 42L ) ) ; + assertThat( recorder.swapper, sameInstance( swapper ) ); + assertThat( recorder.evictionException, sameInstance( ioException ) ); + assertThat( recorder.cachePageId, is( pageRef ) ); + assertThat( recorder.bytesWritten, is( 0L ) ); + assertThat( recorder.flushDone, is( true ) ); + assertThat( recorder.flushException, sameInstance( ioException ) ); + assertThat( recorder.pagesFlushed, is( 0 ) ); + } + + @Test + public void tryEvictThatSucceedsMustNotInterfereWithAdjacentPages() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ); + int swapperId = swappers.allocate( swapper, 313 ); + long prevStamp = pageList.tryOptimisticReadLock( prevPageRef ); + long nextStamp = pageList.tryOptimisticReadLock( nextPageRef ); + doFault( swapperId, 42 ); + pageList.unlockExclusive( pageRef ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); + assertTrue( pageList.validateReadLock( prevPageRef, prevStamp ) ); + assertTrue( pageList.validateReadLock( nextPageRef, nextStamp ) ); + } + + @Test + public void tryEvictThatFlushesAndSucceedsMustNotInterfereWithAdjacentPages() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ); + int swapperId = swappers.allocate( swapper, 313 ); + long prevStamp = pageList.tryOptimisticReadLock( prevPageRef ); + long nextStamp = pageList.tryOptimisticReadLock( nextPageRef ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modifed + assertTrue( pageList.isModified( pageRef ) ); + assertTrue( pageList.tryEvict( pageRef, EvictionRunEvent.NULL ) ); + assertTrue( pageList.validateReadLock( prevPageRef, prevStamp ) ); + assertTrue( pageList.validateReadLock( nextPageRef, nextStamp ) ); + } + + @Test + public void tryEvictThatFailsMustNotInterfereWithAdjacentPages() throws Exception + { + PageSwapper swapper = new DummyPageSwapper( "a" ) + { + @Override + public long write( long filePageId, long bufferAddress, int bufferSize ) throws IOException + { + throw new IOException(); + } + }; + int swapperId = swappers.allocate( swapper, 313 ); + long prevStamp = pageList.tryOptimisticReadLock( prevPageRef ); + long nextStamp = pageList.tryOptimisticReadLock( nextPageRef ); + doFault( swapperId, 42 ); + pageList.unlockExclusiveAndTakeWriteLock( pageRef ); + pageList.unlockWrite( pageRef ); // page is now modifed + assertTrue( pageList.isModified( pageRef ) ); + try + { + pageList.tryEvict( pageRef, EvictionRunEvent.NULL ); + fail( "tryEvict should have thrown" ); + } + catch ( IOException e ) + { + // ok + } + assertTrue( pageList.validateReadLock( prevPageRef, prevStamp ) ); + assertTrue( pageList.validateReadLock( nextPageRef, nextStamp ) ); + } + // todo flush // todo freelist? (entries chained via file page ids in a linked list? should work as free pages are always exclusively locked, and thus don't really need an isLoaded check) } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/StubPageFaultEvent.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/StubPageFaultEvent.java index fbd68b1f462ca..412c3401256b7 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/StubPageFaultEvent.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/StubPageFaultEvent.java @@ -25,7 +25,7 @@ class StubPageFaultEvent implements PageFaultEvent { long bytesRead; - int cachePageId; + long cachePageId; @Override public void addBytesRead( long bytes ) @@ -34,7 +34,7 @@ public void addBytesRead( long bytes ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { this.cachePageId = cachePageId; } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DummyPageSwapper.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DummyPageSwapper.java index 01d6b3eb34366..7a83d5278cd22 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DummyPageSwapper.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DummyPageSwapper.java @@ -37,13 +37,13 @@ public DummyPageSwapper( String filename ) @Override public long read( long filePageId, long bufferAddress, int bufferSize ) throws IOException { - return 0; + return bufferSize; } @Override public long write( long filePageId, long bufferAddress, int bufferSize ) throws IOException { - return 0; + return bufferSize; } @Override diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/linear/HEvents.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/linear/HEvents.java index ca03501a647d9..e69f88ee9c412 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/linear/HEvents.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/linear/HEvents.java @@ -139,13 +139,13 @@ void printBody( PrintStream out, String exceptionLinePrefix ) public static class FlushHEvent extends IntervalHEvent implements FlushEvent { private long filePageId; - private int cachePageId; + private long cachePageId; private int pageCount; private File file; private int bytesWritten; private IOException exception; - FlushHEvent( LinearHistoryTracer tracer, long filePageId, int cachePageId, PageSwapper swapper ) + FlushHEvent( LinearHistoryTracer tracer, long filePageId, long cachePageId, PageSwapper swapper ) { super( tracer ); this.filePageId = filePageId; @@ -212,7 +212,7 @@ public FlushEventOpportunity flushEventOpportunity() } @Override - public FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ) + public FlushEvent beginFlush( long filePageId, long cachePageId, PageSwapper swapper ) { return tracer.add( new FlushHEvent( tracer, filePageId, cachePageId, swapper ) ); } @@ -229,7 +229,7 @@ public static class PinHEvent extends IntervalHEvent implements PinEvent private boolean exclusiveLock; private long filePageId; private File file; - private int cachePageId; + private long cachePageId; private boolean hit; PinHEvent( LinearHistoryTracer tracer, boolean exclusiveLock, long filePageId, PageSwapper swapper ) @@ -242,7 +242,7 @@ public static class PinHEvent extends IntervalHEvent implements PinEvent } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { this.cachePageId = cachePageId; } @@ -283,7 +283,7 @@ void printBody( PrintStream out, String exceptionLinePrefix ) public static class PageFaultHEvent extends IntervalHEvent implements PageFaultEvent { private int bytesRead; - private int cachePageId; + private long cachePageId; private boolean pageEvictedByFaulter; private Throwable exception; @@ -299,7 +299,7 @@ public void addBytesRead( long bytes ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { this.cachePageId = cachePageId; } @@ -342,7 +342,7 @@ public static class EvictionHEvent extends IntervalHEvent implements EvictionEve private long filePageId; private File file; private IOException exception; - private int cachePageId; + private long cachePageId; EvictionHEvent( LinearHistoryTracer linearHistoryTracer ) { @@ -374,13 +374,13 @@ public void threwException( IOException exception ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { this.cachePageId = cachePageId; } @Override - public FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ) + public FlushEvent beginFlush( long filePageId, long cachePageId, PageSwapper swapper ) { return tracer.add( new FlushHEvent( tracer, filePageId, cachePageId, swapper ) ); } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java index 2ddf1e9824e7c..85e8f3f7027ae 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java @@ -240,7 +240,7 @@ public void threwException( IOException exception ) } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java index 8a77e332b7846..7d4d7d8dd1574 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java @@ -113,7 +113,7 @@ public PinEvent beginPin( boolean writeLock, final long filePageId, final PageSw private boolean hit = true; @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } @@ -146,7 +146,7 @@ public EvictionEvent beginEviction() } @Override - public void setCachePageId( int cachePageId ) + public void setCachePageId( long cachePageId ) { } };