diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java index 8b95b9549d95..44cea937c30c 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/DefaultPageCacheTracer.java @@ -21,16 +21,10 @@ import java.io.File; import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.invoke.SwitchPoint; import java.util.concurrent.atomic.AtomicLong; import org.neo4j.io.pagecache.PageSwapper; -import static org.neo4j.unsafe.impl.internal.dragons.FeatureToggles.packageFlag; - /** * The default PageCacheTracer implementation, that just increments counters. */ @@ -73,14 +67,7 @@ public void addPagesFlushed( int pageCount ) } }; - private final FlushEventOpportunity flushEventOpportunity = new FlushEventOpportunity() - { - @Override - public FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ) - { - return flushEvent; - } - }; + private final FlushEventOpportunity flushEventOpportunity = ( filePageId, cachePageId, swapper ) -> flushEvent; private final EvictionEvent evictionEvent = new EvictionEvent() { @@ -259,4 +246,22 @@ public void bytesRead( long bytesRead ) { this.bytesRead.getAndAdd( bytesRead ); } + + @Override + public void evictions( long evictions ) + { + this.evictions.getAndAdd( evictions ); + } + + @Override + public void bytesWritten( long bytesWritten ) + { + this.bytesWritten.getAndAdd( bytesWritten ); + } + + @Override + public void flushes( long flushes ) + { + this.flushes.getAndAdd( flushes ); + } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageCacheTracer.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageCacheTracer.java index 527f1bc5af36..30ef94238353 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageCacheTracer.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/PageCacheTracer.java @@ -144,6 +144,21 @@ public void bytesRead( long bytesRead ) { } + @Override + public void evictions( long evictions ) + { + } + + @Override + public void bytesWritten( long bytesWritten ) + { + } + + @Override + public void flushes( long flushes ) + { + } + @Override public String toString() { @@ -187,4 +202,10 @@ public String toString() void faults( long faults ); void bytesRead( long bytesRead ); + + void evictions( long evictions ); + + void bytesWritten( long bytesWritten ); + + void flushes( long flushes ); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java index ef271bfbe08b..575fde47841f 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/DefaultPageCursorTracer.java @@ -19,6 +19,7 @@ */ package org.neo4j.io.pagecache.tracing.cursor; +import java.io.IOException; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; @@ -27,6 +28,8 @@ import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.EvictionEvent; +import org.neo4j.io.pagecache.tracing.FlushEvent; +import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.PinEvent; @@ -39,11 +42,17 @@ public class DefaultPageCursorTracer implements PageCursorTracer private long unpins = 0L; private long faults = 0L; private long bytesRead = 0L; + private long bytesWritten = 0L; + private long evictions = 0L; + private long flushes; private long cyclePinsStart; private long cycleUnpinsStart; private long cycleFaultsStart; private long cycleBytesReadStart; + private long cycleBytesWrittenStart; + private long cycleEvictionsStart; + private long cycleFlushesStart; private PageCacheTracer pageCacheTracer; @@ -100,6 +109,9 @@ public void init( PageCacheTracer pageCacheTracer ) this.cycleUnpinsStart = unpins; this.cycleFaultsStart = faults; this.cycleBytesReadStart = bytesRead; + this.cycleBytesWrittenStart = bytesWritten; + this.cycleEvictionsStart = evictions; + this.cycleFlushesStart = flushes; } public void reportEvents() @@ -109,6 +121,9 @@ public void reportEvents() pageCacheTracer.unpins( Math.abs( pins - cycleUnpinsStart ) ); pageCacheTracer.faults( Math.abs( faults - cycleFaultsStart ) ); pageCacheTracer.bytesRead( Math.abs( bytesRead - cycleBytesReadStart ) ); + pageCacheTracer.evictions( Math.abs( evictions - cycleEvictionsStart ) ); + pageCacheTracer.bytesWritten( Math.abs( bytesWritten - cycleBytesWrittenStart ) ); + pageCacheTracer.flushes( Math.abs( flushes - cycleFlushesStart ) ); } @Override @@ -206,12 +221,47 @@ public void done() } }; + private final EvictionEvent evictionEvent = new EvictionEvent() + { + @Override + public void setFilePageId( long filePageId ) + { + } + + @Override + public void setSwapper( PageSwapper swapper ) + { + } + + @Override + public FlushEventOpportunity flushEventOpportunity() + { + return flushEventOpportunity; + } + + @Override + public void threwException( IOException exception ) + { + } + + @Override + public void setCachePageId( int cachePageId ) + { + } + + @Override + public void close() + { + evictions++; + } + }; + private final PageFaultEvent pageFaultEvent = new PageFaultEvent() { @Override public void addBytesRead( long bytes ) { - bytesRead = +bytes; + bytesRead += bytes; } @Override @@ -229,8 +279,7 @@ public void done( Throwable throwable ) @Override public EvictionEvent beginEviction() { - //TODO: is that correct to assume that it will never be the case and can be ignored? - return EvictionEvent.NULL; + return evictionEvent; } @Override @@ -239,4 +288,39 @@ public void setCachePageId( int cachePageId ) } }; + private final FlushEventOpportunity flushEventOpportunity = new FlushEventOpportunity() + { + @Override + public FlushEvent beginFlush( long filePageId, int cachePageId, PageSwapper swapper ) + { + return flushEvent; + } + }; + + private final FlushEvent flushEvent = new FlushEvent() + { + @Override + public void addBytesWritten( long bytes ) + { + bytesWritten += bytes; + } + + @Override + public void done() + { + flushes++; + } + + @Override + public void done( IOException exception ) + { + done(); + } + + @Override + public void addPagesFlushed( int pageCount ) + { + } + }; + } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheSlowTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheSlowTest.java index 15160dea17f3..576c37e476e7 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheSlowTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheSlowTest.java @@ -39,8 +39,10 @@ import org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; import org.neo4j.test.LinearHistoryPageCacheTracer; import org.neo4j.test.rule.RepeatRule; +import org.neo4j.test.LinearHistoryPageCursorTracer; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -137,7 +139,7 @@ public void mustNotLoseUpdates() throws Exception final int threadCount = 8; final int pageSize = threadCount * 4; - getPageCache( fs, cachePages, pageSize, PageCacheTracer.NULL ); + getPageCache( fs, cachePages, pageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); final PagedFile pagedFile = pageCache.map( file( "a" ), pageSize ); ensureAllPagesExists( filePages, pagedFile ); @@ -255,7 +257,7 @@ public void mustNotLoseUpdatesWhenOpeningMultiplePageCursorsPerThread() throws E final int maxCursorsPerThread = cachePages / (1 + threadCount); assertThat( maxCursorsPerThread * threadCount, lessThan( cachePages ) ); - getPageCache( fs, cachePages, pageSize, PageCacheTracer.NULL ); + getPageCache( fs, cachePages, pageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); final PagedFile pagedFile = pageCache.map( file( "a" ), pageSize ); ensureAllPagesExists( filePages, pagedFile ); @@ -344,7 +346,7 @@ public void writeLockingCursorMustThrowWhenLockingPageRacesWithUnmapping() throw File file = file( "a" ); generateFileWithRecords( file, recordsPerFilePage * 2, recordSize ); - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); final PagedFile pf = pageCache.map( file, filePageSize ); final CountDownLatch hasLockLatch = new CountDownLatch( 1 ); @@ -453,7 +455,8 @@ public void pageCacheMustRemainInternallyConsistentWhenGettingRandomFailures() t // Because our test failures are non-deterministic, we use this tracer to capture a full history of the // events leading up to any given failure. LinearHistoryPageCacheTracer tracer = new LinearHistoryPageCacheTracer(); - getPageCache( fs, maxPages, pageCachePageSize, tracer ); + //TODO:sdfasdf + getPageCache( fs, maxPages, pageCachePageSize, tracer, LinearHistoryPageCursorTracer::new ); PagedFile pfA = pageCache.map( existingFile( "a" ), filePageSize ); PagedFile pfB = pageCache.map( existingFile( "b" ), filePageSize / 2 + 1 ); diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java index 3860434b1def..51f94c820b59 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java @@ -69,11 +69,15 @@ import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.randomharness.Record; import org.neo4j.io.pagecache.randomharness.StandardRecordFormat; +import org.neo4j.io.pagecache.tracing.ConfigurablePageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PinEvent; import org.neo4j.test.rule.RepeatRule; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; + import static java.lang.Long.toHexString; import static java.lang.System.currentTimeMillis; @@ -133,20 +137,20 @@ public void mustReportConfiguredCachePageSize() throws IOException public void cachePageSizeMustBePowerOfTwo() throws IOException { expectedException.expect( IllegalArgumentException.class ); - getPageCache( fs, maxPages, 31, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, 31, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); } @Test public void mustHaveAtLeastTwoPages() throws Exception { expectedException.expect( IllegalArgumentException.class ); - getPageCache( fs, 1, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, 1, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); } @Test public void mustAcceptTwoPagesAsMinimumConfiguration() throws Exception { - getPageCache( fs, 2, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, 2, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); } @Test @@ -161,7 +165,8 @@ public void close() closed.set( true ); } }; - PageCache cache = createPageCache( swapperFactory, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createPageCache( swapperFactory, maxPages, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); Exception exception = null; try { @@ -209,7 +214,8 @@ public void close() throw new RuntimeException( "boo" ); } }; - PageCache cache = createPageCache( swapperFactory, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createPageCache( swapperFactory, maxPages, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); try { cache.close(); @@ -229,7 +235,7 @@ public void mustReadExistingData() throws IOException { generateFileWithRecords( file( "a" ), recordCount, recordSize ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); int recordId = 0; try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); @@ -252,7 +258,7 @@ public void mustScanInTheMiddleOfTheFile() throws IOException long endPage = (recordCount / recordsPerFilePage) - 10; generateFileWithRecords( file( "a" ), recordCount, recordSize ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); int recordId = (int) (startPage * recordsPerFilePage); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); @@ -271,7 +277,7 @@ public void mustScanInTheMiddleOfTheFile() throws IOException @Test( timeout = SEMI_LONG_TIMEOUT_MILLIS ) public void writesFlushedFromPageFileMustBeExternallyObservable() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); long startPageId = 0; @@ -293,7 +299,7 @@ public void writesFlushedFromPageFileMustBeExternallyObservable() throws IOExcep @Test public void pageCacheFlushAndForceMustThrowOnNullIOPSLimiter() throws Exception { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); expectedException.expect( IllegalArgumentException.class ); cache.flushAndForce( null ); } @@ -301,7 +307,7 @@ public void pageCacheFlushAndForceMustThrowOnNullIOPSLimiter() throws Exception @Test public void pagedFileFlushAndForceMustThrowOnNullIOPSLimiter() throws Exception { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pf = cache.map( file( "a" ), filePageSize ) ) { expectedException.expect( IllegalArgumentException.class ); @@ -313,7 +319,8 @@ public void pagedFileFlushAndForceMustThrowOnNullIOPSLimiter() throws Exception public void pageCacheFlushAndForceMustQueryTheGivenIOPSLimiter() throws Exception { int pagesToDirty = 10_000; - PageCache cache = getPageCache( fs, nextPowerOf2( 2 * pagesToDirty ), pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = getPageCache( fs, nextPowerOf2( 2 * pagesToDirty ), pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); PagedFile pfA = cache.map( existingFile( "a" ), filePageSize ); PagedFile pfB = cache.map( existingFile( "b" ), filePageSize ); @@ -337,7 +344,8 @@ public void pageCacheFlushAndForceMustQueryTheGivenIOPSLimiter() throws Exceptio public void pagedFileFlushAndForceMustQueryTheGivenIOPSLimiter() throws Exception { int pagesToDirty = 10_000; - PageCache cache = getPageCache( fs, nextPowerOf2( pagesToDirty ), pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = getPageCache( fs, nextPowerOf2( pagesToDirty ), pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); PagedFile pf = cache.map( file( "a" ), filePageSize ); // Dirty a bunch of data @@ -391,7 +399,8 @@ public void repeatablyWritesFlushedFromPageFileMustBeExternallyObservable() thro @Test( timeout = LONG_TIMEOUT_MILLIS ) public void writesFlushedFromPageFileMustBeObservableEvenWhenRacingWithEviction() throws IOException { - PageCache cache = getPageCache( fs, 20, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = getPageCache( fs, 20, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); long startPageId = 0; long endPageId = 21; @@ -437,7 +446,7 @@ public void writesFlushedFromPageFileMustBeObservableEvenWhenRacingWithEviction( @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void writesFlushedFromPageCacheMustBeExternallyObservable() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); long startPageId = 0; long endPageId = recordCount / recordsPerFilePage; @@ -492,7 +501,7 @@ public void channelMustBeForcedAfterPagedFileFlushAndForce() throws Exception final AtomicInteger forceCounter = new AtomicInteger(); FileSystemAbstraction fs = writeAndForceCountingFs( writeCounter, forceCounter ); - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); try ( PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ) ) { @@ -518,7 +527,7 @@ public void channelsMustBeForcedAfterPageCacheFlushAndForce() throws Exception final AtomicInteger forceCounter = new AtomicInteger(); FileSystemAbstraction fs = writeAndForceCountingFs( writeCounter, forceCounter ); - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); try ( PagedFile pagedFileA = pageCache.map( existingFile( "a" ), filePageSize ); PagedFile pagedFileB = pageCache.map( existingFile( "b" ), filePageSize ) ) @@ -704,7 +713,7 @@ public void mustCloseFileChannelWhenTheLastHandleIsUnmapped() throws Exception assumeTrue( "This depends on EphemeralFSA specific features", fs.getClass() == EphemeralFileSystemAbstraction.class ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); PagedFile a = cache.map( file( "a" ), filePageSize ); PagedFile b = cache.map( file( "a" ), filePageSize ); a.close(); @@ -763,7 +772,7 @@ public void writeAll( ByteBuffer src, long position ) throws IOException }; } }; - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); PrintStream oldSystemErr = System.err; try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize ); @@ -786,7 +795,7 @@ public void writeAll( ByteBuffer src, long position ) throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void mappingFilesInClosedCacheMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); cache.close(); expectedException.expect( IllegalStateException.class ); cache.map( file( "a" ), filePageSize ); @@ -795,7 +804,7 @@ public void mappingFilesInClosedCacheMustThrow() throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void flushingClosedCacheMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); cache.close(); expectedException.expect( IllegalStateException.class ); cache.flushAndForce(); @@ -804,7 +813,7 @@ public void flushingClosedCacheMustThrow() throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void mappingFileWithPageSizeGreaterThanCachePageSizeMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); expectedException.expect( IllegalArgumentException.class ); cache.map( file( "a" ), pageCachePageSize + 1 ); // this must throw } @@ -813,7 +822,7 @@ public void mappingFileWithPageSizeGreaterThanCachePageSizeMustThrow() throws IO public void mappingFileWithPageSizeSmallerThanLongSizeBytesMustThrow() throws IOException { // Because otherwise we cannot ensure that our branch-free bounds checking always lands within a page boundary. - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); expectedException.expect( IllegalArgumentException.class ); cache.map( file( "a" ), Long.BYTES - 1 ); // this must throw } @@ -823,7 +832,7 @@ public void mappingFileWithPageSizeSmallerThanLongSizeBytesMustThrowEvenWithAnyP throws IOException { // Because otherwise we cannot ensure that our branch-free bounds checking always lands within a page boundary. - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); expectedException.expect( IllegalArgumentException.class ); cache.map( file( "a" ), Long.BYTES - 1, PageCacheOpenOptions.ANY_PAGE_SIZE ); // this must throw } @@ -858,7 +867,7 @@ public void mappingFileWithPageZeroPageSizeAndAnyPageSizeOpenOptionMustNotThrowG @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void mappingFileWithPageSizeEqualToCachePageSizeMustNotThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); PagedFile pagedFile = cache.map( file( "a" ), pageCachePageSize );// this must NOT throw pagedFile.close(); } @@ -866,7 +875,7 @@ public void mappingFileWithPageSizeEqualToCachePageSizeMustNotThrow() throws IOE @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void notSpecifyingAnyPfFlagsMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { expectedException.expect( IllegalArgumentException.class ); @@ -877,7 +886,7 @@ public void notSpecifyingAnyPfFlagsMustThrow() throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void notSpecifyingAnyPfLockFlagsMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { expectedException.expect( IllegalArgumentException.class ); @@ -888,7 +897,7 @@ public void notSpecifyingAnyPfLockFlagsMustThrow() throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void specifyingBothReadAndWriteLocksMustThrow() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { expectedException.expect( IllegalArgumentException.class ); @@ -903,7 +912,7 @@ public void mustNotPinPagesAfterNextReturnsFalse() throws Exception final CountDownLatch unpinLatch = new CountDownLatch( 1 ); final AtomicReference exceptionRef = new AtomicReference<>(); generateFileWithRecords( file( "a" ), recordsPerFilePage, recordSize ); - final PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + final PageCache cache = createStandardPageCache(); final PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); Runnable runnable = () -> { @@ -943,7 +952,7 @@ public void mustNotPinPagesAfterNextReturnsFalse() throws Exception @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void nextMustResetTheCursorOffset() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); try ( PageCursor cursor = pagedFile.io( 0L, PF_SHARED_WRITE_LOCK ) ) @@ -980,7 +989,7 @@ public void nextMustResetTheCursorOffset() throws IOException @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void nextMustAdvanceCurrentPageId() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); PageCursor cursor = pagedFile.io( 0L, PF_SHARED_WRITE_LOCK ) ) @@ -1126,7 +1135,7 @@ public void writeToPreviouslyBoundCursorAfterNextReturnsFalseMustThrow() throws @Test public void tryMappedPagedFileShouldReportMappedFilePresent() throws Exception { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); final File file = file( "a" ); try ( PagedFile pf = cache.map( file, filePageSize ) ) { @@ -1141,7 +1150,7 @@ public void tryMappedPagedFileShouldReportMappedFilePresent() throws Exception @Test public void tryMappedPagedFileShouldReportNonMappedFileNotPresent() throws Exception { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); final Optional dont_exist = cache.getExistingMapping( new File( "dont_exist" ) ); assertFalse( dont_exist.isPresent() ); } @@ -1170,7 +1179,7 @@ private void verifyOnWriteCursor( testTemplate.accept( ( cursor ) -> cursor.putInt( 0, 1 ) ); testTemplate.accept( ( cursor ) -> cursor.putLong( 0, 1 ) ); testTemplate.accept( ( cursor ) -> cursor.putShort( 0, (short) 1 ) ); - testTemplate.accept( ( cursor ) -> cursor.zapPage() ); + testTemplate.accept( PageCursor::zapPage ); } private void checkUnboundReadCursorAccess( PageCursorAction action ) throws IOException @@ -1400,7 +1409,7 @@ public void firstPageMustBeAccessibleWithNoGrowSpecifiedIfItIsTheOnlyPage() thro { generateFileWithRecords( file( "a" ), recordsPerFilePage, recordSize ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { @@ -1433,7 +1442,7 @@ public void firstPageMustBeAccessibleEvenIfTheFileIsNonEmptyButSmallerThanFilePa { generateFileWithRecords( file( "a" ), 1, recordSize ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { @@ -1464,7 +1473,7 @@ public void firstPageMustBeAccessibleEvenIfTheFileIsNonEmptyButSmallerThanFilePa @Test( timeout = SHORT_TIMEOUT_MILLIS ) public void firstPageMustNotBeAccessibleIfFileIsEmptyAndNoGrowSpecified() throws IOException { - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); try ( PagedFile pagedFile = cache.map( file( "a" ), filePageSize ) ) { @@ -1499,7 +1508,7 @@ public void newlyWrittenPagesMustBeAccessibleWithNoGrow() throws IOException int pagesToAdd = 3; generateFileWithRecords( file( "a" ), recordsPerFilePage * initialPages, recordSize ); - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); try ( PageCursor cursor = pagedFile.io( 1L, PF_SHARED_WRITE_LOCK ) ) @@ -1569,7 +1578,7 @@ public void retryMustResetCursorOffset() throws Exception // We then check that every retry iteration will read the special value in the 0th position. // We repeat the experiment a couple of times to make sure we didn't succeed by chance. - PageCache cache = getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + PageCache cache = createStandardPageCache(); final PagedFile pagedFile = cache.map( file( "a" ), filePageSize ); final AtomicReference caughtWriterException = new AtomicReference<>(); final CountDownLatch startLatch = new CountDownLatch( 1 ); @@ -2010,7 +2019,7 @@ public long write( ByteBuffer[] srcs ) throws IOException }; } }; - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); try( PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ); PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) @@ -2114,7 +2123,7 @@ public void tracerMustBeNotifiedAboutPinUnpinFaultAndEvictEventsWhenReading() th DefaultPageCacheTracer tracer = new DefaultPageCacheTracer(); generateFileWithRecords( file( "a" ), recordCount, recordSize ); - getPageCache( fs, maxPages, pageCachePageSize, tracer ); + getPageCache( fs, maxPages, pageCachePageSize, tracer, DefaultPageCursorTracerSupplier.INSTANCE ); long countedPages = 0; long countedFaults = 0; @@ -2171,7 +2180,7 @@ public void tracerMustBeNotifiedAboutPinUnpinFaultFlushAndEvictionEventsWhenWrit long pagesToGenerate = 142; DefaultPageCacheTracer tracer = new DefaultPageCacheTracer(); - getPageCache( fs, maxPages, pageCachePageSize, tracer ); + getPageCache( fs, maxPages, pageCachePageSize, tracer, DefaultPageCursorTracerSupplier.INSTANCE ); try ( PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ); PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) @@ -2229,19 +2238,20 @@ public void tracerMustBeNotifiedOfReadAndWritePins() throws Exception final AtomicInteger writeCount = new AtomicInteger(); final AtomicInteger readCount = new AtomicInteger(); - DefaultPageCacheTracer tracer = new DefaultPageCacheTracer() + DefaultPageCacheTracer tracer = new DefaultPageCacheTracer(); + DefaultPageCursorTracer pageCursorTracer = new DefaultPageCursorTracer() { - //TODO:: -// @Override -// public PinEvent beginPin( boolean writeLock, long filePageId, PageSwapper swapper ) -// { -// (writeLock? writeCount : readCount).getAndIncrement(); -// return super.beginPin( writeLock, filePageId, swapper ); -// } + @Override + public PinEvent beginPin( boolean writeLock, long filePageId, PageSwapper swapper ) + { + (writeLock? writeCount : readCount).getAndIncrement(); + return super.beginPin( writeLock, filePageId, swapper ); + } }; + ConfigurablePageCursorTracerSupplier cursorTracerSupplier = new ConfigurablePageCursorTracerSupplier( pageCursorTracer ); generateFileWithRecords( file( "a" ), recordCount, recordSize ); - getPageCache( fs, maxPages, pageCachePageSize, tracer ); + getPageCache( fs, maxPages, pageCachePageSize, tracer, cursorTracerSupplier ); int pinsForRead = 13; int pinsForWrite = 42; @@ -3135,7 +3145,7 @@ public void writeAll( ByteBuffer src, long position ) throws IOException fs.create( file( "a" ) ).close(); - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ); try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) @@ -3181,7 +3191,7 @@ public void writeAll( ByteBuffer src, long position ) throws IOException } }; - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ); // Create 1 dirty page @@ -3241,7 +3251,7 @@ public void writeAll( ByteBuffer src, long position ) throws IOException fs.create( file( "a" ) ).close(); - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ); try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) @@ -3516,7 +3526,7 @@ public void mustSupportUnalignedWordAccesses() throws Exception // as large as 1 GB - at least I have not heard of anyone trying to // configure it to be more than that. int pageSize = 1024 * 1024 * 8; - getPageCache( fs, 10, pageSize, PageCacheTracer.NULL ); + getPageCache( fs, 10, pageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); ThreadLocalRandom rng = ThreadLocalRandom.current(); @@ -3586,7 +3596,7 @@ public void mustReadZerosFromBeyondEndOfFile() throws Exception } int pageSize = nextPowerOf2( recordFormat.getRecordSize() * (files.length + 1) ); - getPageCache( fs, 2, pageSize, PageCacheTracer.NULL ); + getPageCache( fs, 2, pageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); int fileId = files.length; while ( fileId --> 0 ) @@ -3664,7 +3674,8 @@ public void mustSyncDeviceWhenFlushAndForcingPagedFile() throws Exception 0, // at `p1.flushAndForce` no `syncDevice` has happened before the force 1, 2 ); // closing+forcing the files one by one, we get 2 more `syncDevice` PageSwapperFactory factory = factoryCountingSyncDevice( syncDeviceCounter, expectedCountsInForce ); - try ( PageCache cache = createPageCache( factory, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + try ( PageCache cache = createPageCache( factory, maxPages, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); PagedFile p1 = cache.map( existingFile( "a" ), filePageSize ); PagedFile p2 = cache.map( existingFile( "b" ), filePageSize ) ) { @@ -3692,7 +3703,8 @@ public void mustSyncDeviceWhenFlushAndForcingPageCache() throws Exception 0, 0, // `cache.flushAndForce` forces the individual files, no `syncDevice` yet 1, 2 ); // after test, files are closed+forced one by one PageSwapperFactory factory = factoryCountingSyncDevice( syncDeviceCounter, expectedCountsInForce ); - try ( PageCache cache = createPageCache( factory, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + try ( PageCache cache = createPageCache( factory, maxPages, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); PagedFile p1 = cache.map( existingFile( "a" ), filePageSize ); PagedFile p2 = cache.map( existingFile( "b" ), filePageSize ) ) { @@ -5486,4 +5498,10 @@ public void isWriteLockingMustBeFalseForCursorOpenedWithSharedReadLock() throws assertFalse( cursor.isWriteLocked() ); } } + + private T createStandardPageCache() throws IOException + { + return getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, + PageCursorTracerSupplier.NULL ); + } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java index 12a2a1d493ba..8fde026045a8 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java @@ -43,6 +43,7 @@ import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.test.rule.RepeatRule; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; import static org.junit.Assert.assertThat; import static org.neo4j.test.matchers.ByteArrayMatcher.byteArray; @@ -106,43 +107,34 @@ public void tearDown() throws Exception fs.close(); } - protected final T createPageCache( - PageSwapperFactory swapperFactory, - int maxPages, - int pageSize, - PageCacheTracer tracer ) + protected final T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) { - return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer ); + return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); } - protected T createPageCache( - FileSystemAbstraction fs, - int maxPages, - int pageSize, - PageCacheTracer tracer ) + protected T createPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, + PageCursorTracerSupplier cursorTracerSupplier ) { PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(); swapperFactory.setFileSystemAbstraction( fs ); - return createPageCache( swapperFactory, maxPages, pageSize, tracer ); + return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); } - protected final T getPageCache( - FileSystemAbstraction fs, - int maxPages, - int pageSize, - PageCacheTracer tracer ) throws IOException + protected final T getPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, + PageCursorTracerSupplier cursorTracerSupplier ) throws IOException { if ( pageCache != null ) { tearDownPageCache( pageCache ); } - pageCache = createPageCache( fs, maxPages, pageSize, tracer ); + pageCache = createPageCache( fs, maxPages, pageSize, tracer, cursorTracerSupplier ); return pageCache; } protected void configureStandardPageCache() throws IOException { - getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL ); + getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ); } protected final void tearDownPageCache( T pageCache ) throws IOException @@ -325,11 +317,8 @@ protected void verifyRecordsInFile( ReadableByteChannel channel, int recordCount public abstract static class Fixture { - public abstract T createPageCache( - PageSwapperFactory swapperFactory, - int maxPages, - int pageSize, - PageCacheTracer tracer ); + public abstract T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ); public abstract void tearDownPageCache( T pageCache ) throws IOException; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java index 718048845ac2..26a06ce3ec65 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java @@ -25,7 +25,7 @@ import org.neo4j.io.pagecache.PageCacheTestSupport; import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; -import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture { @@ -33,9 +33,9 @@ public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture { @@ -89,15 +93,19 @@ public void mustEvictCleanPageWithoutFlushing() throws Exception { writeInitialDataTo( file( "a" ) ); RecordingPageCacheTracer tracer = new RecordingPageCacheTracer(); + RecordingPageCursorTracer cursorTracer = new RecordingPageCursorTracer(); + ConfigurablePageCursorTracerSupplier cursorTracerSupplier = new ConfigurablePageCursorTracerSupplier( cursorTracer ); - MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ) ); + MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ), cursorTracerSupplier ); PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) { assertTrue( cursor.next() ); } - assertNotNull( tracer.observe( Fault.class ) ); + assertNotNull( cursorTracer.observe( Fault.class ) ); + assertEquals( 1, cursorTracer.faults() ); + assertEquals( 1, tracer.faults() ); int clockArm = pageCache.evictPages( 1, 0, tracer.beginPageEvictions( 1 ) ); assertThat( clockArm, is( 1 ) ); @@ -120,8 +128,11 @@ public void mustFlushDirtyPagesOnEvictingFirstPage() throws Exception { writeInitialDataTo( file( "a" ) ); RecordingPageCacheTracer tracer = new RecordingPageCacheTracer(); + RecordingPageCursorTracer cursorTracer = new RecordingPageCursorTracer(); + ConfigurablePageCursorTracerSupplier cursorTracerSupplier = new ConfigurablePageCursorTracerSupplier( cursorTracer ); - MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ) ); + MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ), + cursorTracerSupplier ); PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) @@ -129,7 +140,9 @@ public void mustFlushDirtyPagesOnEvictingFirstPage() throws Exception assertTrue( cursor.next() ); cursor.putLong( 0L ); } - assertNotNull( tracer.observe( Fault.class ) ); + assertNotNull( cursorTracer.observe( Fault.class ) ); + assertEquals( 1, cursorTracer.faults() ); + assertEquals( 1, tracer.faults() ); int clockArm = pageCache.evictPages( 1, 0, tracer.beginPageEvictions( 1 ) ); assertThat( clockArm, is( 1 ) ); @@ -148,8 +161,11 @@ public void mustFlushDirtyPagesOnEvictingLastPage() throws Exception { writeInitialDataTo( file( "a" ) ); RecordingPageCacheTracer tracer = new RecordingPageCacheTracer(); + RecordingPageCursorTracer cursorTracer = new RecordingPageCursorTracer(); + ConfigurablePageCursorTracerSupplier cursorTracerSupplier = new ConfigurablePageCursorTracerSupplier( cursorTracer ); - MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ) ); + MuninnPageCache pageCache = createPageCache( fs, 2, 8, blockCacheFlush( tracer ), + cursorTracerSupplier ); PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); try ( PageCursor cursor = pagedFile.io( 1, PF_SHARED_WRITE_LOCK ) ) @@ -157,7 +173,9 @@ public void mustFlushDirtyPagesOnEvictingLastPage() throws Exception assertTrue( cursor.next() ); cursor.putLong( 0L ); } - assertNotNull( tracer.observe( Fault.class ) ); + assertNotNull( cursorTracer.observe( Fault.class ) ); + assertEquals( 1, cursorTracer.faults() ); + assertEquals( 1, tracer.faults() ); int clockArm = pageCache.evictPages( 1, 0, tracer.beginPageEvictions( 1 ) ); assertThat( clockArm, is( 1 ) ); @@ -176,8 +194,11 @@ public void mustFlushDirtyPagesOnEvictingAllPages() throws Exception { writeInitialDataTo( file( "a" ) ); RecordingPageCacheTracer tracer = new RecordingPageCacheTracer(); + RecordingPageCursorTracer cursorTracer = new RecordingPageCursorTracer( Fault.class ); + ConfigurablePageCursorTracerSupplier cursorTracerSupplier = new ConfigurablePageCursorTracerSupplier( cursorTracer ); - MuninnPageCache pageCache = createPageCache( fs, 4, 8, blockCacheFlush( tracer ) ); + MuninnPageCache pageCache = createPageCache( fs, 4, 8, blockCacheFlush( tracer ), + cursorTracerSupplier ); PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK | PF_NO_GROW ) ) @@ -188,8 +209,10 @@ public void mustFlushDirtyPagesOnEvictingAllPages() throws Exception cursor.putLong( 0L ); assertFalse( cursor.next() ); } - assertNotNull( tracer.observe( Fault.class ) ); - assertNotNull( tracer.observe( Fault.class ) ); + assertNotNull( cursorTracer.observe( Fault.class ) ); + assertNotNull( cursorTracer.observe( Fault.class ) ); + assertEquals( 2, cursorTracer.faults() ); + assertEquals( 2, tracer.faults() ); int clockArm = pageCache.evictPages( 2, 0, tracer.beginPageEvictions( 2 ) ); assertThat( clockArm, is( 2 ) ); @@ -209,7 +232,8 @@ public void closingTheCursorMustUnlockModifiedPage() throws Exception { writeInitialDataTo( file( "a" ) ); - final MuninnPageCache pageCache = createPageCache( fs, 2, 8, PageCacheTracer.NULL ); + final MuninnPageCache pageCache = createPageCache( fs, 2, 8, PageCacheTracer.NULL, + DefaultPageCursorTracerSupplier.INSTANCE ); final PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); Future task = executor.submit( () -> { @@ -265,7 +289,8 @@ public void writeAll( ByteBuffer src, long position ) throws IOException } }; - MuninnPageCache pageCache = createPageCache( fs, 2, 8, PageCacheTracer.NULL ); + MuninnPageCache pageCache = createPageCache( fs, 2, 8, PageCacheTracer.NULL, + DefaultPageCursorTracerSupplier.INSTANCE ); final PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); // The basic idea is that this loop, which will encounter a lot of page faults, must not block forever even @@ -290,7 +315,8 @@ public void mustThrowIfMappingFileWouldOverflowReferenceCount() throws Exception { File file = file( "a" ); writeInitialDataTo( file ); - MuninnPageCache pageCache = createPageCache( fs, 30, pageCachePageSize, PageCacheTracer.NULL ); + MuninnPageCache pageCache = createPageCache( fs, 30, pageCachePageSize, PageCacheTracer.NULL, + DefaultPageCursorTracerSupplier.NULL ); PagedFile pf = null; int i = 0; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/ConfigurablePageCursorTracerSupplier.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/ConfigurablePageCursorTracerSupplier.java new file mode 100644 index 000000000000..b480497cb97a --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/ConfigurablePageCursorTracerSupplier.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing; + +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; + +public class ConfigurablePageCursorTracerSupplier implements PageCursorTracerSupplier +{ + private T tracer; + + public ConfigurablePageCursorTracerSupplier( T tracer ) + { + this.tracer = tracer; + } + + @Override + public T get() + { + return tracer; + } +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DelegatingPageCacheTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DelegatingPageCacheTracer.java index eb78cce1032a..89a3c1573edb 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DelegatingPageCacheTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/DelegatingPageCacheTracer.java @@ -116,6 +116,24 @@ public void bytesRead( long bytesRead ) delegate.bytesRead( bytesRead ); } + @Override + public void evictions( long evictions ) + { + delegate.evictions( evictions ); + } + + @Override + public void bytesWritten( long bytesWritten ) + { + delegate.bytesWritten( bytesWritten ); + } + + @Override + public void flushes( long flushes ) + { + delegate.flushes( flushes ); + } + public long filesMapped() { return delegate.filesMapped(); diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/Event.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/Event.java new file mode 100644 index 000000000000..830abf2460c8 --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/Event.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.recording; + +import org.neo4j.io.pagecache.PageSwapper; + +public abstract class Event +{ + public final PageSwapper io; + public final long pageId; + + public Event( PageSwapper io, long pageId ) + { + this.io = io; + this.pageId = pageId; + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + + Event event = (Event) o; + + return pageId == event.pageId && !(io != null ? !io.equals( event.io ) : event.io != null); + + } + + @Override + public int hashCode() + { + int result = io != null ? io.hashCode() : 0; + result = 31 * result + (int) (pageId ^ (pageId >>> 32)); + return result; + } + + @Override + public String toString() + { + return String.format( "%s{io=%s, pageId=%s}", getClass().getSimpleName(), io, pageId ); + } +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCacheTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java similarity index 51% rename from community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCacheTracer.java rename to community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java index 8a94b4215b70..5dc1e789f0ff 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCacheTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCacheTracer.java @@ -17,57 +17,28 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.io.pagecache; - -import org.hamcrest.Matcher; +package org.neo4j.io.pagecache.tracing.recording; import java.io.File; import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.EvictionEvent; import org.neo4j.io.pagecache.tracing.EvictionRunEvent; import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.MajorFlushEvent; import org.neo4j.io.pagecache.tracing.PageCacheTracer; -import org.neo4j.io.pagecache.tracing.PageFaultEvent; -import org.neo4j.io.pagecache.tracing.PinEvent; -public class RecordingPageCacheTracer implements PageCacheTracer +public class RecordingPageCacheTracer extends RecordingTracer implements PageCacheTracer { - private final Set> eventTypesToTrace = new HashSet<>(); - private final BlockingQueue record = new LinkedBlockingQueue<>(); - private CountDownLatch trapLatch; - private Matcher trap; + private AtomicLong pins = new AtomicLong(); + private AtomicLong faults = new AtomicLong(); + private AtomicLong evictions = new AtomicLong(); public RecordingPageCacheTracer() { - this( Evict.class, Fault.class ); - } - - @SafeVarargs - public RecordingPageCacheTracer( Class... eventTypesToTrace ) - { - Collections.addAll( this.eventTypesToTrace, eventTypesToTrace ); - } - - private void evicted( long filePageId, PageSwapper swapper ) - { - record( new Evict( swapper, filePageId ) ); - } - - private void record( Event event ) - { - if ( eventTypesToTrace.contains( event.getClass() ) ) - { - record.add( event ); - trip( event ); - } + super( Evict.class ); } @Override @@ -115,17 +86,17 @@ public MajorFlushEvent beginCacheFlush() @Override public long faults() { - return 0; + return faults.get(); } @Override - public long evictions() + public long pins() { - return 0; + return pins.get(); } @Override - public long pins() + public long evictions() { return 0; } @@ -175,6 +146,7 @@ public long evictionExceptions() @Override public void pins( long pins ) { + this.pins.getAndAdd( pins ); } @Override @@ -185,6 +157,7 @@ public void unpins( long unpins ) @Override public void faults( long faults ) { + this.faults.getAndAdd( faults ); } @Override @@ -192,107 +165,25 @@ public void bytesRead( long bytesRead ) { } - public T observe( Class type ) throws InterruptedException - { - return type.cast( record.take() ); - } - - public T tryObserve( Class type ) - { - return type.cast( record.poll() ); - } - - /** - * Set a trap for the eviction thread, and return a CountDownLatch with a counter set to 1. - * When the eviction thread performs the given trap-event, it will block on the latch after - * making the event observable. - */ - public synchronized CountDownLatch trap( Matcher trap ) + @Override + public void evictions( long evictions ) { - assert trap != null; - trapLatch = new CountDownLatch( 1 ); - this.trap = trap; - return trapLatch; + this.evictions.getAndAdd( evictions ); } - private void trip( Event event ) + @Override + public void bytesWritten( long bytesWritten ) { - Matcher theTrap; - CountDownLatch theTrapLatch; - - // The synchronized block is in here, so we don't risk calling await on - // the trapLatch while holding the monitor lock. - synchronized ( this ) - { - theTrap = trap; - theTrapLatch = trapLatch; - } - - if ( theTrap != null && theTrap.matches( event ) ) - { - try - { - theTrapLatch.await(); - } - catch ( InterruptedException e ) - { - Thread.currentThread().interrupt(); - throw new RuntimeException( "Unexpected interrupt in RecordingMonitor", e ); - } - } } - public abstract static class Event + @Override + public void flushes( long flushes ) { - public final PageSwapper io; - public final long pageId; - - public Event( PageSwapper io, long pageId ) - { - this.io = io; - this.pageId = pageId; - } - - @Override - public boolean equals( Object o ) - { - if ( this == o ) - { - return true; - } - if ( o == null || getClass() != o.getClass() ) - { - return false; - } - - Event event = (Event) o; - - return pageId == event.pageId && !(io != null ? !io.equals( event.io ) : event.io != null); - - } - - @Override - public int hashCode() - { - int result = io != null ? io.hashCode() : 0; - result = 31 * result + (int) (pageId ^ (pageId >>> 32)); - return result; - } - - @Override - public String toString() - { - return String.format( "%s{io=%s, pageId=%s}", - getClass().getSimpleName(), io, pageId ); - } } - public static class Fault extends Event + private void evicted( long filePageId, PageSwapper swapper ) { - private Fault( PageSwapper io, long pageId ) - { - super( io, pageId ); - } + record( new Evict( swapper, filePageId ) ); } public static class Evict extends Event @@ -303,14 +194,6 @@ private Evict( PageSwapper io, long pageId ) } } - public static class Pin extends Event - { - private Pin( PageSwapper io, long pageId ) - { - super( io, pageId ); - } - } - private class RecordingEvictionEvent implements EvictionEvent { private long filePageId; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCursorTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java similarity index 69% rename from community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCursorTracer.java rename to community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java index e1f54ab5ce05..3fd995757604 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/RecordingPageCursorTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java @@ -17,27 +17,49 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.io.pagecache; +package org.neo4j.io.pagecache.tracing.recording; +import java.util.Objects; + +import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.EvictionEvent; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.PinEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; -//TODO: -public class RecordingPageCursorTracer implements PageCursorTracer +/** + * Recording tracer of page cursor events. + * report observed events to supplied page cache tracer. + */ +public class RecordingPageCursorTracer extends RecordingTracer implements PageCursorTracer { + + private int pins = 0; + private int faults = 0; + private PageCacheTracer tracer; + + public RecordingPageCursorTracer() + { + super( Pin.class, Fault.class ); + } + + @SafeVarargs + public RecordingPageCursorTracer( Class... eventTypesToTrace ) + { + super( eventTypesToTrace ); + } + @Override public long faults() { - return 0; + return faults; } @Override public long pins() { - return 0; + return pins; } @Override @@ -107,22 +129,43 @@ public void done() @Override public void init( PageCacheTracer tracer ) { - + this.tracer = tracer; } @Override public void reportEvents() { - + Objects.nonNull( tracer ); + tracer.pins( pins ); + tracer.faults( faults ); } private void pageFaulted( long filePageId, PageSwapper swapper ) { -// record( new Fault( swapper, filePageId ) ); + faults++; + record( new Fault( swapper, filePageId ) ); } private void pinned( long filePageId, PageSwapper swapper ) { -// record( new Pin( swapper, filePageId ) ); + pins++; + record( new Pin( swapper, filePageId ) ); + } + + public static class Fault extends Event + { + private Fault( PageSwapper io, long pageId ) + { + super( io, pageId ); + } } + + public static class Pin extends Event + { + private Pin( PageSwapper io, long pageId ) + { + super( io, pageId ); + } + } + } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingTracer.java new file mode 100644 index 000000000000..ac23323b638b --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingTracer.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.recording; + +import org.hamcrest.Matcher; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; + +public class RecordingTracer +{ + private final Set> eventTypesToTrace = new HashSet<>(); + private final BlockingQueue record = new LinkedBlockingQueue<>(); + private CountDownLatch trapLatch; + private Matcher trap; + + @SafeVarargs + public RecordingTracer( Class... eventTypesToTrace ) + { + Collections.addAll( this.eventTypesToTrace, eventTypesToTrace ); + } + + public T observe( Class type ) throws InterruptedException + { + return type.cast( record.take() ); + } + + public T tryObserve( Class type ) + { + return type.cast( record.poll() ); + } + + protected void record( Event event ) + { + if ( eventTypesToTrace.contains( event.getClass() ) ) + { + record.add( event ); + trip( event ); + } + } + + /** + * Set a trap for the eviction thread, and return a CountDownLatch with a counter set to 1. + * When the eviction thread performs the given trap-event, it will block on the latch after + * making the event observable. + */ + public synchronized CountDownLatch trap( Matcher trap ) + { + assert trap != null; + trapLatch = new CountDownLatch( 1 ); + this.trap = trap; + return trapLatch; + } + + private void trip( Event event ) + { + Matcher theTrap; + CountDownLatch theTrapLatch; + + // The synchronized block is in here, so we don't risk calling await on + // the trapLatch while holding the monitor lock. + synchronized ( this ) + { + theTrap = trap; + theTrapLatch = trapLatch; + } + + if ( theTrap != null && theTrap.matches( event ) ) + { + try + { + theTrapLatch.await(); + } + catch ( InterruptedException e ) + { + Thread.currentThread().interrupt(); + throw new RuntimeException( "Unexpected interrupt in RecordingMonitor", e ); + } + } + } + +} diff --git a/community/io/src/test/java/org/neo4j/test/LinearHistoryPageCacheTracer.java b/community/io/src/test/java/org/neo4j/test/LinearHistoryPageCacheTracer.java index 9c7143d8460e..321ec3ed3196 100644 --- a/community/io/src/test/java/org/neo4j/test/LinearHistoryPageCacheTracer.java +++ b/community/io/src/test/java/org/neo4j/test/LinearHistoryPageCacheTracer.java @@ -641,6 +641,24 @@ public void bytesRead( long bytesRead ) } + @Override + public void evictions( long evictions ) + { + + } + + @Override + public void bytesWritten( long bytesWritten ) + { + + } + + @Override + public void flushes( long flushes ) + { + + } + private class HistoryPrinter implements Consumer { private final List concurrentIntervals; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/CommonAbstractStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/CommonAbstractStoreTest.java index 78710d78227f..6bdbf73b7ae8 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/CommonAbstractStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/CommonAbstractStoreTest.java @@ -37,8 +37,10 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.io.pagecache.RecordingPageCacheTracer; -import org.neo4j.io.pagecache.RecordingPageCacheTracer.Pin; +import org.neo4j.io.pagecache.tracing.ConfigurablePageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.recording.Event; +import org.neo4j.io.pagecache.tracing.recording.RecordingPageCacheTracer; +import org.neo4j.io.pagecache.tracing.recording.RecordingPageCursorTracer; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.store.format.RecordFormat; @@ -84,7 +86,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; -import static org.neo4j.io.pagecache.RecordingPageCacheTracer.Event; +import static org.neo4j.io.pagecache.tracing.recording.RecordingPageCursorTracer.Pin; import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_PROPERTY; import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP; import static org.neo4j.test.rule.TestDirectory.testDirectory; @@ -191,7 +193,8 @@ public void failStoreInitializationWhenHeaderRecordCantBeRead() throws IOExcepti public void recordCursorPinsEachPageItReads() throws Exception { File storeFile = dir.file( "a" ); - RecordingPageCacheTracer tracer = new RecordingPageCacheTracer( Pin.class ); + RecordingPageCacheTracer tracer = new RecordingPageCacheTracer(); + RecordingPageCursorTracer pageCursorTracer = new RecordingPageCursorTracer( Pin.class ); PageCache pageCache = pageCacheRule.getPageCache( fileSystemRule.get(), PageCacheRule.config().withTracer( tracer ), Config.empty() ); @@ -349,6 +352,12 @@ private long insertNodeRecordAndObservePinEvent( RecordingPageCacheTracer tracer return nodeId; } + private static ConfigurablePageCursorTracerSupplier pageCursorTracerSupplier( + RecordingPageCursorTracer pageCursorTracer ) + { + return new ConfigurablePageCursorTracerSupplier( pageCursorTracer ); + } + private static class TheStore extends CommonAbstractStore { TheStore( File fileName, Config configuration, IdType idType, IdGeneratorFactory idGeneratorFactory,