Collapse PagedFile ref count and last page id

This reduces the number of volatile reads performed when pinning pages in the page cache.
The numbers were initially separate variables, but have now been combined into a single volatile long field called the header state.

So the header state includes both the reference count of the PagedFile – 15 bits – and the ID of the last page in
the file – 48 bits, plus an empty file marker bit. Because our pages are usually 2^13 bytes, this means that we
only loose 3 bits to the reference count, in terms of keeping large files byte addressable.

The layout looks like this:

    ┏━ Empty file marker bit. When 1, the file is empty.
    ┃    ┏━ Reference count, 15 bites.
    ┃    ┃                ┏━ 48 bits for the last page id.
    ┃┏━━━┻━━━━━━━━━━┓ ┏━━━┻━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
    MRRRRRRR RRRRRRRR IIIIIIII IIIIIIII IIIIIIII IIIIIIII IIIIIIII IIIIIIII
    1        2        3        4        5        6        7        8        byte

We then use masks and shifts when querying for either the reference count or the last page id.
The page swapper will return -1 for the last page id, so during initialisation, we check for that, and initialises the header state to Long.MIN_VALUE to raise only the M bit.
Because the reference count can now only hold a much smaller value than it could before, overflow checks have been added to the increments and decrements.
This is fine, because those methods are only called infrequently.

community/io/src/main/java/org/neo4j/io/pagecache/PagedFile.java

@@ -149,7 +149,9 @@ public interface PagedFile extends AutoCloseable
     /**
      * Get the file-page-id of the last page in the file.
      *
-     * This will return -1 if the file is completely empty.
+     * This will return <em>a negative number</em> (not necessarily -1) if the file is completely empty.
+     *
+     * @throws IllegalStateException if this file has been unmapped
      */
     long getLastPageId() throws IOException;
 

community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java

@@ -38,7 +38,6 @@ abstract class MuninnPageCursor implements PageCursor
     protected int pf_flags;
     protected long currentPageId;
     protected long nextPageId;
-    protected long lastPageId;
     protected long lockStamp;
 
     private boolean claimed;
@@ -71,7 +70,6 @@ public final void rewind()
     {
         nextPageId = pageId;
         currentPageId = UNBOUND_PAGE_ID;
-        lastPageId = pagedFile.getLastPageId();
     }
 
     public final void reset( MuninnPage page )
@@ -226,7 +224,7 @@ private MuninnPage pageFault(
             // here, so the unmapping would have already happened. We do this
             // check before page.fault(), because that would otherwise reopen
             // the file channel.
-            assertPagedFileStillMapped();
+            assertPagedFileStillMappedAndGetIdOfLastPage();
             page.initBuffer();
             page.fault( swapper, filePageId, faultEvent );
         }
@@ -248,9 +246,9 @@ private MuninnPage pageFault(
         return page;
     }
 
-    protected void assertPagedFileStillMapped()
+    protected long assertPagedFileStillMappedAndGetIdOfLastPage()
     {
-        pagedFile.assertStillMapped();
+        return pagedFile.getLastPageId();
     }
 
     protected abstract void unpinCurrentPage();

community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java

@@ -43,10 +43,12 @@ final class MuninnPagedFile implements PagedFile
     private static final int translationTableChunkArrayBase = UnsafeUtil.arrayBaseOffset( MuninnPage[].class );
     private static final int translationTableChunkArrayScale = UnsafeUtil.arrayIndexScale( MuninnPage[].class );
 
-    private static final long referenceCounterOffset =
-            UnsafeUtil.getFieldOffset( MuninnPagedFile.class, "referenceCounter" );
-    private static final long lastPageIdOffset =
-            UnsafeUtil.getFieldOffset( MuninnPagedFile.class, "lastPageId" );
+    private static final long headerStateOffset =
+            UnsafeUtil.getFieldOffset( MuninnPagedFile.class, "headerState" );
+    private static final int headerStateRefCountShift = 48;
+    private static final int headerStateRefCountMax = 0x7FFF;
+    private static final long headerStateRefCountMask = 0x7FFF_0000_0000_0000L;
+    private static final long headerStateLastPageIdMask = 0x8000_FFFF_FFFF_FFFFL;
 
     final MuninnPageCache pageCache;
     final int filePageSize;
@@ -59,10 +61,22 @@ final class MuninnPagedFile implements PagedFile
     final PageSwapper swapper;
     private final CursorPool cursorPool;
 
+    /**
+     * The header state includes both the reference count of the PagedFile – 15 bits – and the ID of the last page in
+     * the file – 48 bits, plus an empty file marker bit. Because our pages are usually 2^13 bytes, this means that we
+     * only loose 3 bits to the reference count, in terms of keeping large files byte addressable.
+     *
+     * The layout looks like this:
+     *
+     * ┏━ Empty file marker bit. When 1, the file is empty.
+     * ┃    ┏━ Reference count, 15 bites.
+     * ┃    ┃                ┏━ 48 bits for the last page id.
+     * ┃┏━━━┻━━━━━━━━━━┓ ┏━━━┻━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+     * MRRRRRRR RRRRRRRR IIIIIIII IIIIIIII IIIIIIII IIIIIIII IIIIIIII IIIIIIII
+     * 1        2        3        4        5        6        7        8        byte
+     */
     @SuppressWarnings( "unused" ) // Accessed via Unsafe
-    private volatile int referenceCounter;
-    @SuppressWarnings( "unused" ) // Accessed via Unsafe
-    private volatile long lastPageId;
+    private volatile long headerState;
 
     MuninnPagedFile(
             File file,
@@ -145,14 +159,6 @@ public PageCursor io( long pageId, int pf_flags )
         return cursor;
     }
 
-    void assertStillMapped()
-    {
-        if ( getRefCount() == 0 )
-        {
-            throw new IllegalStateException( "File has been unmapped: " + file().getPath() );
-        }
-    }
-
     @Override
     public int pageSize()
     {
@@ -291,35 +297,72 @@ private void syncDevice() throws IOException
     @Override
     public long getLastPageId()
     {
-        return lastPageId;
+        long state = getHeaderState();
+        if ( refCountOf( state ) == 0 )
+        {
+            throw new IllegalStateException( "File has been unmapped: " + file().getPath() );
+        }
+        return state & headerStateLastPageIdMask;
+    }
+
+    private long getHeaderState()
+    {
+        return UnsafeUtil.getLongVolatile( this, headerStateOffset );
+    }
+
+    private long refCountOf( long state )
+    {
+        return (state & headerStateRefCountMask) >>> headerStateRefCountShift;
     }
 
     private void initialiseLastPageId( long lastPageIdFromFile )
     {
-        UnsafeUtil.putLong( this, lastPageIdOffset, lastPageIdFromFile );
+        if ( lastPageIdFromFile < 0 )
+        {
+            // MIN_VALUE only has the sign bit raised, and the rest of the bits are zeros.
+            UnsafeUtil.putLongVolatile( this, headerStateOffset, Long.MIN_VALUE );
+        }
+        else
+        {
+            UnsafeUtil.putLongVolatile( this, headerStateOffset, lastPageIdFromFile );
+        }
     }
 
     /**
      * Make sure that the lastPageId is at least the given pageId
      */
-    long increaseLastPageIdTo( long newLastPageId )
+    void increaseLastPageIdTo( long newLastPageId )
     {
-        long current;
+        long current, update, lastPageId;
         do
         {
-            current = lastPageId;
+            current = getHeaderState();
+            update = newLastPageId + (current & headerStateRefCountMask);
+            lastPageId = current & headerStateLastPageIdMask;
         }
-        while ( current < newLastPageId
-                && !UnsafeUtil.compareAndSwapLong( this, lastPageIdOffset, current, newLastPageId ) );
-        return lastPageId;
+        while ( lastPageId < newLastPageId
+                && !UnsafeUtil.compareAndSwapLong( this, headerStateOffset, current, update ) );
     }
 
     /**
      * Atomically increment the reference count for this mapped file.
      */
     void incrementRefCount()
     {
-        UnsafeUtil.getAndAddInt( this, referenceCounterOffset, 1 );
+        long current, update;
+        do
+        {
+            current = getHeaderState();
+            long count = refCountOf( current ) + 1;
+            if ( count > headerStateRefCountMax )
+            {
+                throw new IllegalStateException( "Cannot map file because reference counter would overflow. " +
+                                                 "Maximum reference count is " + headerStateRefCountMax + ". " +
+                                                 "File is " + swapper.file().getAbsolutePath() );
+            }
+            update = (current & headerStateLastPageIdMask) + (count << headerStateRefCountShift);
+        }
+        while ( !UnsafeUtil.compareAndSwapLong( this, headerStateOffset, current, update ) );
     }
 
     /**
@@ -328,9 +371,20 @@ void incrementRefCount()
      */
     boolean decrementRefCount()
     {
-        // compares with 1 because getAndAdd returns the old value, and a 1
-        // means the value is now 0.
-        return UnsafeUtil.getAndAddInt( this, referenceCounterOffset, -1 ) <= 1;
+        long current, update, count;
+        do
+        {
+            current = getHeaderState();
+            count = refCountOf( current ) - 1;
+            if ( count < 0 )
+            {
+                throw new IllegalStateException( "File has already been closed and unmapped. " +
+                                                 "It cannot be closed any further." );
+            }
+            update = (current & headerStateLastPageIdMask) + (count << headerStateRefCountShift);
+        }
+        while ( !UnsafeUtil.compareAndSwapLong( this, headerStateOffset, current, update ) );
+        return count == 0;
     }
 
     /**
@@ -339,7 +393,7 @@ boolean decrementRefCount()
      */
     int getRefCount()
     {
-        return UnsafeUtil.getIntVolatile( this, referenceCounterOffset );
+        return (int) refCountOf( getHeaderState() );
     }
 
     /**

community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java

@@ -50,7 +50,7 @@ protected void unpinCurrentPage()
     @Override
     public boolean next() throws IOException
     {
-        assertPagedFileStillMapped();
+        long lastPageId = assertPagedFileStillMappedAndGetIdOfLastPage();
         if ( nextPageId > lastPageId )
         {
             return false;

community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java

@@ -42,7 +42,7 @@ protected void unpinCurrentPage()
     @Override
     public boolean next() throws IOException
     {
-        assertPagedFileStillMapped();
+        long lastPageId = assertPagedFileStillMappedAndGetIdOfLastPage();
         if ( nextPageId > lastPageId )
         {
             if ( (pf_flags & PagedFile.PF_NO_GROW) != 0 )
@@ -82,7 +82,7 @@ protected void pinCursorToPage( MuninnPage page, long filePageId, PageSwapper sw
         // files have been unmapped, the page cache can be closed. And when
         // that happens, dirty contents in memory will no longer have a chance
         // to get flushed.
-        assertPagedFileStillMapped();
+        assertPagedFileStillMappedAndGetIdOfLastPage();
         page.incrementUsage();
         page.markAsDirty();
     }

community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java

@@ -86,6 +86,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
@@ -2315,13 +2316,13 @@ public PinEvent beginPin( boolean exclusiveLock, long filePageId, PageSwapper sw
     }
 
     @Test
-    public void lastPageIdOfEmptyFileIsMinusOne() throws IOException
+    public void lastPageIdOfEmptyFileIsLessThanZero() throws IOException
     {
         getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL );
 
         try ( PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ) )
         {
-            assertThat( pagedFile.getLastPageId(), is( -1L ) );
+            assertThat( pagedFile.getLastPageId(), lessThan( 0L ) );
         }
     }
 
@@ -2334,9 +2335,10 @@ public void lastPageIdOfFileWithOneByteIsZero() throws IOException
 
         getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL );
 
-        PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize );
-        assertThat( pagedFile.getLastPageId(), is( 0L ) );
-        pagedFile.close();
+        try ( PagedFile pagedFile = pageCache.map( file( "a" ), filePageSize ) )
+        {
+            assertThat( pagedFile.getLastPageId(), is( 0L ) );
+        }
     }
 
     @Test
@@ -2469,6 +2471,20 @@ public void lastPageIdMustIncreaseWhenJumpingPastEndWithExclusiveLock()
         }
     }
 
+    @Test( expected = IllegalStateException.class )
+    public void lastPageIdFromUnmappedFileMustThrow() throws IOException
+    {
+        getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL );
+
+        PagedFile file;
+        try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize, StandardOpenOption.CREATE ) )
+        {
+            file = pf;
+        }
+
+        file.getLastPageId();
+    }
+
     @Test
     public void cursorOffsetMustBeUpdatedReadAndWrite() throws IOException
     {
@@ -3721,7 +3737,7 @@ public void pageCacheMustRemainInternallyConsistentWhenGettingRandomFailures() t
             PagedFile pagedFile = rng.nextBoolean()? pfA : pfB;
             long maxPageId = pagedFile.getLastPageId();
             boolean performingRead = rng.nextBoolean() && maxPageId != -1;
-            long startingPage = maxPageId == -1? 0 : rng.nextLong( maxPageId + 1 );
+            long startingPage = maxPageId < 0? 0 : rng.nextLong( maxPageId + 1 );
             int pf_flags = performingRead ? PF_SHARED_LOCK : PF_EXCLUSIVE_LOCK;
             int pageSize = pagedFile.pageSize();
 
@@ -4171,14 +4187,14 @@ public void mappingFileWithTruncateOptionMustTruncateFile() throws Exception
         try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize );
               PageCursor cursor = pf.io( 10, PF_EXCLUSIVE_LOCK ) )
         {
-            assertThat( pf.getLastPageId(), is( -1L ) );
+            assertThat( pf.getLastPageId(), lessThan( 0L ) );
             assertTrue( cursor.next() );
             cursor.putInt( 0xcafebabe );
         }
         try ( PagedFile pf = pageCache.map( file( "a" ), filePageSize, StandardOpenOption.TRUNCATE_EXISTING );
               PageCursor cursor = pf.io( 0, PF_SHARED_LOCK ) )
         {
-            assertThat( pf.getLastPageId(), is( -1L ) );
+            assertThat( pf.getLastPageId(), lessThan( 0L ) );
             assertFalse( cursor.next() );
         }
     }
@@ -4194,4 +4210,13 @@ public void mappingAlreadyMappedFileWithTruncateOptionMustThrow() throws Excepti
             fail( "the second map call should have thrown" );
         }
     }
+
+    @Test( expected = IllegalStateException.class )
+    public void mustThrowIfFileIsClosedMoreThanItIsMapped() throws Exception
+    {
+        getPageCache( fs, maxPages, pageCachePageSize, PageCacheTracer.NULL );
+        PagedFile pf = pageCache.map( file( "a" ), filePageSize );
+        pf.close();
+        pf.close();
+    }
 }