Skip to content

Commit

Permalink
Avid using finalization in GrabAllocator. Switch to use cleaner instead.
Browse files Browse the repository at this point in the history 
Update grabs to be cleaned using Cleaner.
Add possibility to close page list.
Small renaming in page cache tests.
  • Loading branch information
@MishaDemianenko
MishaDemianenko committed May 17, 2018
1 parent 72dadff commit 6812613
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 120 deletions.
273 changes: 160 additions & 113 deletions community/io/src/main/java/org/neo4j/io/mem/GrabAllocator.java
View file
Expand Up @@ -19,6 +19,8 @@
*/ */
package org.neo4j.io.mem; package org.neo4j.io.mem;


import sun.misc.Cleaner;

import org.neo4j.memory.MemoryAllocationTracker; import org.neo4j.memory.MemoryAllocationTracker;
import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil;


Expand All @@ -31,18 +33,9 @@
*/ */
public final class GrabAllocator implements MemoryAllocator public final class GrabAllocator implements MemoryAllocator
{ {
/** private final Grabs grabs;
* The amount of memory, in bytes, to grab in each Grab. @SuppressWarnings( {"unused", "FieldCanBeLocal"} )
*/ private final Cleaner cleaner;
private static final long GRAB_SIZE = getInteger( GrabAllocator.class, "GRAB_SIZE", (int) kibiBytes( 512 ) );

/**
* The amount of memory that this memory manager can still allocate.
*/
private long memoryReserve;
private final MemoryAllocationTracker memoryTracker;

private Grab grabs;


/** /**
* Create a new GrabAllocator that will allocate the given amount of memory, to pointers that are aligned to the * Create a new GrabAllocator that will allocate the given amount of memory, to pointers that are aligned to the
Expand All @@ -53,126 +46,32 @@ public final class GrabAllocator implements MemoryAllocator
*/ */
GrabAllocator( long expectedMaxMemory, MemoryAllocationTracker memoryTracker ) GrabAllocator( long expectedMaxMemory, MemoryAllocationTracker memoryTracker )
{ {
this.memoryReserve = expectedMaxMemory; this.grabs = new Grabs( expectedMaxMemory, memoryTracker );
this.memoryTracker = memoryTracker; this.cleaner = Cleaner.create( this, new GrabsDeallocator( grabs ) );
} }


@Override @Override
public synchronized long usedMemory() public synchronized long usedMemory()
{ {
long sum = 0; return grabs.usedMemory();
Grab grab = grabs;
while ( grab != null )
{
sum += grab.nextPointer - grab.address;
grab = grab.next;
}
return sum;
} }


@Override @Override
public synchronized long availableMemory() public synchronized long availableMemory()
{ {
Grab grab = grabs; return grabs.availableMemory();
long availableInCurrentGrab = 0;
if ( grab != null )
{
availableInCurrentGrab = grab.limit - grab.nextPointer;
}
return Math.max( memoryReserve, 0L ) + availableInCurrentGrab;
} }


@Override @Override
public synchronized long allocateAligned( long bytes, long alignment ) public synchronized long allocateAligned( long bytes, long alignment )
{ {
if ( alignment <= 0 ) return grabs.allocateAligned( bytes, alignment );
{
throw new IllegalArgumentException( "Invalid alignment: " + alignment + ". Alignment must be positive." );
}
long grabSize = Math.min( GRAB_SIZE, memoryReserve );
try
{
if ( bytes > GRAB_SIZE )
{
// This is a huge allocation. Put it in its own grab and keep any existing grab at the head.
grabSize = bytes;
Grab nextGrab = grabs == null ? null : grabs.next;
Grab allocationGrab = new Grab( nextGrab, grabSize, memoryTracker );
if ( !allocationGrab.canAllocate( bytes, alignment ) )
{
allocationGrab.free();
grabSize = bytes + alignment;
allocationGrab = new Grab( nextGrab, grabSize, memoryTracker );
}
long allocation = allocationGrab.allocate( bytes, alignment );
grabs = grabs == null ? allocationGrab : grabs.setNext( allocationGrab );
memoryReserve -= bytes;
return allocation;
}

if ( grabs == null || !grabs.canAllocate( bytes, alignment ) )
{
if ( grabSize < bytes )
{
grabSize = bytes;
Grab grab = new Grab( grabs, grabSize, memoryTracker );
if ( grab.canAllocate( bytes, alignment ) )
{
memoryReserve -= grabSize;
grabs = grab;
return grabs.allocate( bytes, alignment );
}
grab.free();
grabSize = bytes + alignment;
}
grabs = new Grab( grabs, grabSize, memoryTracker );
memoryReserve -= grabSize;
}
return grabs.allocate( bytes, alignment );
}
catch ( OutOfMemoryError oome )
{
NativeMemoryAllocationRefusedError error =
new NativeMemoryAllocationRefusedError( grabSize, usedMemory() );
initCause( error, oome );
throw error;
}
}

private void initCause( NativeMemoryAllocationRefusedError error, OutOfMemoryError cause )
{
try
{
error.initCause( cause );
}
catch ( Throwable ignore )
{
// This can only happen if our NMARE somehow already has a cause initialised, which should not
// be the case, but it could if the JDK decided to inject a default cause in some future version.
// To avoid loosing the ability to trace this cause back, we'll add it as a suppressed exception
// instead.
try
{
error.addSuppressed( cause );
}
catch ( Throwable ignore2 )
{
// Okay, we tried.
}
}
} }


@Override @Override
protected synchronized void finalize() throws Throwable public void close()
{ {
super.finalize(); cleaner.clean();
Grab current = grabs;

while ( current != null )
{
current.free();
current = current.next;
}
} }


private static class Grab private static class Grab
Expand Down Expand Up @@ -246,4 +145,152 @@ public String toString()
return String.format( "Grab[size = %d bytes, reserve = %d bytes, use = %5.2f %%]", size, reserve, use ); return String.format( "Grab[size = %d bytes, reserve = %d bytes, use = %5.2f %%]", size, reserve, use );
} }
} }

private static final class Grabs
{
/**
* The amount of memory, in bytes, to grab in each Grab.
*/
private static final long GRAB_SIZE = getInteger( GrabAllocator.class, "GRAB_SIZE", (int) kibiBytes( 512 ) );

private final MemoryAllocationTracker memoryTracker;
private long expectedMaxMemory;
private Grab head;

Grabs( long expectedMaxMemory, MemoryAllocationTracker memoryTracker )
{
this.expectedMaxMemory = expectedMaxMemory;
this.memoryTracker = memoryTracker;
}

long usedMemory()
{
long sum = 0;
Grab grab = head;
while ( grab != null )
{
sum += grab.nextPointer - grab.address;
grab = grab.next;
}
return sum;
}

long availableMemory()
{
Grab grab = head;
long availableInCurrentGrab = 0;
if ( grab != null )
{
availableInCurrentGrab = grab.limit - grab.nextPointer;
}
return Math.max( expectedMaxMemory, 0L ) + availableInCurrentGrab;
}

public void close()
{
Grab current = head;

while ( current != null )
{
current.free();
current = current.next;
}
head = null;
}

long allocateAligned( long bytes, long alignment )
{
if ( alignment <= 0 )
{
throw new IllegalArgumentException( "Invalid alignment: " + alignment + ". Alignment must be positive." );
}
long grabSize = Math.min( GRAB_SIZE, expectedMaxMemory );
try
{
if ( bytes > GRAB_SIZE )
{
// This is a huge allocation. Put it in its own grab and keep any existing grab at the head.
grabSize = bytes;
Grab nextGrab = head == null ? null : head.next;
Grab allocationGrab = new Grab( nextGrab, grabSize, memoryTracker );
if ( !allocationGrab.canAllocate( bytes, alignment ) )
{
allocationGrab.free();
grabSize = bytes + alignment;
allocationGrab = new Grab( nextGrab, grabSize, memoryTracker );
}
long allocation = allocationGrab.allocate( bytes, alignment );
head = head == null ? allocationGrab : head.setNext( allocationGrab );
expectedMaxMemory -= bytes;
return allocation;
}

if ( head == null || !head.canAllocate( bytes, alignment ) )
{
if ( grabSize < bytes )
{
grabSize = bytes;
Grab grab = new Grab( head, grabSize, memoryTracker );
if ( grab.canAllocate( bytes, alignment ) )
{
expectedMaxMemory -= grabSize;
head = grab;
return head.allocate( bytes, alignment );
}
grab.free();
grabSize = bytes + alignment;
}
head = new Grab( head, grabSize, memoryTracker );
expectedMaxMemory -= grabSize;
}
return head.allocate( bytes, alignment );
}
catch ( OutOfMemoryError oome )
{
NativeMemoryAllocationRefusedError error =
new NativeMemoryAllocationRefusedError( grabSize, usedMemory() );
initCause( error, oome );
throw error;
}
}

private static void initCause( NativeMemoryAllocationRefusedError error, OutOfMemoryError cause )
{
try
{
error.initCause( cause );
}
catch ( Throwable ignore )
{
// This can only happen if our NMARE somehow already has a cause initialised, which should not
// be the case, but it could if the JDK decided to inject a default cause in some future version.
// To avoid loosing the ability to trace this cause back, we'll add it as a suppressed exception
// instead.
try
{
error.addSuppressed( cause );
}
catch ( Throwable ignore2 )
{
// Okay, we tried.
}
}
}
}

private static final class GrabsDeallocator implements Runnable
{
private final Grabs grabs;

GrabsDeallocator( Grabs grabs )
{
this.grabs = grabs;
}

@Override
public void run()
{
grabs.close();
}
}
} }
5 changes: 5 additions & 0 deletions community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java
View file
Expand Up @@ -50,4 +50,9 @@ static MemoryAllocator createAllocator( String expectedMemory, MemoryAllocationT
* @throws OutOfMemoryError if the requested memory could not be allocated. * @throws OutOfMemoryError if the requested memory could not be allocated.
*/ */
long allocateAligned( long bytes, long alignment ); long allocateAligned( long bytes, long alignment );

/**
* Close all allocated resources and free all allocated memory.
*/
void close();
} }
5 changes: 5 additions & 0 deletions community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/PageList.java
View file
Expand Up @@ -535,4 +535,9 @@ void toString( long pageRef, StringBuilder sb )
sb.append( ", usageCounter = " ).append( getUsageCounter( pageRef ) ); sb.append( ", usageCounter = " ).append( getUsageCounter( pageRef ) );
sb.append( " ] " ).append( OffHeapPageLock.toString( offLock( pageRef ) ) ); sb.append( " ] " ).append( OffHeapPageLock.toString( offLock( pageRef ) ) );
} }

public void close()
{
memoryAllocator.close();
}
} }
2 changes: 1 addition & 1 deletion community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java
View file
Expand Up @@ -170,7 +170,7 @@ public void trackMemoryAllocations() throws Throwable
assertEquals( ByteUnit.mebiBytes( 1 ), memoryTracker.usedDirectMemory() ); assertEquals( ByteUnit.mebiBytes( 1 ), memoryTracker.usedDirectMemory() );


//noinspection FinalizeCalledExplicitly //noinspection FinalizeCalledExplicitly
allocator.finalize(); allocator.close();
assertEquals( 0, memoryTracker.usedDirectMemory() ); assertEquals( 0, memoryTracker.usedDirectMemory() );
} }
} }
10 changes: 5 additions & 5 deletions community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTest.java
View file
Expand Up @@ -2798,7 +2798,7 @@ public void writeLockedPageMustBlockFileUnmapping() throws Exception
PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ); PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK );
assertTrue( cursor.next() ); assertTrue( cursor.next() );


Thread unmapper = fork( $close( pagedFile ) ); Thread unmapper = fork( closePageFile( pagedFile ) );
unmapper.join( 100 ); unmapper.join( 100 );


cursor.close(); cursor.close();
Expand All @@ -2816,7 +2816,7 @@ public void optimisticReadLockedPageMustNotBlockFileUnmapping() throws Exception
PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ); PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK );
assertTrue( cursor.next() ); // Got a read lock assertTrue( cursor.next() ); // Got a read lock


fork( $close( pagedFile ) ).join(); fork( closePageFile( pagedFile ) ).join();


cursor.close(); cursor.close();
} }
Expand All @@ -2832,7 +2832,7 @@ public void advancingPessimisticReadLockingCursorAfterUnmappingMustThrow() throw
PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ); PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK );
assertTrue( cursor.next() ); // Got a pessimistic read lock assertTrue( cursor.next() ); // Got a pessimistic read lock


fork( $close( pagedFile ) ).join(); fork( closePageFile( pagedFile ) ).join();


expectedException.expect( FileIsNotMappedException.class ); expectedException.expect( FileIsNotMappedException.class );
cursor.next(); cursor.next();
Expand All @@ -2851,7 +2851,7 @@ public void advancingOptimisticReadLockingCursorAfterUnmappingMustThrow() throws
assertTrue( cursor.next() ); // fault + unpin page 0 assertTrue( cursor.next() ); // fault + unpin page 0
assertTrue( cursor.next( 0 ) ); // potentially optimistic read lock page 0 assertTrue( cursor.next( 0 ) ); // potentially optimistic read lock page 0


fork( $close( pagedFile ) ).join(); fork( closePageFile( pagedFile ) ).join();


try try
{ {
Expand All @@ -2877,7 +2877,7 @@ public void readingAndRetryingOnPageWithOptimisticReadLockingAfterUnmappingMustN
assertTrue( cursor.next() ); // fault + unpin page 0 assertTrue( cursor.next() ); // fault + unpin page 0
assertTrue( cursor.next( 0 ) ); // potentially optimistic read lock page 0 assertTrue( cursor.next( 0 ) ); // potentially optimistic read lock page 0


fork( $close( pagedFile ) ).join(); fork( closePageFile( pagedFile ) ).join();
pageCache.close(); pageCache.close();
pageCache = null; pageCache = null;


Expand Down

0 comments on commit 6812613

Please sign in to comment.