Skip to content

Commit

Permalink
PR comments:
Browse files Browse the repository at this point in the history
restore finalization call and use it in tests (fight for next time),
additional helper methods with default global tracker
cleanup explicit passing of global tracker
  • Loading branch information
MishaDemianenko committed Feb 1, 2018
1 parent 1398728 commit 975b6de
Show file tree
Hide file tree
Showing 10 changed files with 36 additions and 35 deletions.
Expand Up @@ -166,13 +166,8 @@ private void initCause( NativeMemoryAllocationRefusedError error, OutOfMemoryErr
protected synchronized void finalize() throws Throwable protected synchronized void finalize() throws Throwable
{ {
super.finalize(); super.finalize();
close();
}

@Override
public void close()
{
Grab current = grabs; Grab current = grabs;

while ( current != null ) while ( current != null )
{ {
current.free(); current.free();
Expand Down
Expand Up @@ -50,9 +50,4 @@ static MemoryAllocator createAllocator( String expectedMemory, MemoryAllocationT
* @throws OutOfMemoryError if the requested memory could not be allocated. * @throws OutOfMemoryError if the requested memory could not be allocated.
*/ */
long allocateAligned( long bytes, long alignment ); long allocateAligned( long bytes, long alignment );

/**
* Close allocator and release all allocated resources
*/
void close();
} }
Expand Up @@ -119,18 +119,19 @@ public void allocatingMustIncreaseMemoryUsedAndDecreaseAvailableMemory() throws
} }


@Test @Test
public void trackMemoryAllocations() public void trackMemoryAllocations() throws Throwable
{ {
LocalMemoryTracker memoryTracker = new LocalMemoryTracker(); LocalMemoryTracker memoryTracker = new LocalMemoryTracker();
MemoryAllocator allocator = MemoryAllocator.createAllocator( "2m", memoryTracker ); GrabAllocator allocator = (GrabAllocator) MemoryAllocator.createAllocator( "2m", memoryTracker );


assertEquals( 0, memoryTracker.usedDirectMemory() ); assertEquals( 0, memoryTracker.usedDirectMemory() );


long pointer = allocator.allocateAligned( ByteUnit.mebiBytes( 1 ), 1 ); long pointer = allocator.allocateAligned( ByteUnit.mebiBytes( 1 ), 1 );


assertEquals( ByteUnit.mebiBytes( 1 ), memoryTracker.usedDirectMemory() ); assertEquals( ByteUnit.mebiBytes( 1 ), memoryTracker.usedDirectMemory() );


allocator.close(); //noinspection FinalizeCalledExplicitly
allocator.finalize();
assertEquals( 0, memoryTracker.usedDirectMemory() ); assertEquals( 0, memoryTracker.usedDirectMemory() );
} }
} }
Expand Up @@ -26,7 +26,6 @@
import org.neo4j.io.ByteUnit; import org.neo4j.io.ByteUnit;
import org.neo4j.io.mem.MemoryAllocator; import org.neo4j.io.mem.MemoryAllocator;
import org.neo4j.memory.GlobalMemoryTracker; import org.neo4j.memory.GlobalMemoryTracker;
import org.neo4j.memory.LocalMemoryTracker;


import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
Expand All @@ -45,7 +44,7 @@ public void veryLargePageListsMustBeFullyAccessible() throws Exception
long pageCacheSize = ByteUnit.gibiBytes( 513 ) + pageSize; long pageCacheSize = ByteUnit.gibiBytes( 513 ) + pageSize;
int pages = Math.toIntExact( pageCacheSize / pageSize ); int pages = Math.toIntExact( pageCacheSize / pageSize );


MemoryAllocator mman = MemoryAllocator.createAllocator( "2 GiB", new LocalMemoryTracker() ); MemoryAllocator mman = MemoryAllocator.createAllocator( "2 GiB", GlobalMemoryTracker.INSTANCE );
SwapperSet swappers = new SwapperSet(); SwapperSet swappers = new SwapperSet();
long victimPage = VictimPageReference.getVictimPage( pageSize, GlobalMemoryTracker.INSTANCE ); long victimPage = VictimPageReference.getVictimPage( pageSize, GlobalMemoryTracker.INSTANCE );


Expand Down
Expand Up @@ -51,7 +51,6 @@
import org.neo4j.io.pagecache.tracing.FlushEventOpportunity; import org.neo4j.io.pagecache.tracing.FlushEventOpportunity;
import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.PageFaultEvent;
import org.neo4j.memory.GlobalMemoryTracker; import org.neo4j.memory.GlobalMemoryTracker;
import org.neo4j.memory.LocalMemoryTracker;
import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil;


import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -89,7 +88,7 @@ public static Iterable<Object[]> parameters()
public static void setUpStatics() public static void setUpStatics()
{ {
executor = Executors.newCachedThreadPool( new DaemonThreadFactory() ); executor = Executors.newCachedThreadPool( new DaemonThreadFactory() );
mman = MemoryAllocator.createAllocator( "1 MiB", new LocalMemoryTracker() ); mman = MemoryAllocator.createAllocator( "1 MiB", GlobalMemoryTracker.INSTANCE );
} }


@AfterClass @AfterClass
Expand Down
Expand Up @@ -35,7 +35,6 @@
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;


import org.neo4j.memory.GlobalMemoryTracker;
import org.neo4j.test.rule.RepeatRule; import org.neo4j.test.rule.RepeatRule;
import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil;


Expand All @@ -57,7 +56,7 @@ public static void shutDownExecutor()
@Before @Before
public void allocateLock() public void allocateLock()
{ {
lockAddr = UnsafeUtil.allocateMemory( Long.BYTES, GlobalMemoryTracker.INSTANCE ); lockAddr = UnsafeUtil.allocateMemory( Long.BYTES );
UnsafeUtil.putLong( lockAddr, 0 ); UnsafeUtil.putLong( lockAddr, 0 );
} }


Expand Down
Expand Up @@ -36,6 +36,7 @@
import org.neo4j.collection.primitive.hopscotch.PrimitiveLongIntHashMap; import org.neo4j.collection.primitive.hopscotch.PrimitiveLongIntHashMap;
import org.neo4j.collection.primitive.hopscotch.PrimitiveLongLongHashMap; import org.neo4j.collection.primitive.hopscotch.PrimitiveLongLongHashMap;
import org.neo4j.collection.primitive.hopscotch.PrimitiveLongObjectHashMap; import org.neo4j.collection.primitive.hopscotch.PrimitiveLongObjectHashMap;
import org.neo4j.memory.GlobalMemoryTracker;
import org.neo4j.memory.MemoryAllocationTracker; import org.neo4j.memory.MemoryAllocationTracker;


import static org.neo4j.collection.primitive.hopscotch.HopScotchHashingAlgorithm.NO_MONITOR; import static org.neo4j.collection.primitive.hopscotch.HopScotchHashingAlgorithm.NO_MONITOR;
Expand Down Expand Up @@ -83,6 +84,11 @@ public static PrimitiveLongSet longSet( int initialCapacity )
VALUE_MARKER, NO_MONITOR ); VALUE_MARKER, NO_MONITOR );
} }


public static PrimitiveLongSet offHeapLongSet()
{
return offHeapLongSet( GlobalMemoryTracker.INSTANCE );
}

public static PrimitiveLongSet offHeapLongSet( MemoryAllocationTracker allocationTracker ) public static PrimitiveLongSet offHeapLongSet( MemoryAllocationTracker allocationTracker )
{ {
return offHeapLongSet( DEFAULT_OFFHEAP_CAPACITY, allocationTracker ); return offHeapLongSet( DEFAULT_OFFHEAP_CAPACITY, allocationTracker );
Expand Down Expand Up @@ -114,6 +120,11 @@ public static PrimitiveLongLongMap longLongMap( int initialCapacity )
return new PrimitiveLongLongHashMap( new LongKeyLongValueTable( initialCapacity ), NO_MONITOR ); return new PrimitiveLongLongHashMap( new LongKeyLongValueTable( initialCapacity ), NO_MONITOR );
} }


public static PrimitiveLongLongMap offHeapLongLongMap()
{
return offHeapLongLongMap( GlobalMemoryTracker.INSTANCE );
}

public static PrimitiveLongLongMap offHeapLongLongMap( MemoryAllocationTracker allocationTracker ) public static PrimitiveLongLongMap offHeapLongLongMap( MemoryAllocationTracker allocationTracker )
{ {
return offHeapLongLongMap( DEFAULT_OFFHEAP_CAPACITY, allocationTracker ); return offHeapLongLongMap( DEFAULT_OFFHEAP_CAPACITY, allocationTracker );
Expand Down Expand Up @@ -145,6 +156,11 @@ public static PrimitiveIntSet intSet( int initialCapacity )
VALUE_MARKER, NO_MONITOR ); VALUE_MARKER, NO_MONITOR );
} }


public static PrimitiveIntSet offHeapIntSet()
{
return offHeapIntSet( GlobalMemoryTracker.INSTANCE );
}

public static PrimitiveIntSet offHeapIntSet( MemoryAllocationTracker allocationTracker ) public static PrimitiveIntSet offHeapIntSet( MemoryAllocationTracker allocationTracker )
{ {
return new PrimitiveIntHashSet( new IntKeyUnsafeTable<>( DEFAULT_OFFHEAP_CAPACITY, VALUE_MARKER, allocationTracker ), return new PrimitiveIntHashSet( new IntKeyUnsafeTable<>( DEFAULT_OFFHEAP_CAPACITY, VALUE_MARKER, allocationTracker ),
Expand Down
Expand Up @@ -59,7 +59,7 @@ protected UnsafeTable( int capacity, int bytesPerKey, VALUE valueMarker, MemoryA
if ( UnsafeUtil.allowUnalignedMemoryAccess ) if ( UnsafeUtil.allowUnalignedMemoryAccess )
{ {
allocatedBytes = dataSize; allocatedBytes = dataSize;
this.allocatedAddress = this.address = allocateMemory( allocatedBytes ); this.allocatedAddress = this.address = UnsafeUtil.allocateMemory( allocatedBytes, this.allocationTracker );
} }
else else
{ {
Expand All @@ -75,7 +75,7 @@ protected UnsafeTable( int capacity, int bytesPerKey, VALUE valueMarker, MemoryA
} }


allocatedBytes = dataSize + Integer.BYTES - 1; allocatedBytes = dataSize + Integer.BYTES - 1;
this.allocatedAddress = allocateMemory( allocatedBytes ); this.allocatedAddress = UnsafeUtil.allocateMemory( allocatedBytes, this.allocationTracker );
this.address = UnsafeUtil.alignedMemory( allocatedAddress, Integer.BYTES ); this.address = UnsafeUtil.alignedMemory( allocatedAddress, Integer.BYTES );
} }


Expand Down Expand Up @@ -196,7 +196,7 @@ public void removeHopBit( int index, int hd )
@Override @Override
public void close() public void close()
{ {
deallocateMemory(); UnsafeUtil.free( allocatedAddress, allocatedBytes, allocationTracker );
} }


protected static void alignmentSafePutLongAsTwoInts( long address, long value ) protected static void alignmentSafePutLongAsTwoInts( long address, long value )
Expand Down Expand Up @@ -226,13 +226,4 @@ protected static long alignmentSafeGetLongAsTwoInts( long address )
return lsb | (msb << Integer.SIZE); return lsb | (msb << Integer.SIZE);
} }


private long allocateMemory( long bytesToAllocate )
{
return UnsafeUtil.allocateMemory( bytesToAllocate, allocationTracker );
}

private void deallocateMemory()
{
UnsafeUtil.free( allocatedAddress, allocatedBytes, allocationTracker );
}
} }
Expand Up @@ -38,6 +38,7 @@
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors; import java.util.stream.Collectors;


import org.neo4j.memory.GlobalMemoryTracker;
import org.neo4j.memory.MemoryAllocationTracker; import org.neo4j.memory.MemoryAllocationTracker;


import static java.lang.String.format; import static java.lang.String.format;
Expand Down Expand Up @@ -346,6 +347,11 @@ public static String newSharedArrayString( char[] chars )
} }
} }


public static long allocateMemory( long sizeInBytes )
{
return allocateMemory( sizeInBytes, GlobalMemoryTracker.INSTANCE );
}

/** /**
* Allocate a block of memory of the given size in bytes, and return a pointer to that memory. * Allocate a block of memory of the given size in bytes, and return a pointer to that memory.
* <p> * <p>
Expand Down
Expand Up @@ -276,7 +276,7 @@ public void mustSupportReadingFromAndWritingToFields()
public void mustSupportReadingAndWritingOfPrimitivesToMemory() public void mustSupportReadingAndWritingOfPrimitivesToMemory()
{ {
int sizeInBytes = 8; int sizeInBytes = 8;
long address = allocateMemory( sizeInBytes, GlobalMemoryTracker.INSTANCE ); long address = allocateMemory( sizeInBytes );
try try
{ {
putByte( address, (byte) 1 ); putByte( address, (byte) 1 );
Expand Down Expand Up @@ -483,7 +483,7 @@ public void unsafeArrayElementAccess()
public void directByteBufferCreationAndInitialisation() throws Exception public void directByteBufferCreationAndInitialisation() throws Exception
{ {
int sizeInBytes = 313; int sizeInBytes = 313;
long address = allocateMemory( sizeInBytes, GlobalMemoryTracker.INSTANCE ); long address = allocateMemory( sizeInBytes );
try try
{ {
setMemory( address, sizeInBytes, (byte) 0 ); setMemory( address, sizeInBytes, (byte) 0 );
Expand All @@ -504,7 +504,7 @@ public void directByteBufferCreationAndInitialisation() throws Exception
a.limit( 202 ); a.limit( 202 );


int sizeInBytes2 = 424; int sizeInBytes2 = 424;
long address2 = allocateMemory( sizeInBytes2, GlobalMemoryTracker.INSTANCE ); long address2 = allocateMemory( sizeInBytes2 );
try try
{ {
setMemory( address2, sizeInBytes2, (byte) 0 ); setMemory( address2, sizeInBytes2, (byte) 0 );
Expand Down

0 comments on commit 975b6de

Please sign in to comment.