Skip to content

Commit

Permalink
Create memory allocators with capacity specified as a string
Browse files Browse the repository at this point in the history 
This involved adding a method to ByteUnit, that parses string representations
of byte values, such as '32 KiB'.
  • Loading branch information
@chrisvest
chrisvest committed Dec 11, 2017
1 parent 0fdbedd commit 9cbd88a
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 15 deletions.
68 changes: 60 additions & 8 deletions community/io/src/main/java/org/neo4j/io/ByteUnit.java
View file
Expand Up @@ -19,7 +19,11 @@
*/
package org.neo4j.io;

import java.util.Arrays;
import java.util.Locale;
import java.util.stream.Stream;

import org.neo4j.helpers.collection.Pair;

import static java.lang.String.format;

Expand All @@ -42,12 +46,12 @@ public enum ByteUnit
*/

Byte( 0, "B" ),
KibiByte( 1, "KiB" ),
MebiByte( 2, "MiB" ),
GibiByte( 3, "GiB" ),
TebiByte( 4, "TiB" ),
PebiByte( 5, "PiB" ),
ExbiByte( 6, "EiB" );
KibiByte( 1, "KiB", "KB", "K", "kB", "kb", "k" ),
MebiByte( 2, "MiB", "MB", "M", "mB", "mb", "m" ),
GibiByte( 3, "GiB", "GB", "G", "gB", "gb", "g" ),
TebiByte( 4, "TiB", "TB" ),
PebiByte( 5, "PiB", "PB" ),
ExbiByte( 6, "EiB", "EB" );

public static final long ONE_KIBI_BYTE = ByteUnit.KibiByte.toBytes( 1 );
public static final long ONE_MEBI_BYTE = ByteUnit.MebiByte.toBytes( 1 );
Expand All @@ -57,11 +61,13 @@ public enum ByteUnit

private final long factor;
private final String shortName;
private final String[] names;

ByteUnit( long power, String shortName )
ByteUnit( long power, String... names )
{
this.factor = factorFromPower( power );
this.shortName = shortName;
this.shortName = names[0];
this.names = names;
}

/**
Expand Down Expand Up @@ -194,4 +200,50 @@ else if ( bytes > ONE_KIBI_BYTE )
return bytes + Byte.shortName;
}
}

public static long parse( String text )
{
long result = 0;
int len = text.length();
int unitCharacter = 0;
Stream<Pair<String,ByteUnit>> unitsStream = listUnits();

for ( int i = 0; i < len; i++ )
{
char ch = text.charAt( i );
int digit = Character.digit( ch, 10 );
if ( digit != -1 )
{
if ( result != 0 )
{
result *= 10;
}
result += digit;
}
else if ( !Character.isWhitespace( ch ) )
{
int idx = unitCharacter;
unitsStream = unitsStream.filter( p -> p.first().length() > idx && p.first().charAt( idx ) == ch );
unitCharacter++;
}
}

if ( unitCharacter > 0 )
{
ByteUnit byteUnit = unitsStream.map( Pair::other ).findFirst().orElse( null );
if ( byteUnit == null )
{
throw new IllegalArgumentException( "Unknown byte unit in '" + text + "'" );
}
result = byteUnit.toBytes( result );
}

return result;
}

private static Stream<Pair<String,ByteUnit>> listUnits()
{
return Arrays.stream( values() ).flatMap(
b -> Stream.of( b.names ).map( n -> Pair.of( n, b ) ) );
}
}
7 changes: 7 additions & 0 deletions community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java
View file
Expand Up @@ -19,11 +19,18 @@
*/
package org.neo4j.io.mem;

import org.neo4j.io.ByteUnit;

/**
* A MemoryAllocator is simple: it only allocates memory, until it itself is finalizable and frees it all in one go.
*/
public interface MemoryAllocator
{
static MemoryAllocator createAllocator( String expectedMemory )
{
return new GrabAllocator( ByteUnit.parse( expectedMemory ) );
}

static MemoryAllocator createAllocator( long expectedMaxMemory )
{
return new GrabAllocator( expectedMaxMemory );
Expand Down
65 changes: 65 additions & 0 deletions community/io/src/test/java/org/neo4j/io/ByteUnitTest.java
View file
Expand Up @@ -149,6 +149,71 @@ public void unitsAsBytes() throws Exception
assertThat( ByteUnit.tebiBytes( 1 ), is( 1099511627776L ) );
assertThat( ByteUnit.pebiBytes( 1 ), is( 1125899906842624L ) );
assertThat( ByteUnit.exbiBytes( 1 ), is( 1152921504606846976L ) );

assertThat( ByteUnit.parse( "1" ), is( 1L ) );
assertThat( ByteUnit.parse( "1 KiB" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1KiB" ), is( 1024L ) );
assertThat( ByteUnit.parse( " 1 Ki B" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1 KB" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1KB" ), is( 1024L ) );
assertThat( ByteUnit.parse( " 1 KB " ), is( 1024L ) );
assertThat( ByteUnit.parse( "1 kB" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1kB" ), is( 1024L ) );
assertThat( ByteUnit.parse( " 1 kB " ), is( 1024L ) );
assertThat( ByteUnit.parse( "1 kb" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1kb" ), is( 1024L ) );
assertThat( ByteUnit.parse( " 1 kb " ), is( 1024L ) );
assertThat( ByteUnit.parse( "1 k" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1k" ), is( 1024L ) );
assertThat( ByteUnit.parse( " 1 k" ), is( 1024L ) );
assertThat( ByteUnit.parse( "1 MiB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1MiB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( " 1 Mi B" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1 MB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1MB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( " 1 MB " ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1 mB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1mB" ), is( 1048576L ) );
assertThat( ByteUnit.parse( " 1 mB " ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1 mb" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1mb" ), is( 1048576L ) );
assertThat( ByteUnit.parse( " 1 mb " ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1 m" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1m" ), is( 1048576L ) );
assertThat( ByteUnit.parse( " 1 m" ), is( 1048576L ) );
assertThat( ByteUnit.parse( "1 GiB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1GiB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( " 1 Gi B" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1 GB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1GB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( " 1 GB " ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1 gB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1gB" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( " 1 gB " ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1 gb" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1gb" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( " 1 gb " ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1 g" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1g" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( " 1 g" ), is( 1073741824L ) );
assertThat( ByteUnit.parse( "1 TiB" ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( "1TiB" ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( " 1 Ti B" ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( "1 TB" ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( "1TB" ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( " 1 TB " ), is( 1099511627776L ) );
assertThat( ByteUnit.parse( "1 PiB" ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( "1PiB" ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( " 1 Pi B" ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( "1 PB" ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( "1PB" ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( " 1 PB " ), is( 1125899906842624L ) );
assertThat( ByteUnit.parse( "1 EiB" ), is( 1152921504606846976L ) );
assertThat( ByteUnit.parse( "1EiB" ), is( 1152921504606846976L ) );
assertThat( ByteUnit.parse( " 1 Ei B" ), is( 1152921504606846976L ) );
assertThat( ByteUnit.parse( "1 EB" ), is( 1152921504606846976L ) );
assertThat( ByteUnit.parse( "1EB" ), is( 1152921504606846976L ) );
assertThat( ByteUnit.parse( " 1 EB " ), is( 1152921504606846976L ) );
}

@Test
Expand Down
17 changes: 10 additions & 7 deletions community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java
View file
Expand Up @@ -32,31 +32,34 @@

public class MemoryAllocatorTest
{
protected MemoryAllocator createAllocator( long expectedMaxMemory )
protected static final String ONE_PAGE = PageCache.PAGE_SIZE + "";
protected static final String EIGHT_PAGES = (8 * PageCache.PAGE_SIZE) + "";

protected MemoryAllocator createAllocator( String expectedMaxMemory )
{
return MemoryAllocator.createAllocator( expectedMaxMemory );
}

@Test
public void allocatedPointerMustNotBeNull() throws Exception
{
MemoryAllocator mman = createAllocator( 8 * PageCache.PAGE_SIZE );
MemoryAllocator mman = createAllocator( EIGHT_PAGES );
long address = mman.allocateAligned( PageCache.PAGE_SIZE, 8 );
assertThat( address, is( not( 0L ) ) );
}

@Test
public void allocatedPointerMustBePageAligned() throws Exception
{
MemoryAllocator mman = createAllocator( 8 * PageCache.PAGE_SIZE );
MemoryAllocator mman = createAllocator( EIGHT_PAGES );
long address = mman.allocateAligned( PageCache.PAGE_SIZE, UnsafeUtil.pageSize() );
assertThat( address % UnsafeUtil.pageSize(), is( 0L ) );
}

@Test
public void mustBeAbleToAllocatePastMemoryLimit() throws Exception
{
MemoryAllocator mman = createAllocator( PageCache.PAGE_SIZE );
MemoryAllocator mman = createAllocator( ONE_PAGE );
for ( int i = 0; i < 4100; i++ )
{
assertThat( mman.allocateAligned( 1, 2 ) % 2, is( 0L ) );
Expand All @@ -67,13 +70,13 @@ public void mustBeAbleToAllocatePastMemoryLimit() throws Exception
@Test( expected = IllegalArgumentException.class )
public void alignmentCannotBeZero() throws Exception
{
createAllocator( PageCache.PAGE_SIZE ).allocateAligned( 8, 0 );
createAllocator( ONE_PAGE ).allocateAligned( 8, 0 );
}

@Test
public void mustBeAbleToAllocateSlabsLargerThanGrabSize() throws Exception
{
MemoryAllocator mman = createAllocator( 32 * 1024 * 1024 );
MemoryAllocator mman = createAllocator( "32 KiB" );
long page1 = mman.allocateAligned( UnsafeUtil.pageSize(), 1 );
long largeBlock = mman.allocateAligned( 1024 * 1024, 1 ); // 1 MiB
long page2 = mman.allocateAligned( UnsafeUtil.pageSize(), 1 );
Expand All @@ -85,7 +88,7 @@ public void mustBeAbleToAllocateSlabsLargerThanGrabSize() throws Exception
@Test
public void allocatingMustIncreaseMemoryUsedAndDecreaseAvailableMemory() throws Exception
{
MemoryAllocator mman = createAllocator( PageCache.PAGE_SIZE );
MemoryAllocator mman = createAllocator( ONE_PAGE );
assertThat( mman.usedMemory(), is( 0L ) );
assertThat( mman.availableMemory(), is( (long) PageCache.PAGE_SIZE ) );
assertThat( mman.usedMemory() + mman.availableMemory(), is( (long) PageCache.PAGE_SIZE ) );
Expand Down

0 comments on commit 9cbd88a

Please sign in to comment.