From 9cbd88a7e6204d4a9bb1e069747bbbe1ddd78d41 Mon Sep 17 00:00:00 2001 From: Chris Vest Date: Wed, 26 Jul 2017 16:15:29 +0200 Subject: [PATCH] Create memory allocators with capacity specified as a string This involved adding a method to ByteUnit, that parses string representations of byte values, such as '32 KiB'. --- .../src/main/java/org/neo4j/io/ByteUnit.java | 68 ++++++++++++++++--- .../org/neo4j/io/mem/MemoryAllocator.java | 7 ++ .../test/java/org/neo4j/io/ByteUnitTest.java | 65 ++++++++++++++++++ .../org/neo4j/io/mem/MemoryAllocatorTest.java | 17 +++-- 4 files changed, 142 insertions(+), 15 deletions(-) diff --git a/community/io/src/main/java/org/neo4j/io/ByteUnit.java b/community/io/src/main/java/org/neo4j/io/ByteUnit.java index 88605df481357..02c675a623415 100644 --- a/community/io/src/main/java/org/neo4j/io/ByteUnit.java +++ b/community/io/src/main/java/org/neo4j/io/ByteUnit.java @@ -19,7 +19,11 @@ */ package org.neo4j.io; +import java.util.Arrays; import java.util.Locale; +import java.util.stream.Stream; + +import org.neo4j.helpers.collection.Pair; import static java.lang.String.format; @@ -42,12 +46,12 @@ public enum ByteUnit */ Byte( 0, "B" ), - KibiByte( 1, "KiB" ), - MebiByte( 2, "MiB" ), - GibiByte( 3, "GiB" ), - TebiByte( 4, "TiB" ), - PebiByte( 5, "PiB" ), - ExbiByte( 6, "EiB" ); + KibiByte( 1, "KiB", "KB", "K", "kB", "kb", "k" ), + MebiByte( 2, "MiB", "MB", "M", "mB", "mb", "m" ), + GibiByte( 3, "GiB", "GB", "G", "gB", "gb", "g" ), + TebiByte( 4, "TiB", "TB" ), + PebiByte( 5, "PiB", "PB" ), + ExbiByte( 6, "EiB", "EB" ); public static final long ONE_KIBI_BYTE = ByteUnit.KibiByte.toBytes( 1 ); public static final long ONE_MEBI_BYTE = ByteUnit.MebiByte.toBytes( 1 ); @@ -57,11 +61,13 @@ public enum ByteUnit private final long factor; private final String shortName; + private final String[] names; - ByteUnit( long power, String shortName ) + ByteUnit( long power, String... names ) { this.factor = factorFromPower( power ); - this.shortName = shortName; + this.shortName = names[0]; + this.names = names; } /** @@ -194,4 +200,50 @@ else if ( bytes > ONE_KIBI_BYTE ) return bytes + Byte.shortName; } } + + public static long parse( String text ) + { + long result = 0; + int len = text.length(); + int unitCharacter = 0; + Stream> unitsStream = listUnits(); + + for ( int i = 0; i < len; i++ ) + { + char ch = text.charAt( i ); + int digit = Character.digit( ch, 10 ); + if ( digit != -1 ) + { + if ( result != 0 ) + { + result *= 10; + } + result += digit; + } + else if ( !Character.isWhitespace( ch ) ) + { + int idx = unitCharacter; + unitsStream = unitsStream.filter( p -> p.first().length() > idx && p.first().charAt( idx ) == ch ); + unitCharacter++; + } + } + + if ( unitCharacter > 0 ) + { + ByteUnit byteUnit = unitsStream.map( Pair::other ).findFirst().orElse( null ); + if ( byteUnit == null ) + { + throw new IllegalArgumentException( "Unknown byte unit in '" + text + "'" ); + } + result = byteUnit.toBytes( result ); + } + + return result; + } + + private static Stream> listUnits() + { + return Arrays.stream( values() ).flatMap( + b -> Stream.of( b.names ).map( n -> Pair.of( n, b ) ) ); + } } diff --git a/community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java b/community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java index fb46275477b51..e5e2700219562 100644 --- a/community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java +++ b/community/io/src/main/java/org/neo4j/io/mem/MemoryAllocator.java @@ -19,11 +19,18 @@ */ package org.neo4j.io.mem; +import org.neo4j.io.ByteUnit; + /** * A MemoryAllocator is simple: it only allocates memory, until it itself is finalizable and frees it all in one go. */ public interface MemoryAllocator { + static MemoryAllocator createAllocator( String expectedMemory ) + { + return new GrabAllocator( ByteUnit.parse( expectedMemory ) ); + } + static MemoryAllocator createAllocator( long expectedMaxMemory ) { return new GrabAllocator( expectedMaxMemory ); diff --git a/community/io/src/test/java/org/neo4j/io/ByteUnitTest.java b/community/io/src/test/java/org/neo4j/io/ByteUnitTest.java index 035d452375c98..d70bc18582421 100644 --- a/community/io/src/test/java/org/neo4j/io/ByteUnitTest.java +++ b/community/io/src/test/java/org/neo4j/io/ByteUnitTest.java @@ -149,6 +149,71 @@ public void unitsAsBytes() throws Exception assertThat( ByteUnit.tebiBytes( 1 ), is( 1099511627776L ) ); assertThat( ByteUnit.pebiBytes( 1 ), is( 1125899906842624L ) ); assertThat( ByteUnit.exbiBytes( 1 ), is( 1152921504606846976L ) ); + + assertThat( ByteUnit.parse( "1" ), is( 1L ) ); + assertThat( ByteUnit.parse( "1 KiB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1KiB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( " 1 Ki B" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1 KB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1KB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( " 1 KB " ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1 kB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1kB" ), is( 1024L ) ); + assertThat( ByteUnit.parse( " 1 kB " ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1 kb" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1kb" ), is( 1024L ) ); + assertThat( ByteUnit.parse( " 1 kb " ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1 k" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1k" ), is( 1024L ) ); + assertThat( ByteUnit.parse( " 1 k" ), is( 1024L ) ); + assertThat( ByteUnit.parse( "1 MiB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1MiB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( " 1 Mi B" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1 MB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1MB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( " 1 MB " ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1 mB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1mB" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( " 1 mB " ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1 mb" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1mb" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( " 1 mb " ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1 m" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1m" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( " 1 m" ), is( 1048576L ) ); + assertThat( ByteUnit.parse( "1 GiB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1GiB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( " 1 Gi B" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1 GB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1GB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( " 1 GB " ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1 gB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1gB" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( " 1 gB " ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1 gb" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1gb" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( " 1 gb " ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1 g" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1g" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( " 1 g" ), is( 1073741824L ) ); + assertThat( ByteUnit.parse( "1 TiB" ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( "1TiB" ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( " 1 Ti B" ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( "1 TB" ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( "1TB" ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( " 1 TB " ), is( 1099511627776L ) ); + assertThat( ByteUnit.parse( "1 PiB" ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( "1PiB" ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( " 1 Pi B" ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( "1 PB" ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( "1PB" ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( " 1 PB " ), is( 1125899906842624L ) ); + assertThat( ByteUnit.parse( "1 EiB" ), is( 1152921504606846976L ) ); + assertThat( ByteUnit.parse( "1EiB" ), is( 1152921504606846976L ) ); + assertThat( ByteUnit.parse( " 1 Ei B" ), is( 1152921504606846976L ) ); + assertThat( ByteUnit.parse( "1 EB" ), is( 1152921504606846976L ) ); + assertThat( ByteUnit.parse( "1EB" ), is( 1152921504606846976L ) ); + assertThat( ByteUnit.parse( " 1 EB " ), is( 1152921504606846976L ) ); } @Test diff --git a/community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java b/community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java index 696d2d1628b5b..f419f9e271de4 100644 --- a/community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java +++ b/community/io/src/test/java/org/neo4j/io/mem/MemoryAllocatorTest.java @@ -32,7 +32,10 @@ public class MemoryAllocatorTest { - protected MemoryAllocator createAllocator( long expectedMaxMemory ) + protected static final String ONE_PAGE = PageCache.PAGE_SIZE + ""; + protected static final String EIGHT_PAGES = (8 * PageCache.PAGE_SIZE) + ""; + + protected MemoryAllocator createAllocator( String expectedMaxMemory ) { return MemoryAllocator.createAllocator( expectedMaxMemory ); } @@ -40,7 +43,7 @@ protected MemoryAllocator createAllocator( long expectedMaxMemory ) @Test public void allocatedPointerMustNotBeNull() throws Exception { - MemoryAllocator mman = createAllocator( 8 * PageCache.PAGE_SIZE ); + MemoryAllocator mman = createAllocator( EIGHT_PAGES ); long address = mman.allocateAligned( PageCache.PAGE_SIZE, 8 ); assertThat( address, is( not( 0L ) ) ); } @@ -48,7 +51,7 @@ public void allocatedPointerMustNotBeNull() throws Exception @Test public void allocatedPointerMustBePageAligned() throws Exception { - MemoryAllocator mman = createAllocator( 8 * PageCache.PAGE_SIZE ); + MemoryAllocator mman = createAllocator( EIGHT_PAGES ); long address = mman.allocateAligned( PageCache.PAGE_SIZE, UnsafeUtil.pageSize() ); assertThat( address % UnsafeUtil.pageSize(), is( 0L ) ); } @@ -56,7 +59,7 @@ public void allocatedPointerMustBePageAligned() throws Exception @Test public void mustBeAbleToAllocatePastMemoryLimit() throws Exception { - MemoryAllocator mman = createAllocator( PageCache.PAGE_SIZE ); + MemoryAllocator mman = createAllocator( ONE_PAGE ); for ( int i = 0; i < 4100; i++ ) { assertThat( mman.allocateAligned( 1, 2 ) % 2, is( 0L ) ); @@ -67,13 +70,13 @@ public void mustBeAbleToAllocatePastMemoryLimit() throws Exception @Test( expected = IllegalArgumentException.class ) public void alignmentCannotBeZero() throws Exception { - createAllocator( PageCache.PAGE_SIZE ).allocateAligned( 8, 0 ); + createAllocator( ONE_PAGE ).allocateAligned( 8, 0 ); } @Test public void mustBeAbleToAllocateSlabsLargerThanGrabSize() throws Exception { - MemoryAllocator mman = createAllocator( 32 * 1024 * 1024 ); + MemoryAllocator mman = createAllocator( "32 KiB" ); long page1 = mman.allocateAligned( UnsafeUtil.pageSize(), 1 ); long largeBlock = mman.allocateAligned( 1024 * 1024, 1 ); // 1 MiB long page2 = mman.allocateAligned( UnsafeUtil.pageSize(), 1 ); @@ -85,7 +88,7 @@ public void mustBeAbleToAllocateSlabsLargerThanGrabSize() throws Exception @Test public void allocatingMustIncreaseMemoryUsedAndDecreaseAvailableMemory() throws Exception { - MemoryAllocator mman = createAllocator( PageCache.PAGE_SIZE ); + MemoryAllocator mman = createAllocator( ONE_PAGE ); assertThat( mman.usedMemory(), is( 0L ) ); assertThat( mman.availableMemory(), is( (long) PageCache.PAGE_SIZE ) ); assertThat( mman.usedMemory() + mman.availableMemory(), is( (long) PageCache.PAGE_SIZE ) );