Browse files

More RelIdArray integration

  • Loading branch information...
1 parent a179320 commit 0e8a662e2f04528a20cf4051974d78169eae160b @tinwelint committed Mar 15, 2013
View
250 community/kernel/src/main/java/org/neo4j/kernel/impl/util/RelIdArray.java
@@ -30,6 +30,15 @@
import org.neo4j.helpers.Enc128;
import org.neo4j.kernel.impl.cache.SizeOf;
+/**
+ * About the state of the {@link ByteBuffer}s that store the ids in here:
+ *
+ * A buffer that is currently assigned to the fields has:
+ * + its position where to write new ids.
+ *
+ * @author Mattias
+ *
+ */
public class RelIdArray implements SizeOf
{
private static final Enc128 enc128 = new Enc128();
@@ -92,8 +101,6 @@ public static RelIdArray empty( int type )
private final int type;
private ByteBuffer outIds;
private ByteBuffer inIds;
-// private IdBlock lastOutBlock;
-// private IdBlock lastInBlock;
public RelIdArray( int type )
{
@@ -143,18 +150,39 @@ public void add( long id, DirectionWrapper direction )
direction.setLastBlock( this, newLastBlock );
lastBlock = newLastBlock;
}
- add( lastBlock, id );
+ add( lastBlock, id, direction );
}
- private void add( ByteBuffer buffer, long id )
+ private void add( ByteBuffer buffer, long id, DirectionWrapper direction )
{
+ buffer = ensureBufferSpace( buffer, 1, direction );
enc128.encode( buffer, id );
}
- private void addAll( ByteBuffer target, ByteBuffer source )
+ private int remainingToWriter( ByteBuffer buffer )
+ {
+ return buffer.capacity()-buffer.position();
+ }
+
+ private ByteBuffer ensureBufferSpace( ByteBuffer buffer, int bytesToAdd, DirectionWrapper direction )
+ {
+ int remaining = remainingToWriter( buffer );
+ if ( bytesToAdd > remaining )
+ {
+ int candidateSize = buffer.capacity()*3;
+ if ( candidateSize < remaining+bytesToAdd )
+ candidateSize = (remaining+bytesToAdd)*2;
+
+ buffer = copy( buffer, candidateSize );
+ direction.setLastBlock( this, buffer );
+ }
+ return buffer;
+ }
+
+ private void addAll( ByteBuffer target, ByteBuffer source, DirectionWrapper direction )
{
- // TODO source position?
- target.put( source );
+ target = ensureBufferSpace( target, source.capacity(), direction );
+ target.put( newBufferForReading( source ) );
}
public RelIdArray addAll( RelIdArray source )
@@ -188,19 +216,20 @@ public RelIdArray shrink()
new RelIdArray( type, shrunkOut, shrunkIn );
}
- private ByteBuffer shrink( ByteBuffer buffer )
+ protected ByteBuffer shrink( ByteBuffer buffer )
{
- if ( buffer.position() == buffer.capacity() )
+ if ( buffer == null || buffer.position() == buffer.capacity() )
return buffer;
- return copy( buffer );
+ return copy( buffer, buffer.position() );
}
- private ByteBuffer copy( ByteBuffer buffer )
+ private ByteBuffer copy( ByteBuffer buffer, int newSize )
{
- ByteBuffer newBuffer = ByteBuffer.allocate( buffer.position() );
- buffer.position( 0 );
- newBuffer.put( buffer );
+ ByteBuffer newBuffer = ByteBuffer.allocate( newSize );
+ ByteBuffer reader = buffer.duplicate();
+ reader.flip();
+ newBuffer.put( reader );
return newBuffer;
}
@@ -227,11 +256,11 @@ protected void append( RelIdArray source, DirectionWrapper direction )
{
if ( toBlock == null )
{
- direction.setLastBlock( this, copy( fromBlock ) );
+ direction.setLastBlock( this, copy( fromBlock, fromBlock.position() ) );
}
else
{
- addAll( toBlock, fromBlock );
+ addAll( toBlock, fromBlock, direction );
}
}
}
@@ -451,117 +480,14 @@ public static DirectionWrapper wrap( Direction direction )
//
// abstract long getHighBits();
// }
-//
-// private static class LowIdBlock extends IdBlock
-// {
-// @Override
-// void setPrev( IdBlock prev )
-// {
-// throw new UnsupportedOperationException();
-// }
-//
-// @Override
-// IdBlock upgradeIfNeeded()
-// {
-// IdBlock highBlock = new HighIdBlock( 0 );
-// highBlock.ids = ((IdBlock)this).ids;
-// return highBlock;
-// }
-//
-// @Override
-// long transform( int id )
-// {
-// return (long)(id&0xFFFFFFFFL);
-// }
-//
-// @Override
-// protected IdBlock copyInstance()
-// {
-// return new LowIdBlock();
-// }
-//
-// @Override
-// long getHighBits()
-// {
-// return 0;
-// }
-// }
-//
-// private static class HighIdBlock extends IdBlock
-// {
-// private final long highBits;
-// private IdBlock prev;
-//
-// HighIdBlock( long highBits )
-// {
-// this.highBits = highBits;
-// }
-//
-// public int size()
-// {
-// int size = super.size() + 8 + SizeOfs.REFERENCE_SIZE;
-// if ( prev != null )
-// {
-// size += prev.size();
-// }
-// return size;
-// }
-//
-// @Override
-// IdBlock upgradeIfNeeded()
-// {
-// return this;
-// }
-//
-// @Override
-// IdBlock copy()
-// {
-// IdBlock copy = super.copy();
-// if ( prev != null )
-// {
-// copy.setPrev( prev.copy() );
-// }
-// return copy;
-// }
-//
-// @Override
-// IdBlock getPrev()
-// {
-// return prev;
-// }
-//
-// @Override
-// void setPrev( IdBlock prev )
-// {
-// this.prev = prev;
-// }
-//
-// @Override
-// long transform( int id )
-// {
-// return (((long)(id&0xFFFFFFFFL))|(highBits));
-// }
-//
-// @Override
-// protected IdBlock copyInstance()
-// {
-// return new HighIdBlock( highBits );
-// }
-//
-// @Override
-// long getHighBits()
-// {
-// return highBits;
-// }
-// }
private static class IteratorState
{
private ByteBuffer block;
public IteratorState( ByteBuffer block, int relativePosition )
{
- this.block = block;
+ this.block = newBufferForReading( block );
// TODO use relativePosition?
}
@@ -584,6 +510,16 @@ public void update( ByteBuffer lastBlock )
}
}
+ private static ByteBuffer newBufferForReading( ByteBuffer buffer )
+ {
+ if ( buffer == null )
+ return buffer;
+ ByteBuffer result = buffer.duplicate();
+ result.position( 0 );
+ result.limit( buffer.position() );
+ return result;
+ }
+
public static class RelIdIteratorImpl implements RelIdIterator
{
private final DirectionWrapper[] directions;
@@ -682,32 +618,15 @@ public boolean hasNext()
nextElement = -1;
return false;
}
-
- protected boolean nextBlock()
- {
- // Try next block in the chain
- if ( currentState != null && currentState.nextBlock() )
- {
- return true;
- }
-
- // It's ok to return null here... which will result in hasNext
- // returning false. IntArrayIterator will try to get more relationships
- // and call hasNext again.
- return findNextBlock();
- }
- /* (non-Javadoc)
- * @see org.neo4j.kernel.impl.util.RelIdIterator#doAnotherRound()
- */
@Override
public void doAnotherRound()
{
directionPosition = -1;
- findNextBlock();
+ nextBlock();
}
- protected boolean findNextBlock()
+ protected boolean nextBlock()
{
while ( directionPosition+1 < directions.length )
{
@@ -718,7 +637,7 @@ protected boolean findNextBlock()
currentState = nextState;
return true;
}
- IdBlock block = currentDirection.getLastBlock( ids );
+ ByteBuffer block = currentDirection.getLastBlock( ids );
if ( block != null )
{
currentState = new IteratorState( block, 0 );
@@ -729,9 +648,6 @@ protected boolean findNextBlock()
return false;
}
- /* (non-Javadoc)
- * @see org.neo4j.kernel.impl.util.RelIdIterator#next()
- */
@Override
public long next()
{
@@ -769,8 +685,8 @@ public static RelIdArray from( RelIdArray src, RelIdArray add, Collection<Long>
if ( src != null )
{
newArray = src.newSimilarInstance();
- newArray.addAll( src );
- evictExcluded( newArray, remove );
+ newArray = newArray.addAll( src );
+ newArray.removeAll( remove );
}
else
{
@@ -792,33 +708,28 @@ public static RelIdArray from( RelIdArray src, RelIdArray add, Collection<Long>
}
}
- private static void evictExcluded( RelIdArray ids, Collection<Long> excluded )
+ private void removeAll( Collection<Long> excluded )
{
- for ( RelIdIteratorImpl iterator = (RelIdIteratorImpl) DirectionWrapper.BOTH.iterator( ids ); iterator.hasNext(); )
+ removeAll( excluded, DirectionWrapper.OUTGOING );
+ removeAll( excluded, DirectionWrapper.INCOMING );
+ removeAll( excluded, DirectionWrapper.BOTH );
+ }
+
+ private void removeAll( Collection<Long> excluded, DirectionWrapper direction )
+ {
+ ByteBuffer buffer = direction.getLastBlock( this );
+ if ( buffer == null )
+ return;
+
+ ByteBuffer newBuffer = ByteBuffer.allocate( buffer.capacity() );
+ for ( RelIdIteratorImpl iterator = (RelIdIteratorImpl) direction.iterator( this ); iterator.hasNext(); )
{
long value = iterator.next();
- if ( excluded.contains( value ) )
- {
- boolean swapSuccessful = false;
- IteratorState state = iterator.currentState;
- IdBlock block = state.block;
- for ( int j = block.length() - 1; j >= state.relativePosition; j--)
- {
- long backValue = block.get( j );
- block.ids[0] = block.ids[0]-1;
- if ( !excluded.contains( backValue) )
- {
- block.set( backValue, state.relativePosition-1 );
- swapSuccessful = true;
- break;
- }
- }
- if ( !swapSuccessful ) // all elements from pos in remove
- {
- block.ids[0] = block.ids[0]-1;
- }
- }
+ if ( !excluded.contains( value ) )
+ add( newBuffer, value, iterator.currentDirection );
}
+ newBuffer = copy( newBuffer, newBuffer.position() );
+ direction.setLastBlock( this, newBuffer );
}
/**
@@ -830,7 +741,6 @@ private static void evictExcluded( RelIdArray ids, Collection<Long> excluded )
*/
public boolean couldBeNeedingUpdate()
{
- return (lastOutBlock != null && lastOutBlock.getPrev() != null) ||
- (lastInBlock != null && lastInBlock.getPrev() != null);
+ return false;
}
}
View
23 community/kernel/src/main/java/org/neo4j/kernel/impl/util/RelIdArrayWithLoops.java
@@ -19,9 +19,11 @@
*/
package org.neo4j.kernel.impl.util;
+import java.nio.ByteBuffer;
+
public class RelIdArrayWithLoops extends RelIdArray
{
- private IdBlock lastLoopBlock;
+ private ByteBuffer lastLoopBlock;
public RelIdArrayWithLoops( int type )
{
@@ -40,20 +42,20 @@ protected RelIdArrayWithLoops( RelIdArray from )
lastLoopBlock = from.getLastLoopBlock();
}
- protected RelIdArrayWithLoops( int type, IdBlock out, IdBlock in, IdBlock loop )
+ protected RelIdArrayWithLoops( int type, ByteBuffer out, ByteBuffer in, ByteBuffer loop )
{
super( type, out, in );
this.lastLoopBlock = loop;
}
@Override
- protected IdBlock getLastLoopBlock()
+ protected ByteBuffer getLastLoopBlock()
{
return this.lastLoopBlock;
}
@Override
- protected void setLastLoopBlock( IdBlock block )
+ protected void setLastLoopBlock( ByteBuffer block )
{
this.lastLoopBlock = block;
}
@@ -83,6 +85,7 @@ public RelIdArray addAll( RelIdArray source )
return this;
}
+ @Override
public RelIdArray newSimilarInstance()
{
return new RelIdArrayWithLoops( getType() );
@@ -97,12 +100,12 @@ public boolean couldBeNeedingUpdate()
@Override
public RelIdArray shrink()
{
- IdBlock lastOutBlock = DirectionWrapper.OUTGOING.getLastBlock( this );
- IdBlock lastInBlock = DirectionWrapper.INCOMING.getLastBlock( this );
- IdBlock shrunkOut = lastOutBlock != null ? lastOutBlock.shrink() : null;
- IdBlock shrunkIn = lastInBlock != null ? lastInBlock.shrink() : null;
- IdBlock shrunkLoop = lastLoopBlock != null ? lastLoopBlock.shrink() : null;
- return shrunkOut == lastOutBlock && shrunkIn == lastInBlock && shrunkLoop == lastLoopBlock ? this :
+ ByteBuffer outBuffer = DirectionWrapper.OUTGOING.getLastBlock( this );
+ ByteBuffer shrunkOut = shrink( outBuffer );
+ ByteBuffer inBuffer = DirectionWrapper.INCOMING.getLastBlock( this );
+ ByteBuffer shrunkIn = shrink( inBuffer );
+ ByteBuffer shrunkLoop = shrink( lastLoopBlock );
+ return shrunkOut == outBuffer && shrunkIn == inBuffer && shrunkLoop == lastLoopBlock ? this :
new RelIdArrayWithLoops( getType(), shrunkOut, shrunkIn, shrunkLoop );
}
}
View
2 community/kernel/src/test/java/org/neo4j/helpers/Enc128Test.java
@@ -73,7 +73,7 @@ public void testPerformance() throws Exception
{
int idCount = 10_000_000;
ByteBuffer buffer = ByteBuffer.allocate( idCount*4 );
- for ( int i = 0; i < 1000; i++ )
+ for ( int i = 0; i < 10; i++ )
timeEncodeDecode( buffer, 10_000_000, 100000 );
}
View
16 community/kernel/src/test/java/org/neo4j/kernel/impl/core/TestRelIdArray.java
@@ -22,12 +22,12 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.neo4j.helpers.collection.IteratorUtil.asSet;
import static org.neo4j.kernel.impl.util.RelIdArray.DirectionWrapper.BOTH;
import static org.neo4j.kernel.impl.util.RelIdArray.DirectionWrapper.INCOMING;
import static org.neo4j.kernel.impl.util.RelIdArray.DirectionWrapper.OUTGOING;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -56,7 +56,7 @@ public void testBasic() throws Exception
assertEquals( 1L, itr.next() );
assertTrue( itr.hasNext() );
assertEquals( 2L, itr.next() );
- assertFalse( itr.hasNext() );
+ assertFalse( "Should have no more", itr.hasNext() );
assertFalse( itr.hasNext() );
// Iterate INCOMING
@@ -95,7 +95,7 @@ public void testWithAddRemove() throws Exception
remove.add( 6L );
List<Long> allIds = asList( RelIdArray.from( source, add, remove ) );
Collections.sort( allIds );
- assertEquals( Arrays.asList( 1L, 3L, 4L, 5L, 7L ), allIds );
+ assertEquals( asSet( 1L, 3L, 4L, 5L, 7L ), asSet( allIds ) );
}
@Test
@@ -115,12 +115,13 @@ public void testDifferentBlocks() throws Exception
array.add( verySmall, OUTGOING );
array.add( verySmall+1, OUTGOING );
- Collection<Long> allIds = new HashSet<Long>( asList( array ) );
- assertEquals( new HashSet<Long>( Arrays.asList(
+ assertEquals( asSet(
justUnderIntMax, justUnderIntMax+1,
justOverIntMax, justOverIntMax+1,
aBitOverIntMax, aBitOverIntMax+1,
- verySmall, verySmall+1 ) ), allIds );
+ verySmall, verySmall+1 ),
+
+ asSet( asList( array ) ) );
}
@Test
@@ -139,8 +140,7 @@ public void testAddDifferentBlocks() throws Exception
all.addAll( array1 );
all.addAll( array2 );
- assertEquals( new HashSet<Long>( Arrays.asList(
- 0L, 1L, justOverIntMax, justOverIntMax+1 ) ), new HashSet<Long>( asList( all ) ) );
+ assertEquals( asSet( 0L, 1L, justOverIntMax, justOverIntMax+1 ), asSet( asList( all ) ) );
}
private List<Long> asList( RelIdArray ids )

0 comments on commit 0e8a662

Please sign in to comment.