Skip to content

Commit

Permalink
Move standard update data to LayoutTestUtil
Browse files Browse the repository at this point in the history 
Because this is the component that know about
Unique vs Non-Unique and can serve correct
update data for tests, with or without duplicates.
  • Loading branch information
@burqen @tinwelint
burqen authored and tinwelint committed Jun 26, 2017
1 parent 9c362f4 commit b98b97e
Show file tree
Hide file tree
Showing 10 changed files with 208 additions and 223 deletions.
11 changes: 9 additions & 2 deletions ...nel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexAccessor.java
View file
Expand Up @@ -39,7 +39,6 @@
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.schema.IndexQuery;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.kernel.impl.util.Cursors;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSampler;

Expand Down Expand Up @@ -138,7 +137,15 @@ public long countIndexedNodes( long nodeId, Object... propertyValues )
treeKeyTo.from( nodeId, propertyValues );
try ( RawCursor<Hit<KEY,VALUE>,IOException> seeker = tree.seek( treeKeyFrom, treeKeyTo ) )
{
return Cursors.count( seeker );
long count = 0;
while ( seeker.next() )
{
if ( seeker.get().key().entityId == nodeId )
{
count++;
}
}
return count;
}
catch ( IOException e )
{
Expand Down
2 changes: 1 addition & 1 deletion ...l/src/test/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSamplerTest.java
View file
Expand Up @@ -93,7 +93,7 @@ static int countUniqueValues( List<Number> values )
private List<Number> generateNumberValues()
{
List<Number> result = new ArrayList<>();
for ( IndexEntryUpdate<?> update : someDuplicateIndexEntryUpdates() )
for ( IndexEntryUpdate<?> update : layoutUtil.someUpdates() )
{
result.add( (Number) update.values()[0].asObject() );
}
Expand Down
117 changes: 117 additions & 0 deletions community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/LayoutTestUtil.java
View file
Expand Up @@ -19,16 +19,40 @@
*/
package org.neo4j.kernel.impl.index.schema;

import org.apache.commons.lang3.ArrayUtils;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.neo4j.helpers.collection.PrefetchingIterator;
import org.neo4j.index.internal.gbptree.Layout;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.test.rule.RandomRule;
import org.neo4j.values.Values;

import static org.neo4j.kernel.impl.index.schema.NumberValue.DOUBLE;
import static org.neo4j.kernel.impl.index.schema.NumberValue.FLOAT;
import static org.neo4j.kernel.impl.index.schema.NumberValue.LONG;

abstract class LayoutTestUtil<KEY extends NumberKey, VALUE extends NumberValue>
{
private final IndexDescriptor indexDescriptor;

LayoutTestUtil( IndexDescriptor indexDescriptor )
{
this.indexDescriptor = indexDescriptor;
}

abstract Layout<KEY,VALUE> createLayout();

abstract IndexEntryUpdate[] someUpdates();

protected abstract double fractionDuplicates();

void copyValue( VALUE value, VALUE intoValue )
{
intoValue.type = value.type;
Expand Down Expand Up @@ -65,4 +89,97 @@ int compareIndexedPropertyValue( NumberValue value1, NumberValue value2 )
}
return typeCompare;
}

Iterator<IndexEntryUpdate<IndexDescriptor>> randomUniqueUpdateGenerator( RandomRule random )
{
double fractionDuplicates = fractionDuplicates();
return new PrefetchingIterator<IndexEntryUpdate<IndexDescriptor>>()
{
private final Set<Double> uniqueCompareValues = new HashSet<>();
private final List<Number> uniqueValues = new ArrayList<>();
private long currentEntityId;

@Override
protected IndexEntryUpdate<IndexDescriptor> fetchNextOrNull()
{
Number value;
if ( fractionDuplicates > 0 && !uniqueValues.isEmpty() &&
random.nextFloat() < fractionDuplicates )
{
value = existingNonUniqueValue( random );
}
else
{
value = newUniqueValue( random );
}

return add( currentEntityId++, value );
}

private Number newUniqueValue( RandomRule randomRule )
{
Number value;
Double compareValue;
do
{
value = randomRule.numberPropertyValue();
compareValue = value.doubleValue();
}
while ( !uniqueCompareValues.add( compareValue ) );
uniqueValues.add( value );
return value;
}

private Number existingNonUniqueValue( RandomRule randomRule )
{
return uniqueValues.get( randomRule.nextInt( uniqueValues.size() ) );
}
};
}

@SuppressWarnings( "rawtypes" )
IndexEntryUpdate[] someUpdatesNoDuplicateValues()
{
return generateAddUpdatesFor( ALL_EXTREME_VALUES );
}

@SuppressWarnings( "rawtypes" )
IndexEntryUpdate[] someUpdatesWithDuplicateValues()
{
return generateAddUpdatesFor( ArrayUtils.addAll( ALL_EXTREME_VALUES, ALL_EXTREME_VALUES ) );
}

private IndexEntryUpdate[] generateAddUpdatesFor( Number[] values )
{
IndexEntryUpdate[] indexEntryUpdates = new IndexEntryUpdate[values.length];
for ( int i = 0; i < indexEntryUpdates.length; i++ )
{
indexEntryUpdates[i] = add( i, values[i] );
}
return indexEntryUpdates;
}

private static final Number[] ALL_EXTREME_VALUES = new Number[]
{
Byte.MAX_VALUE,
Byte.MIN_VALUE,
Short.MAX_VALUE,
Short.MIN_VALUE,
Integer.MAX_VALUE,
Integer.MIN_VALUE,
Long.MAX_VALUE,
Long.MIN_VALUE,
Float.MAX_VALUE,
-Float.MAX_VALUE,
Double.MAX_VALUE,
-Double.MAX_VALUE,
Double.POSITIVE_INFINITY,
Double.NEGATIVE_INFINITY,
0
};

protected IndexEntryUpdate<IndexDescriptor> add( long nodeId, Object value )
{
return IndexEntryUpdate.add( nodeId, indexDescriptor, Values.of( value ) );
}
}
44 changes: 18 additions & 26 deletions ...ava/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaNumberIndexPopulatorTest.java
View file
Expand Up @@ -23,8 +23,8 @@

import java.io.File;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Random;
import java.util.HashSet;
import java.util.Set;

import org.neo4j.index.internal.gbptree.Layout;
import org.neo4j.io.pagecache.PageCache;
Expand Down Expand Up @@ -57,7 +57,7 @@ public void addShouldApplyDuplicateValues() throws Exception
// given
populator.create();
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someDuplicateIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdatesWithDuplicateValues();

// when
populator.add( Arrays.asList( updates ) );
Expand All @@ -74,7 +74,7 @@ public void updaterShouldApplyDuplicateValues() throws Exception
populator.create();
IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor );
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someDuplicateIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdatesWithDuplicateValues();

// when
for ( IndexEntryUpdate<IndexDescriptor> update : updates )
Expand All @@ -87,24 +87,6 @@ public void updaterShouldApplyDuplicateValues() throws Exception
verifyUpdates( updates );
}

@Test
public void shouldApplyLargeAmountOfInterleavedRandomUpdatesWithDuplicates() throws Exception
{
// given
populator.create();
random.reset();
Random updaterRandom = new Random( random.seed() );
Iterator<IndexEntryUpdate<IndexDescriptor>> updates = randomUniqueUpdateGenerator( 0.1f );

// when
int count = interleaveLargeAmountOfUpdates( updaterRandom, updates );

// then
populator.close( true );
random.reset();
verifyUpdates( randomUniqueUpdateGenerator( 0.1f ), count );
}

@Test
public void shouldFailOnSampleBeforeConfiguredSampling() throws Exception
{
Expand All @@ -131,35 +113,45 @@ public void shouldSampleWholeIndexIfConfiguredForPopulatingSampling() throws Exc
populator.create();
populator.configureSampling( false );
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdates();
populator.add( Arrays.asList( updates ) );

// WHEN
IndexSample sample = populator.sampleResult();

// THEN
assertEquals( updates.length, sample.sampleSize() );
assertEquals( updates.length, sample.uniqueValues() );
assertEquals( countUniqueValuesAmongUpdates( updates ), sample.uniqueValues() );
assertEquals( updates.length, sample.indexSize() );
populator.close( true );
}

private long countUniqueValuesAmongUpdates( IndexEntryUpdate<IndexDescriptor>[] updates )
{
Set<Double> uniqueValues = new HashSet<>();
for ( IndexEntryUpdate<IndexDescriptor> update : updates )
{
uniqueValues.add( ((Number)update.values()[0]).doubleValue() );
}
return uniqueValues.size();
}

@Test
public void shouldSampleUpdatesIfConfiguredForOnlineSampling() throws Exception
{
// GIVEN
populator.create();
populator.configureSampling( true );
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] scanUpdates = someIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] scanUpdates = layoutUtil.someUpdates();
populator.add( Arrays.asList( scanUpdates ) );
Number[] updates = array( 101, 102, 102, 103, 103 );
try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) )
{
long nodeId = 1000;
for ( Number number : updates )
{
IndexEntryUpdate<IndexDescriptor> update = add( nodeId++, number );
IndexEntryUpdate<IndexDescriptor> update = layoutUtil.add( nodeId++, number );
updater.process( update );
populator.includeSample( update );
}
Expand Down
12 changes: 6 additions & 6 deletions ...rnel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulatorTest.java
View file
Expand Up @@ -159,7 +159,7 @@ public void addShouldApplyAllUpdatesOnce() throws Exception
// given
populator.create();
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdates();

// when
populator.add( Arrays.asList( updates ) );
Expand All @@ -176,7 +176,7 @@ public void updaterShouldApplyUpdates() throws Exception
populator.create();
IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor );
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdates();

// when
for ( IndexEntryUpdate<IndexDescriptor> update : updates )
Expand All @@ -202,7 +202,7 @@ public void updaterMustThrowIfProcessAfterClose() throws Exception
// then
try
{
updater.process( add( 1, Long.MAX_VALUE ) );
updater.process( layoutUtil.add( 1, Long.MAX_VALUE ) );
fail( "Expected process to throw on closed updater" );
}
catch ( IllegalStateException e )
Expand All @@ -219,7 +219,7 @@ public void shouldApplyInterleavedUpdatesFromAddAndUpdater() throws Exception
populator.create();
IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor );
@SuppressWarnings( "unchecked" )
IndexEntryUpdate<IndexDescriptor>[] updates = someIndexEntryUpdates();
IndexEntryUpdate<IndexDescriptor>[] updates = layoutUtil.someUpdates();

// when
applyInterleaved( updates, updater, populator );
Expand Down Expand Up @@ -370,15 +370,15 @@ public void shouldApplyLargeAmountOfInterleavedRandomUpdates() throws Exception
populator.create();
random.reset();
Random updaterRandom = new Random( random.seed() );
Iterator<IndexEntryUpdate<IndexDescriptor>> updates = randomUniqueUpdateGenerator( 0 );
Iterator<IndexEntryUpdate<IndexDescriptor>> updates = layoutUtil.randomUniqueUpdateGenerator( random );

// when
int count = interleaveLargeAmountOfUpdates( updaterRandom, updates );

// then
populator.close( true );
random.reset();
verifyUpdates( randomUniqueUpdateGenerator( 0 ), count );
verifyUpdates( layoutUtil.randomUniqueUpdateGenerator( random ), count );
}

@Test
Expand Down

0 comments on commit b98b97e

Please sign in to comment.