Skip to content

Commit

Permalink
Randomized range query test for both simple and composite index
Browse files Browse the repository at this point in the history
  • Loading branch information
@burqen
burqen committed Sep 20, 2018
1 parent 926451f commit 3ae721b
Show file tree
Hide file tree
Showing 6 changed files with 261 additions and 67 deletions.
8 changes: 4 additions & 4 deletions ...rnel-it/src/test/java/org/neo4j/kernel/api/index/CompositeIndexAccessorCompatibility.java
View file
Expand Up @@ -30,10 +30,10 @@
import java.time.LocalTime; import java.time.LocalTime;
import java.time.OffsetTime; import java.time.OffsetTime;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.ArrayList;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
Expand Down Expand Up @@ -632,7 +632,7 @@ public void testIndexSeekRangeWithExistsBySpatialArray() throws Exception
public void testExactMatchOnRandomCompositeValues() throws Exception public void testExactMatchOnRandomCompositeValues() throws Exception
{ {
// given // given
List<RandomValues.Type> types = testSuite.supportedValueTypes(); RandomValues.Type[] types = randomSetOfSupportedTypes();
List<IndexEntryUpdate<?>> updates = new ArrayList<>(); List<IndexEntryUpdate<?>> updates = new ArrayList<>();
Set<ValueTuple> duplicateChecker = new HashSet<>(); Set<ValueTuple> duplicateChecker = new HashSet<>();
for ( long id = 0; id < 10_000; id++ ) for ( long id = 0; id < 10_000; id++ )
Expand All @@ -641,8 +641,8 @@ public void testExactMatchOnRandomCompositeValues() throws Exception
do do
{ {
update = add( id, descriptor.schema(), update = add( id, descriptor.schema(),
random.nextValue( random.among( types ) ), random.randomValues().nextValueOfTypes( types ),
random.nextValue( random.among( types ) ) ); random.randomValues().nextValueOfTypes( types ) );
} }
while ( !duplicateChecker.add( ValueTuple.of( update.values() ) ) ); while ( !duplicateChecker.add( ValueTuple.of( update.values() ) ) );
updates.add( update ); updates.add( update );
Expand Down
171 changes: 138 additions & 33 deletions ...c/test/java/org/neo4j/kernel/api/index/CompositeRandomizedIndexAccessorCompatibility.java
View file
Expand Up @@ -19,19 +19,25 @@
*/ */
package org.neo4j.kernel.api.index; package org.neo4j.kernel.api.index;


import org.junit.Assume;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;


import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;


import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptor; import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.kernel.api.schema.index.TestIndexDescriptorFactory; import org.neo4j.kernel.api.schema.index.TestIndexDescriptorFactory;
import org.neo4j.storageengine.api.schema.IndexDescriptor;
import org.neo4j.values.storable.RandomValues; import org.neo4j.values.storable.RandomValues;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueTuple; import org.neo4j.values.storable.ValueTuple;
import org.neo4j.values.storable.Values;


import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
Expand All @@ -47,48 +53,147 @@
" errors or warnings in some IDEs about test classes needing a public zero-arg constructor." ) " errors or warnings in some IDEs about test classes needing a public zero-arg constructor." )
public class CompositeRandomizedIndexAccessorCompatibility extends IndexAccessorCompatibility public class CompositeRandomizedIndexAccessorCompatibility extends IndexAccessorCompatibility
{ {
public CompositeRandomizedIndexAccessorCompatibility( IndexProviderCompatibilityTestSuite testSuite ) public CompositeRandomizedIndexAccessorCompatibility( IndexProviderCompatibilityTestSuite testSuite, IndexDescriptor descriptor )
{ {
// composite index of 4 properties super( testSuite, descriptor );
super( testSuite, TestIndexDescriptorFactory.forLabel( 1000, 100, 101, 102, 103 ) );
} }


@Test @Ignore( "Not a test. This is a compatibility suite" )
public void testExactMatchOnRandomCompositeValues() throws Exception public static class Exact extends CompositeRandomizedIndexAccessorCompatibility
{ {
// given public Exact( IndexProviderCompatibilityTestSuite testSuite )
List<RandomValues.Type> types = testSuite.supportedValueTypes();
Collections.shuffle( types, random.random() );
types = types.subList( 0, random.nextInt( 2, types.size() ) );
List<IndexEntryUpdate<?>> updates = new ArrayList<>();
Set<ValueTuple> duplicateChecker = new HashSet<>();
for ( long id = 0; id < 30_000; id++ )
{ {
IndexEntryUpdate<SchemaDescriptor> update; // composite index of 4 properties
do super( testSuite, TestIndexDescriptorFactory.forLabel( 1000, 100, 101, 102, 103 ) );
}

@Test
public void testExactMatchOnRandomCompositeValues() throws Exception
{
// given
RandomValues.Type[] types = randomSetOfSupportedTypes();
List<IndexEntryUpdate<?>> updates = new ArrayList<>();
Set<ValueTuple> duplicateChecker = new HashSet<>();
for ( long id = 0; id < 30_000; id++ )
{
IndexEntryUpdate<SchemaDescriptor> update;
do
{
update = add( id, descriptor.schema(),
random.randomValues().nextValueOfTypes( types ),
random.randomValues().nextValueOfTypes( types ),
random.randomValues().nextValueOfTypes( types ),
random.randomValues().nextValueOfTypes( types ) );
}
while ( !duplicateChecker.add( ValueTuple.of( update.values() ) ) );
updates.add( update );
}
updateAndCommit( updates );

// when
for ( IndexEntryUpdate<?> update : updates )
{
// then
List<Long> hits = query(
exact( 100, update.values()[0] ),
exact( 101, update.values()[1] ),
exact( 102, update.values()[2] ),
exact( 103, update.values()[3] ) );
assertEquals( update + " " + hits.toString(), 1, hits.size() );
assertThat( single( hits ), equalTo( update.getEntityId() ) );
}
}
}

@Ignore( "Not a test. This is a compatibility suite" )
public static class Range extends CompositeRandomizedIndexAccessorCompatibility
{
public Range( IndexProviderCompatibilityTestSuite testSuite )
{
// composite index of 2 properties
super( testSuite, TestIndexDescriptorFactory.forLabel( 1000, 100, 101 ) );
}

/**
* All entries in composite index look like (booleanValue, randomValue ).
* Range queries in composite only work if all predicates before it is exact.
* We use boolean values for exact part so that we get some real ranges to work
* on in second composite slot where the random values are.
*/
@Test
public void testRangeMatchOnRandomValues() throws Exception
{
Assume.assumeTrue( "Assume support for granular composite queries", testSuite.supportsGranularCompositeQueries() );
// given
RandomValues.Type[] types = randomSetOfSupportedAndSortableTypes();
List<ValueTuple> values = generateValuesFromType( types );
List<IndexEntryUpdate<?>> updates = generateUpdatesFromValues( values );
updateAndCommit( updates );
TreeSet<IndexEntryUpdate> sortedValues = new TreeSet<>( ( u1, u2 ) -> ValueTuple.COMPARATOR.compare(
ValueTuple.of( u1.values()[0], u1.values()[1] ),
ValueTuple.of( u2.values()[0], u2.values()[1] ) ) );
sortedValues.addAll( updates );

for ( int i = 0; i < 100; i++ )
{ {
update = add( id, descriptor.schema(), Value booleanValue = random.randomValues().nextBooleanValue();
random.nextValue( random.among( types ) ), RandomValues.Type type = random.among( types );
random.nextValue( random.among( types ) ), Value from = random.randomValues().nextValueOfType( type );
random.nextValue( random.among( types ) ), Value to = random.randomValues().nextValueOfType( type );
random.nextValue( random.among( types ) ) ); if ( Values.COMPARATOR.compare( from, to ) > 0 )
{
Value tmp = from;
from = to;
to = tmp;
}
boolean fromInclusive = random.nextBoolean();
boolean toInclusive = random.nextBoolean();

// when
List<Long> expectedIds = sortedValues.subSet(
add( 0, descriptor.schema(), booleanValue, from ), fromInclusive,
add( 0, descriptor.schema(), booleanValue, to ), toInclusive )
.stream()
.map( IndexEntryUpdate::getEntityId )
.collect( Collectors.toList() );
List<Long> actualIds = query( IndexQuery.exact( 100, booleanValue ), IndexQuery.range( 101, from, fromInclusive, to, toInclusive ) );
expectedIds.sort( Long::compare );
actualIds.sort( Long::compare );

// then
assertThat( actualIds, equalTo( expectedIds ) );
} }
while ( !duplicateChecker.add( ValueTuple.of( update.values() ) ) );
updates.add( update );
} }
updateAndCommit( updates );


// when private List<ValueTuple> generateValuesFromType( RandomValues.Type[] types )
for ( IndexEntryUpdate<?> update : updates )
{ {
// then List<ValueTuple> values = new ArrayList<>();
List<Long> hits = query( Set<ValueTuple> duplicateChecker = new HashSet<>();
exact( 100, update.values()[0] ), for ( long i = 0; i < 30_000; i++ )
exact( 101, update.values()[1] ), {
exact( 102, update.values()[2] ), ValueTuple value;
exact( 103, update.values()[3] ) ); do
assertEquals( update + " " + hits.toString(), 1, hits.size() ); {
assertThat( single( hits ), equalTo( update.getEntityId() ) ); value = ValueTuple.of(
// Use boolean for first slot in composite because we will use exact match on this part.x
random.randomValues().nextBooleanValue(),
random.randomValues().nextValueOfTypes( types ) );
}
while ( !duplicateChecker.add( value ) );
values.add( value );
}
return values;
}

private List<IndexEntryUpdate<?>> generateUpdatesFromValues( List<ValueTuple> values )
{
List<IndexEntryUpdate<?>> updates = new ArrayList<>();
int id = 0;
for ( ValueTuple value : values )
{
updates.add( add( id++, descriptor.schema(), (Object[]) value.getValues() ) );
}
return updates;
} }
} }
} }
24 changes: 23 additions & 1 deletion ...ity-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java
View file
Expand Up @@ -23,8 +23,8 @@
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;


import java.util.Collection;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
Expand All @@ -40,6 +40,7 @@
import org.neo4j.storageengine.api.schema.IndexDescriptor; import org.neo4j.storageengine.api.schema.IndexDescriptor;
import org.neo4j.storageengine.api.schema.IndexReader; import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.SimpleNodeValueClient; import org.neo4j.storageengine.api.schema.SimpleNodeValueClient;
import org.neo4j.values.storable.RandomValues;
import org.neo4j.values.storable.Value; import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.ValueCategory;
import org.neo4j.values.storable.ValueGroup; import org.neo4j.values.storable.ValueGroup;
Expand Down Expand Up @@ -84,6 +85,27 @@ public void after()
} }
} }


RandomValues.Type[] randomSetOfSupportedTypes()
{
RandomValues.Type[] supportedTypes = testSuite.supportedValueTypes();
return random.randomValues().selection( supportedTypes, 2, supportedTypes.length, false );
}

RandomValues.Type[] randomSetOfSupportedAndSortableTypes()
{
RandomValues.Type[] types = randomSetOfSupportedTypes();
types = removeSpatialTypes( types ); // <- don't use spatial values
types = RandomValues.excluding( types, RandomValues.Type.STRING, RandomValues.Type.STRING_ARRAY ); // <- don't use strings outside of BMP
return types;
}

private RandomValues.Type[] removeSpatialTypes( RandomValues.Type[] types )
{
return Arrays.stream( types )
.filter( t -> !t.name().contains( "POINT" ) )
.toArray( RandomValues.Type[]::new );
}

protected List<Long> query( IndexQuery... predicates ) throws Exception protected List<Long> query( IndexQuery... predicates ) throws Exception
{ {
try ( IndexReader reader = accessor.newReader(); ) try ( IndexReader reader = accessor.newReader(); )
Expand Down
21 changes: 13 additions & 8 deletions ...rnel-it/src/test/java/org/neo4j/kernel/api/index/IndexProviderCompatibilityTestSuite.java
View file
Expand Up @@ -69,7 +69,8 @@
CompositeIndexAccessorCompatibility.Unique.class, CompositeIndexAccessorCompatibility.Unique.class,
UniqueConstraintCompatibility.class, UniqueConstraintCompatibility.class,
SimpleRandomizedIndexAccessorCompatibility.class, SimpleRandomizedIndexAccessorCompatibility.class,
CompositeRandomizedIndexAccessorCompatibility.class CompositeRandomizedIndexAccessorCompatibility.Exact.class,
CompositeRandomizedIndexAccessorCompatibility.Range.class
} ) } )
public abstract class IndexProviderCompatibilityTestSuite public abstract class IndexProviderCompatibilityTestSuite
{ {
Expand Down Expand Up @@ -98,17 +99,21 @@ public boolean supportFullValuePrecisionForNumbers()
return true; return true;
} }


public List<RandomValues.Type> supportedValueTypes() public RandomValues.Type[] supportedValueTypes()
{ {
List<RandomValues.Type> types = new ArrayList<>( Arrays.asList( RandomValues.Type.values() ) );
if ( !supportsSpatial() ) if ( !supportsSpatial() )
{ {
types.remove( RandomValues.Type.CARTESIAN_POINT ); return RandomValues.excluding(
types.remove( RandomValues.Type.CARTESIAN_POINT_3D ); RandomValues.Type.CARTESIAN_POINT,
types.remove( RandomValues.Type.GEOGRAPHIC_POINT ); RandomValues.Type.CARTESIAN_POINT_ARRAY,
types.remove( RandomValues.Type.GEOGRAPHIC_POINT_3D ); RandomValues.Type.CARTESIAN_POINT_3D,
RandomValues.Type.CARTESIAN_POINT_3D_ARRAY,
RandomValues.Type.GEOGRAPHIC_POINT,
RandomValues.Type.GEOGRAPHIC_POINT_ARRAY,
RandomValues.Type.GEOGRAPHIC_POINT_3D,
RandomValues.Type.GEOGRAPHIC_POINT_3D_ARRAY );
} }
return types; return RandomValues.Type.values();
} }


public void consistencyCheck( IndexAccessor accessor ) public void consistencyCheck( IndexAccessor accessor )
Expand Down

0 comments on commit 3ae721b

Please sign in to comment.