Skip to content

Commit

Permalink
Add test for index populators and accessors (spatial/temporal)
Browse files Browse the repository at this point in the history 
This adds test for both unique and non-unique schema index
accessors and populators for all spatial and temporal types.

To fix tests it moves the sampling down from Spatial/Temporal
populators to the PartPopulators.
  • Loading branch information
@sherfert
sherfert committed Apr 5, 2018
1 parent 52b75b3 commit 56c9e37
Show file tree
Hide file tree
Showing 55 changed files with 2,276 additions and 173 deletions.
74 changes: 74 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/IndexSamplerWrapper.java
View file
@@ -0,0 +1,74 @@
/*
* Copyright (c) 2002-2018 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.index.schema;

import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler;
import org.neo4j.storageengine.api.schema.IndexSample;
import org.neo4j.values.storable.Value;

class IndexSamplerWrapper
{
private final DefaultNonUniqueIndexSampler generalSampler;
private final UniqueIndexSampler uniqueSampler;

IndexSamplerWrapper( SchemaIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
{
switch ( descriptor.type() )
{
case GENERAL:
generalSampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() );
uniqueSampler = null;
break;
case UNIQUE:
generalSampler = null;
uniqueSampler = new UniqueIndexSampler();
break;
default:
throw new UnsupportedOperationException( "Unexpected index type " + descriptor.type() );
}
}

void includeSample( Value[] values )
{
if ( uniqueSampler != null )
{
uniqueSampler.increment( 1 );
}
else
{
generalSampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) values ) );
}
}

IndexSample sampleResult()
{
if ( uniqueSampler != null )
{
return uniqueSampler.result();
}
else
{
return generalSampler.result();
}
}
}
1 change: 1 addition & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexAccessor.java
View file
Expand Up @@ -206,6 +206,7 @@ static class PartAccessor extends NativeSchemaIndexAccessor<SpatialSchemaKey, Na
@Override
public SpatialIndexPartReader<NativeSchemaValue> newReader()
{
assertOpen();
return new SpatialIndexPartReader<>( tree, layout, samplingConfig, descriptor, searchConfiguration );
}
}
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexFiles.java
View file
Expand Up @@ -96,7 +96,7 @@ static class SpatialFileLayout
private final CoordinateReferenceSystem crs;
Layout<SpatialSchemaKey,NativeSchemaValue> layout;

private SpatialFileLayout( CoordinateReferenceSystem crs, SpaceFillingCurveSettings settings, File indexDirectory )
SpatialFileLayout( CoordinateReferenceSystem crs, SpaceFillingCurveSettings settings, File indexDirectory )
{
this.crs = crs;
this.settings = settings;
Expand Down
69 changes: 13 additions & 56 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexPopulator.java
View file
Expand Up @@ -24,6 +24,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.stream.StreamSupport;
import java.util.Map;

import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration;
Expand All @@ -37,9 +38,7 @@
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler;
import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettings;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSample;
Expand All @@ -48,11 +47,10 @@
import org.neo4j.values.storable.Value;

import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexBase.forAll;
import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexSampler.combineSamples;

class SpatialIndexPopulator extends SpatialIndexCache<SpatialIndexPopulator.PartPopulator> implements IndexPopulator
{
private final IndexSamplerWrapper sampler;

SpatialIndexPopulator( long indexId,
SchemaIndexDescriptor descriptor,
IndexSamplingConfig samplingConfig,
Expand All @@ -63,7 +61,6 @@ class SpatialIndexPopulator extends SpatialIndexCache<SpatialIndexPopulator.Part
SpaceFillingCurveConfiguration configuration )
{
super( new PartFactory( pageCache, fs, spatialIndexFiles, indexId, descriptor, monitor, samplingConfig, configuration ) );
this.sampler = new IndexSamplerWrapper( descriptor, samplingConfig );
}

@Override
Expand Down Expand Up @@ -129,66 +126,25 @@ public synchronized void markAsFailed( String failure )
@Override
public void includeSample( IndexEntryUpdate<?> update )
{
sampler.includeSample( update.values() );
Value[] values = update.values();
assert values.length == 1;
uncheckedSelect( ((PointValue) values[0]).getCoordinateReferenceSystem() ).includeSample( update );
}

@Override
public IndexSample sampleResult()
{
return sampler.sampleResult();
}

private static class IndexSamplerWrapper
{
private final DefaultNonUniqueIndexSampler generalSampler;
private final UniqueIndexSampler uniqueSampler;

IndexSamplerWrapper( SchemaIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
{
switch ( descriptor.type() )
{
case GENERAL:
generalSampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() );
uniqueSampler = null;
break;
case UNIQUE:
generalSampler = null;
uniqueSampler = new UniqueIndexSampler();
break;
default:
throw new UnsupportedOperationException( "Unexpected index type " + descriptor.type() );
}
}

void includeSample( Value[] values )
{
if ( uniqueSampler != null )
{
uniqueSampler.increment( 1 );
}
else
{
generalSampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) values ) );
}
}

IndexSample sampleResult()
{
if ( uniqueSampler != null )
{
return uniqueSampler.result();
}
else
{
return generalSampler.result();
}
}
IndexSample[] indexSamples = StreamSupport.stream( this.spliterator(), false )
.map( PartPopulator::sampleResult )
.toArray( IndexSample[]::new );
return combineSamples( indexSamples );
}

static class PartPopulator extends NativeSchemaIndexPopulator<SpatialSchemaKey, NativeSchemaValue>
{
private final SpaceFillingCurveConfiguration configuration;
private final SpaceFillingCurveSettings settings;
private final IndexSamplerWrapper sampler;

PartPopulator( PageCache pageCache, FileSystemAbstraction fs, SpatialIndexFiles.SpatialFileLayout fileLayout,
IndexProvider.Monitor monitor, SchemaIndexDescriptor descriptor, long indexId, IndexSamplingConfig samplingConfig,
Expand All @@ -197,6 +153,7 @@ static class PartPopulator extends NativeSchemaIndexPopulator<SpatialSchemaKey,
super( pageCache, fs, fileLayout.indexFile, fileLayout.layout, monitor, descriptor, indexId, samplingConfig );
this.configuration = configuration;
this.settings = fileLayout.settings;
this.sampler = new IndexSamplerWrapper( descriptor, samplingConfig );
}

@Override
Expand All @@ -208,13 +165,13 @@ IndexReader newReader()
@Override
public void includeSample( IndexEntryUpdate<?> update )
{
throw new UnsupportedOperationException( "please to not get here!" );
sampler.includeSample( update.values() );
}

@Override
public IndexSample sampleResult()
{
throw new UnsupportedOperationException( "this sampling code needs a rewrite." );
return sampler.sampleResult();
}

@Override
Expand Down
1 change: 1 addition & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexAccessor.java
View file
Expand Up @@ -193,6 +193,7 @@ static class PartAccessor<KEY extends NativeSchemaKey<KEY>> extends NativeSchema
@Override
public TemporalIndexPartReader<KEY> newReader()
{
assertOpen();
return new TemporalIndexPartReader<>( tree, layout, samplingConfig, descriptor );
}
}
Expand Down
70 changes: 14 additions & 56 deletions ...unity/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPopulator.java
View file
Expand Up @@ -24,6 +24,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.stream.StreamSupport;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;

Expand All @@ -36,20 +37,17 @@
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSample;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueGroup;

import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexBase.forAll;
import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexSampler.combineSamples;

class TemporalIndexPopulator extends TemporalIndexCache<TemporalIndexPopulator.PartPopulator<?>> implements IndexPopulator
{
private final IndexSamplerWrapper sampler;

TemporalIndexPopulator( long indexId,
SchemaIndexDescriptor descriptor,
IndexSamplingConfig samplingConfig,
Expand All @@ -59,7 +57,6 @@ class TemporalIndexPopulator extends TemporalIndexCache<TemporalIndexPopulator.P
IndexProvider.Monitor monitor )
{
super( new PartFactory( pageCache, fs, temporalIndexFiles, indexId, descriptor, samplingConfig, monitor ) );
this.sampler = new IndexSamplerWrapper( descriptor, samplingConfig );
}

@Override
Expand Down Expand Up @@ -124,68 +121,29 @@ public synchronized void markAsFailed( String failure )
@Override
public void includeSample( IndexEntryUpdate<?> update )
{
sampler.includeSample( update.values() );
Value[] values = update.values();
assert values.length == 1;
uncheckedSelect( values[0].valueGroup() ).includeSample( update );
}

@Override
public IndexSample sampleResult()
{
return sampler.sampleResult();
}

private static class IndexSamplerWrapper
{
private final DefaultNonUniqueIndexSampler generalSampler;
private final UniqueIndexSampler uniqueSampler;

IndexSamplerWrapper( SchemaIndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
{
switch ( descriptor.type() )
{
case GENERAL:
generalSampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() );
uniqueSampler = null;
break;
case UNIQUE:
generalSampler = null;
uniqueSampler = new UniqueIndexSampler();
break;
default:
throw new UnsupportedOperationException( "Unexpected index type " + descriptor.type() );
}
}

void includeSample( Value[] values )
{
if ( uniqueSampler != null )
{
uniqueSampler.increment( 1 );
}
else
{
generalSampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) values ) );
}
}

IndexSample sampleResult()
{
if ( uniqueSampler != null )
{
return uniqueSampler.result();
}
else
{
return generalSampler.result();
}
}
IndexSample[] indexSamples = StreamSupport.stream( this.spliterator(), false )
.map( PartPopulator::sampleResult )
.toArray( IndexSample[]::new );
return combineSamples( indexSamples );
}

static class PartPopulator<KEY extends NativeSchemaKey<KEY>> extends NativeSchemaIndexPopulator<KEY, NativeSchemaValue>
{
private final IndexSamplerWrapper sampler;

PartPopulator( PageCache pageCache, FileSystemAbstraction fs, TemporalIndexFiles.FileLayout<KEY> fileLayout,
IndexProvider.Monitor monitor, SchemaIndexDescriptor descriptor, long indexId, IndexSamplingConfig samplingConfig )
{
super( pageCache, fs, fileLayout.indexFile, fileLayout.layout, monitor, descriptor, indexId, samplingConfig );
this.sampler = new IndexSamplerWrapper( descriptor, samplingConfig );
}

@Override
Expand All @@ -197,13 +155,13 @@ IndexReader newReader()
@Override
public void includeSample( IndexEntryUpdate<?> update )
{
throw new UnsupportedOperationException( "please to not get here!" );
sampler.includeSample( update.values() );
}

@Override
public IndexSample sampleResult()
{
throw new UnsupportedOperationException( "this sampling code needs a rewrite." );
return sampler.sampleResult();
}
}

Expand Down

0 comments on commit 56c9e37

Please sign in to comment.