Skip to content

Commit

Permalink
Consults property store for distinguishing distinct spatial values
Browse files Browse the repository at this point in the history
  • Loading branch information
tinwelint committed Feb 6, 2019
1 parent affe1db commit a2a903f
Show file tree
Hide file tree
Showing 88 changed files with 479 additions and 96 deletions.
Expand Up @@ -72,7 +72,7 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
Expand Down
Expand Up @@ -46,7 +46,7 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.schema.SchemaDescriptorFactory;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
Expand Down
Expand Up @@ -44,7 +44,7 @@
import org.neo4j.internal.kernel.api.exceptions.schema.SchemaKernelException;
import org.neo4j.internal.kernel.api.schema.LabelSchemaDescriptor;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.schema.MultiTokenSchemaDescriptor;
import org.neo4j.kernel.api.schema.SchemaDescriptorFactory;
import org.neo4j.kernel.api.schema.constraints.IndexBackedConstraintDescriptor;
Expand Down
Expand Up @@ -52,7 +52,7 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
import org.neo4j.kernel.api.schema.SchemaDescriptorFactory;
import org.neo4j.kernel.configuration.Config;
Expand Down
Expand Up @@ -46,7 +46,7 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.extension.ExtensionType;
import org.neo4j.kernel.extension.KernelExtensionFactory;
Expand Down
Expand Up @@ -64,7 +64,7 @@
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.api.schema.index.TestIndexDescriptorFactory;
import org.neo4j.kernel.extension.KernelExtensionFactory;
Expand Down
Expand Up @@ -27,7 +27,7 @@
import java.util.List;

import org.neo4j.helpers.collection.Visitor;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.store.NodeStore;
import org.neo4j.kernel.impl.store.PropertyStore;
import org.neo4j.kernel.impl.store.StoreAccess;
Expand Down
Expand Up @@ -27,7 +27,7 @@
import org.neo4j.helpers.collection.BoundedIterable;
import org.neo4j.kernel.api.impl.index.AbstractLuceneIndexAccessor;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.values.storable.Value;

Expand Down
Expand Up @@ -30,7 +30,7 @@
import org.neo4j.kernel.api.impl.schema.populator.LuceneIndexPopulator;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.IndexSample;

public class FulltextIndexPopulator extends LuceneIndexPopulator<DatabaseIndex<FulltextIndexReader>>
Expand Down
Expand Up @@ -25,6 +25,7 @@
import org.neo4j.internal.kernel.api.IndexOrder;
import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotApplicableKernelException;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.IndexProgressor;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSampler;
Expand Down Expand Up @@ -59,7 +60,7 @@ public void query( IndexProgressor.NodeValueClient client, IndexOrder indexOrder
}

@Override
public void distinctValues( IndexProgressor.NodeValueClient client )
public void distinctValues( IndexProgressor.NodeValueClient client, NodePropertyAccessor propertyAccessor )
{
throw new UnsupportedOperationException( "Fulltext indexes does not support distinctValues queries" );
}
Expand Down
Expand Up @@ -17,14 +17,11 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.index;
package org.neo4j.storageengine.api;

import org.neo4j.internal.kernel.api.exceptions.EntityNotFoundException;
import org.neo4j.values.storable.Value;

/**
* Used by the {@link IndexPopulator} for verifying constraints, if need be.
*/
public interface NodePropertyAccessor
{
Value getNodePropertyValue( long nodeId, int propertyKeyId ) throws EntityNotFoundException;
Expand Down
Expand Up @@ -25,6 +25,7 @@
import org.neo4j.internal.kernel.api.IndexOrder;
import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotApplicableKernelException;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.values.storable.Value;

/**
Expand Down Expand Up @@ -78,8 +79,9 @@ void query( IndexProgressor.NodeValueClient client, IndexOrder indexOrder, boole
* regardless of the actual values.
*
* @param client {@link IndexProgressor.NodeValueClient} to get initialized with this progression.
* @param propertyAccessor used for distinguishing between lossy indexed values.
*/
void distinctValues( IndexProgressor.NodeValueClient client );
void distinctValues( IndexProgressor.NodeValueClient client, NodePropertyAccessor propertyAccessor );

IndexReader EMPTY = new IndexReader()
{
Expand Down Expand Up @@ -120,7 +122,7 @@ public boolean hasFullValuePrecision( IndexQuery... predicates )
}

@Override
public void distinctValues( IndexProgressor.NodeValueClient client )
public void distinctValues( IndexProgressor.NodeValueClient client, NodePropertyAccessor propertyAccessor )
{
// do nothing
}
Expand Down
Expand Up @@ -25,8 +25,10 @@
import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet;
import org.junit.Test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
Expand All @@ -37,11 +39,13 @@
import org.neo4j.internal.kernel.api.exceptions.KernelException;
import org.neo4j.values.storable.CoordinateReferenceSystem;
import org.neo4j.values.storable.DateValue;
import org.neo4j.values.storable.PointValue;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueCategory;
import org.neo4j.values.storable.ValueGroup;
import org.neo4j.values.storable.Values;

import static java.lang.Math.toIntExact;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
Expand Down Expand Up @@ -1444,6 +1448,7 @@ public void shouldCountDistinctValues() throws Exception
IndexReference index = schemaRead.index( label, key );
Map<Value,Set<Long>> expected = new HashMap<>();
int expectedTotalCount = 100;
List<Long> createdNodes = new ArrayList<>();
try ( org.neo4j.internal.kernel.api.Transaction tx = beginTransaction() )
{
ThreadLocalRandom random = ThreadLocalRandom.current();
Expand All @@ -1455,6 +1460,7 @@ public void shouldCountDistinctValues() throws Exception
Value value = Values.of( random.nextBoolean() ? String.valueOf( i % 10 ) : (i % 10) );
write.nodeSetProperty( nodeId, key, value );
expected.computeIfAbsent( value, v -> new HashSet<>() ).add( nodeId );
createdNodes.add( nodeId );
}
tx.success();
}
Expand Down Expand Up @@ -1490,6 +1496,78 @@ public void shouldCountDistinctValues() throws Exception
}
assertEquals( expectedTotalCount, totalCount );
}
// the tests in this class share and modify the very same graph, so restore it
deleteNodes( createdNodes );
}

@Test
public void shouldCountDistinctButSimilarPointValues() throws Exception
{
// given
int label = token.nodeLabel( "Node" );
int key = token.propertyKey( "prop2" );
IndexReference index = schemaRead.index( label, key );
Value p1 = PointValue.parse( "{latitude: 40.7128, longitude: -74.0060, crs: 'wgs-84'}" );
Value p2 = PointValue.parse( "{latitude: 40.7128, longitude: -74.006000001, crs: 'wgs-84'}" );
List<Long> createdNodes = new ArrayList<>();
try ( org.neo4j.internal.kernel.api.Transaction tx = beginTransaction() )
{
Write write = tx.dataWrite();
// point 1
{
long node1 = write.nodeCreate();
write.nodeAddLabel( node1, label );
write.nodeSetProperty( node1, key, p1 );
createdNodes.add( node1 );
}
// point 2
{
long node2 = write.nodeCreate();
write.nodeAddLabel( node2, label );
write.nodeSetProperty( node2, key, p2 );
createdNodes.add( node2 );
}
// point 2
{
long node3 = write.nodeCreate();
write.nodeAddLabel( node3, label );
write.nodeSetProperty( node3, key, p2 );
createdNodes.add( node3 );
}
tx.success();
}

// when
Map<Value,Integer> expected = new HashMap<>();
expected.put( p1, 1 );
expected.put( p2, 2 );
try ( org.neo4j.internal.kernel.api.Transaction tx = beginTransaction();
NodeValueIndexCursor node = cursors.allocateNodeValueIndexCursor() )
{
tx.dataRead().nodeIndexDistinctValues( index, node );

// then
while ( node.next() )
{
assertTrue( node.hasValue() );
assertTrue( expected.containsKey( node.propertyValue( 0 ) ) );
assertEquals( expected.remove( node.propertyValue( 0 ) ).intValue(), toIntExact( node.nodeReference() ) );
}
assertTrue( expected.isEmpty() );
}
deleteNodes( createdNodes );
}

private void deleteNodes( List<Long> createdNodes ) throws Exception
{
try ( org.neo4j.internal.kernel.api.Transaction tx = beginTransaction() )
{
for ( Long createdNode : createdNodes )
{
tx.dataWrite().nodeDelete( createdNode );
}
tx.success();
}
}

private long nodeWithProp( GraphDatabaseService graphDb, Object value )
Expand Down
Expand Up @@ -153,6 +153,10 @@ public boolean isDense()
@Override
public boolean next()
{
if ( offset == Integer.MAX_VALUE )
{
return false;
}
return ++offset < nodes.size();
}

Expand Down
Expand Up @@ -43,7 +43,7 @@
import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier;
import org.neo4j.kernel.api.InwardKernel;
import org.neo4j.kernel.api.explicitindex.AutoIndexing;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.availability.AvailabilityGuard;
import org.neo4j.kernel.availability.DatabaseAvailability;
Expand Down
Expand Up @@ -30,6 +30,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.kernel.impl.api.index.updater.SwallowingIndexUpdater;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.values.storable.Value;

Expand Down
Expand Up @@ -26,6 +26,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.impl.api.index.UpdateMode;
import org.neo4j.kernel.impl.api.index.updater.SwallowingIndexUpdater;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.IndexSample;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;

Expand Down
Expand Up @@ -27,7 +27,7 @@
import org.neo4j.collection.PrimitiveLongCollections;
import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.internal.kernel.api.exceptions.EntityNotFoundException;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueGroup;
import org.neo4j.values.storable.Values;
Expand Down
Expand Up @@ -31,7 +31,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException;
import org.neo4j.kernel.api.exceptions.schema.UniquePropertyValueValidationException;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.CapableIndexDescriptor;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.PopulationProgress;
Expand Down
Expand Up @@ -38,7 +38,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexProxyAlreadyClosedKernelException;
import org.neo4j.kernel.api.exceptions.schema.UniquePropertyValueValidationException;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.api.index.updater.DelegatingIndexUpdater;
import org.neo4j.storageengine.api.schema.CapableIndexDescriptor;
import org.neo4j.storageengine.api.schema.IndexReader;
Expand Down
Expand Up @@ -33,7 +33,7 @@
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.schema.CapableIndexDescriptor;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.PopulationProgress;
Expand Down
Expand Up @@ -25,7 +25,7 @@

import org.neo4j.helpers.collection.Visitor;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
import org.neo4j.register.Register.DoubleLongRegister;
import org.neo4j.storageengine.api.EntityType;
Expand Down
Expand Up @@ -63,7 +63,7 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.api.SchemaState;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingController;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingMode;
Expand Down
Expand Up @@ -45,7 +45,7 @@
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.api.SchemaState;
import org.neo4j.logging.Log;
import org.neo4j.logging.LogProvider;
Expand Down
Expand Up @@ -28,7 +28,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.impl.api.index.updater.UpdateCountingIndexUpdater;
import org.neo4j.storageengine.api.schema.CapableIndexDescriptor;
import org.neo4j.storageengine.api.schema.IndexReader;
Expand Down
Expand Up @@ -21,7 +21,7 @@

import org.eclipse.collections.api.set.primitive.MutableIntSet;

import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.EntityType;
import org.neo4j.values.storable.Value;

Expand Down
Expand Up @@ -33,7 +33,7 @@
import org.neo4j.kernel.api.exceptions.schema.UniquePropertyValueValidationException;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.schema.constraints.ConstraintDescriptorFactory;
import org.neo4j.kernel.impl.api.index.updater.DelegatingIndexUpdater;
import org.neo4j.kernel.impl.index.schema.DeferredConflictCheckingIndexUpdater;
Expand Down
Expand Up @@ -41,7 +41,7 @@
import org.neo4j.kernel.api.exceptions.schema.AlreadyConstrainedException;
import org.neo4j.kernel.api.exceptions.schema.AlreadyIndexedException;
import org.neo4j.kernel.api.exceptions.schema.UniquePropertyValueValidationException;
import org.neo4j.kernel.api.index.NodePropertyAccessor;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.api.schema.constraints.ConstraintDescriptorFactory;
import org.neo4j.kernel.api.schema.constraints.UniquenessConstraintDescriptor;
import org.neo4j.kernel.impl.api.KernelTransactionImplementation;
Expand Down

0 comments on commit a2a903f

Please sign in to comment.