Skip to content

Commit

Permalink
Let kernel augment the generic native index capability to support all…
Browse files Browse the repository at this point in the history
… values
  • Loading branch information
fickludd committed Sep 20, 2018
1 parent 7d1c4ff commit 18a2e05
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 24 deletions.
Expand Up @@ -23,6 +23,7 @@
import org.eclipse.collections.api.set.primitive.MutableLongSet;
import org.eclipse.collections.impl.factory.primitive.LongLists;
import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet;
import org.junit.Assume;
import org.junit.Test;

import org.neo4j.graphdb.GraphDatabaseService;
Expand Down Expand Up @@ -543,7 +544,7 @@ public void shouldPerformSpatialRangeSearch() throws KernelException
public void shouldPerformBooleanSearch() throws KernelException
{
// given
boolean needsValues = false;
boolean needsValues = indexProvidesAllValues();
int label = token.nodeLabel( "Node" );
int prop = token.propertyKey( "prop" );
IndexReference index = schemaRead.index( label, prop );
Expand All @@ -570,7 +571,7 @@ public void shouldPerformBooleanSearch() throws KernelException
public void shouldPerformTextArraySearch() throws KernelException
{
// given
boolean needsValues = false;
boolean needsValues = indexProvidesAllValues();
int label = token.nodeLabel( "Node" );
int prop = token.propertyKey( "prop" );
IndexReference index = schemaRead.index( label, prop );
Expand Down Expand Up @@ -606,11 +607,11 @@ public void shouldPerformIndexScan() throws Exception
MutableLongSet uniqueIds = new LongHashSet();

// when
read.nodeIndexScan( index, node, IndexOrder.NONE, false );
read.nodeIndexScan( index, node, IndexOrder.NONE, indexProvidesAllValues() );

// then
assertThat( node.numberOfProperties(), equalTo( 1 ) );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT, uniqueIds, wildcardCapability, false );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT, uniqueIds, wildcardCapability, indexProvidesAllValues() );
}
}

Expand Down Expand Up @@ -753,39 +754,45 @@ public void shouldRespectOrderCapabilitiesForWildcard() throws Exception
public void shouldProvideValuesForPoints() throws Exception
{
// given
boolean needsValues = indexProvidesAllValues();
Assume.assumeTrue( indexProvidesAllValues() );

int label = token.nodeLabel( "What" );
int prop = token.propertyKey( "ever" );
IndexReference index = schemaRead.index( label, prop );
assertEquals( IndexValueCapability.YES, index.valueCapability( ValueCategory.GEOMETRY ) );

try ( NodeValueIndexCursor node = cursors.allocateNodeValueIndexCursor() )
{
MutableLongSet uniqueIds = new LongHashSet();

// when
read.nodeIndexSeek( index, node, IndexOrder.NONE, needsValues, IndexQuery.range( prop, Cartesian ) );
read.nodeIndexSeek( index, node, IndexOrder.NONE, true, IndexQuery.range( prop, Cartesian ) );

// then
assertFoundNodesAndValue( node, uniqueIds, index.valueCapability( ValueCategory.GEOMETRY ),needsValues, whateverPoint );
assertFoundNodesAndValue( node, uniqueIds, index.valueCapability( ValueCategory.GEOMETRY ), true, whateverPoint );
}
}

@Test
public void shouldProvideValuesForAllTypes() throws Exception
{
// given
boolean needsValues = indexProvidesAllValues();
Assume.assumeTrue( indexProvidesAllValues() );

int label = token.nodeLabel( "What" );
int prop = token.propertyKey( "ever" );
IndexReference index = schemaRead.index( label, prop );
assertEquals( IndexValueCapability.YES, index.valueCapability( ValueCategory.UNKNOWN ) );

try ( NodeValueIndexCursor node = cursors.allocateNodeValueIndexCursor() )
{
MutableLongSet uniqueIds = new LongHashSet();

// when
read.nodeIndexSeek( index, node, IndexOrder.NONE, needsValues, IndexQuery.exists( prop ) );
read.nodeIndexSeek( index, node, IndexOrder.NONE, true, IndexQuery.exists( prop ) );

// then
assertFoundNodesAndValue( node, uniqueIds, index.valueCapability( ValueCategory.UNKNOWN ),needsValues, nodesOfAllPropertyTypes );
assertFoundNodesAndValue( node, uniqueIds, index.valueCapability( ValueCategory.UNKNOWN ), true, nodesOfAllPropertyTypes );
}
}

Expand Down Expand Up @@ -819,8 +826,11 @@ private void assertFoundNodesInOrder( NodeValueIndexCursor node, IndexOrder inde
}
}

private void assertFoundNodesAndValue( NodeValueIndexCursor node, int nodes, MutableLongSet uniqueIds, IndexValueCapability expectValue,
boolean indexProvidesValues )
private void assertFoundNodesAndValue( NodeValueIndexCursor node,
int nodes,
MutableLongSet uniqueIds,
IndexValueCapability expectValue,
boolean indexProvidesValues )
{
uniqueIds.clear();
for ( int i = 0; i < nodes; i++ )
Expand Down Expand Up @@ -942,6 +952,7 @@ public void shouldGetVersionAndKeyFromIndexReference()
public void shouldNotFindDeletedNodeInIndexScan() throws Exception
{
// Given
boolean needsValues = indexProvidesAllValues();
int label = token.nodeLabel( "Node" );
int prop = token.propertyKey( "prop" );
IndexReference index = schemaRead.index( label, prop );
Expand All @@ -952,14 +963,14 @@ public void shouldNotFindDeletedNodeInIndexScan() throws Exception
MutableLongSet uniqueIds = new LongHashSet();

// when
tx.dataRead().nodeIndexScan( index, node, IndexOrder.NONE, false );
tx.dataRead().nodeIndexScan( index, node, IndexOrder.NONE, needsValues );
assertThat( node.numberOfProperties(), equalTo( 1 ) );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT, uniqueIds, wildcardCapability, false );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT, uniqueIds, wildcardCapability, needsValues );

// then
tx.dataWrite().nodeDelete( strOne );
tx.dataRead().nodeIndexScan( index, node, IndexOrder.NONE, false );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT - 1, uniqueIds, wildcardCapability, false );
tx.dataRead().nodeIndexScan( index, node, IndexOrder.NONE, needsValues );
assertFoundNodesAndValue( node, TOTAL_NODE_COUNT - 1, uniqueIds, wildcardCapability, needsValues );
}
}

Expand Down
Expand Up @@ -24,6 +24,7 @@
import org.neo4j.internal.kernel.api.IndexCapability;
import org.neo4j.internal.kernel.api.schema.IndexProviderDescriptor;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.impl.newapi.KernelIndexAugmentation;
import org.neo4j.storageengine.api.schema.CapableIndexDescriptor;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;

Expand Down Expand Up @@ -73,8 +74,10 @@ public interface IndexProviderMap
*/
default CapableIndexDescriptor withCapabilities( StoreIndexDescriptor descriptor )
{
IndexCapability capability = lookup( descriptor.providerDescriptor() ).getCapability();
return new CapableIndexDescriptor( descriptor, capability );
IndexProviderDescriptor providerDescriptor = descriptor.providerDescriptor();
IndexCapability capability = lookup( providerDescriptor ).getCapability();
IndexCapability augmented = KernelIndexAugmentation.augmentIndexCapability( capability, providerDescriptor );
return new CapableIndexDescriptor( descriptor, augmented );
}

IndexProviderMap EMPTY = new IndexProviderMap()
Expand Down
@@ -0,0 +1,74 @@
/*
* Copyright (c) 2002-2018 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.newapi;

import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.internal.kernel.api.IndexCapability;
import org.neo4j.internal.kernel.api.IndexOrder;
import org.neo4j.internal.kernel.api.IndexReference;
import org.neo4j.internal.kernel.api.IndexValueCapability;
import org.neo4j.internal.kernel.api.schema.IndexProviderDescriptor;
import org.neo4j.values.storable.ValueCategory;

/**
* Gathers the business logic on what indexes to augment with node property value injection.
*
* Currently, the new generic native index provider can provide values directly from the index for
* all value types except points, which are stored in a lossy fashion in the index. Even thought
* that does not initially seems like a big deal, this stops Cypher completely from being able to
* rely on index-backed property values from index scans, even though the index does not contain any
* points - the reason being that it might get points later, and in that case the plan must still
* hold. Therefore augmenting the index with property lookup backed values makes sense.
*/
public class KernelIndexAugmentation
{
private static final String NATIVE_PROVIDER = GraphDatabaseSettings.SchemaIndex.NATIVE_BTREE10.providerName();

public static IndexCapability augmentIndexCapability( IndexCapability capability,
IndexProviderDescriptor providerDescriptor )
{
if ( NATIVE_PROVIDER.equals( providerDescriptor.getKey() ) )
{
return new IndexCapability()
{
@Override
public IndexOrder[] orderCapability( ValueCategory... valueCategories )
{
return capability.orderCapability( valueCategories );
}

@Override
public IndexValueCapability valueCapability( ValueCategory... valueCategories )
{
return IndexValueCapability.YES;
}
};
}
else
{
return capability;
}
}

public static boolean shouldInjectValues( IndexReference index, boolean needsValues )
{
return needsValues && NATIVE_PROVIDER.equals( index.providerKey() );
}
}
Expand Up @@ -95,16 +95,16 @@ public final void nodeIndexSeek( IndexReference index, NodeValueIndexCursor curs
DefaultNodeValueIndexCursor cursorImpl = (DefaultNodeValueIndexCursor) cursor;
IndexReader reader = indexReader( index, false );
cursorImpl.setRead( this, null );
IndexProgressor.NodeValueClient withValues = injectValues( cursorImpl, needsValues, index, reader );
IndexProgressor.NodeValueClient withValues = injectValues( cursorImpl, needsValues, index );
IndexProgressor.NodeValueClient withFullPrecision = injectFullValuePrecision( withValues, query, reader );
reader.query( withFullPrecision, indexOrder, needsValues, query );
}

private IndexProgressor.NodeValueClient injectValues( IndexProgressor.NodeValueClient cursor,
boolean needsValues,
IndexReference index, IndexReader reader )
IndexReference index )
{
if ( needsValues && GraphDatabaseSettings.SchemaIndex.NATIVE_BTREE10.providerName().equals( index.providerKey() ) )
if ( KernelIndexAugmentation.shouldInjectValues( index, needsValues ) )
{
return new NodeValueInjector( cursor,
cursors.allocateNodeCursor(),
Expand Down Expand Up @@ -217,7 +217,10 @@ void nodeIndexSeekWithFreshIndexReader(
}

@Override
public final void nodeIndexScan( IndexReference index, NodeValueIndexCursor cursor, IndexOrder indexOrder, boolean needsValues ) throws KernelException
public final void nodeIndexScan( IndexReference index,
NodeValueIndexCursor cursor,
IndexOrder indexOrder,
boolean needsValues ) throws KernelException
{
ktx.assertOpen();
if ( hasForbiddenProperties( index ) )
Expand All @@ -228,8 +231,11 @@ public final void nodeIndexScan( IndexReference index, NodeValueIndexCursor curs

// for a scan, we simply query for existence of the first property, which covers all entries in an index
int firstProperty = index.properties()[0];
((DefaultNodeValueIndexCursor) cursor).setRead( this, null );
indexReader( index, false ).query( (DefaultNodeValueIndexCursor) cursor, indexOrder, needsValues, IndexQuery.exists( firstProperty ) );

DefaultNodeValueIndexCursor cursorImpl = (DefaultNodeValueIndexCursor) cursor;
cursorImpl.setRead( this, null );
IndexProgressor.NodeValueClient withValues = injectValues( cursorImpl, needsValues, index );
indexReader( index, false ).query( withValues, indexOrder, needsValues, IndexQuery.exists( firstProperty ) );
}

private boolean hasForbiddenProperties( IndexReference index )
Expand Down

0 comments on commit 18a2e05

Please sign in to comment.