Support for scan and index sampling on composite indexes

Also makes the index structure backwards compatible

community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/PropertyAndNodeIndexedCheck.java

@@ -25,6 +25,7 @@
 
 import org.neo4j.collection.primitive.Primitive;
 import org.neo4j.collection.primitive.PrimitiveIntSet;
+import org.neo4j.collection.primitive.PrimitiveLongCollections;
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.collection.primitive.PrimitiveLongPeekingIterator;
 import org.neo4j.consistency.checking.ChainCheck;
@@ -34,6 +35,7 @@
 import org.neo4j.consistency.checking.index.IndexAccessors;
 import org.neo4j.consistency.report.ConsistencyReport;
 import org.neo4j.consistency.store.RecordAccess;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.schema_new.IndexQuery;
 import org.neo4j.kernel.impl.api.LookupFilter;
 import org.neo4j.kernel.impl.store.record.IndexRule;
@@ -122,7 +124,15 @@ private void verifyNodeCorrectlyIndexedUniquely( long nodeId, int propertyKeyId,
             CheckerEngine<NodeRecord,ConsistencyReport.NodeConsistencyReport> engine, IndexRule indexRule,
             IndexReader reader )
     {
-        PrimitiveLongIterator indexedNodeIds = reader.query( IndexQuery.exact( propertyKeyId, propertyValue ) );
+        PrimitiveLongIterator indexedNodeIds = null;
+        try
+        {
+            indexedNodeIds = reader.query( IndexQuery.exact( propertyKeyId, propertyValue ) );
+        }
+        catch ( IndexNotApplicableKernelException e )
+        {
+            indexedNodeIds = PrimitiveLongCollections.emptyIterator();
+        }
 
         // For verifying node indexed uniquely in offline CC, if one match found in the first stage match,
         // then there is no need to filter the result. The result is a exact match.

community/kernel/src/main/java/org/neo4j/kernel/api/ReadOperations.java

@@ -37,6 +37,7 @@
 import org.neo4j.kernel.api.exceptions.ProcedureException;
 import org.neo4j.kernel.api.exceptions.PropertyKeyIdNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.RelationshipTypeIdNotFoundKernelException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.LegacyIndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.schema.DuplicateSchemaRuleException;
@@ -126,7 +127,8 @@ public interface ReadOperations
      * @return ids of the matching nodes
      * @throws org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException if no such index is found.
      */
-    PrimitiveLongIterator indexQuery( NewIndexDescriptor index, IndexQuery... predicates ) throws IndexNotFoundKernelException;
+    PrimitiveLongIterator indexQuery( NewIndexDescriptor index, IndexQuery... predicates )
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException;
 
     /**
      * @return an iterator over all nodes in the database.
@@ -155,8 +157,8 @@ RelationshipIterator nodeGetRelationships( long nodeId, Direction direction, int
      *
      * @throws org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException if no such index found.
      */
-    long nodeGetFromUniqueIndexSeek( NewIndexDescriptor index, Object value ) throws IndexNotFoundKernelException,
-            IndexBrokenKernelException;
+    long nodeGetFromUniqueIndexSeek( NewIndexDescriptor index, Object value )
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException;
 
     long nodesCountIndexed( NewIndexDescriptor index, long nodeId, Object value )
             throws IndexNotFoundKernelException, IndexBrokenKernelException;

community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java

@@ -20,6 +20,7 @@
 package org.neo4j.kernel.api.index;
 
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.schema_new.IndexQuery;
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.IndexSampler;
@@ -46,7 +47,7 @@ public IndexSampler createSampler()
     }
 
     @Override
-    public PrimitiveLongIterator query( IndexQuery... predicates )
+    public PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotApplicableKernelException
     {
         return delegate.query( predicates );
     }

community/kernel/src/main/java/org/neo4j/kernel/impl/api/ConstraintEnforcingEntityOperations.java

@@ -32,7 +32,7 @@
 import org.neo4j.kernel.api.exceptions.EntityNotFoundException;
 import org.neo4j.kernel.api.exceptions.InvalidTransactionTypeKernelException;
 import org.neo4j.kernel.api.exceptions.KernelException;
-import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.AutoIndexingKernelException;
 import org.neo4j.kernel.api.exceptions.schema.AlreadyConstrainedException;
@@ -181,7 +181,7 @@ private void validateNoExistingNodeWithLabelAndProperty(
                         new IndexEntryConflictException( existing, NO_SUCH_NODE, value ) );
             }
         }
-        catch ( IndexNotFoundKernelException | IndexBrokenKernelException e )
+        catch ( IndexNotFoundKernelException | IndexBrokenKernelException | IndexNotApplicableKernelException e )
         {
             throw new UnableToValidateConstraintException( constraint, e );
         }
@@ -281,7 +281,8 @@ public PrimitiveLongIterator nodesGetForLabel( KernelStatement state, int labelI
 
     @Override
     public PrimitiveLongIterator indexQuery( KernelStatement statement, NewIndexDescriptor index,
-                                             IndexQuery[] predicates ) throws IndexNotFoundKernelException
+            IndexQuery[] predicates )
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         return entityReadOperations.indexQuery( statement, index, predicates );
     }
@@ -291,7 +292,7 @@ public long nodeGetFromUniqueIndexSeek(
             KernelStatement state,
             NewIndexDescriptor index,
             Object value )
-            throws IndexNotFoundKernelException, IndexBrokenKernelException
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException
     {
         assertIndexOnline( state, index );
 

community/kernel/src/main/java/org/neo4j/kernel/impl/api/GuardingStatementOperations.java

@@ -26,6 +26,7 @@
 import org.neo4j.kernel.api.exceptions.EntityNotFoundException;
 import org.neo4j.kernel.api.exceptions.InvalidTransactionTypeKernelException;
 import org.neo4j.kernel.api.exceptions.KernelException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.AutoIndexingKernelException;
 import org.neo4j.kernel.api.exceptions.schema.ConstraintValidationException;
@@ -173,15 +174,16 @@ public PrimitiveLongIterator nodesGetForLabel( KernelStatement statement, int la
 
     @Override
     public PrimitiveLongIterator indexQuery( KernelStatement statement, NewIndexDescriptor index,
-                                             IndexQuery[] predicates ) throws IndexNotFoundKernelException
+            IndexQuery[] predicates )
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         guard.check( statement );
         return entityReadDelegate.indexQuery( statement, index, predicates );
     }
 
     @Override
     public long nodeGetFromUniqueIndexSeek( KernelStatement statement, NewIndexDescriptor index, Object value )
-            throws IndexNotFoundKernelException, IndexBrokenKernelException
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException
     {
         guard.check( statement );
         return entityReadDelegate.nodeGetFromUniqueIndexSeek( statement, index, value );

community/kernel/src/main/java/org/neo4j/kernel/impl/api/OperationsFacade.java

@@ -52,6 +52,7 @@
 import org.neo4j.kernel.api.exceptions.ProcedureException;
 import org.neo4j.kernel.api.exceptions.PropertyKeyIdNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.RelationshipTypeIdNotFoundKernelException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.AutoIndexingKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.LegacyIndexNotFoundKernelException;
@@ -228,15 +229,15 @@ public PrimitiveLongIterator nodesGetForLabel( int labelId )
 
     @Override
     public PrimitiveLongIterator indexQuery( NewIndexDescriptor index, IndexQuery... predicates )
-            throws IndexNotFoundKernelException
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         statement.assertOpen();
         return dataRead().indexQuery( statement, index, predicates );
     }
 
     @Override
     public long nodeGetFromUniqueIndexSeek( NewIndexDescriptor index, Object value )
-            throws IndexNotFoundKernelException, IndexBrokenKernelException
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException
     {
         statement.assertOpen();
         return dataRead().nodeGetFromUniqueIndexSeek( statement, index, value );

community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java

@@ -40,6 +40,7 @@
 import org.neo4j.kernel.api.exceptions.PropertyKeyIdNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.RelationshipTypeIdNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.TransactionFailureException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.AutoIndexingKernelException;
 import org.neo4j.kernel.api.exceptions.legacyindex.LegacyIndexNotFoundKernelException;
@@ -681,7 +682,7 @@ public Iterator<NewIndexDescriptor> uniqueIndexesGetAll( KernelStatement state )
 
     @Override
     public long nodeGetFromUniqueIndexSeek( KernelStatement state, NewIndexDescriptor index, Object value )
-            throws IndexNotFoundKernelException, IndexBrokenKernelException
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException
     {
         IndexReader reader = state.getStoreStatement().getFreshIndexReader( index );
 
@@ -700,8 +701,8 @@ public long nodeGetFromUniqueIndexSeek( KernelStatement state, NewIndexDescripto
     }
 
     @Override
-    public PrimitiveLongIterator indexQuery( KernelStatement state, NewIndexDescriptor index,
-                                             IndexQuery...predicates ) throws IndexNotFoundKernelException
+    public PrimitiveLongIterator indexQuery( KernelStatement state, NewIndexDescriptor index, IndexQuery... predicates )
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         StorageStatement storeStatement = state.getStoreStatement();
         IndexReader reader = storeStatement.getIndexReader( index );

community/kernel/src/main/java/org/neo4j/kernel/impl/api/operations/EntityReadOperations.java

@@ -24,6 +24,7 @@
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.cursor.Cursor;
 import org.neo4j.kernel.api.exceptions.EntityNotFoundException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.schema.IndexBrokenKernelException;
 import org.neo4j.kernel.api.schema_new.IndexQuery;
@@ -52,7 +53,7 @@ public interface EntityReadOperations
      * @throws IndexNotFoundKernelException if no such index is found.
      */
     PrimitiveLongIterator indexQuery( KernelStatement statement, NewIndexDescriptor index, IndexQuery... predicates )
-            throws IndexNotFoundKernelException;
+            throws IndexNotFoundKernelException, IndexNotApplicableKernelException;
 
     /**
      * Returns an iterable with the matched node.
@@ -61,7 +62,7 @@ PrimitiveLongIterator indexQuery( KernelStatement statement, NewIndexDescriptor
      * @throws IndexBrokenKernelException   if we found an index that was corrupt or otherwise in a failed state.
      */
     long nodeGetFromUniqueIndexSeek( KernelStatement state, NewIndexDescriptor index, Object value )
-            throws IndexNotFoundKernelException, IndexBrokenKernelException;
+            throws IndexNotFoundKernelException, IndexBrokenKernelException, IndexNotApplicableKernelException;
 
     long nodesCountIndexed( KernelStatement statement, NewIndexDescriptor index, long nodeId, Object value )
             throws IndexNotFoundKernelException, IndexBrokenKernelException;

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java

@@ -63,6 +63,7 @@
 import org.neo4j.kernel.api.Statement;
 import org.neo4j.kernel.api.exceptions.EntityNotFoundException;
 import org.neo4j.kernel.api.exceptions.InvalidTransactionTypeKernelException;
+import org.neo4j.kernel.api.exceptions.KernelException;
 import org.neo4j.kernel.api.exceptions.Status;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.schema.SchemaKernelException;
@@ -602,7 +603,7 @@ private ResourceIterator<Node> nodesByLabelAndProperty( Label myLabel, String ke
                 return map2nodes( readOps.indexQuery( descriptor, query ), statement );
             }
         }
-        catch ( IndexNotFoundKernelException e )
+        catch ( KernelException e )
         {
             // weird at this point but ignore and fallback to a label scan
         }

community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java

@@ -23,6 +23,7 @@
 import org.neo4j.collection.primitive.PrimitiveLongCollections;
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.graphdb.Resource;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.schema_new.IndexQuery;
 
 
@@ -47,7 +48,7 @@ public interface IndexReader extends Resource
      * @param predicates the predicates to query for.
      * @return the matching entity IDs.
      */
-    PrimitiveLongIterator query( IndexQuery... predicates );
+    PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotApplicableKernelException;
 
     IndexReader EMPTY = new IndexReader()
     {

community/kernel/src/test/java/org/neo4j/graphdb/IndexingAcceptanceTest.java

@@ -34,6 +34,7 @@
 import org.neo4j.helpers.collection.Iterators;
 import org.neo4j.kernel.api.ReadOperations;
 import org.neo4j.kernel.api.Statement;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.schema.SchemaRuleNotFoundException;
 import org.neo4j.kernel.api.schema.IndexDescriptorFactory;
@@ -474,7 +475,7 @@ public void shouldAddIndexedPropertyToNodeWithDynamicLabels()
 
     @Test
     public void shouldSupportIndexSeekByPrefix()
-            throws SchemaRuleNotFoundException, IndexNotFoundKernelException
+            throws SchemaRuleNotFoundException, IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         // GIVEN
         GraphDatabaseService db = dbRule.getGraphDatabaseAPI();
@@ -499,7 +500,7 @@ public void shouldSupportIndexSeekByPrefix()
 
     @Test
     public void shouldIncludeNodesCreatedInSameTxInIndexSeekByPrefix()
-            throws SchemaRuleNotFoundException, IndexNotFoundKernelException
+            throws SchemaRuleNotFoundException, IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         // GIVEN
         GraphDatabaseService db = dbRule.getGraphDatabaseAPI();
@@ -524,7 +525,7 @@ public void shouldIncludeNodesCreatedInSameTxInIndexSeekByPrefix()
 
     @Test
     public void shouldNotIncludeNodesDeletedInSameTxInIndexSeekByPrefix()
-            throws SchemaRuleNotFoundException, IndexNotFoundKernelException
+            throws SchemaRuleNotFoundException, IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         // GIVEN
         GraphDatabaseService db = dbRule.getGraphDatabaseAPI();
@@ -555,7 +556,7 @@ public void shouldNotIncludeNodesDeletedInSameTxInIndexSeekByPrefix()
 
     @Test
     public void shouldConsiderNodesChangedInSameTxInIndexPrefixSearch()
-            throws SchemaRuleNotFoundException, IndexNotFoundKernelException
+            throws SchemaRuleNotFoundException, IndexNotFoundKernelException, IndexNotApplicableKernelException
     {
         // GIVEN
         GraphDatabaseService db = dbRule.getGraphDatabaseAPI();

community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java

@@ -136,12 +136,12 @@ public void testIndexSeekByPrefixOnNonStrings() throws Exception
         assertThat( query( IndexQuery.stringPrefix( 1, "2" ) ), equalTo( EMPTY_LIST ) );
     }
 
-    protected List<Long> query( IndexQuery... predicates )
+    protected List<Long> query( IndexQuery... predicates ) throws Exception
     {
         return metaGet( reader -> reader.query( predicates ) );
     }
 
-    private List<Long> metaGet( ReaderInteraction interaction )
+    private List<Long> metaGet( ReaderInteraction interaction ) throws Exception
     {
         try ( IndexReader reader = accessor.newReader() )
         {
@@ -157,7 +157,7 @@ private List<Long> metaGet( ReaderInteraction interaction )
 
     private interface ReaderInteraction
     {
-        PrimitiveLongIterator results( IndexReader reader );
+        PrimitiveLongIterator results( IndexReader reader ) throws Exception;
     }
 
     protected void updateAndCommit( List<IndexEntryUpdate> updates )

community/kernel/src/test/java/org/neo4j/kernel/impl/api/StatementOperationsTestHelper.java

@@ -25,6 +25,7 @@
 
 import org.neo4j.collection.primitive.PrimitiveLongCollections;
 import org.neo4j.kernel.api.ReadOperations;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.schema_new.IndexQuery;
 import org.neo4j.kernel.api.txstate.TransactionState;
@@ -84,7 +85,7 @@ public static KernelStatement mockedState( final TransactionState txState )
             when( storageStatement.getIndexReader( Matchers.any() ) ).thenReturn( indexReader );
             when( state.getStoreStatement() ).thenReturn( storageStatement );
         }
-        catch ( IndexNotFoundKernelException e )
+        catch ( IndexNotFoundKernelException | IndexNotApplicableKernelException e )
         {
             throw new Error( e );
         }

community/kernel/src/test/java/org/neo4j/kernel/impl/api/integrationtest/NodeGetUniqueFromIndexSeekIT.java

@@ -22,26 +22,20 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import org.neo4j.graphdb.DependencyResolver;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.kernel.api.DataWriteOperations;
 import org.neo4j.kernel.api.ReadOperations;
 import org.neo4j.kernel.api.Statement;
 import org.neo4j.kernel.api.StatementConstants;
 import org.neo4j.kernel.api.TokenWriteOperations;
 import org.neo4j.kernel.api.exceptions.KernelException;
+import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
 import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
 import org.neo4j.kernel.api.exceptions.schema.IndexBrokenKernelException;
 import org.neo4j.kernel.api.properties.Property;
-import org.neo4j.kernel.api.schema.IndexDescriptor;
 import org.neo4j.kernel.api.schema.NodePropertyDescriptor;
-import org.neo4j.kernel.api.schema_new.LabelSchemaDescriptor;
-import org.neo4j.kernel.api.schema_new.SchemaBoundary;
-import org.neo4j.kernel.api.schema_new.SchemaDescriptorFactory;
-import org.neo4j.kernel.api.security.SecurityContext;
-import org.neo4j.kernel.api.schema_new.index.IndexBoundary;
 import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
-import org.neo4j.kernel.impl.locking.Locks;
+import org.neo4j.kernel.api.security.SecurityContext;
 import org.neo4j.test.DoubleLatch;
 
 import static org.junit.Assert.assertTrue;
@@ -152,7 +146,7 @@ public void shouldBlockUniqueIndexSeekFromCompetingTransaction() throws Exceptio
                 }
                 tx.success();
             }
-            catch ( IndexNotFoundKernelException | IndexBrokenKernelException e )
+            catch ( IndexNotFoundKernelException | IndexNotApplicableKernelException | IndexBrokenKernelException e )
             {
                 throw new RuntimeException( e );
             }