Makes index value validation customizable per IndexAccessor

Instead of having on global static tailored to lucene.
This allows native indexes to have their own validation.

community/index/src/main/java/org/neo4j/index/internal/gbptree/DynamicSizeUtil.java

@@ -95,7 +95,7 @@ class DynamicSizeUtil
     // mask for one-byte key size to map to the k's in [_,_,_,k,k,k,k,k]
     static final int MASK_ONE_BYTE_KEY_SIZE = 0x1F;
     // max two-byte key size to map to the k's in [_,_,_,k,k,k,k,k][_,k,k,k,k,k,k,k]
-    private static final int MAX_TWO_BYTE_KEY_SIZE = 0xFFF;
+    static final int MAX_TWO_BYTE_KEY_SIZE = 0xFFF;
     // mask for one-byte value size to map to the v's in [_,v,v,v,v,v,v,v]
     static final int MASK_ONE_BYTE_VALUE_SIZE = 0x7F;
     // max two-byte value size to map to the v's in [_,v,v,v,v,v,v,v][v,v,v,v,v,v,v,v]

community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java

@@ -63,6 +63,8 @@
  */
 public class TreeNodeDynamicSize<KEY, VALUE> extends TreeNode<KEY,VALUE>
 {
+    public static final int MAX_KEY_SIZE = DynamicSizeUtil.MAX_TWO_BYTE_KEY_SIZE;
+
     static final byte FORMAT_IDENTIFIER = 3;
     static final byte FORMAT_VERSION = 0;
 

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -681,7 +681,7 @@ private NeoStoreKernelModule buildKernel( LogFiles logFiles, TransactionAppender
         StatementOperationParts statementOperationParts = dependencies.satisfyDependency(
                 buildStatementOperations( storeLayer, autoIndexing,
                         constraintIndexCreator, databaseSchemaState, explicitIndexStore, cpuClockRef,
-                        heapAllocationRef ) );
+                        heapAllocationRef, indexingService ) );
 
         TransactionHooks hooks = new TransactionHooks();
 
@@ -691,7 +691,7 @@ private NeoStoreKernelModule buildKernel( LogFiles logFiles, TransactionAppender
                 availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock,
                 cpuClockRef, heapAllocationRef, accessCapability, DefaultCursors::new, autoIndexing,
                 explicitIndexStore, versionContextSupplier, collectionsFactorySupplier, constraintSemantics,
-                databaseSchemaState ) );
+                databaseSchemaState, indexingService ) );
 
         buildTransactionMonitor( kernelTransactions, clock, config );
 
@@ -866,13 +866,14 @@ public DependencyResolver getDependencyResolver()
     private StatementOperationParts buildStatementOperations( StoreReadLayer storeReadLayer, AutoIndexing autoIndexing,
             ConstraintIndexCreator constraintIndexCreator, DatabaseSchemaState databaseSchemaState,
             ExplicitIndexStore explicitIndexStore, AtomicReference<CpuClock> cpuClockRef,
-            AtomicReference<HeapAllocation> heapAllocationRef )
+            AtomicReference<HeapAllocation> heapAllocationRef,
+            IndexingService indexingService )
     {
         // The passed in StoreReadLayer is the bottom most layer: Read-access to committed data.
         // To it we add:
         // + Transaction state handling
         StateHandlingStatementOperations stateHandlingContext = new StateHandlingStatementOperations( storeReadLayer,
-                autoIndexing, constraintIndexCreator, explicitIndexStore );
+                autoIndexing, constraintIndexCreator, explicitIndexStore, indexingService );
 
         QueryRegistrationOperations queryRegistrationOperations =
                 new StackingQueryRegistrationOperations( clock, cpuClockRef, heapAllocationRef );

community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java

@@ -30,7 +30,9 @@
 import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
 import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
 import org.neo4j.kernel.impl.api.index.updater.SwallowingIndexUpdater;
+import org.neo4j.kernel.impl.util.Validator;
 import org.neo4j.storageengine.api.schema.IndexReader;
+import org.neo4j.values.storable.Value;
 
 import static java.util.Collections.emptyIterator;
 import static org.neo4j.helpers.collection.Iterators.emptyResourceIterator;
@@ -53,7 +55,7 @@ public interface IndexAccessor extends Closeable
     /**
      * Return an updater for applying a set of changes to this index.
      * Updates must be visible in {@link #newReader() readers} created after this update.
-     *
+     * <p>
      * This is called with IndexUpdateMode.RECOVERY when starting up after
      * a crash or similar. Updates given then may have already been applied to this index, so
      * additional checks must be in place so that data doesn't get duplicated, but is idempotent.
@@ -120,6 +122,14 @@ public interface IndexAccessor extends Closeable
      */
     boolean isDirty();
 
+    /**
+     * @return a {@link Validator} capable of validating {@link Value values} before transaction determines that it can commit.
+     */
+    default void validateBeforeCommit( Value[] tuple )
+    {
+        // For most types values there are no specific validations to be made.
+    }
+
     class Adapter implements IndexAccessor
     {
         @Override
@@ -271,5 +281,11 @@ public boolean isDirty()
         {
             return delegate.isDirty();
         }
+
+        @Override
+        public void validateBeforeCommit( Value[] tuple )
+        {
+            delegate.validateBeforeCommit( tuple );
+        }
     }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexTextValueLengthValidator.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.api;
+
+import org.neo4j.kernel.impl.util.Validator;
+import org.neo4j.values.storable.TextValue;
+import org.neo4j.values.storable.Value;
+import org.neo4j.values.storable.Values;
+
+/**
+ * Validates {@link TextValue text values} so that they are within a certain length, byte-wise.
+ */
+public class IndexTextValueLengthValidator implements Validator<Value>
+{
+    private final int maxByteLength;
+    private final int checkThreshold;
+
+    public IndexTextValueLengthValidator( int maxByteLength )
+    {
+        this.maxByteLength = maxByteLength;
+
+        // This check threshold is for not having to check every value that comes in, only those that may have a chance to exceed the max length.
+        // The value 5 comes from a safer 4, which is the number of bytes that a max size UTF-8 code point needs.
+        this.checkThreshold = maxByteLength / 5;
+    }
+
+    @Override
+    public void validate( Value value )
+    {
+        if ( value == null || value == Values.NO_VALUE )
+        {
+            throw new IllegalArgumentException( "Null value" );
+        }
+        if ( Values.isTextValue( value ) && ((TextValue)value).length() >= checkThreshold )
+        {
+            validate( ((TextValue)value).stringValue().getBytes() );
+        }
+    }
+
+    public void validate( byte[] encodedValue )
+    {
+        if ( encodedValue == null )
+        {
+            throw new IllegalArgumentException( "Null value" );
+        }
+
+        int byteLength = encodedValue.length;
+        if ( byteLength > maxByteLength )
+        {
+            throw new IllegalArgumentException( "Property value bytes length: " + byteLength +
+                    " is longer than " + maxByteLength + ", which is maximum supported length" +
+                    " of indexed property value." );
+        }
+    }
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java

@@ -68,6 +68,7 @@
 import org.neo4j.kernel.api.txstate.ExplicitIndexTransactionState;
 import org.neo4j.kernel.api.txstate.TransactionState;
 import org.neo4j.kernel.api.txstate.TxStateHolder;
+import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.api.state.ConstraintIndexCreator;
 import org.neo4j.kernel.impl.api.state.TxState;
 import org.neo4j.kernel.impl.constraints.ConstraintSemantics;
@@ -193,7 +194,7 @@ public KernelTransactionImplementation( StatementOperationParts statementOperati
             StorageEngine storageEngine, AccessCapability accessCapability, DefaultCursors cursors, AutoIndexing autoIndexing,
             ExplicitIndexStore explicitIndexStore, VersionContextSupplier versionContextSupplier,
             CollectionsFactorySupplier collectionsFactorySupplier, ConstraintSemantics constraintSemantics,
-            SchemaState schemaState )
+            SchemaState schemaState, IndexingService indexingService )
     {
         this.statementOperations = statementOperations;
         this.schemaWriteGuard = schemaWriteGuard;
@@ -223,7 +224,7 @@ public KernelTransactionImplementation( StatementOperationParts statementOperati
         this.operations =
                 new Operations(
                         allStoreHolder,
-                        new IndexTxStateUpdater( storageEngine.storeReadLayer(), allStoreHolder ),
+                        new IndexTxStateUpdater( storageEngine.storeReadLayer(), allStoreHolder, indexingService ),
                         storageStatement,
                         this, new KernelToken( storeLayer, this ), cursors, autoIndexing, constraintIndexCreator,
                         constraintSemantics );

community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java

@@ -40,6 +40,7 @@
 import org.neo4j.kernel.api.exceptions.Status;
 import org.neo4j.kernel.api.explicitindex.AutoIndexing;
 import org.neo4j.kernel.api.txstate.ExplicitIndexTransactionState;
+import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.api.state.ConstraintIndexCreator;
 import org.neo4j.kernel.impl.api.state.ExplicitIndexTransactionStateImpl;
 import org.neo4j.kernel.impl.constraints.ConstraintSemantics;
@@ -100,6 +101,7 @@ public class KernelTransactions extends LifecycleAdapter implements Supplier<Ker
     private final Supplier<DefaultCursors> cursorsSupplier;
     private final AutoIndexing autoIndexing;
     private final ExplicitIndexStore explicitIndexStore;
+    private final IndexingService indexingService;
     private final CollectionsFactorySupplier collectionsFactorySupplier;
     private final SchemaState schemaState;
 
@@ -147,7 +149,8 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory,
             VersionContextSupplier versionContextSupplier,
             CollectionsFactorySupplier collectionsFactorySupplier,
             ConstraintSemantics constraintSemantics,
-            SchemaState schemaState )
+            SchemaState schemaState,
+            IndexingService indexingService )
     {
         this.statementLocksFactory = statementLocksFactory;
         this.constraintIndexCreator = constraintIndexCreator;
@@ -167,6 +170,7 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory,
         this.accessCapability = accessCapability;
         this.autoIndexing = autoIndexing;
         this.explicitIndexStore = explicitIndexStore;
+        this.indexingService = indexingService;
         this.explicitIndexTxStateSupplier = () ->
                 new CachingExplicitIndexTransactionState(
                         new ExplicitIndexTransactionStateImpl( indexConfigStore, explicitIndexProviderLookup ) );
@@ -370,7 +374,7 @@ public KernelTransactionImplementation newInstance()
                             tracers.pageCursorTracerSupplier, storageEngine, accessCapability,
                             cursorsSupplier.get(), autoIndexing,
                             explicitIndexStore, versionContextSupplier, collectionsFactorySupplier, constraintSemantics,
-                            schemaState );
+                            schemaState, indexingService );
             this.transactions.add( tx );
             return tx;
         }

community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java

@@ -70,6 +70,7 @@
 import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptorFactory;
 import org.neo4j.kernel.api.txstate.TransactionCountingStateVisitor;
 import org.neo4j.kernel.api.txstate.TransactionState;
+import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.api.operations.CountsOperations;
 import org.neo4j.kernel.impl.api.operations.EntityOperations;
 import org.neo4j.kernel.impl.api.operations.ExplicitIndexReadOperations;
@@ -136,13 +137,14 @@ public class StateHandlingStatementOperations implements
     public StateHandlingStatementOperations(
             StoreReadLayer storeLayer, AutoIndexing propertyTrackers,
             ConstraintIndexCreator constraintIndexCreator,
-            ExplicitIndexStore explicitIndexStore )
+            ExplicitIndexStore explicitIndexStore,
+            IndexingService indexingService )
     {
         this.storeLayer = storeLayer;
         this.autoIndexing = propertyTrackers;
         this.constraintIndexCreator = constraintIndexCreator;
         this.explicitIndexStore = explicitIndexStore;
-        this.indexTxStateUpdater = new IndexTxStateUpdater( storeLayer, this );
+        this.indexTxStateUpdater = new IndexTxStateUpdater( storeLayer, this, indexingService );
     }
 
     // <Cursors>

community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/AbstractDelegatingIndexProxy.java

@@ -38,6 +38,7 @@
 import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptor;
 import org.neo4j.storageengine.api.schema.IndexReader;
 import org.neo4j.storageengine.api.schema.PopulationProgress;
+import org.neo4j.values.storable.Value;
 
 public abstract class AbstractDelegatingIndexProxy implements IndexProxy
 {
@@ -133,6 +134,12 @@ public void validate() throws IndexPopulationFailedKernelException, UniqueProper
         getDelegate().validate();
     }
 
+    @Override
+    public void validateBeforeCommit( Value[] tuple )
+    {
+        getDelegate().validateBeforeCommit( tuple );
+    }
+
     @Override
     public IndexPopulationFailure getPopulationFailure() throws IllegalStateException
     {