Merge remote-tracking branch 'upstream/3.4' into 3.5

community/community-it/index-it/src/test/java/org/neo4j/kernel/impl/index/schema/NativeStringIndexingIT.java

@@ -120,7 +120,7 @@ public void shouldFailBeforeCommitOnSizesLargerThanLimit()
         catch ( IllegalArgumentException e )
         {
             // then good
-            assertThat( e.getMessage(), containsString( "Property value bytes length: " + length + " is longer than" ) );
+            assertThat( e.getMessage(), containsString( "Property value size is too large for index. Please see index documentation for limitations." ) );
         }
     }
 
@@ -188,7 +188,7 @@ public void shouldFailBeforeCommitOnCompositeSizesLargerThanLimit()
         catch ( IllegalArgumentException e )
         {
             // then good
-            assertThat( e.getMessage(), containsString( "Property value bytes length: " + length + " is longer than" ) );
+            assertThat( e.getMessage(), containsString( "Property value size is too large for index. Please see index documentation for limitations." ) );
         }
     }
 

community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java

@@ -1368,4 +1368,14 @@ public boolean wasDirtyOnStartup()
     {
         return dirtyOnStartup;
     }
+
+    /**
+     * Total size limit for key and value.
+     * This limit includes storage overhead that is specific to key implementation for example entity id or meta data about type.
+     * @return Total size limit for key and value or {@link TreeNode#NO_KEY_VALUE_SIZE_CAP} if no such value exists.
+     */
+    public int keyValueSizeCap()
+    {
+        return bTreeNode.keyValueSizeCap();
+    }
 }

community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNode.java

@@ -64,6 +64,8 @@ enum Overflow
     static final byte INTERNAL_FLAG = 0;
     static final long NO_NODE_FLAG = 0;
 
+    static final int NO_KEY_VALUE_SIZE_CAP = -1;
+
     final Layout<KEY,VALUE> layout;
     final int pageSize;
 
@@ -211,7 +213,6 @@ long pointerGeneration( PageCursor cursor, long readResult )
      * @param baseOffset Offset to slot in logical position 0.
      * @param slotSize Size of one single slot.
      */
-
     static void insertSlotsAt( PageCursor cursor, int pos, int numberOfSlots, int totalSlotCount, int baseOffset,
             int slotSize )
     {
@@ -274,6 +275,11 @@ static void writeChild( PageCursor cursor, long child, long stableGeneration, lo
 
     // HELPERS
 
+    public int keyValueSizeCap()
+    {
+        return NO_KEY_VALUE_SIZE_CAP;
+    }
+
     abstract boolean reasonableKeyCount( int keyCount );
 
     abstract boolean reasonableChildCount( int childCount );

community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java

@@ -27,6 +27,7 @@
 import java.util.StringJoiner;
 
 import org.neo4j.io.pagecache.PageCursor;
+import org.neo4j.util.VisibleForTesting;
 
 import static java.lang.String.format;
 import static org.neo4j.index.internal.gbptree.DynamicSizeUtil.SIZE_KEY_SIZE;
@@ -66,8 +67,6 @@
  */
 public class TreeNodeDynamicSize<KEY, VALUE> extends TreeNode<KEY,VALUE>
 {
-    public static final int MAX_KEY_SIZE = DynamicSizeUtil.MAX_TWO_BYTE_KEY_SIZE;
-
     static final byte FORMAT_IDENTIFIER = 3;
     static final byte FORMAT_VERSION = 0;
 
@@ -108,7 +107,7 @@ public class TreeNodeDynamicSize<KEY, VALUE> extends TreeNode<KEY,VALUE>
         super( pageSize, layout );
         totalSpace = pageSize - HEADER_LENGTH_DYNAMIC;
         halfSpace = totalSpace / 2;
-        keyValueSizeCap = totalSpace / LEAST_NUMBER_OF_ENTRIES_PER_PAGE - SIZE_TOTAL_OVERHEAD;
+        keyValueSizeCap = keyValueSizeCapFromPageSize( pageSize );
 
         if ( keyValueSizeCap < MINIMUM_ENTRY_SIZE_CAP )
         {
@@ -122,6 +121,12 @@ public class TreeNodeDynamicSize<KEY, VALUE> extends TreeNode<KEY,VALUE>
         tmpKeyRight = layout.newKey();
     }
 
+    @VisibleForTesting
+    public static int keyValueSizeCapFromPageSize( int pageSize )
+    {
+        return (pageSize - HEADER_LENGTH_DYNAMIC) / LEAST_NUMBER_OF_ENTRIES_PER_PAGE - SIZE_TOTAL_OVERHEAD;
+    }
+
     @Override
     void writeAdditionalHeader( PageCursor cursor )
     {
@@ -360,6 +365,12 @@ void setChildAt( PageCursor cursor, long child, int pos, long stableGeneration,
         writeChild( cursor, child, stableGeneration, unstableGeneration );
     }
 
+    @Override
+    public int keyValueSizeCap()
+    {
+        return keyValueSizeCap;
+    }
+
     @Override
     boolean reasonableKeyCount( int keyCount )
     {
@@ -939,6 +950,12 @@ private void recordDeadAndAliveInternal( PageCursor cursor, MutableIntStack dead
     private void moveKeysAndChildren( PageCursor fromCursor, int fromPos, PageCursor toCursor, int toPos, int count,
             boolean includeLeftMostChild )
     {
+        if ( count == 0 && !includeLeftMostChild )
+        {
+            // Nothing to move
+            return;
+        }
+
         // All children
         // This will also copy key offsets but those will be overwritten below.
         int childFromOffset = includeLeftMostChild ? childOffset( fromPos ) : childOffset( fromPos + 1 );

community/index/src/test/java/org/neo4j/index/internal/gbptree/LargeDynamicKeysIT.java

@@ -31,6 +31,7 @@
 
 import org.neo4j.cursor.RawCursor;
 import org.neo4j.helpers.collection.Pair;
+import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.string.UTF8;
 import org.neo4j.test.rule.PageCacheAndDependenciesRule;
 import org.neo4j.test.rule.RandomRule;
@@ -39,6 +40,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.neo4j.test.rule.PageCacheRule.config;
 
 public class LargeDynamicKeysIT
 {
@@ -48,16 +50,34 @@ public class LargeDynamicKeysIT
     @Rule
     public final RandomRule random = new RandomRule();
 
+    @Test
+    public void mustStayCorrectWhenInsertingValuesOfIncreasingLength() throws IOException
+    {
+        Layout<RawBytes,RawBytes> layout = new SimpleByteArrayLayout();
+        try ( GBPTree<RawBytes,RawBytes> index = createIndex( layout );
+              Writer<RawBytes,RawBytes> writer = index.writer() )
+        {
+            RawBytes emptyValue = layout.newValue();
+            emptyValue.bytes = new byte[0];
+            for ( int keySize = 1; keySize < index.keyValueSizeCap(); keySize++ )
+            {
+                RawBytes key = layout.newKey();
+                key.bytes = new byte[keySize];
+                writer.put( key, emptyValue );
+            }
+        }
+    }
+
     @Test
     public void shouldWriteAndReadKeysOfSizesCloseToTheLimits() throws IOException
     {
         // given
-        try ( GBPTree<RawBytes,RawBytes> tree =
-                new GBPTreeBuilder<>( storage.pageCache(), storage.directory().file( "index" ), new SimpleByteArrayLayout() ).build() )
+        try ( GBPTree<RawBytes,RawBytes> tree = createIndex( new SimpleByteArrayLayout() ) )
         {
             // when
             Set<String> generatedStrings = new HashSet<>();
             List<Pair<RawBytes,RawBytes>> entries = new ArrayList<>();
+            int keyValueSizeCap = tree.keyValueSizeCap();
             try ( Writer<RawBytes,RawBytes> writer = tree.writer() )
             {
                 for ( int i = 0; i < 1_000; i++ )
@@ -71,7 +91,8 @@ public void shouldWriteAndReadKeysOfSizesCloseToTheLimits() throws IOException
                     String string;
                     do
                     {
-                        string = random.nextAlphaNumericString( 3_000, 4_043 );
+                        int keySizeCap = keyValueSizeCap - value.bytes.length;
+                        string = random.nextAlphaNumericString( keySizeCap - 1000, keySizeCap );
                     }
                     while ( !generatedStrings.add( string ) );
                     RawBytes key = new RawBytes();
@@ -96,4 +117,11 @@ public void shouldWriteAndReadKeysOfSizesCloseToTheLimits() throws IOException
             }
         }
     }
+
+    private GBPTree<RawBytes,RawBytes> createIndex( Layout<RawBytes,RawBytes> layout ) throws IOException
+    {
+        // some random padding
+        PageCache pageCache = storage.pageCacheRule().getPageCache( storage.fileSystem(), config().withAccessChecks( true ) );
+        return new GBPTreeBuilder<>( pageCache, storage.directory().file( "index" ), layout ).build();
+    }
 }

community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java

@@ -605,12 +605,12 @@ public int priority( Config config )
             "This improves read and write performance for non-composite indexed numbers. " +
             "lucene+native-2.0: Store strings in a native index and remaining value types like lucene+native-1.0. " +
             "This improves write performance for non-composite indexed strings. " +
-            "This version of the native string index has a value limit of 4047 bytes, such that byte-representation " +
+            "This version of the native string index has a value limit of 4039 bytes, such that byte-representation " +
             "of a string to index cannot be larger than that limit, or the transaction trying to index such a value will fail. " +
             "This version of the native string index also has reduced performance for CONTAINS and ENDS WITH queries, " +
             "due to resorting to index scan+filter internally. " +
             "native-btree-1.0: Store all value, including composite value, in native index. " +
-            "Just like lucene+native-2.0 there is a value size limit of 4047 bytes. For composite indexes this limit is for all of the values in " +
+            "Just like lucene+native-2.0 there is a value size limit of 4039 bytes. For composite indexes this limit is for all of the values in " +
             "the indexed key combined. The same limitations for CONTAINS and ENDS WITH queries as for lucene+native-2.0 also applies" +
             "to native-btree-1.0." +
             "Native indexes generally has these benefits over Lucene:\n" +

community/kernel/src/main/java/org/neo4j/kernel/impl/api/AbstractIndexKeyLengthValidator.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.api;
+
+import org.neo4j.kernel.impl.util.Validator;
+import org.neo4j.values.storable.TextValue;
+import org.neo4j.values.storable.Value;
+import org.neo4j.values.storable.Values;
+
+public abstract class AbstractIndexKeyLengthValidator implements Validator<Value>
+{
+    protected final int maxByteLength;
+    private final int checkThreshold;
+
+    protected AbstractIndexKeyLengthValidator( int maxByteLength )
+    {
+        this.maxByteLength = maxByteLength;
+
+        // This check threshold is for not having to check every value that comes in, only those that may have a chance to exceed the max length.
+        // The value 5 comes from a safer 4, which is the number of bytes that a max size UTF-8 code point needs.
+        this.checkThreshold = maxByteLength / 5;
+    }
+
+    @Override
+    public void validate( Value value )
+    {
+        if ( value == null || value == Values.NO_VALUE )
+        {
+            throw new IllegalArgumentException( "Null value" );
+        }
+        if ( Values.isTextValue( value ) && ((TextValue)value).length() >= checkThreshold )
+        {
+            int length = indexKeyLength( value );
+            validateLength( length );
+        }
+    }
+
+    void validateLength( int byteLength )
+    {
+        if ( byteLength > maxByteLength )
+        {
+            throw new IllegalArgumentException( "Property value size is too large for index. Please see index documentation for limitations." );
+        }
+    }
+
+    protected abstract int indexKeyLength( Value value );
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexTextValueLengthValidator.java

@@ -19,39 +19,23 @@
  */
 package org.neo4j.kernel.impl.api;
 
-import org.neo4j.kernel.impl.util.Validator;
 import org.neo4j.values.storable.TextValue;
 import org.neo4j.values.storable.Value;
-import org.neo4j.values.storable.Values;
 
 /**
  * Validates {@link TextValue text values} so that they are within a certain length, byte-wise.
  */
-public class IndexTextValueLengthValidator implements Validator<Value>
+public class IndexTextValueLengthValidator extends AbstractIndexKeyLengthValidator
 {
-    private final int maxByteLength;
-    private final int checkThreshold;
-
-    public IndexTextValueLengthValidator( int maxByteLength )
+    IndexTextValueLengthValidator( int maxByteLength )
     {
-        this.maxByteLength = maxByteLength;
-
-        // This check threshold is for not having to check every value that comes in, only those that may have a chance to exceed the max length.
-        // The value 5 comes from a safer 4, which is the number of bytes that a max size UTF-8 code point needs.
-        this.checkThreshold = maxByteLength / 5;
+        super( maxByteLength );
     }
 
     @Override
-    public void validate( Value value )
+    protected int indexKeyLength( Value value )
     {
-        if ( value == null || value == Values.NO_VALUE )
-        {
-            throw new IllegalArgumentException( "Null value" );
-        }
-        if ( Values.isTextValue( value ) && ((TextValue)value).length() >= checkThreshold )
-        {
-            validate( ((TextValue)value).stringValue().getBytes() );
-        }
+        return ((TextValue)value).stringValue().getBytes().length;
     }
 
     public void validate( byte[] encodedValue )
@@ -60,13 +44,6 @@ public void validate( byte[] encodedValue )
         {
             throw new IllegalArgumentException( "Null value" );
         }
-
-        int byteLength = encodedValue.length;
-        if ( byteLength > maxByteLength )
-        {
-            throw new IllegalArgumentException( "Property value bytes length: " + byteLength +
-                    " is longer than " + maxByteLength + ", which is maximum supported length" +
-                    " of indexed property value." );
-        }
+        validateLength( encodedValue.length );
     }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndex.java

@@ -64,6 +64,11 @@ void instantiateTree( RecoveryCleanupWorkCollector recoveryCleanupWorkCollector,
         ensureDirectoryExist();
         GBPTree.Monitor monitor = treeMonitor();
         tree = new GBPTree<>( pageCache, storeFile, layout, 0, monitor, NO_HEADER_READER, headerWriter, recoveryCleanupWorkCollector );
+        afterTreeInstantiation( tree );
+    }
+
+    protected void afterTreeInstantiation( GBPTree<KEY,VALUE> tree )
+    {   // no-op per default
     }
 
     private GBPTree.Monitor treeMonitor( )

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexKeyLengthValidator.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import org.neo4j.index.internal.gbptree.Layout;
+import org.neo4j.kernel.impl.api.AbstractIndexKeyLengthValidator;
+import org.neo4j.values.storable.Value;
+
+public class NativeIndexKeyLengthValidator<KEY extends NativeIndexKey,VALUE extends NativeIndexValue> extends AbstractIndexKeyLengthValidator
+{
+    private final Layout<KEY,VALUE> layout;
+
+    NativeIndexKeyLengthValidator( int maxByteLength, Layout<KEY,VALUE> layout )
+    {
+        super( maxByteLength );
+        this.layout = layout;
+    }
+
+    @Override
+    protected int indexKeyLength( Value value )
+    {
+        KEY key = layout.newKey();
+        key.initFromValue( 0, value, NativeIndexKey.Inclusion.NEUTRAL );
+        return layout.keySize( key );
+    }
+}