TreeNodeDynamicSize impl part 11 - Pass GBPTreeIT

TreeNodeDynamicSize
- In doDefragment, avoid copyTo with lastBlockSize==0.
  Because then targetOffset is pageSize and that causes OutOfBoundsException
- Don't rebalance from left to right if left has a smaller active space than right.
  This can be the case for the leftmost leaf.
- Update deadSpace and keyCount when moving keys and values.
- Update deadSpace when moving keys and children.

Also
- (GBPTree) Remove can now cause split in root, handle structure changes accordingly
- (GBPTree) printNode to make debugging easier
- (GBPTreeITBase) Remove @after for simplicity
- (SimpleByteArrayLayout) Better safety in getSeed to not throw when reading crap data inside shouldRetry.

community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java

@@ -1046,14 +1046,35 @@ void printTree() throws IOException
      * @param printState whether or not to print the tree state.
      * @throws IOException on I/O error.
      */
-    public void printTree( boolean printValues, boolean printPosition, boolean printState ) throws IOException
+    void printTree( boolean printValues, boolean printPosition, boolean printState ) throws IOException
     {
         try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) )
         {
             new TreePrinter<>( bTreeNode, layout, stableGeneration( generation ), unstableGeneration( generation ) )
                 .printTree( cursor, cursor, System.out, printValues, printPosition, printState );
         }
     }
+    // Utility method
+    /**
+     * Print node with given id to System.out, if node with id exists.
+     * @param id the page id of node to print
+     */
+    void printNode( int id ) throws IOException
+    {
+        if ( id < freeList.lastId() )
+        {
+            // Use write lock to avoid adversary interference
+            try ( PageCursor cursor = pagedFile.io( id, PagedFile.PF_SHARED_WRITE_LOCK ) )
+            {
+                cursor.next();
+                byte nodeType = TreeNode.nodeType( cursor );
+                if ( nodeType == TreeNode.NODE_TYPE_TREE_NODE )
+                {
+                    bTreeNode.printNode( cursor, false, true, stableGeneration( generation ), unstableGeneration( generation ) );
+                }
+            }
+        }
+    }
 
     // Utility method
     boolean consistencyCheck() throws IOException
@@ -1185,25 +1206,7 @@ public void merge( KEY key, VALUE value, ValueMerger<KEY,VALUE> valueMerger ) th
                 throw e;
             }
 
-            if ( structurePropagation.hasRightKeyInsert )
-            {
-                // New root
-                long newRootId = freeList.acquireNewId( stableGeneration, unstableGeneration );
-                PageCursorUtil.goTo( cursor, "new root", newRootId );
-
-                bTreeNode.initializeInternal( cursor, stableGeneration, unstableGeneration );
-                bTreeNode.setChildAt( cursor, structurePropagation.midChild, 0,
-                        stableGeneration, unstableGeneration );
-                bTreeNode.insertKeyAndRightChildAt( cursor, structurePropagation.rightKey, structurePropagation.rightChild, 0, 0,
-                        stableGeneration, unstableGeneration );
-                TreeNode.setKeyCount( cursor, 1 );
-                setRoot( newRootId );
-            }
-            else if ( structurePropagation.hasMidChildUpdate )
-            {
-                setRoot( structurePropagation.midChild );
-            }
-            structurePropagation.clear();
+            handleStructureChanges();
 
             checkOutOfBounds( cursor );
         }
@@ -1230,14 +1233,33 @@ public VALUE remove( KEY key ) throws IOException
                 throw e;
             }
 
-            if ( structurePropagation.hasMidChildUpdate )
+            handleStructureChanges();
+
+            checkOutOfBounds( cursor );
+            return result;
+        }
+
+        private void handleStructureChanges() throws IOException
+        {
+            if ( structurePropagation.hasRightKeyInsert )
+            {
+                // New root
+                long newRootId = freeList.acquireNewId( stableGeneration, unstableGeneration );
+                PageCursorUtil.goTo( cursor, "new root", newRootId );
+
+                bTreeNode.initializeInternal( cursor, stableGeneration, unstableGeneration );
+                bTreeNode.setChildAt( cursor, structurePropagation.midChild, 0,
+                        stableGeneration, unstableGeneration );
+                bTreeNode.insertKeyAndRightChildAt( cursor, structurePropagation.rightKey, structurePropagation.rightChild, 0, 0,
+                        stableGeneration, unstableGeneration );
+                TreeNode.setKeyCount( cursor, 1 );
+                setRoot( newRootId );
+            }
+            else if ( structurePropagation.hasMidChildUpdate )
             {
                 setRoot( structurePropagation.midChild );
             }
             structurePropagation.clear();
-
-            checkOutOfBounds( cursor );
-            return result;
         }
 
         @Override

community/index/src/main/java/org/neo4j/index/internal/gbptree/TreeNodeDynamicSize.java

@@ -483,9 +483,12 @@ For each dead space of size X (can be multiple consecutive dead keys)
             }
             // Move the last piece
             int lastBlockSize = deadRangeOffset - moveOffset;
-            deadRangeOffset -= lastBlockSize;
-            aliveRangeOffset -= lastBlockSize;
-            cursor.copyTo( deadRangeOffset, cursor, aliveRangeOffset, lastBlockSize );
+            if ( lastBlockSize > 0 )
+            {
+                deadRangeOffset -= lastBlockSize;
+                aliveRangeOffset -= lastBlockSize;
+                cursor.copyTo( deadRangeOffset, cursor, aliveRangeOffset, lastBlockSize );
+            }
         }
         while ( !aliveKeysOffset.isEmpty() );
         // Update allocOffset
@@ -541,6 +544,11 @@ int canRebalanceLeaves( PageCursor leftCursor, int leftKeyCount, PageCursor righ
             // We can merge
             return -1;
         }
+        if ( leftActiveSpace < rightActiveSpace )
+        {
+            // Moving keys to the right will only create more imbalance
+            return 0;
+        }
 
         int prevDelta;
         int currentDelta = Math.abs( leftActiveSpace - rightActiveSpace );
@@ -765,18 +773,29 @@ private void recordDeadAndAliveInternal( PageCursor cursor, PrimitiveIntStack de
         }
     }
 
+    // NOTE: Does update keyCount
     private void moveKeysAndValues( PageCursor fromCursor, int fromPos, PageCursor toCursor, int toPos, int count )
     {
         int toAllocOffset = getAllocOffset( toCursor );
+        int firstAllocOffset = toAllocOffset;
         for ( int i = 0; i < count; i++, toPos++ )
         {
             toAllocOffset = transferRawKeyValue( fromCursor, fromPos + i, toCursor, toAllocOffset );
             toCursor.setOffset( keyPosOffsetLeaf( toPos ) );
             putKeyOffset( toCursor, toAllocOffset );
         }
         setAllocOffset( toCursor, toAllocOffset );
+
+        // Update deadspace
+        int deadSpace = getDeadSpace( fromCursor );
+        int totalMovedBytes = firstAllocOffset - toAllocOffset;
+        setDeadSpace( fromCursor, deadSpace + totalMovedBytes );
+
+        // Key count
+        setKeyCount( fromCursor, fromPos );
     }
 
+    // NOTE: Does NOT update keyCount
     private void moveKeysAndChildren( PageCursor fromCursor, int fromPos, PageCursor toCursor, int toPos, int count,
             boolean includeLeftMostChild )
     {
@@ -790,6 +809,7 @@ private void moveKeysAndChildren( PageCursor fromCursor, int fromPos, PageCursor
 
         // Move actual keys and update pointers
         int toAllocOffset = getAllocOffset( toCursor );
+        int firstAllocOffset = toAllocOffset;
         for ( int i = 0; i < count; i++, toPos++ )
         {
             // Key
@@ -799,13 +819,18 @@ private void moveKeysAndChildren( PageCursor fromCursor, int fromPos, PageCursor
         }
         setAllocOffset( toCursor, toAllocOffset );
 
+        // Update deadspace
+        int deadSpace = getDeadSpace( fromCursor );
+        int totalMovedBytes = firstAllocOffset - toAllocOffset;
+        setDeadSpace( fromCursor, deadSpace + totalMovedBytes );
+
         // Zero pad empty area
         zeroPad( fromCursor, childFromOffset, lengthInBytes );
     }
 
-    private void zeroPad( PageCursor fromCursor, int childFromOffset, int lengthInBytes )
+    private void zeroPad( PageCursor fromCursor, int fromOffset, int lengthInBytes )
     {
-        fromCursor.setOffset( childFromOffset );
+        fromCursor.setOffset( fromOffset );
         fromCursor.putBytes( lengthInBytes, (byte) 0 );
     }
 

community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreeDynamixSizeIT.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.index.internal.gbptree;
+
+import org.neo4j.test.rule.RandomRule;
+
+public class GBPTreeDynamixSizeIT extends GBPTreeITBase<RawBytes,RawBytes>
+{
+    @Override
+    TestLayout<RawBytes,RawBytes> getLayout( RandomRule random )
+    {
+        return new SimpleByteArrayLayout();
+    }
+
+    @Override
+    Class<RawBytes> getKeyClass()
+    {
+        return RawBytes.class;
+    }
+}