NativeLSS will redo population if did not complete first time

Before if NativeLabelScanStore was initialised (init) for the first time without being populated (start), then it would not retry population job on next start. Now it will note in GBPTree header that it needs to be rebuilt on next start and only clean this header after population has completed.
neo4j · Jun 13, 2017 · 403d5ff · 403d5ff
1 parent 8ebde02
commit 403d5ff
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 26 deletions.
diff --git a/...nity/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java b/...nity/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java
@@ -93,7 +93,7 @@ public class NativeLabelScanStore implements LabelScanStore
     /**
      * Written in header to indicate native label scan store is rebuilding
      */
-    private static final byte REBUILDING = (byte) 0x01;
+    private static final byte NEEDS_REBUILDING = (byte) 0x01;
 
     /**
      * Native label index tag, to distinguish native label index from other label indexes
@@ -157,7 +157,8 @@ public class NativeLabelScanStore implements LabelScanStore
     /**
      * Write rebuilding bit to header.
      */
-    private static final Consumer<PageCursor> writeRebuilding = pageCursor -> pageCursor.putByte( REBUILDING );
+    private static final Consumer<PageCursor> needsRebuildingWriter =
+            pageCursor -> pageCursor.putByte( NEEDS_REBUILDING );
 
     /**
      * Write clean header.
@@ -352,11 +353,12 @@ private boolean instantiateTree() throws IOException
     {
         monitors.addMonitorListener( treeMonitor(), NATIVE_LABEL_INDEX_TAG );
         GBPTree.Monitor monitor = monitors.newMonitor( GBPTree.Monitor.class, NATIVE_LABEL_INDEX_TAG );
-        MutableBoolean isRebuilding = new MutableBoolean();
-        Header.Reader readRebuilding =
-                (pageCursor, length) -> isRebuilding.setValue( pageCursor.getByte() == REBUILDING );
-        index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout(), pageSize, monitor, readRebuilding );
-        return isRebuilding.getValue();
+        MutableBoolean needsRebuilding = new MutableBoolean();
+        Header.Reader headerReader =
+                (pageCursor, length) -> needsRebuilding.setValue( pageCursor.getByte() == NEEDS_REBUILDING );
+        index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout(), pageSize, monitor, headerReader,
+                needsRebuildingWriter );
+        return needsRebuilding.getValue();
     }
 
     private GBPTree.Monitor treeMonitor()
@@ -412,8 +414,6 @@ public void start() throws IOException
             monitor.rebuilding();
             long numberOfNodes;
 
-            index.checkpoint( IOLimiter.unlimited(), writeRebuilding );
-
             // Intentionally ignore read-only flag here when rebuilding.
             try ( LabelScanWriter writer = writer() )
             {

diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java
@@ -55,6 +55,7 @@
 import org.neo4j.kernel.api.labelscan.LabelScanWriter;
 import org.neo4j.kernel.api.labelscan.NodeLabelRange;
 import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
+import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
 import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.storageengine.api.schema.LabelScanReader;
 import org.neo4j.test.rule.RandomRule;
@@ -73,12 +74,13 @@
 import static org.neo4j.helpers.collection.Iterators.iterator;
 import static org.neo4j.helpers.collection.Iterators.single;
 import static org.neo4j.kernel.api.labelscan.NodeLabelUpdate.labelChanges;
+import static org.neo4j.kernel.impl.api.scan.FullStoreChangeStream.asStream;
 
 public abstract class LabelScanStoreTest
 {
     private final TestDirectory testDirectory = TestDirectory.testDirectory();
     private final ExpectedException expectedException = ExpectedException.none();
-    private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();
+    protected final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();
     final RandomRule random = new RandomRule();
 
     @Rule
@@ -90,7 +92,7 @@ public abstract class LabelScanStoreTest
     private LifeSupport life;
     private TrackingMonitor monitor;
     private LabelScanStore store;
-    private File dir;
+    protected File dir;
 
     @Before
     public void clearDir() throws IOException
@@ -101,11 +103,14 @@ public void clearDir() throws IOException
     @After
     public void shutdown()
     {
-        life.shutdown();
+        if ( life != null )
+        {
+            life.shutdown();
+        }
     }
 
     protected abstract LabelScanStore createLabelScanStore( FileSystemAbstraction fileSystemAbstraction,
-            File rootFolder, List<NodeLabelUpdate> existingData, boolean usePersistentStore, boolean readOnly,
+            File rootFolder, FullStoreChangeStream fullStoreChangeStream, boolean usePersistentStore, boolean readOnly,
             LabelScanStore.Monitor monitor );
 
     @Test
@@ -550,7 +555,8 @@ private void start( List<NodeLabelUpdate> existingData, boolean usePersistentSto
         life = new LifeSupport();
         monitor = new TrackingMonitor();
 
-        store = createLabelScanStore( fileSystemRule.get(), dir, existingData, usePersistentStore, readOnly, monitor );
+        store = createLabelScanStore( fileSystemRule.get(), dir, asStream( existingData ), usePersistentStore, readOnly,
+                monitor );
         life.add( store );
 
         life.start();
@@ -594,10 +600,13 @@ private static void putRandomBytes( Random random, byte[] bytes )
         }
     }
 
-    private static class TrackingMonitor extends LabelScanStore.Monitor.Adaptor
+    public static class TrackingMonitor extends LabelScanStore.Monitor.Adaptor
     {
-        boolean initCalled, rebuildingCalled, rebuiltCalled, noIndexCalled;
-        boolean corruptedIndex = false;
+        boolean initCalled;
+        public boolean rebuildingCalled;
+        public boolean rebuiltCalled;
+        public boolean noIndexCalled;
+        public boolean corruptedIndex = false;
 
         @Override
         public void noIndex()
@@ -633,5 +642,10 @@ public void init()
         {
             initCalled = true;
         }
+
+        public void reset()
+        {
+            initCalled = rebuildingCalled = rebuiltCalled = noIndexCalled = corruptedIndex = false;
+        }
     }
 }
diff --git a/.../kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java b/.../kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStoreTest.java
@@ -22,20 +22,22 @@
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
+import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.List;
 
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.api.impl.labelscan.LabelScanStoreTest;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
-import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
+import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
+import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.kernel.monitoring.Monitors;
 import org.neo4j.test.rule.PageCacheRule;
 
-import static org.neo4j.kernel.impl.api.scan.FullStoreChangeStream.asStream;
+import static org.junit.Assert.assertTrue;
+import static org.neo4j.kernel.impl.api.scan.FullStoreChangeStream.EMPTY;
 
 public class NativeLabelScanStoreTest extends LabelScanStoreTest
 {
@@ -44,14 +46,14 @@ public class NativeLabelScanStoreTest extends LabelScanStoreTest
 
     @Override
     protected LabelScanStore createLabelScanStore( FileSystemAbstraction fileSystemAbstraction, File rootFolder,
-            List<NodeLabelUpdate> existingData, boolean usePersistentStore, boolean readOnly,
+            FullStoreChangeStream fullStoreChangeStream, boolean usePersistentStore, boolean readOnly,
             LabelScanStore.Monitor monitor )
     {
         Monitors monitors = new Monitors();
         monitors.addMonitorListener( monitor );
         PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction );
         return new NativeLabelScanStore( pageCache, rootFolder,
-                asStream( existingData ), readOnly, monitors );
+                fullStoreChangeStream, readOnly, monitors );
     }
 
     @Override
@@ -66,4 +68,32 @@ protected void corruptIndex( FileSystemAbstraction fileSystem, File rootFolder )
         File lssFile = new File( rootFolder, NativeLabelScanStore.FILE_NAME );
         scrambleFile( lssFile );
     }
+
+    @Test
+    public void shouldStartPopulationAgainIfNotCompletedFirstTime() throws Exception
+    {
+        // given
+        // label scan store init but no start
+        LifeSupport life = new LifeSupport();
+        TrackingMonitor monitor = new TrackingMonitor();
+        life.add( createLabelScanStore( fileSystemRule.get(), dir, EMPTY, true, false, monitor ) );
+        life.init();
+        assertTrue( monitor.noIndexCalled );
+        monitor.reset();
+        life.shutdown();
+
+        // when
+        // starting label scan store again
+        life = new LifeSupport();
+        life.add( createLabelScanStore( fileSystemRule.get(), dir, EMPTY, true, false, monitor ) );
+        life.init();
+
+        // then
+        // label scan store should recognize it still needs to be rebuilt
+        assertTrue( monitor.corruptedIndex );
+        life.start();
+        assertTrue( monitor.rebuildingCalled );
+        assertTrue( monitor.rebuiltCalled );
+        life.shutdown();
+    }
 }
diff --git a/...ene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneLabelScanStoreTest.java b/...ene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneLabelScanStoreTest.java
@@ -35,14 +35,13 @@
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 import org.neo4j.kernel.api.impl.labelscan.storestrategy.BitmapDocumentFormat;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
-import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
 import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
 import org.neo4j.kernel.impl.factory.OperationalMode;
 
 import static java.util.Arrays.asList;
 import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.core.IsCollectionContaining.hasItem;
-import static org.neo4j.kernel.impl.api.scan.FullStoreChangeStream.asStream;
 
 @RunWith( Parameterized.class )
 public class LuceneLabelScanStoreTest extends LabelScanStoreTest
@@ -60,7 +59,7 @@ public static List<BitmapDocumentFormat> parameterizedWithStrategies()
 
     @Override
     protected LabelScanStore createLabelScanStore( FileSystemAbstraction fs, File rootFolder,
-            List<NodeLabelUpdate> existingData, boolean usePersistentStore, boolean readOnly,
+            FullStoreChangeStream fullStoreChangeStream, boolean usePersistentStore, boolean readOnly,
             LabelScanStore.Monitor monitor )
     {
         DirectoryFactory directoryFactory = usePersistentStore ? DirectoryFactory.PERSISTENT : inMemoryDirectoryFactory;
@@ -77,7 +76,7 @@ protected LabelScanStore createLabelScanStore( FileSystemAbstraction fs, File ro
                 .withConfig( config )
                 .withDocumentFormat( documentFormat );
 
-        return new LuceneLabelScanStore( indexBuilder, asStream( existingData ), monitor );
+        return new LuceneLabelScanStore( indexBuilder, fullStoreChangeStream, monitor );
     }
 
     @Override