Allow consistency checker to warn about indexes that are dirty

and can return wrong results

Dirty native index is an index that will require recovery on next non
read only startup and potentially any data that is in the index
on a moment when index is dirty are not fully trustable.

This commit allow CC to notify about those indexes during CC check and
report those.

community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/ConsistencyCheckTasks.java

@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.neo4j.consistency.RecordType;
 import org.neo4j.consistency.checking.NodeRecordCheck;
 import org.neo4j.consistency.checking.PropertyChain;
 import org.neo4j.consistency.checking.RelationshipRecordCheck;
@@ -33,11 +34,15 @@
 import org.neo4j.consistency.checking.index.IndexIterator;
 import org.neo4j.consistency.checking.labelscan.LabelScanCheck;
 import org.neo4j.consistency.checking.labelscan.LabelScanDocumentProcessor;
+import org.neo4j.consistency.report.ConsistencyReport;
 import org.neo4j.consistency.report.ConsistencyReporter;
 import org.neo4j.consistency.statistics.Statistics;
+import org.neo4j.consistency.store.synthetic.IndexRecord;
+import org.neo4j.consistency.store.synthetic.LabelScanIndex;
 import org.neo4j.helpers.collection.BoundedIterable;
 import org.neo4j.helpers.progress.ProgressMonitorFactory;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
+import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
 import org.neo4j.kernel.impl.store.RecordStore;
 import org.neo4j.kernel.impl.store.Scanner;
 import org.neo4j.kernel.impl.store.SchemaStorage;
@@ -176,13 +181,15 @@ public List<ConsistencyCheckerTask> createTasksForFullCheck( boolean checkLabelS
         if ( checkLabelScanStore )
         {
             long highId = nativeStores.getNodeStore().getHighId();
+            tasks.add( new LabelIndexDirtyCheckTask() );
             tasks.add( recordScanner( "LabelScanStore",
                     new GapFreeAllEntriesLabelScanReader( labelScanStore.allNodeLabelRanges(), highId ),
                     new LabelScanDocumentProcessor( filteredReporter, new LabelScanCheck() ), Stage.SEQUENTIAL_FORWARD,
                     ROUND_ROBIN ) );
         }
         if ( checkIndexes )
         {
+            tasks.add( new IndexDirtyCheckTask() );
             for ( IndexRule indexRule : indexes.onlineRules() )
             {
                 tasks.add( recordScanner( format( "Index_%d", indexRule.getId() ),
@@ -219,4 +226,47 @@ private <RECORD extends AbstractBaseRecord> StoreProcessorTask<RECORD> create( S
         return new StoreProcessorTask<>( name, statistics, numberOfThreads, input, nativeStores, name, progress,
                 cacheAccess, processor, distribution );
     }
+
+    private class LabelIndexDirtyCheckTask extends ConsistencyCheckerTask
+    {
+        LabelIndexDirtyCheckTask()
+        {
+            super( "Label index dirty check", Statistics.NONE, 1 );
+        }
+
+        @Override
+        public void run()
+        {
+            if ( labelScanStore instanceof NativeLabelScanStore )
+            {
+                if ( ((NativeLabelScanStore)labelScanStore).isDirty() )
+                {
+                    reporter.report( new LabelScanIndex(), ConsistencyReport.LabelScanConsistencyReport.class,
+                            RecordType.LABEL_SCAN_DOCUMENT ).dirtyIndex();
+                }
+            }
+        }
+    }
+
+    private class IndexDirtyCheckTask extends ConsistencyCheckerTask
+    {
+        IndexDirtyCheckTask()
+        {
+            super( "Indexes dirty check", Statistics.NONE, 1 );
+        }
+
+        @Override
+        public void run()
+        {
+            for ( IndexRule indexRule : indexes.onlineRules() )
+            {
+                if ( indexes.accessorFor( indexRule ).isDirty() )
+                {
+                    reporter.report( new IndexRecord( indexRule ), ConsistencyReport.IndexConsistencyReport.class,
+                            RecordType.INDEX ).dirtyIndex();
+                }
+            }
+
+        }
+    }
 }

community/consistency-check/src/main/java/org/neo4j/consistency/report/ConsistencyReport.java

@@ -492,6 +492,10 @@ interface LabelScanConsistencyReport extends NodeInUseWithCorrectLabelsReport
         @Override
         @Documented( "This node record has a label that is not found in the label scan store entry for this node" )
         void nodeLabelNotInIndex( NodeRecord referredNodeRecord, long missingLabelId );
+
+        @Warning
+        @Documented( "Label index was not properly shutdown and rebuild is required." )
+        void dirtyIndex();
     }
 
     interface IndexConsistencyReport extends NodeInUseWithCorrectLabelsReport
@@ -507,6 +511,10 @@ interface IndexConsistencyReport extends NodeInUseWithCorrectLabelsReport
         @Override
         @Documented( "This node record has a label that is not found in the index for this node" )
         void nodeLabelNotInIndex( NodeRecord referredNodeRecord, long missingLabelId );
+
+        @Warning
+        @Documented( "Index was not properly shutdown and rebuild is required." )
+        void dirtyIndex();
     }
 
     interface CountsConsistencyReport extends ConsistencyReport

community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/IndexRecord.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.consistency.store.synthetic;
+
+import org.neo4j.kernel.impl.store.record.AbstractBaseRecord;
+import org.neo4j.kernel.impl.store.record.IndexRule;
+
+public class IndexRecord extends AbstractBaseRecord
+{
+    private final IndexRule indexRule;
+
+    public IndexRecord( IndexRule indexRule )
+    {
+        super( indexRule.getId() );
+        this.indexRule = indexRule;
+    }
+
+    @Override
+    public void clear()
+    {
+        initialize( false );
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Index[ " + indexRule.toString() + " ]";
+    }
+}

community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/LabelScanIndex.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.consistency.store.synthetic;
+
+import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
+import org.neo4j.kernel.impl.store.record.AbstractBaseRecord;
+
+public class LabelScanIndex extends AbstractBaseRecord
+{
+    public LabelScanIndex()
+    {
+        super( NO_ID );
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Label index: " + NativeLabelScanStore.FILE_NAME;
+    }
+}

community/consistency-check/src/test/java/org/neo4j/consistency/checking/AllNodesInStoreExistInLabelIndexTest.java

@@ -22,12 +22,9 @@
 import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStreamReader;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -42,14 +39,20 @@
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
+import org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer;
+import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo;
 import org.neo4j.logging.AssertableLogProvider;
 import org.neo4j.test.rule.DatabaseRule;
 import org.neo4j.test.rule.EmbeddedDatabaseRule;
 import org.neo4j.test.rule.RandomRule;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasItem;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.neo4j.helpers.progress.ProgressMonitorFactory.NONE;
+import static org.neo4j.io.fs.FileUtils.copyFile;
 import static org.neo4j.test.TestLabels.LABEL_ONE;
 import static org.neo4j.test.TestLabels.LABEL_THREE;
 import static org.neo4j.test.TestLabels.LABEL_TWO;
@@ -83,6 +86,31 @@ public void mustReportSuccessfulForConsistentLabelScanStore() throws Exception
         assertTrue( "Expected consistency check to succeed", result.isSuccessful() );
     }
 
+    @Test
+    public void reportNotCleanLabelIndex() throws IOException, ConsistencyCheckIncompleteException
+    {
+        File storeDir = db.getStoreDir();
+        someData();
+        db.resolveDependency( CheckPointer.class ).forceCheckPoint( new SimpleTriggerInfo( "forcedCheckpoint" ) );
+        File labelIndexFileCopy = new File( storeDir, "label_index_copy" );
+        copyFile( new File( storeDir, NativeLabelScanStore.FILE_NAME ), labelIndexFileCopy );
+
+        try ( Transaction tx = db.beginTx() )
+        {
+            db.createNode( LABEL_ONE );
+            tx.success();
+        }
+
+        db.shutdownAndKeepStore();
+
+        copyFile( labelIndexFileCopy, new File( storeDir, NativeLabelScanStore.FILE_NAME ) );
+
+        ConsistencyCheckService.Result result = fullConsistencyCheck();
+        assertFalse( "Expected consistency check to fail", result.isSuccessful() );
+        assertThat( readReport( result ),
+                hasItem( containsString("WARN : Label index was not properly shutdown and rebuild is required.") ) );
+    }
+
     @Test
     public void mustReportMissingNode() throws Exception
     {
@@ -175,27 +203,10 @@ public void mustReportExtraNode() throws Exception
         assertFalse( "Expected consistency check to fail", result.isSuccessful() );
     }
 
-    // Do not remove this, very useful for debugging
-    private void printReport( ConsistencyCheckService.Result result )
+    private List<String> readReport( ConsistencyCheckService.Result result )
             throws IOException
     {
-        try
-        {
-            FileInputStream fileInputStream = new FileInputStream( result.reportFile() );
-
-            BufferedReader bufferedReader = new BufferedReader( new InputStreamReader( fileInputStream ) );
-            String line;
-            while ( (line = bufferedReader.readLine()) != null )
-            {
-                System.out.println( line );
-            }
-        }
-        catch ( FileNotFoundException e )
-        {
-            // Fine dont print it then, just report
-            System.out
-                    .println( "FileNotFoundException, might just mean that consistency checker had nothing to report" );
-        }
+        return Files.readAllLines( result.reportFile().toPath() );
     }
 
     private void removeExistingNode( List<Pair<Long,Label[]>> nodesInStore )