Enhance consistency report parsing in dump logical log tool

Enhance consistency report parsing capabilities by including entity ids
from reported inconsistent indexes; collect inconsistent indexes ids;
skip property warning lines.

tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java

@@ -29,8 +29,6 @@
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
-import org.neo4j.collection.primitive.Primitive;
-import org.neo4j.collection.primitive.PrimitiveLongSet;
 import org.neo4j.cursor.IOCursor;
 import org.neo4j.helpers.Args;
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
@@ -40,6 +38,7 @@
 import org.neo4j.kernel.impl.transaction.command.Command.PropertyCommand;
 import org.neo4j.kernel.impl.transaction.command.Command.RelationshipCommand;
 import org.neo4j.kernel.impl.transaction.command.Command.RelationshipGroupCommand;
+import org.neo4j.kernel.impl.transaction.command.Command.SchemaRuleCommand;
 import org.neo4j.kernel.impl.transaction.log.FilteringIOCursor;
 import org.neo4j.kernel.impl.transaction.log.LogEntryCursor;
 import org.neo4j.kernel.impl.transaction.log.LogVersionBridge;
@@ -56,7 +55,7 @@
 import org.neo4j.kernel.impl.transaction.log.entry.LogHeader;
 import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader;
 import org.neo4j.storageengine.api.StorageCommand;
-import org.neo4j.tools.dump.InconsistencyReportReader.Inconsistencies;
+import org.neo4j.tools.dump.inconsistency.ReportInconsistencies;
 
 import static java.util.TimeZone.getTimeZone;
 import static org.neo4j.helpers.Format.DEFAULT_TIME_ZONE;
@@ -185,40 +184,13 @@ public boolean test( LogEntry[] transaction )
     public static class ConsistencyCheckOutputCriteria implements Predicate<LogEntry[]>, Function<LogEntry,String>
     {
         private final TimeZone timeZone;
-        private final PrimitiveLongSet relationshipIds = Primitive.longSet();
-        private final PrimitiveLongSet nodeIds = Primitive.longSet();
-        private final PrimitiveLongSet propertyIds = Primitive.longSet();
-        private final PrimitiveLongSet relationshipGroupIds = Primitive.longSet();
+        private ReportInconsistencies inconsistencies;
 
         public ConsistencyCheckOutputCriteria( String ccFile, TimeZone timeZone ) throws IOException
         {
             this.timeZone = timeZone;
-            new InconsistencyReportReader( new Inconsistencies()
-            {
-                @Override
-                public void relationshipGroup( long id )
-                {
-                    relationshipGroupIds.add( id );
-                }
-
-                @Override
-                public void relationship( long id )
-                {
-                    relationshipIds.add( id );
-                }
-
-                @Override
-                public void property( long id )
-                {
-                    propertyIds.add( id );
-                }
-
-                @Override
-                public void node( long id )
-                {
-                    nodeIds.add( id );
-                }
-            } ).read( new File( ccFile ) );
+            inconsistencies = new ReportInconsistencies();
+            new InconsistencyReportReader( inconsistencies ).read( new File( ccFile ) );
         }
 
         @Override
@@ -250,19 +222,23 @@ private boolean matches( StorageCommand command )
         {
             if ( command instanceof NodeCommand )
             {
-                return nodeIds.contains( ((NodeCommand) command).getKey() );
+                return inconsistencies.containsNodeId( ((NodeCommand) command).getKey() );
             }
             if ( command instanceof RelationshipCommand )
             {
-                return relationshipIds.contains( ((RelationshipCommand) command).getKey() );
+                return inconsistencies.containsRelationshipId( ((RelationshipCommand) command).getKey() );
             }
             if ( command instanceof PropertyCommand )
             {
-                return propertyIds.contains( ((PropertyCommand) command).getKey() );
+                return inconsistencies.containsPropertyId( ((PropertyCommand) command).getKey() );
             }
             if ( command instanceof RelationshipGroupCommand )
             {
-                return relationshipGroupIds.contains( ((RelationshipGroupCommand) command).getKey() );
+                return inconsistencies.containsRelationshipGroupId( ((RelationshipGroupCommand) command).getKey() );
+            }
+            if ( command instanceof SchemaRuleCommand )
+            {
+                return inconsistencies.containsSchemaIndexId( ((SchemaRuleCommand) command).getKey() );
             }
             return false;
         }

tools/src/main/java/org/neo4j/tools/dump/InconsistencyReportReader.java

@@ -23,7 +23,8 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.Reader;
+
+import org.neo4j.tools.dump.inconsistency.Inconsistencies;
 
 /**
  * Reads CC inconsistency reports. Example of entry:
@@ -38,45 +39,31 @@ public class InconsistencyReportReader
 {
     private final Inconsistencies inconsistencies;
 
-    public interface Inconsistencies
-    {
-        void node( long id );
-
-        void relationship( long id );
-
-        void property( long id );
-
-        void relationshipGroup( long id );
-
-        // TODO: incomplete list/arguments, driven by actual need a.t.m.
-    }
-
     public InconsistencyReportReader( Inconsistencies inconsistencies )
     {
         this.inconsistencies = inconsistencies;
     }
 
     public void read( File file ) throws IOException
     {
-        try ( Reader reader = new FileReader( file ) )
+        try ( BufferedReader reader = new BufferedReader( new FileReader( file )) )
         {
             read( reader );
         }
     }
 
-    public void read( Reader reader ) throws IOException
+    public void read( BufferedReader bufferedReader ) throws IOException
     {
         int state = 0; // 0:inconsistency description, 1:entity, 2:inconsistent with
-        BufferedReader bufferedReader = new BufferedReader( reader );
         String line;
         while ( (line = bufferedReader.readLine()) != null )
         {
             line = line.trim();
             if ( state == 0 )
             {
-                if ( !line.contains( "ERROR" ) && !line.contains( "WARNING" ) )
+                if ( !line.contains( " ERROR " ) && !line.contains( " WARNING " ) )
                 {
-                    break;
+                    continue;
                 }
             }
             else if ( state == 1 )
@@ -122,6 +109,12 @@ private void propagate( String entityType, long id )
         case "RelationshipGroup":
             inconsistencies.relationshipGroup( id );
             break;
+        case "IndexRule":
+            inconsistencies.schemaIndex( id );
+            break;
+        case "IndexEntry":
+            inconsistencies.node( id );
+            break;
         default:
             // it's OK, we just haven't implemented support for this yet
         }
@@ -132,15 +125,40 @@ private long id( String line )
         int bracket = line.indexOf( '[' );
         if ( bracket > -1 )
         {
-            int comma = line.indexOf( ',', bracket );
-            if ( comma > -1 )
+            int separator = min( getSeparatorIndex( ',', line, bracket ),
+                    getSeparatorIndex( ';', line, bracket ),
+                    getSeparatorIndex( ']', line, bracket ) );
+            int equally = line.indexOf( "=", bracket );
+            int startPosition = (isNotPlainId( bracket, separator, equally ) ? equally : bracket) + 1;
+            if ( separator > -1 )
             {
-                return Long.parseLong( line.substring( bracket + 1, comma ) );
+                return Long.parseLong( line.substring( startPosition, separator ) );
             }
         }
         return -1;
     }
 
+    private static int min(int... values)
+    {
+        int min = Integer.MAX_VALUE;
+        for ( int value : values )
+        {
+            min = Math.min( min, value );
+        }
+        return min;
+    }
+
+    private int getSeparatorIndex( char character, String line, int bracket )
+    {
+        int index = line.indexOf( character, bracket );
+        return index >= 0 ? index : Integer.MAX_VALUE;
+    }
+
+    private boolean isNotPlainId( int bracket, int comma, int equally )
+    {
+        return (equally > bracket) && (equally < comma);
+    }
+
     private String entityType( String line )
     {
         int bracket = line.indexOf( '[' );

tools/src/main/java/org/neo4j/tools/dump/inconsistency/Inconsistencies.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.tools.dump.inconsistency;
+
+/**
+ * Container for ids of entities that are considered to be inconsistent.
+ */
+public interface Inconsistencies
+{
+    void node( long id );
+
+    void relationship( long id );
+
+    void property( long id );
+
+    void relationshipGroup( long id );
+
+    void schemaIndex( long id );
+
+    boolean containsNodeId( long id );
+
+    boolean containsRelationshipId( long id );
+
+    boolean containsPropertyId( long id );
+
+    boolean containsRelationshipGroupId( long id );
+
+    boolean containsSchemaIndexId( long id );
+}

tools/src/main/java/org/neo4j/tools/dump/inconsistency/ReportInconsistencies.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.tools.dump.inconsistency;
+
+import org.neo4j.collection.primitive.Primitive;
+import org.neo4j.collection.primitive.PrimitiveLongSet;
+
+/**
+ * Entity ids that reported to be inconsistent in consistency report where they where extracted from.
+ */
+public class ReportInconsistencies implements Inconsistencies
+{
+    private final PrimitiveLongSet schemaIndexesIds = Primitive.longSet();
+    private final PrimitiveLongSet relationshipIds = Primitive.longSet();
+    private final PrimitiveLongSet nodeIds = Primitive.longSet();
+    private final PrimitiveLongSet propertyIds = Primitive.longSet();
+    private final PrimitiveLongSet relationshipGroupIds = Primitive.longSet();
+
+    @Override
+    public void relationshipGroup( long id )
+    {
+        relationshipGroupIds.add( id );
+    }
+
+    @Override
+    public void schemaIndex( long id )
+    {
+        schemaIndexesIds.add( id );
+    }
+
+    @Override
+    public void relationship( long id )
+    {
+        relationshipIds.add( id );
+    }
+
+    @Override
+    public void property( long id )
+    {
+        propertyIds.add( id );
+    }
+
+    @Override
+    public void node( long id )
+    {
+        nodeIds.add( id );
+    }
+
+    @Override
+    public boolean containsNodeId( long id )
+    {
+        return nodeIds.contains( id );
+    }
+
+    @Override
+    public boolean containsRelationshipId( long id )
+    {
+        return relationshipIds.contains( id );
+    }
+
+    @Override
+    public boolean containsPropertyId( long id )
+    {
+        return propertyIds.contains( id );
+    }
+
+    @Override
+    public boolean containsRelationshipGroupId( long id )
+    {
+        return relationshipGroupIds.contains( id );
+    }
+
+    @Override
+    public boolean containsSchemaIndexId( long id )
+    {
+        return schemaIndexesIds.contains( id );
+    }
+}