diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java b/tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java index 89615002eaad5..6bf85a134aa7a 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java @@ -34,6 +34,7 @@ import org.neo4j.helpers.Args; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.transaction.command.Command; import org.neo4j.kernel.impl.transaction.command.Command.NodeCommand; import org.neo4j.kernel.impl.transaction.command.Command.PropertyCommand; import org.neo4j.kernel.impl.transaction.command.Command.RelationshipCommand; @@ -47,8 +48,8 @@ import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; import org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader; import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.tools.dump.InconsistentRecords.Type; import org.neo4j.tools.dump.TransactionLogAnalyzer.Monitor; -import org.neo4j.tools.dump.inconsistency.ReportInconsistencies; import static java.util.TimeZone.getTimeZone; import static org.neo4j.helpers.Format.DEFAULT_TIME_ZONE; @@ -134,12 +135,12 @@ public boolean test( LogEntry[] transaction ) public static class ConsistencyCheckOutputCriteria implements Predicate, Function { private final TimeZone timeZone; - private final ReportInconsistencies inconsistencies; + private final InconsistentRecords inconsistencies; public ConsistencyCheckOutputCriteria( String ccFile, TimeZone timeZone ) throws IOException { this.timeZone = timeZone; - inconsistencies = new ReportInconsistencies(); + inconsistencies = new InconsistentRecords(); new InconsistencyReportReader( inconsistencies ).read( new File( ccFile ) ); } @@ -169,28 +170,35 @@ private boolean matches( LogEntry logEntry ) } private boolean matches( StorageCommand command ) + { + Type type = mapCommandToType( command ); + // For the time being we can assume BaseCommand here + return type != null && inconsistencies.containsId( type, ((Command.BaseCommand) command).getKey() ); + } + + private Type mapCommandToType( StorageCommand command ) { if ( command instanceof NodeCommand ) { - return inconsistencies.containsNodeId( ((NodeCommand) command).getKey() ); + return Type.NODE; } if ( command instanceof RelationshipCommand ) { - return inconsistencies.containsRelationshipId( ((RelationshipCommand) command).getKey() ); + return Type.RELATIONSHIP; } if ( command instanceof PropertyCommand ) { - return inconsistencies.containsPropertyId( ((PropertyCommand) command).getKey() ); + return Type.PROPERTY; } if ( command instanceof RelationshipGroupCommand ) { - return inconsistencies.containsRelationshipGroupId( ((RelationshipGroupCommand) command).getKey() ); + return Type.RELATIONSHIP_GROUP; } if ( command instanceof SchemaRuleCommand ) { - return inconsistencies.containsSchemaIndexId( ((SchemaRuleCommand) command).getKey() ); + return Type.SCHEMA_INDEX; } - return false; + return null; // means ignore this command } @Override diff --git a/tools/src/main/java/org/neo4j/tools/dump/InconsistencyReportReader.java b/tools/src/main/java/org/neo4j/tools/dump/InconsistencyReportReader.java index e95dd8fc2b44f..9adf80797e84d 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/InconsistencyReportReader.java +++ b/tools/src/main/java/org/neo4j/tools/dump/InconsistencyReportReader.java @@ -27,8 +27,7 @@ import java.io.FileReader; import java.io.IOException; -import org.neo4j.consistency.RecordType; -import org.neo4j.tools.dump.inconsistency.Inconsistencies; +import org.neo4j.tools.dump.InconsistentRecords.Type; /** * Reads CC inconsistency reports. Example of entry: @@ -50,9 +49,9 @@ public class InconsistencyReportReader { private static final String INCONSISTENT_WITH = "Inconsistent with: "; - private final Inconsistencies inconsistencies; + private final InconsistentRecords inconsistencies; - public InconsistencyReportReader( Inconsistencies inconsistencies ) + public InconsistencyReportReader( InconsistentRecords inconsistencies ) { this.inconsistencies = inconsistencies; } @@ -68,8 +67,8 @@ public void read( File file ) throws IOException public void read( BufferedReader bufferedReader ) throws IOException { String line = bufferedReader.readLine(); - RecordType inconsistentRecordType; - RecordType inconsistentWithRecordType; + Type inconsistentRecordType; + Type inconsistentWithRecordType; long inconsistentRecordId; long inconsistentWithRecordId; @@ -86,7 +85,13 @@ public void read( BufferedReader bufferedReader ) throws IOException } line = line.trim(); inconsistentRecordType = toRecordType( entityType( line ) ); - inconsistentRecordId = id( line ); + if ( inconsistentRecordType == null ) + { + continue; + } + + inconsistentRecordId = inconsistentRecordType.extractId( line ); + inconsistencies.reportInconsistency( inconsistentRecordType, inconsistentRecordId ); // Then get the Inconsistent With line: line = bufferedReader.readLine(); @@ -100,10 +105,11 @@ public void read( BufferedReader bufferedReader ) throws IOException { line = line.substring( INCONSISTENT_WITH.length() ).trim(); inconsistentWithRecordType = toRecordType( entityType( line ) ); - inconsistentWithRecordId = id( line ); - inconsistencies.reportInconsistency( - inconsistentRecordType, inconsistentRecordId, - inconsistentWithRecordType, inconsistentWithRecordId ); + if ( inconsistentWithRecordType != null ) + { + inconsistentWithRecordId = inconsistentWithRecordType.extractId( line ); + inconsistencies.reportInconsistency( inconsistentWithRecordType, inconsistentWithRecordId ); + } line = bufferedReader.readLine(); // Prepare a line for the next iteration of the loop. } } @@ -116,7 +122,7 @@ public void read( BufferedReader bufferedReader ) throws IOException } } - private RecordType toRecordType( String entityType ) + private Type toRecordType( String entityType ) { if ( entityType == null ) { @@ -127,62 +133,25 @@ private RecordType toRecordType( String entityType ) switch ( entityType ) { case "Relationship": - return RecordType.RELATIONSHIP; + return Type.RELATIONSHIP; case "Node": - return RecordType.NODE; + return Type.NODE; case "Property": - return RecordType.PROPERTY; + return Type.PROPERTY; case "RelationshipGroup": - return RecordType.RELATIONSHIP_GROUP; + return Type.RELATIONSHIP_GROUP; case "IndexRule": - return RecordType.SCHEMA; + return Type.SCHEMA_INDEX; case "IndexEntry": - return RecordType.NODE; + return Type.NODE; + case "NodeLabelRange": + return Type.NODE_LABEL_RANGE; default: // it's OK, we just haven't implemented support for this yet return null; } } - private long id( String line ) - { - int bracket = line.indexOf( '[' ); - if ( bracket > -1 ) - { - int separator = min( getSeparatorIndex( ',', line, bracket ), - getSeparatorIndex( ';', line, bracket ), - getSeparatorIndex( ']', line, bracket ) ); - int equally = line.indexOf( '=', bracket ); - int startPosition = (isNotPlainId( bracket, separator, equally ) ? equally : bracket) + 1; - if ( separator > -1 ) - { - return Long.parseLong( line.substring( startPosition, separator ) ); - } - } - return -1; - } - - private static int min( int... values ) - { - int min = Integer.MAX_VALUE; - for ( int value : values ) - { - min = Math.min( min, value ); - } - return min; - } - - private int getSeparatorIndex( char character, String line, int bracket ) - { - int index = line.indexOf( character, bracket ); - return index >= 0 ? index : Integer.MAX_VALUE; - } - - private boolean isNotPlainId( int bracket, int comma, int equally ) - { - return (equally > bracket) && (equally < comma); - } - private String entityType( String line ) { int bracket = line.indexOf( '[' ); diff --git a/tools/src/main/java/org/neo4j/tools/dump/InconsistentRecords.java b/tools/src/main/java/org/neo4j/tools/dump/InconsistentRecords.java new file mode 100644 index 0000000000000..e3fe3b552554d --- /dev/null +++ b/tools/src/main/java/org/neo4j/tools/dump/InconsistentRecords.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2002-2018 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Enterprise Edition. The included source + * code can be redistributed and/or modified under the terms of the + * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 + * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the + * Commons Clause, as found in the associated LICENSE.txt file. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * Neo4j object code can be licensed independently from the source + * under separate terms from the AGPL. Inquiries can be directed to: + * licensing@neo4j.com + * + * More information is also available at: + * https://neo4j.com/licensing/ + */ +package org.neo4j.tools.dump; + +import java.util.EnumMap; + +import org.neo4j.collection.primitive.Primitive; +import org.neo4j.collection.primitive.PrimitiveLongSet; + +import static org.neo4j.collection.primitive.base.Empty.EMPTY_PRIMITIVE_LONG_SET; + +/** + * Container for ids of entities that are considered to be inconsistent. + */ +public class InconsistentRecords +{ + static long NO_ID = -1; + + enum Type + { + NODE, + RELATIONSHIP, + RELATIONSHIP_GROUP, + PROPERTY, + SCHEMA_INDEX, + NODE_LABEL_RANGE + { + @Override + public long extractId( String line ) + { + // For the main report line there's nothing of interest... it's the next INCONSISTENT WITH line that is + return NO_ID; + } + }; + + public long extractId( String line ) + { + int bracket = line.indexOf( '[' ); + if ( bracket > -1 ) + { + int separator = min( getSeparatorIndex( ',', line, bracket ), + getSeparatorIndex( ';', line, bracket ), + getSeparatorIndex( ']', line, bracket ) ); + int equally = line.indexOf( '=', bracket ); + int startPosition = (isNotPlainId( bracket, separator, equally ) ? equally : bracket) + 1; + if ( separator > -1 ) + { + return Long.parseLong( line.substring( startPosition, separator ) ); + } + } + return NO_ID; + } + + private static int min( int... values ) + { + int min = Integer.MAX_VALUE; + for ( int value : values ) + { + min = Math.min( min, value ); + } + return min; + } + + private int getSeparatorIndex( char character, String line, int bracket ) + { + int index = line.indexOf( character, bracket ); + return index >= 0 ? index : Integer.MAX_VALUE; + } + + private boolean isNotPlainId( int bracket, int comma, int equally ) + { + return (equally > bracket) && (equally < comma); + } + } + + private final EnumMap ids = new EnumMap<>( Type.class ); + + public boolean containsId( Type recordType, long id ) + { + return ids.getOrDefault( recordType, EMPTY_PRIMITIVE_LONG_SET ).contains( id ); + } + + public void reportInconsistency( Type recordType, long recordId ) + { + if ( recordId != NO_ID ) + { + ids.computeIfAbsent( recordType, t -> Primitive.longSet() ).add( recordId ); + } + } +} diff --git a/tools/src/main/java/org/neo4j/tools/dump/inconsistency/Inconsistencies.java b/tools/src/main/java/org/neo4j/tools/dump/inconsistency/Inconsistencies.java deleted file mode 100644 index 1c3e55bda7734..0000000000000 --- a/tools/src/main/java/org/neo4j/tools/dump/inconsistency/Inconsistencies.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j Enterprise Edition. The included source - * code can be redistributed and/or modified under the terms of the - * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 - * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the - * Commons Clause, as found in the associated LICENSE.txt file. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * Neo4j object code can be licensed independently from the source - * under separate terms from the AGPL. Inquiries can be directed to: - * licensing@neo4j.com - * - * More information is also available at: - * https://neo4j.com/licensing/ - */ -package org.neo4j.tools.dump.inconsistency; - -import org.neo4j.consistency.RecordType; - -/** - * Container for ids of entities that are considered to be inconsistent. - */ -public interface Inconsistencies -{ - void node( long id ); - - void relationship( long id ); - - void property( long id ); - - void relationshipGroup( long id ); - - void schemaIndex( long id ); - - boolean containsNodeId( long id ); - - boolean containsRelationshipId( long id ); - - boolean containsPropertyId( long id ); - - boolean containsRelationshipGroupId( long id ); - - boolean containsSchemaIndexId( long id ); - - default void reportInconsistency( RecordType recordType, long recordId ) - { - if ( recordType == null ) - { - // Skip records of unknown type. - return; - } - - switch ( recordType ) - { - case NODE: - node( recordId ); - break; - case RELATIONSHIP: - relationship( recordId ); - break; - case PROPERTY: - property( recordId ); - break; - case RELATIONSHIP_GROUP: - relationshipGroup( recordId ); - break; - case SCHEMA: - schemaIndex( recordId ); - break; - default: - // Ignore unknown record types. - break; - } - } - - default void reportInconsistency( RecordType recordType, long recordId, - RecordType inconsistentWithRecordType, long inconsistentWithRecordId ) - { - reportInconsistency( recordType, recordId ); - reportInconsistency( inconsistentWithRecordType, inconsistentWithRecordId ); - } -} diff --git a/tools/src/main/java/org/neo4j/tools/dump/inconsistency/ReportInconsistencies.java b/tools/src/main/java/org/neo4j/tools/dump/inconsistency/ReportInconsistencies.java deleted file mode 100644 index 31d48a353b2c0..0000000000000 --- a/tools/src/main/java/org/neo4j/tools/dump/inconsistency/ReportInconsistencies.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo4j," - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j Enterprise Edition. The included source - * code can be redistributed and/or modified under the terms of the - * GNU AFFERO GENERAL PUBLIC LICENSE Version 3 - * (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the - * Commons Clause, as found in the associated LICENSE.txt file. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * Neo4j object code can be licensed independently from the source - * under separate terms from the AGPL. Inquiries can be directed to: - * licensing@neo4j.com - * - * More information is also available at: - * https://neo4j.com/licensing/ - */ -package org.neo4j.tools.dump.inconsistency; - -import org.neo4j.collection.primitive.Primitive; -import org.neo4j.collection.primitive.PrimitiveLongSet; - -/** - * Entity ids that reported to be inconsistent in consistency report where they where extracted from. - */ -public class ReportInconsistencies implements Inconsistencies -{ - private final PrimitiveLongSet schemaIndexesIds = Primitive.longSet(); - private final PrimitiveLongSet relationshipIds = Primitive.longSet(); - private final PrimitiveLongSet nodeIds = Primitive.longSet(); - private final PrimitiveLongSet propertyIds = Primitive.longSet(); - private final PrimitiveLongSet relationshipGroupIds = Primitive.longSet(); - - @Override - public void relationshipGroup( long id ) - { - relationshipGroupIds.add( id ); - } - - @Override - public void schemaIndex( long id ) - { - schemaIndexesIds.add( id ); - } - - @Override - public void relationship( long id ) - { - relationshipIds.add( id ); - } - - @Override - public void property( long id ) - { - propertyIds.add( id ); - } - - @Override - public void node( long id ) - { - nodeIds.add( id ); - } - - @Override - public boolean containsNodeId( long id ) - { - return nodeIds.contains( id ); - } - - @Override - public boolean containsRelationshipId( long id ) - { - return relationshipIds.contains( id ); - } - - @Override - public boolean containsPropertyId( long id ) - { - return propertyIds.contains( id ); - } - - @Override - public boolean containsRelationshipGroupId( long id ) - { - return relationshipGroupIds.contains( id ); - } - - @Override - public boolean containsSchemaIndexId( long id ) - { - return schemaIndexesIds.contains( id ); - } -} diff --git a/tools/src/test/java/org/neo4j/tools/dump/InconsistencyReportReaderTest.java b/tools/src/test/java/org/neo4j/tools/dump/InconsistencyReportReaderTest.java index ee81da3da1bae..c030f0b1e8d75 100644 --- a/tools/src/test/java/org/neo4j/tools/dump/InconsistencyReportReaderTest.java +++ b/tools/src/test/java/org/neo4j/tools/dump/InconsistencyReportReaderTest.java @@ -31,7 +31,9 @@ import org.neo4j.consistency.RecordType; import org.neo4j.consistency.report.InconsistencyMessageLogger; import org.neo4j.consistency.store.synthetic.IndexEntry; +import org.neo4j.consistency.store.synthetic.LabelScanDocument; import org.neo4j.kernel.api.index.IndexProvider; +import org.neo4j.kernel.api.labelscan.NodeLabelRange; import org.neo4j.kernel.api.schema.index.SchemaIndexDescriptorFactory; import org.neo4j.kernel.impl.store.record.IndexRule; import org.neo4j.kernel.impl.store.record.NodeRecord; @@ -39,7 +41,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.logging.FormattedLog; -import org.neo4j.tools.dump.inconsistency.ReportInconsistencies; +import org.neo4j.tools.dump.InconsistentRecords.Type; import static org.junit.Assert.assertTrue; @@ -55,6 +57,7 @@ public void shouldReadBasicEntities() throws Exception long nodeId = 5; long indexNodeId = 7; long nodeNotInTheIndexId = 17; + long nodeNotInTheLabelIndexId = 18; long indexId = 99; long relationshipGroupId = 10; long relationshipId = 15; @@ -71,28 +74,31 @@ public void shouldReadBasicEntities() throws Exception logger.error( RecordType.NODE, new NodeRecord( nodeNotInTheIndexId ), "Some index error", IndexRule.indexRule( indexId, SchemaIndexDescriptorFactory.forLabel( 1, 2 ), new IndexProvider.Descriptor( "key", "version" ) ).toString() ); + logger.error( RecordType.LABEL_SCAN_DOCUMENT, new LabelScanDocument( new NodeLabelRange( 0, new long[0][] ) ), + "Some label index error", new NodeRecord( nodeNotInTheLabelIndexId ) ); String text = out.toString(); // WHEN - ReportInconsistencies inconsistencies = new ReportInconsistencies(); + InconsistentRecords inconsistencies = new InconsistentRecords(); InconsistencyReportReader reader = new InconsistencyReportReader( inconsistencies ); reader.read( new BufferedReader( new StringReader( text ) ) ); // THEN - assertTrue( inconsistencies.containsNodeId( nodeId ) ); - assertTrue( inconsistencies.containsNodeId( indexNodeId ) ); - assertTrue( inconsistencies.containsNodeId( nodeNotInTheIndexId ) ); - assertTrue( inconsistencies.containsRelationshipId( relationshipId ) ); - assertTrue( inconsistencies.containsRelationshipGroupId( relationshipGroupId ) ); - assertTrue( inconsistencies.containsPropertyId( propertyId ) ); - assertTrue( inconsistencies.containsSchemaIndexId( indexId ) ); + assertTrue( inconsistencies.containsId( Type.NODE, nodeId ) ); + assertTrue( inconsistencies.containsId( Type.NODE, indexNodeId ) ); + assertTrue( inconsistencies.containsId( Type.NODE, nodeNotInTheIndexId ) ); + assertTrue( inconsistencies.containsId( Type.NODE, nodeNotInTheLabelIndexId ) ); + assertTrue( inconsistencies.containsId( Type.RELATIONSHIP, relationshipId ) ); + assertTrue( inconsistencies.containsId( Type.RELATIONSHIP_GROUP, relationshipGroupId ) ); + assertTrue( inconsistencies.containsId( Type.PROPERTY, propertyId ) ); + assertTrue( inconsistencies.containsId( Type.SCHEMA_INDEX, indexId ) ); } @Test public void shouldParseRelationshipGroupInconsistencies() throws Exception { // Given - ReportInconsistencies inconsistencies = new ReportInconsistencies(); + InconsistentRecords inconsistencies = new InconsistentRecords(); String text = "ERROR: The first outgoing relationship is not the first in its chain.\n" + "\tRelationshipGroup[1337,type=1,out=2,in=-1,loop=-1,prev=-1,next=3,used=true,owner=4,secondaryUnitId=-1]\n" + @@ -104,7 +110,7 @@ public void shouldParseRelationshipGroupInconsistencies() throws Exception reader.read( new BufferedReader( new StringReader( text ) ) ); // Then - assertTrue( inconsistencies.containsRelationshipGroupId( 1337 ) ); - assertTrue( inconsistencies.containsRelationshipGroupId( 4242 ) ); + assertTrue( inconsistencies.containsId( Type.RELATIONSHIP_GROUP, 1337 ) ); + assertTrue( inconsistencies.containsId( Type.RELATIONSHIP_GROUP, 4242 ) ); } }