Skip to content

Commit

Permalink
DumpLogicalLog support for NodeLabelRange in --ccfilter
Browse files Browse the repository at this point in the history
Also made some refactorings to allow for a more maintainable impl
  • Loading branch information
tinwelint committed Dec 11, 2018
1 parent 96a909b commit 54d33f7
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 265 deletions.
26 changes: 17 additions & 9 deletions tools/src/main/java/org/neo4j/tools/dump/DumpLogicalLog.java
Expand Up @@ -34,6 +34,7 @@
import org.neo4j.helpers.Args;
import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.impl.transaction.command.Command;
import org.neo4j.kernel.impl.transaction.command.Command.NodeCommand;
import org.neo4j.kernel.impl.transaction.command.Command.PropertyCommand;
import org.neo4j.kernel.impl.transaction.command.Command.RelationshipCommand;
Expand All @@ -47,8 +48,8 @@
import org.neo4j.kernel.impl.transaction.log.entry.LogHeader;
import org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader;
import org.neo4j.storageengine.api.StorageCommand;
import org.neo4j.tools.dump.InconsistentRecords.Type;
import org.neo4j.tools.dump.TransactionLogAnalyzer.Monitor;
import org.neo4j.tools.dump.inconsistency.ReportInconsistencies;

import static java.util.TimeZone.getTimeZone;
import static org.neo4j.helpers.Format.DEFAULT_TIME_ZONE;
Expand Down Expand Up @@ -134,12 +135,12 @@ public boolean test( LogEntry[] transaction )
public static class ConsistencyCheckOutputCriteria implements Predicate<LogEntry[]>, Function<LogEntry,String>
{
private final TimeZone timeZone;
private final ReportInconsistencies inconsistencies;
private final InconsistentRecords inconsistencies;

public ConsistencyCheckOutputCriteria( String ccFile, TimeZone timeZone ) throws IOException
{
this.timeZone = timeZone;
inconsistencies = new ReportInconsistencies();
inconsistencies = new InconsistentRecords();
new InconsistencyReportReader( inconsistencies ).read( new File( ccFile ) );
}

Expand Down Expand Up @@ -169,28 +170,35 @@ private boolean matches( LogEntry logEntry )
}

private boolean matches( StorageCommand command )
{
Type type = mapCommandToType( command );
// For the time being we can assume BaseCommand here
return type != null && inconsistencies.containsId( type, ((Command.BaseCommand) command).getKey() );
}

private Type mapCommandToType( StorageCommand command )
{
if ( command instanceof NodeCommand )
{
return inconsistencies.containsNodeId( ((NodeCommand) command).getKey() );
return Type.NODE;
}
if ( command instanceof RelationshipCommand )
{
return inconsistencies.containsRelationshipId( ((RelationshipCommand) command).getKey() );
return Type.RELATIONSHIP;
}
if ( command instanceof PropertyCommand )
{
return inconsistencies.containsPropertyId( ((PropertyCommand) command).getKey() );
return Type.PROPERTY;
}
if ( command instanceof RelationshipGroupCommand )
{
return inconsistencies.containsRelationshipGroupId( ((RelationshipGroupCommand) command).getKey() );
return Type.RELATIONSHIP_GROUP;
}
if ( command instanceof SchemaRuleCommand )
{
return inconsistencies.containsSchemaIndexId( ((SchemaRuleCommand) command).getKey() );
return Type.SCHEMA_INDEX;
}
return false;
return null; // means ignore this command
}

@Override
Expand Down
Expand Up @@ -27,8 +27,7 @@
import java.io.FileReader;
import java.io.IOException;

import org.neo4j.consistency.RecordType;
import org.neo4j.tools.dump.inconsistency.Inconsistencies;
import org.neo4j.tools.dump.InconsistentRecords.Type;

/**
* Reads CC inconsistency reports. Example of entry:
Expand All @@ -50,9 +49,9 @@
public class InconsistencyReportReader
{
private static final String INCONSISTENT_WITH = "Inconsistent with: ";
private final Inconsistencies inconsistencies;
private final InconsistentRecords inconsistencies;

public InconsistencyReportReader( Inconsistencies inconsistencies )
public InconsistencyReportReader( InconsistentRecords inconsistencies )
{
this.inconsistencies = inconsistencies;
}
Expand All @@ -68,8 +67,8 @@ public void read( File file ) throws IOException
public void read( BufferedReader bufferedReader ) throws IOException
{
String line = bufferedReader.readLine();
RecordType inconsistentRecordType;
RecordType inconsistentWithRecordType;
Type inconsistentRecordType;
Type inconsistentWithRecordType;
long inconsistentRecordId;
long inconsistentWithRecordId;

Expand All @@ -86,7 +85,13 @@ public void read( BufferedReader bufferedReader ) throws IOException
}
line = line.trim();
inconsistentRecordType = toRecordType( entityType( line ) );
inconsistentRecordId = id( line );
if ( inconsistentRecordType == null )
{
continue;
}

inconsistentRecordId = inconsistentRecordType.extractId( line );
inconsistencies.reportInconsistency( inconsistentRecordType, inconsistentRecordId );

// Then get the Inconsistent With line:
line = bufferedReader.readLine();
Expand All @@ -100,10 +105,11 @@ public void read( BufferedReader bufferedReader ) throws IOException
{
line = line.substring( INCONSISTENT_WITH.length() ).trim();
inconsistentWithRecordType = toRecordType( entityType( line ) );
inconsistentWithRecordId = id( line );
inconsistencies.reportInconsistency(
inconsistentRecordType, inconsistentRecordId,
inconsistentWithRecordType, inconsistentWithRecordId );
if ( inconsistentWithRecordType != null )
{
inconsistentWithRecordId = inconsistentWithRecordType.extractId( line );
inconsistencies.reportInconsistency( inconsistentWithRecordType, inconsistentWithRecordId );
}
line = bufferedReader.readLine(); // Prepare a line for the next iteration of the loop.
}
}
Expand All @@ -116,7 +122,7 @@ public void read( BufferedReader bufferedReader ) throws IOException
}
}

private RecordType toRecordType( String entityType )
private Type toRecordType( String entityType )
{
if ( entityType == null )
{
Expand All @@ -127,62 +133,25 @@ private RecordType toRecordType( String entityType )
switch ( entityType )
{
case "Relationship":
return RecordType.RELATIONSHIP;
return Type.RELATIONSHIP;
case "Node":
return RecordType.NODE;
return Type.NODE;
case "Property":
return RecordType.PROPERTY;
return Type.PROPERTY;
case "RelationshipGroup":
return RecordType.RELATIONSHIP_GROUP;
return Type.RELATIONSHIP_GROUP;
case "IndexRule":
return RecordType.SCHEMA;
return Type.SCHEMA_INDEX;
case "IndexEntry":
return RecordType.NODE;
return Type.NODE;
case "NodeLabelRange":
return Type.NODE_LABEL_RANGE;
default:
// it's OK, we just haven't implemented support for this yet
return null;
}
}

private long id( String line )
{
int bracket = line.indexOf( '[' );
if ( bracket > -1 )
{
int separator = min( getSeparatorIndex( ',', line, bracket ),
getSeparatorIndex( ';', line, bracket ),
getSeparatorIndex( ']', line, bracket ) );
int equally = line.indexOf( '=', bracket );
int startPosition = (isNotPlainId( bracket, separator, equally ) ? equally : bracket) + 1;
if ( separator > -1 )
{
return Long.parseLong( line.substring( startPosition, separator ) );
}
}
return -1;
}

private static int min( int... values )
{
int min = Integer.MAX_VALUE;
for ( int value : values )
{
min = Math.min( min, value );
}
return min;
}

private int getSeparatorIndex( char character, String line, int bracket )
{
int index = line.indexOf( character, bracket );
return index >= 0 ? index : Integer.MAX_VALUE;
}

private boolean isNotPlainId( int bracket, int comma, int equally )
{
return (equally > bracket) && (equally < comma);
}

private String entityType( String line )
{
int bracket = line.indexOf( '[' );
Expand Down
110 changes: 110 additions & 0 deletions tools/src/main/java/org/neo4j/tools/dump/InconsistentRecords.java
@@ -0,0 +1,110 @@
/*
* Copyright (c) 2002-2018 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j Enterprise Edition. The included source
* code can be redistributed and/or modified under the terms of the
* GNU AFFERO GENERAL PUBLIC LICENSE Version 3
* (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the
* Commons Clause, as found in the associated LICENSE.txt file.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* Neo4j object code can be licensed independently from the source
* under separate terms from the AGPL. Inquiries can be directed to:
* licensing@neo4j.com
*
* More information is also available at:
* https://neo4j.com/licensing/
*/
package org.neo4j.tools.dump;

import java.util.EnumMap;

import org.neo4j.collection.primitive.Primitive;
import org.neo4j.collection.primitive.PrimitiveLongSet;

import static org.neo4j.collection.primitive.base.Empty.EMPTY_PRIMITIVE_LONG_SET;

/**
* Container for ids of entities that are considered to be inconsistent.
*/
public class InconsistentRecords
{
static long NO_ID = -1;

enum Type
{
NODE,
RELATIONSHIP,
RELATIONSHIP_GROUP,
PROPERTY,
SCHEMA_INDEX,
NODE_LABEL_RANGE
{
@Override
public long extractId( String line )
{
// For the main report line there's nothing of interest... it's the next INCONSISTENT WITH line that is
return NO_ID;
}
};

public long extractId( String line )
{
int bracket = line.indexOf( '[' );
if ( bracket > -1 )
{
int separator = min( getSeparatorIndex( ',', line, bracket ),
getSeparatorIndex( ';', line, bracket ),
getSeparatorIndex( ']', line, bracket ) );
int equally = line.indexOf( '=', bracket );
int startPosition = (isNotPlainId( bracket, separator, equally ) ? equally : bracket) + 1;
if ( separator > -1 )
{
return Long.parseLong( line.substring( startPosition, separator ) );
}
}
return NO_ID;
}

private static int min( int... values )
{
int min = Integer.MAX_VALUE;
for ( int value : values )
{
min = Math.min( min, value );
}
return min;
}

private int getSeparatorIndex( char character, String line, int bracket )
{
int index = line.indexOf( character, bracket );
return index >= 0 ? index : Integer.MAX_VALUE;
}

private boolean isNotPlainId( int bracket, int comma, int equally )
{
return (equally > bracket) && (equally < comma);
}
}

private final EnumMap<Type,PrimitiveLongSet> ids = new EnumMap<>( Type.class );

public boolean containsId( Type recordType, long id )
{
return ids.getOrDefault( recordType, EMPTY_PRIMITIVE_LONG_SET ).contains( id );
}

public void reportInconsistency( Type recordType, long recordId )
{
if ( recordId != NO_ID )
{
ids.computeIfAbsent( recordType, t -> Primitive.longSet() ).add( recordId );
}
}
}

0 comments on commit 54d33f7

Please sign in to comment.