Skip to content

Commit

Permalink
Increases tx-local name/key id space of legacy index commands
Browse files Browse the repository at this point in the history 
Legacy index commands don't take advantage of the token system in Neo4j.
Perhaps that will change, but until then each transaction having one or
more legacy index commands will also have an IndexDefineCommand which
contains a String->id mapping where all other legacy index commands in
that transaction will refer to ids instead of strings when it comes to
names/keys. Previously this id space was only 6 bits, yielding a maximum
of 63 names and 63 keys within the same transaction.

This has turned out to be a problem in some scenarios so this commit
extends that id space to 16 bits, now ranging up to 65535 names/keys in
the same transaction. This means that the log format has changed. Although
don't panic since the following commit will deal with this.
  • Loading branch information
@tinwelint @digitalstain
tinwelint authored and digitalstain committed Jul 27, 2015
1 parent 292822e commit ffcd017
Show file tree
Hide file tree
Showing 17 changed files with 886 additions and 142 deletions.
24 changes: 12 additions & 12 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexCommand.java
View file
Expand Up @@ -19,8 +19,6 @@
*/
package org.neo4j.kernel.impl.index;

import static java.lang.String.format;

import java.io.IOException;
import java.util.Map;

Expand All @@ -30,6 +28,8 @@
import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler;
import org.neo4j.kernel.impl.transaction.command.NeoCommandType;

import static java.lang.String.format;

/**
* Created from {@link IndexDefineCommand} or read from a logical log.
* Contains all the different types of commands that an {@link Index} need
Expand All @@ -46,14 +46,14 @@ public abstract class IndexCommand extends Command
public static final byte VALUE_TYPE_STRING = (byte) 6;

private byte commandType;
protected byte indexNameId;
protected int indexNameId;
protected byte entityType;
protected long entityId;
protected byte keyId;
protected int keyId;
protected byte valueType;
protected Object value;

protected void init( byte commandType, byte indexNameId, byte entityType, long entityId, byte keyId, Object value )
protected void init( byte commandType, int indexNameId, byte entityType, long entityId, int keyId, Object value )
{
this.commandType = commandType ;
this.indexNameId = indexNameId;
Expand All @@ -64,7 +64,7 @@ protected void init( byte commandType, byte indexNameId, byte entityType, long e
this.valueType = valueTypeOf( value );
}

public byte getIndexNameId()
public int getIndexNameId()
{
return indexNameId;
}
Expand All @@ -79,7 +79,7 @@ public long getEntityId()
return entityId;
}

public byte getKeyId()
public int getKeyId()
{
return keyId;
}
Expand Down Expand Up @@ -144,7 +144,7 @@ else if ( value instanceof Short )

public static class AddNodeCommand extends IndexCommand
{
public void init( byte indexNameId, long entityId, byte keyId, Object value )
public void init( int indexNameId, long entityId, int keyId, Object value )
{
super.init( NeoCommandType.INDEX_ADD_COMMAND, indexNameId, IndexEntityType.Node.id(),
entityId, keyId, value );
Expand Down Expand Up @@ -173,7 +173,7 @@ public static class AddRelationshipCommand extends IndexCommand
private long startNode;
private long endNode;

public void init( byte indexNameId, long entityId, byte keyId,
public void init( int indexNameId, long entityId, int keyId,
Object value, long startNode, long endNode )
{
super.init( NeoCommandType.INDEX_ADD_RELATIONSHIP_COMMAND, indexNameId, IndexEntityType.Relationship.id(),
Expand Down Expand Up @@ -242,7 +242,7 @@ public String toString()

public static class RemoveCommand extends IndexCommand
{
public void init( byte indexNameId, byte entityType, long entityId, byte keyId, Object value )
public void init( int indexNameId, byte entityType, long entityId, int keyId, Object value )
{
super.init( NeoCommandType.INDEX_REMOVE_COMMAND, indexNameId, entityType, entityId, keyId, value );
}
Expand All @@ -263,7 +263,7 @@ public String toString()

public static class DeleteCommand extends IndexCommand
{
public void init( byte indexNameId, byte entityType )
public void init( int indexNameId, byte entityType )
{
super.init( NeoCommandType.INDEX_DELETE_COMMAND, indexNameId, entityType, 0L, (byte)0, null );
}
Expand All @@ -285,7 +285,7 @@ public static class CreateCommand extends IndexCommand
{
private Map<String, String> config;

public void init( byte indexNameId, byte entityType, Map<String, String> config )
public void init( int indexNameId, byte entityType, Map<String, String> config )
{
super.init( NeoCommandType.INDEX_CREATE_COMMAND, indexNameId, entityType, 0L, (byte)0, null );
this.config = config;
Expand Down
65 changes: 41 additions & 24 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexDefineCommand.java
View file
Expand Up @@ -24,11 +24,16 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import org.neo4j.collection.primitive.Primitive;
import org.neo4j.collection.primitive.PrimitiveIntObjectMap;
import org.neo4j.kernel.impl.transaction.command.Command;
import org.neo4j.kernel.impl.transaction.command.CommandRecordVisitor;
import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler;

import static org.neo4j.helpers.collection.MapUtil.reverse;
import static java.lang.Math.pow;
import static java.lang.String.format;

import static org.neo4j.collection.primitive.Primitive.intObjectMap;

/**
* A command which have to be first in the transaction. It will map index names
Expand All @@ -44,28 +49,38 @@ public class IndexDefineCommand extends Command
{
private final AtomicInteger nextIndexNameId = new AtomicInteger();
private final AtomicInteger nextKeyId = new AtomicInteger();
private Map<String, Byte> indexNameIdRange;
private Map<String, Byte> keyIdRange;
private Map<Byte, String> idToIndexName;
private Map<Byte, String> idToKey;
private Map<String,Integer> indexNameIdRange;
private Map<String,Integer> keyIdRange;
private PrimitiveIntObjectMap<String> idToIndexName;
private PrimitiveIntObjectMap<String> idToKey;

public IndexDefineCommand()
{
setIndexNameIdRange( new HashMap<String, Byte>() );
setKeyIdRange( new HashMap<String, Byte>() );
idToIndexName = new HashMap<>();
idToKey = new HashMap<>();
setIndexNameIdRange( new HashMap<String,Integer>() );
setKeyIdRange( new HashMap<String,Integer>() );
idToIndexName = intObjectMap( 16 );
idToKey = intObjectMap( 16 );
}

public void init( Map<String, Byte> indexNames, Map<String, Byte> keys )
public void init( Map<String,Integer> indexNames, Map<String,Integer> keys )
{
this.setIndexNameIdRange( indexNames );
this.setKeyIdRange( keys );
idToIndexName = reverse( indexNames );
idToKey = reverse( keys );
}

private static String getFromMap( Map<Byte, String> map, byte id )
private static PrimitiveIntObjectMap<String> reverse( Map<String,Integer> map )
{
PrimitiveIntObjectMap<String> result = Primitive.intObjectMap( map.size() );
for ( Map.Entry<String,Integer> entry : map.entrySet() )
{
result.put( entry.getValue().intValue(), entry.getKey() );
}
return result;
}

private static String getFromMap( PrimitiveIntObjectMap<String> map, int id )
{
if ( id == -1 )
{
Expand All @@ -79,46 +94,48 @@ private static String getFromMap( Map<Byte, String> map, byte id )
return result;
}

public String getIndexName( byte id )
public String getIndexName( int id )
{
return getFromMap( idToIndexName, id );
}

public String getKey( byte id )
public String getKey( int id )
{
return getFromMap( idToKey, id );
}

public byte getOrAssignIndexNameId( String indexName )
public int getOrAssignIndexNameId( String indexName )
{
return getOrAssignId( indexNameIdRange, idToIndexName, nextIndexNameId, indexName );
}

public byte getOrAssignKeyId( String key )
public int getOrAssignKeyId( String key )
{
return getOrAssignId( keyIdRange, idToKey, nextKeyId, key );
}

private byte getOrAssignId( Map<String, Byte> stringToId, Map<Byte, String> idToString,
private int getOrAssignId( Map<String,Integer> stringToId, PrimitiveIntObjectMap<String> idToString,
AtomicInteger nextId, String string )
{
if ( string == null )
{
return -1;
}

Byte id = stringToId.get( string );
Integer id = stringToId.get( string );
if ( id != null )
{
return id;
}

int nextIdInt = nextId.incrementAndGet();
if ( nextIdInt > 63 )
if ( (nextIdInt & ~0xFFFF) != 0 )
{
throw new IllegalStateException( "Modifying more than 63 indexes in a single transaction is not supported" );
throw new IllegalStateException( format(
"Modifying more than %d indexes in a single transaction is not supported",
(int)(pow( 2, 16 ) - 1) ) );
}
id = (byte) nextIdInt;
id = nextIdInt;

stringToId.put( string, id );
idToString.put( id, string );
Expand Down Expand Up @@ -158,22 +175,22 @@ public boolean handle( NeoCommandHandler visitor ) throws IOException
return visitor.visitIndexDefineCommand( this );
}

public Map<String, Byte> getIndexNameIdRange()
public Map<String,Integer> getIndexNameIdRange()
{
return indexNameIdRange;
}

public void setIndexNameIdRange( Map<String, Byte> indexNameIdRange )
public void setIndexNameIdRange( Map<String,Integer> indexNameIdRange )
{
this.indexNameIdRange = indexNameIdRange;
}

public Map<String, Byte> getKeyIdRange()
public Map<String,Integer> getKeyIdRange()
{
return keyIdRange;
}

public void setKeyIdRange( Map<String, Byte> keyIdRange )
public void setKeyIdRange( Map<String,Integer> keyIdRange )
{
this.keyIdRange = keyIdRange;
}
Expand Down
43 changes: 23 additions & 20 deletions .../kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/CommandReaderFactory.java
View file
Expand Up @@ -35,6 +35,7 @@
import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_1;
import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_2;
import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_3;
import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_2_4;

public abstract class CommandReaderFactory
{
Expand Down Expand Up @@ -66,26 +67,28 @@ private CommandReader figureOutCorrectReader( byte logFormatVersion, byte logEnt
{
switch ( logEntryVersion )
{
// These are not thread safe, so if they are to be cached it has to be done in an object pool
case LEGACY_LOG_ENTRY_VERSION:
switch ( logFormatVersion )
{
case LOG_VERSION_2_0:
return new PhysicalLogNeoCommandReaderV0_20();
case LOG_VERSION_1_9:
return new PhysicalLogNeoCommandReaderV0_19();
}
case LOG_ENTRY_VERSION_2_1:
case LOG_ENTRY_VERSION_2_2:
case LOG_ENTRY_VERSION_2_3:
switch ( logFormatVersion )
{
case LOG_VERSION_2_1:
return new PhysicalLogNeoCommandReaderV1();
case LOG_VERSION_2_2:
case LOG_VERSION_2_3:
return new PhysicalLogNeoCommandReaderV2();
}
// These are not thread safe, so if they are to be cached it has to be done in an object pool
case LEGACY_LOG_ENTRY_VERSION:
switch ( logFormatVersion )
{
case LOG_VERSION_2_0:
return new PhysicalLogNeoCommandReaderV0_20();
case LOG_VERSION_1_9:
return new PhysicalLogNeoCommandReaderV0_19();
}
case LOG_ENTRY_VERSION_2_1:
case LOG_ENTRY_VERSION_2_2:
case LOG_ENTRY_VERSION_2_3:
switch ( logFormatVersion )
{
case LOG_VERSION_2_1:
return new PhysicalLogNeoCommandReaderV1();
case LOG_VERSION_2_2:
case LOG_VERSION_2_3:
return new PhysicalLogNeoCommandReaderV2();
case LOG_VERSION_2_2_4:
return new PhysicalLogNeoCommandReaderV2_2_4();
}
}
throw new IllegalArgumentException( "Unknown log format version (" + logFormatVersion + ") and " +
"log entry version (" + logEntryVersion + ")" );
Expand Down
10 changes: 5 additions & 5 deletions ...rc/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1.java
View file
Expand Up @@ -696,8 +696,8 @@ public boolean visitIndexCreateCommand( CreateCommand createCommand ) throws IOE
public boolean visitIndexDefineCommand( IndexDefineCommand indexDefineCommand ) throws IOException
{
readIndexCommandHeader();
Map<String,Byte> indexNames = readMap( channel );
Map<String,Byte> keys = readMap( channel );
Map<String,Integer> indexNames = readMap( channel );
Map<String,Integer> keys = readMap( channel );
indexDefineCommand.init( indexNames, keys );
return false;
}
Expand All @@ -722,14 +722,14 @@ public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command
return false;
}

private Map<String,Byte> readMap( ReadableLogChannel channel ) throws IOException
private Map<String,Integer> readMap( ReadableLogChannel channel ) throws IOException
{
byte size = channel.get();
Map<String,Byte> result = new HashMap<>();
Map<String,Integer> result = new HashMap<>();
for ( int i = 0; i < size; i++ )
{
String key = read2bLengthAndString( channel );
byte id = channel.get();
int id = channel.get();
if ( key == null )
{
return null;
Expand Down
10 changes: 5 additions & 5 deletions ...rc/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2.java
View file
Expand Up @@ -702,8 +702,8 @@ public boolean visitIndexCreateCommand( CreateCommand createCommand ) throws IOE
public boolean visitIndexDefineCommand( IndexDefineCommand indexDefineCommand ) throws IOException
{
readIndexCommandHeader();
Map<String,Byte> indexNames = readMap( channel );
Map<String,Byte> keys = readMap( channel );
Map<String,Integer> indexNames = readMap( channel );
Map<String,Integer> keys = readMap( channel );
indexDefineCommand.init( indexNames, keys );
return false;
}
Expand All @@ -728,14 +728,14 @@ public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command
return false;
}

private Map<String,Byte> readMap( ReadableLogChannel channel ) throws IOException
private Map<String,Integer> readMap( ReadableLogChannel channel ) throws IOException
{
byte size = channel.get();
Map<String,Byte> result = new HashMap<>();
Map<String,Integer> result = new HashMap<>();
for ( int i = 0; i < size; i++ )
{
String key = read2bLengthAndString( channel );
byte id = channel.get();
int id = channel.get();
if ( key == null )
{
return null;
Expand Down

0 comments on commit ffcd017

Please sign in to comment.