From ffcd017e13d60cc68ce0eb5818b4ab7f1cdc87f6 Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Thu, 23 Jul 2015 09:04:06 +0200 Subject: [PATCH] Increases tx-local name/key id space of legacy index commands Legacy index commands don't take advantage of the token system in Neo4j. Perhaps that will change, but until then each transaction having one or more legacy index commands will also have an IndexDefineCommand which contains a String->id mapping where all other legacy index commands in that transaction will refer to ids instead of strings when it comes to names/keys. Previously this id space was only 6 bits, yielding a maximum of 63 names and 63 keys within the same transaction. This has turned out to be a problem in some scenarios so this commit extends that id space to 16 bits, now ranging up to 65535 names/keys in the same transaction. This means that the log format has changed. Although don't panic since the following commit will deal with this. --- .../neo4j/kernel/impl/index/IndexCommand.java | 24 +- .../kernel/impl/index/IndexDefineCommand.java | 65 +- .../command/CommandReaderFactory.java | 43 +- .../PhysicalLogNeoCommandReaderV1.java | 10 +- .../PhysicalLogNeoCommandReaderV2.java | 10 +- .../PhysicalLogNeoCommandReaderV2_2_4.java | 754 ++++++++++++++++++ .../impl/transaction/log/CommandWriter.java | 16 +- .../entry/DefaultLogEntryParserFactory.java | 1 + .../transaction/log/entry/LogVersions.java | 5 +- .../impl/api/LegacyIndexApplierTest.java | 10 +- ...sactionRepresentationStoreApplierTest.java | 7 +- .../impl/index/IndexDefineCommandTest.java | 12 +- .../PhysicalLogNeoCommandReaderV1Test.java | 38 - .../PhysicalLogNeoCommandReaderV2Test.java | 4 +- .../log/BatchingTransactionAppenderTest.java | 6 +- .../transaction/state/LogTruncationTest.java | 13 +- .../impl/lucene/LuceneCommandApplierTest.java | 10 +- 17 files changed, 886 insertions(+), 142 deletions(-) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2_2_4.java diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexCommand.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexCommand.java index 80097049a5aa0..d7e1fd2b313a3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexCommand.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexCommand.java @@ -19,8 +19,6 @@ */ package org.neo4j.kernel.impl.index; -import static java.lang.String.format; - import java.io.IOException; import java.util.Map; @@ -30,6 +28,8 @@ import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler; import org.neo4j.kernel.impl.transaction.command.NeoCommandType; +import static java.lang.String.format; + /** * Created from {@link IndexDefineCommand} or read from a logical log. * Contains all the different types of commands that an {@link Index} need @@ -46,14 +46,14 @@ public abstract class IndexCommand extends Command public static final byte VALUE_TYPE_STRING = (byte) 6; private byte commandType; - protected byte indexNameId; + protected int indexNameId; protected byte entityType; protected long entityId; - protected byte keyId; + protected int keyId; protected byte valueType; protected Object value; - protected void init( byte commandType, byte indexNameId, byte entityType, long entityId, byte keyId, Object value ) + protected void init( byte commandType, int indexNameId, byte entityType, long entityId, int keyId, Object value ) { this.commandType = commandType ; this.indexNameId = indexNameId; @@ -64,7 +64,7 @@ protected void init( byte commandType, byte indexNameId, byte entityType, long e this.valueType = valueTypeOf( value ); } - public byte getIndexNameId() + public int getIndexNameId() { return indexNameId; } @@ -79,7 +79,7 @@ public long getEntityId() return entityId; } - public byte getKeyId() + public int getKeyId() { return keyId; } @@ -144,7 +144,7 @@ else if ( value instanceof Short ) public static class AddNodeCommand extends IndexCommand { - public void init( byte indexNameId, long entityId, byte keyId, Object value ) + public void init( int indexNameId, long entityId, int keyId, Object value ) { super.init( NeoCommandType.INDEX_ADD_COMMAND, indexNameId, IndexEntityType.Node.id(), entityId, keyId, value ); @@ -173,7 +173,7 @@ public static class AddRelationshipCommand extends IndexCommand private long startNode; private long endNode; - public void init( byte indexNameId, long entityId, byte keyId, + public void init( int indexNameId, long entityId, int keyId, Object value, long startNode, long endNode ) { super.init( NeoCommandType.INDEX_ADD_RELATIONSHIP_COMMAND, indexNameId, IndexEntityType.Relationship.id(), @@ -242,7 +242,7 @@ public String toString() public static class RemoveCommand extends IndexCommand { - public void init( byte indexNameId, byte entityType, long entityId, byte keyId, Object value ) + public void init( int indexNameId, byte entityType, long entityId, int keyId, Object value ) { super.init( NeoCommandType.INDEX_REMOVE_COMMAND, indexNameId, entityType, entityId, keyId, value ); } @@ -263,7 +263,7 @@ public String toString() public static class DeleteCommand extends IndexCommand { - public void init( byte indexNameId, byte entityType ) + public void init( int indexNameId, byte entityType ) { super.init( NeoCommandType.INDEX_DELETE_COMMAND, indexNameId, entityType, 0L, (byte)0, null ); } @@ -285,7 +285,7 @@ public static class CreateCommand extends IndexCommand { private Map config; - public void init( byte indexNameId, byte entityType, Map config ) + public void init( int indexNameId, byte entityType, Map config ) { super.init( NeoCommandType.INDEX_CREATE_COMMAND, indexNameId, entityType, 0L, (byte)0, null ); this.config = config; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexDefineCommand.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexDefineCommand.java index 62b1acdad4917..8a6ba16081108 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexDefineCommand.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexDefineCommand.java @@ -24,11 +24,16 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.neo4j.collection.primitive.Primitive; +import org.neo4j.collection.primitive.PrimitiveIntObjectMap; import org.neo4j.kernel.impl.transaction.command.Command; import org.neo4j.kernel.impl.transaction.command.CommandRecordVisitor; import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler; -import static org.neo4j.helpers.collection.MapUtil.reverse; +import static java.lang.Math.pow; +import static java.lang.String.format; + +import static org.neo4j.collection.primitive.Primitive.intObjectMap; /** * A command which have to be first in the transaction. It will map index names @@ -44,20 +49,20 @@ public class IndexDefineCommand extends Command { private final AtomicInteger nextIndexNameId = new AtomicInteger(); private final AtomicInteger nextKeyId = new AtomicInteger(); - private Map indexNameIdRange; - private Map keyIdRange; - private Map idToIndexName; - private Map idToKey; + private Map indexNameIdRange; + private Map keyIdRange; + private PrimitiveIntObjectMap idToIndexName; + private PrimitiveIntObjectMap idToKey; public IndexDefineCommand() { - setIndexNameIdRange( new HashMap() ); - setKeyIdRange( new HashMap() ); - idToIndexName = new HashMap<>(); - idToKey = new HashMap<>(); + setIndexNameIdRange( new HashMap() ); + setKeyIdRange( new HashMap() ); + idToIndexName = intObjectMap( 16 ); + idToKey = intObjectMap( 16 ); } - public void init( Map indexNames, Map keys ) + public void init( Map indexNames, Map keys ) { this.setIndexNameIdRange( indexNames ); this.setKeyIdRange( keys ); @@ -65,7 +70,17 @@ public void init( Map indexNames, Map keys ) idToKey = reverse( keys ); } - private static String getFromMap( Map map, byte id ) + private static PrimitiveIntObjectMap reverse( Map map ) + { + PrimitiveIntObjectMap result = Primitive.intObjectMap( map.size() ); + for ( Map.Entry entry : map.entrySet() ) + { + result.put( entry.getValue().intValue(), entry.getKey() ); + } + return result; + } + + private static String getFromMap( PrimitiveIntObjectMap map, int id ) { if ( id == -1 ) { @@ -79,27 +94,27 @@ private static String getFromMap( Map map, byte id ) return result; } - public String getIndexName( byte id ) + public String getIndexName( int id ) { return getFromMap( idToIndexName, id ); } - public String getKey( byte id ) + public String getKey( int id ) { return getFromMap( idToKey, id ); } - public byte getOrAssignIndexNameId( String indexName ) + public int getOrAssignIndexNameId( String indexName ) { return getOrAssignId( indexNameIdRange, idToIndexName, nextIndexNameId, indexName ); } - public byte getOrAssignKeyId( String key ) + public int getOrAssignKeyId( String key ) { return getOrAssignId( keyIdRange, idToKey, nextKeyId, key ); } - private byte getOrAssignId( Map stringToId, Map idToString, + private int getOrAssignId( Map stringToId, PrimitiveIntObjectMap idToString, AtomicInteger nextId, String string ) { if ( string == null ) @@ -107,18 +122,20 @@ private byte getOrAssignId( Map stringToId, Map idTo return -1; } - Byte id = stringToId.get( string ); + Integer id = stringToId.get( string ); if ( id != null ) { return id; } int nextIdInt = nextId.incrementAndGet(); - if ( nextIdInt > 63 ) + if ( (nextIdInt & ~0xFFFF) != 0 ) { - throw new IllegalStateException( "Modifying more than 63 indexes in a single transaction is not supported" ); + throw new IllegalStateException( format( + "Modifying more than %d indexes in a single transaction is not supported", + (int)(pow( 2, 16 ) - 1) ) ); } - id = (byte) nextIdInt; + id = nextIdInt; stringToId.put( string, id ); idToString.put( id, string ); @@ -158,22 +175,22 @@ public boolean handle( NeoCommandHandler visitor ) throws IOException return visitor.visitIndexDefineCommand( this ); } - public Map getIndexNameIdRange() + public Map getIndexNameIdRange() { return indexNameIdRange; } - public void setIndexNameIdRange( Map indexNameIdRange ) + public void setIndexNameIdRange( Map indexNameIdRange ) { this.indexNameIdRange = indexNameIdRange; } - public Map getKeyIdRange() + public Map getKeyIdRange() { return keyIdRange; } - public void setKeyIdRange( Map keyIdRange ) + public void setKeyIdRange( Map keyIdRange ) { this.keyIdRange = keyIdRange; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/CommandReaderFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/CommandReaderFactory.java index bbac64d85094a..4044d17390247 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/CommandReaderFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/CommandReaderFactory.java @@ -35,6 +35,7 @@ import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_1; import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_2; import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_3; +import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.LOG_VERSION_2_2_4; public abstract class CommandReaderFactory { @@ -66,26 +67,28 @@ private CommandReader figureOutCorrectReader( byte logFormatVersion, byte logEnt { switch ( logEntryVersion ) { - // These are not thread safe, so if they are to be cached it has to be done in an object pool - case LEGACY_LOG_ENTRY_VERSION: - switch ( logFormatVersion ) - { - case LOG_VERSION_2_0: - return new PhysicalLogNeoCommandReaderV0_20(); - case LOG_VERSION_1_9: - return new PhysicalLogNeoCommandReaderV0_19(); - } - case LOG_ENTRY_VERSION_2_1: - case LOG_ENTRY_VERSION_2_2: - case LOG_ENTRY_VERSION_2_3: - switch ( logFormatVersion ) - { - case LOG_VERSION_2_1: - return new PhysicalLogNeoCommandReaderV1(); - case LOG_VERSION_2_2: - case LOG_VERSION_2_3: - return new PhysicalLogNeoCommandReaderV2(); - } + // These are not thread safe, so if they are to be cached it has to be done in an object pool + case LEGACY_LOG_ENTRY_VERSION: + switch ( logFormatVersion ) + { + case LOG_VERSION_2_0: + return new PhysicalLogNeoCommandReaderV0_20(); + case LOG_VERSION_1_9: + return new PhysicalLogNeoCommandReaderV0_19(); + } + case LOG_ENTRY_VERSION_2_1: + case LOG_ENTRY_VERSION_2_2: + case LOG_ENTRY_VERSION_2_3: + switch ( logFormatVersion ) + { + case LOG_VERSION_2_1: + return new PhysicalLogNeoCommandReaderV1(); + case LOG_VERSION_2_2: + case LOG_VERSION_2_3: + return new PhysicalLogNeoCommandReaderV2(); + case LOG_VERSION_2_2_4: + return new PhysicalLogNeoCommandReaderV2_2_4(); + } } throw new IllegalArgumentException( "Unknown log format version (" + logFormatVersion + ") and " + "log entry version (" + logEntryVersion + ")" ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1.java index 7ac16981b81ed..35259b6aad95b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1.java @@ -696,8 +696,8 @@ public boolean visitIndexCreateCommand( CreateCommand createCommand ) throws IOE public boolean visitIndexDefineCommand( IndexDefineCommand indexDefineCommand ) throws IOException { readIndexCommandHeader(); - Map indexNames = readMap( channel ); - Map keys = readMap( channel ); + Map indexNames = readMap( channel ); + Map keys = readMap( channel ); indexDefineCommand.init( indexNames, keys ); return false; } @@ -722,14 +722,14 @@ public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command return false; } - private Map readMap( ReadableLogChannel channel ) throws IOException + private Map readMap( ReadableLogChannel channel ) throws IOException { byte size = channel.get(); - Map result = new HashMap<>(); + Map result = new HashMap<>(); for ( int i = 0; i < size; i++ ) { String key = read2bLengthAndString( channel ); - byte id = channel.get(); + int id = channel.get(); if ( key == null ) { return null; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2.java index b54140b39e673..57268a922c502 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2.java @@ -702,8 +702,8 @@ public boolean visitIndexCreateCommand( CreateCommand createCommand ) throws IOE public boolean visitIndexDefineCommand( IndexDefineCommand indexDefineCommand ) throws IOException { readIndexCommandHeader(); - Map indexNames = readMap( channel ); - Map keys = readMap( channel ); + Map indexNames = readMap( channel ); + Map keys = readMap( channel ); indexDefineCommand.init( indexNames, keys ); return false; } @@ -728,14 +728,14 @@ public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command return false; } - private Map readMap( ReadableLogChannel channel ) throws IOException + private Map readMap( ReadableLogChannel channel ) throws IOException { byte size = channel.get(); - Map result = new HashMap<>(); + Map result = new HashMap<>(); for ( int i = 0; i < size; i++ ) { String key = read2bLengthAndString( channel ); - byte id = channel.get(); + int id = channel.get(); if ( key == null ) { return null; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2_2_4.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2_2_4.java new file mode 100644 index 0000000000000..528da7fc82bc0 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2_2_4.java @@ -0,0 +1,754 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.command; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.neo4j.kernel.api.exceptions.schema.MalformedSchemaRuleException; +import org.neo4j.kernel.impl.index.IndexCommand; +import org.neo4j.kernel.impl.index.IndexCommand.AddNodeCommand; +import org.neo4j.kernel.impl.index.IndexCommand.AddRelationshipCommand; +import org.neo4j.kernel.impl.index.IndexCommand.CreateCommand; +import org.neo4j.kernel.impl.index.IndexCommand.DeleteCommand; +import org.neo4j.kernel.impl.index.IndexCommand.RemoveCommand; +import org.neo4j.kernel.impl.index.IndexDefineCommand; +import org.neo4j.kernel.impl.store.AbstractDynamicStore; +import org.neo4j.kernel.impl.store.PropertyType; +import org.neo4j.kernel.impl.store.record.DynamicRecord; +import org.neo4j.kernel.impl.store.record.LabelTokenRecord; +import org.neo4j.kernel.impl.store.record.NeoStoreRecord; +import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.PropertyBlock; +import org.neo4j.kernel.impl.store.record.PropertyKeyTokenRecord; +import org.neo4j.kernel.impl.store.record.PropertyRecord; +import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; +import org.neo4j.kernel.impl.store.record.RelationshipRecord; +import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRule; +import org.neo4j.kernel.impl.transaction.command.Command.NodeCountsCommand; +import org.neo4j.kernel.impl.transaction.command.Command.RelationshipCountsCommand; +import org.neo4j.kernel.impl.transaction.command.CommandReaderFactory.DynamicRecordAdder; +import org.neo4j.kernel.impl.transaction.log.LogPositionMarker; +import org.neo4j.kernel.impl.transaction.log.ReadPastEndException; +import org.neo4j.kernel.impl.transaction.log.ReadableLogChannel; + +import static org.neo4j.helpers.collection.IteratorUtil.first; +import static org.neo4j.kernel.impl.transaction.command.CommandReaderFactory.COLLECTION_DYNAMIC_RECORD_ADDER; +import static org.neo4j.kernel.impl.transaction.command.CommandReaderFactory.PROPERTY_BLOCK_DYNAMIC_RECORD_ADDER; +import static org.neo4j.kernel.impl.transaction.command.CommandReaderFactory.PROPERTY_DELETED_DYNAMIC_RECORD_ADDER; +import static org.neo4j.kernel.impl.transaction.command.CommandReaderFactory.PROPERTY_INDEX_DYNAMIC_RECORD_ADDER; +import static org.neo4j.kernel.impl.util.Bits.bitFlag; +import static org.neo4j.kernel.impl.util.Bits.notFlag; +import static org.neo4j.kernel.impl.util.IoPrimitiveUtils.read2bLengthAndString; +import static org.neo4j.kernel.impl.util.IoPrimitiveUtils.read2bMap; +import static org.neo4j.kernel.impl.util.IoPrimitiveUtils.read3bLengthAndString; + +public class PhysicalLogNeoCommandReaderV2_2_4 implements CommandReader, NeoCommandHandler +{ + private ReadableLogChannel channel; + private IndexCommandHeader indexCommandHeader; + + @Override + public Command read( ReadableLogChannel channel ) throws IOException + { + // for the reader to pick up + this.channel = channel; + byte commandType = 0; + while ( commandType == 0 ) + { + commandType = channel.get(); + } + Command command = instantiateCommand( channel, commandType ); + if ( command.handle( this ) ) + { + return null; + } + return command; + } + + private Command instantiateCommand( ReadableLogChannel channel, byte commandType ) throws IOException + { + switch ( commandType ) + { + case NeoCommandType.NODE_COMMAND: + return new Command.NodeCommand(); + case NeoCommandType.PROP_COMMAND: + return new Command.PropertyCommand(); + case NeoCommandType.PROP_INDEX_COMMAND: + return new Command.PropertyKeyTokenCommand(); + case NeoCommandType.REL_COMMAND: + return new Command.RelationshipCommand(); + case NeoCommandType.REL_TYPE_COMMAND: + return new Command.RelationshipTypeTokenCommand(); + case NeoCommandType.LABEL_KEY_COMMAND: + return new Command.LabelTokenCommand(); + case NeoCommandType.NEOSTORE_COMMAND: + return new Command.NeoStoreCommand(); + case NeoCommandType.SCHEMA_RULE_COMMAND: + return new Command.SchemaRuleCommand(); + case NeoCommandType.REL_GROUP_COMMAND: + return new Command.RelationshipGroupCommand(); + case NeoCommandType.INDEX_DEFINE_COMMAND: + return new IndexDefineCommand(); + case NeoCommandType.INDEX_ADD_COMMAND: + return new IndexCommand.AddNodeCommand(); + case NeoCommandType.INDEX_ADD_RELATIONSHIP_COMMAND: + return new IndexCommand.AddRelationshipCommand(); + case NeoCommandType.INDEX_REMOVE_COMMAND: + return new IndexCommand.RemoveCommand(); + case NeoCommandType.INDEX_DELETE_COMMAND: + return new IndexCommand.DeleteCommand(); + case NeoCommandType.INDEX_CREATE_COMMAND: + return new IndexCommand.CreateCommand(); + case NeoCommandType.UPDATE_RELATIONSHIP_COUNTS_COMMAND: + return new RelationshipCountsCommand(); + case NeoCommandType.UPDATE_NODE_COUNTS_COMMAND: + return new NodeCountsCommand(); + default: + LogPositionMarker position = new LogPositionMarker(); + channel.getCurrentPosition( position ); + throw new IOException( "Unknown command type[" + commandType + "] near " + position.newPosition() ); + } + } + + private static final class IndexCommandHeader + { + byte valueType; + byte entityType; + boolean entityIdNeedsLong; + int indexNameId; + boolean startNodeNeedsLong; + boolean endNodeNeedsLong; + int keyId; + + IndexCommandHeader set( byte valueType, byte entityType, boolean entityIdNeedsLong, + int indexNameId, boolean startNodeNeedsLong, boolean endNodeNeedsLong, int keyId ) + { + this.valueType = valueType; + this.entityType = entityType; + this.entityIdNeedsLong = entityIdNeedsLong; + this.indexNameId = indexNameId; + this.startNodeNeedsLong = startNodeNeedsLong; + this.endNodeNeedsLong = endNodeNeedsLong; + this.keyId = keyId; + return this; + } + } + + @Override + public boolean visitNodeCommand( Command.NodeCommand command ) throws IOException + { + long id = channel.getLong(); + NodeRecord before = readNodeRecord( id ); + if ( before == null ) + { + return true; + } + NodeRecord after = readNodeRecord( id ); + if ( after == null ) + { + return true; + } + if ( !before.inUse() && after.inUse() ) + { + after.setCreated(); + } + command.init( before, after ); + return false; + } + + @Override + public boolean visitRelationshipCommand( Command.RelationshipCommand command ) throws IOException + { + long id = channel.getLong(); + byte flags = channel.get(); + boolean inUse = false; + if ( notFlag( notFlag( flags, Record.IN_USE.byteValue() ), Record.CREATED_IN_TX ) != 0 ) + { + throw new IOException( "Illegal in use flag: " + flags ); + } + if ( bitFlag( flags, Record.IN_USE.byteValue() ) ) + { + inUse = true; + } + RelationshipRecord record; + if ( inUse ) + { + record = new RelationshipRecord( id, channel.getLong(), channel.getLong(), channel.getInt() ); + record.setInUse( true ); + record.setFirstPrevRel( channel.getLong() ); + record.setFirstNextRel( channel.getLong() ); + record.setSecondPrevRel( channel.getLong() ); + record.setSecondNextRel( channel.getLong() ); + record.setNextProp( channel.getLong() ); + byte extraByte = channel.get(); + record.setFirstInFirstChain( (extraByte & 0x1) > 0 ); + record.setFirstInSecondChain( (extraByte & 0x2) > 0 ); + } + else + { + record = new RelationshipRecord( id, -1, -1, channel.getInt() ); + record.setInUse( false ); + } + if ( bitFlag( flags, Record.CREATED_IN_TX ) ) + { + record.setCreated(); + } + command.init( record ); + return false; + } + + @Override + public boolean visitPropertyCommand( Command.PropertyCommand command ) throws IOException + { + // ID + long id = channel.getLong(); // 8 + // BEFORE + PropertyRecord before = readPropertyRecord( id ); + if ( before == null ) + { + return true; + } + // AFTER + PropertyRecord after = readPropertyRecord( id ); + if ( after == null ) + { + return true; + } + command.init( before, after ); + return false; + } + + @Override + public boolean visitRelationshipGroupCommand( Command.RelationshipGroupCommand command ) throws IOException + { + long id = channel.getLong(); + byte inUseByte = channel.get(); + boolean inUse = inUseByte == Record.IN_USE.byteValue(); + if ( inUseByte != Record.IN_USE.byteValue() && inUseByte != Record.NOT_IN_USE.byteValue() ) + { + throw new IOException( "Illegal in use flag: " + inUseByte ); + } + int type = channel.getShort(); + RelationshipGroupRecord record = new RelationshipGroupRecord( id, type ); + record.setInUse( inUse ); + record.setNext( channel.getLong() ); + record.setFirstOut( channel.getLong() ); + record.setFirstIn( channel.getLong() ); + record.setFirstLoop( channel.getLong() ); + record.setOwningNode( channel.getLong() ); + command.init( record ); + return false; + } + + @Override + public boolean visitRelationshipTypeTokenCommand( Command.RelationshipTypeTokenCommand command ) + throws IOException + { + // id+in_use(byte)+type_blockId(int)+nr_type_records(int) + int id = channel.getInt(); + byte inUseFlag = channel.get(); + boolean inUse = false; + if ( (inUseFlag & Record.IN_USE.byteValue()) == Record.IN_USE.byteValue() ) + { + inUse = true; + } + else if ( inUseFlag != Record.NOT_IN_USE.byteValue() ) + { + throw new IOException( "Illegal in use flag: " + inUseFlag ); + } + RelationshipTypeTokenRecord record = new RelationshipTypeTokenRecord( id ); + record.setInUse( inUse ); + record.setNameId( channel.getInt() ); + int nrTypeRecords = channel.getInt(); + for ( int i = 0; i < nrTypeRecords; i++ ) + { + DynamicRecord dr = readDynamicRecord(); + if ( dr == null ) + { + return true; + } + record.addNameRecord( dr ); + } + command.init( record ); + return false; + } + + @Override + public boolean visitLabelTokenCommand( Command.LabelTokenCommand command ) throws IOException + { + // id+in_use(byte)+type_blockId(int)+nr_type_records(int) + int id = channel.getInt(); + byte inUseFlag = channel.get(); + boolean inUse = false; + if ( (inUseFlag & Record.IN_USE.byteValue()) == Record.IN_USE.byteValue() ) + { + inUse = true; + } + else if ( inUseFlag != Record.NOT_IN_USE.byteValue() ) + { + throw new IOException( "Illegal in use flag: " + inUseFlag ); + } + LabelTokenRecord record = new LabelTokenRecord( id ); + record.setInUse( inUse ); + record.setNameId( channel.getInt() ); + int nrTypeRecords = channel.getInt(); + for ( int i = 0; i < nrTypeRecords; i++ ) + { + DynamicRecord dr = readDynamicRecord(); + if ( dr == null ) + { + return true; + } + record.addNameRecord( dr ); + } + command.init( record ); + return false; + } + + @Override + public boolean visitPropertyKeyTokenCommand( Command.PropertyKeyTokenCommand command ) throws IOException + { + // id+in_use(byte)+count(int)+key_blockId(int) + int id = channel.getInt(); + byte inUseFlag = channel.get(); + boolean inUse = false; + if ( (inUseFlag & Record.IN_USE.byteValue()) == Record.IN_USE.byteValue() ) + { + inUse = true; + } + else if ( inUseFlag != Record.NOT_IN_USE.byteValue() ) + { + throw new IOException( "Illegal in use flag: " + inUseFlag ); + } + PropertyKeyTokenRecord record = new PropertyKeyTokenRecord( id ); + record.setInUse( inUse ); + record.setPropertyCount( channel.getInt() ); + record.setNameId( channel.getInt() ); + if ( readDynamicRecords( record, PROPERTY_INDEX_DYNAMIC_RECORD_ADDER ) == -1 ) + { + return true; + } + command.init( record ); + return false; + } + + @Override + public boolean visitSchemaRuleCommand( Command.SchemaRuleCommand command ) throws IOException + { + Collection recordsBefore = new ArrayList<>(); + readDynamicRecords( recordsBefore, COLLECTION_DYNAMIC_RECORD_ADDER ); + Collection recordsAfter = new ArrayList<>(); + readDynamicRecords( recordsAfter, COLLECTION_DYNAMIC_RECORD_ADDER ); + byte isCreated = channel.get(); + if ( 1 == isCreated ) + { + for ( DynamicRecord record : recordsAfter ) + { + record.setCreated(); + } + } + SchemaRule rule = first( recordsAfter ).inUse() ? readSchemaRule( recordsAfter ) + : readSchemaRule( recordsBefore ); + command.init( recordsBefore, recordsAfter, rule ); + return false; + } + + @Override + public boolean visitNeoStoreCommand( Command.NeoStoreCommand command ) throws IOException + { + long nextProp = channel.getLong(); + NeoStoreRecord record = new NeoStoreRecord(); + record.setNextProp( nextProp ); + command.init( record ); + return false; + } + + private NodeRecord readNodeRecord( long id ) throws IOException + { + byte inUseFlag = channel.get(); + boolean inUse = false; + if ( inUseFlag == Record.IN_USE.byteValue() ) + { + inUse = true; + } + else if ( inUseFlag != Record.NOT_IN_USE.byteValue() ) + { + throw new IOException( "Illegal in use flag: " + inUseFlag ); + } + NodeRecord record; + Collection dynamicLabelRecords = new ArrayList<>(); + long labelField = Record.NO_LABELS_FIELD.intValue(); + if ( inUse ) + { + boolean dense = channel.get() == 1; + record = new NodeRecord( id, dense, channel.getLong(), channel.getLong() ); + // labels + labelField = channel.getLong(); + } + else + { + record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), + Record.NO_NEXT_PROPERTY.intValue() ); + } + + readDynamicRecords( dynamicLabelRecords, COLLECTION_DYNAMIC_RECORD_ADDER ); + record.setLabelField( labelField, dynamicLabelRecords ); + + record.setInUse( inUse ); + return record; + } + + DynamicRecord readDynamicRecord() throws IOException + { + // id+type+in_use(byte)+nr_of_bytes(int)+next_block(long) + long id = channel.getLong(); + assert id >= 0 && id <= (1l << 36) - 1 : id + " is not a valid dynamic record id"; + int type = channel.getInt(); + byte inUseFlag = channel.get(); + boolean inUse = (inUseFlag & Record.IN_USE.byteValue()) != 0; + DynamicRecord record = new DynamicRecord( id ); + record.setInUse( inUse, type ); + if ( inUse ) + { + record.setStartRecord( (inUseFlag & Record.FIRST_IN_CHAIN.byteValue()) != 0 ); + int nrOfBytes = channel.getInt(); + assert nrOfBytes >= 0 && nrOfBytes < ((1 << 24) - 1) : nrOfBytes + + + " is not valid for a number of bytes field of " + + "a dynamic record"; + long nextBlock = channel.getLong(); + assert (nextBlock >= 0 && nextBlock <= (1l << 36 - 1)) + || (nextBlock == Record.NO_NEXT_BLOCK.intValue()) : nextBlock + + + " is not valid for a next record field of " + + "a dynamic record"; + record.setNextBlock( nextBlock ); + byte data[] = new byte[nrOfBytes]; + channel.get( data, nrOfBytes ); + record.setData( data ); + } + return record; + } + + int readDynamicRecords( T target, DynamicRecordAdder adder ) throws IOException + { + int numberOfRecords = channel.getInt(); + assert numberOfRecords >= 0; + while ( numberOfRecords > 0 ) + { + DynamicRecord read = readDynamicRecord(); + if ( read == null ) + { + return -1; + } + adder.add( target, read ); + numberOfRecords--; + } + return numberOfRecords; + } + + private PropertyRecord readPropertyRecord( long id ) throws IOException + { + // in_use(byte)+type(int)+key_indexId(int)+prop_blockId(long)+ + // prev_prop_id(long)+next_prop_id(long) + PropertyRecord record = new PropertyRecord( id ); + byte inUseFlag = channel.get(); // 1 + long nextProp = channel.getLong(); // 8 + long prevProp = channel.getLong(); // 8 + record.setNextProp( nextProp ); + record.setPrevProp( prevProp ); + boolean inUse = false; + if ( (inUseFlag & Record.IN_USE.byteValue()) == Record.IN_USE.byteValue() ) + { + inUse = true; + } + boolean nodeProperty = true; + if ( (inUseFlag & Record.REL_PROPERTY.byteValue()) == Record.REL_PROPERTY.byteValue() ) + { + nodeProperty = false; + } + long primitiveId = channel.getLong(); // 8 + if ( primitiveId != -1 && nodeProperty ) + { + record.setNodeId( primitiveId ); + } + else if ( primitiveId != -1 ) + { + record.setRelId( primitiveId ); + } + int nrPropBlocks = channel.get(); + assert nrPropBlocks >= 0; + if ( nrPropBlocks > 0 ) + { + record.setInUse( true ); + } + while ( nrPropBlocks-- > 0 ) + { + PropertyBlock block = readPropertyBlock(); + if ( block == null ) + { + return null; + } + record.addPropertyBlock( block ); + } + int deletedRecords = readDynamicRecords( record, PROPERTY_DELETED_DYNAMIC_RECORD_ADDER ); + if ( deletedRecords == -1 ) + { + return null; + } + assert deletedRecords >= 0; + while ( deletedRecords-- > 0 ) + { + DynamicRecord read = readDynamicRecord(); + if ( read == null ) + { + return null; + } + record.addDeletedRecord( read ); + } + if ( (inUse && !record.inUse()) || (!inUse && record.inUse()) ) + { + throw new IllegalStateException( "Weird, inUse was read in as " + inUse + " but the record is " + + record ); + } + return record; + } + + PropertyBlock readPropertyBlock() throws IOException + { + PropertyBlock toReturn = new PropertyBlock(); + byte blockSize = channel.get(); // the size is stored in bytes // 1 + assert blockSize > 0 && blockSize % 8 == 0 : blockSize + " is not a valid block size value"; + // Read in blocks + long[] blocks = readLongs( blockSize / 8 ); + assert blocks.length == blockSize / 8 : blocks.length + + " longs were read in while i asked for what corresponds to " + + blockSize; + assert PropertyType.getPropertyType( blocks[0], false ).calculateNumberOfBlocksUsed( blocks[0] ) == + blocks.length : blocks.length + + " is not a valid number of blocks for type " + + PropertyType.getPropertyType( blocks[0], false ); + /* + * Ok, now we may be ready to return, if there are no DynamicRecords. So + * we start building the Object + */ + toReturn.setValueBlocks( blocks ); + /* + * Read in existence of DynamicRecords. Remember, this has already been + * read in the buffer with the blocks, above. + */ + if ( readDynamicRecords( toReturn, PROPERTY_BLOCK_DYNAMIC_RECORD_ADDER ) == -1 ) + { + return null; + } + return toReturn; + } + + private long[] readLongs( int count ) throws IOException + { + long[] result = new long[count]; + for ( int i = 0; i < count; i++ ) + { + result[i] = channel.getLong(); + } + return result; + } + + private SchemaRule readSchemaRule( Collection recordsBefore ) + { + // TODO: Why was this assertion here? + // assert first(recordsBefore).inUse() : "Asked to deserialize schema records that were not in + // use."; + SchemaRule rule; + ByteBuffer deserialized = AbstractDynamicStore.concatData( recordsBefore, new byte[100] ); + try + { + rule = SchemaRule.Kind.deserialize( first( recordsBefore ).getId(), deserialized ); + } + catch ( MalformedSchemaRuleException e ) + { + return null; + } + return rule; + } + + @Override + public boolean visitIndexAddNodeCommand( AddNodeCommand command ) throws IOException + { + IndexCommandHeader header = readIndexCommandHeader(); + Number entityId = header.entityIdNeedsLong ? channel.getLong() : channel.getInt(); + Object value = readIndexValue( header.valueType ); + command.init( header.indexNameId, entityId.longValue(), header.keyId, value ); + return false; + } + + @Override + public boolean visitIndexAddRelationshipCommand( AddRelationshipCommand command ) throws IOException + { + IndexCommandHeader header = readIndexCommandHeader(); + Number entityId = header.entityIdNeedsLong ? channel.getLong() : channel.getInt(); + Object value = readIndexValue( header.valueType ); + Number startNode = header.startNodeNeedsLong ? channel.getLong() : channel.getInt(); + Number endNode = header.endNodeNeedsLong ? channel.getLong() : channel.getInt(); + command.init( header.indexNameId, entityId.longValue(), header.keyId, value, + startNode.longValue(), endNode.longValue() ); + return false; + } + + @Override + public boolean visitIndexRemoveCommand( RemoveCommand command ) throws IOException + { + IndexCommandHeader header = readIndexCommandHeader(); + Number entityId = header.entityIdNeedsLong ? channel.getLong() : channel.getInt(); + Object value = readIndexValue( header.valueType ); + command.init( header.indexNameId, header.entityType, entityId.longValue(), header.keyId, value ); + return false; + } + + @Override + public boolean visitIndexDeleteCommand( DeleteCommand deleteCommand ) throws IOException + { + IndexCommandHeader header = readIndexCommandHeader(); + deleteCommand.init( header.indexNameId, header.entityType ); + return false; + } + + @Override + public boolean visitIndexCreateCommand( CreateCommand createCommand ) throws IOException + { + IndexCommandHeader header = readIndexCommandHeader(); + Map config = read2bMap( channel ); + createCommand.init( header.indexNameId, header.entityType, config ); + return false; + } + + @Override + public boolean visitIndexDefineCommand( IndexDefineCommand indexDefineCommand ) throws IOException + { + readIndexCommandHeader(); + Map indexNames = readMap( channel ); + Map keys = readMap( channel ); + indexDefineCommand.init( indexNames, keys ); + return false; + } + + @Override + public boolean visitNodeCountsCommand( NodeCountsCommand command ) throws IOException + { + int labelId = channel.getInt(); + long delta = channel.getLong(); + command.init( labelId, delta ); + return false; + } + + @Override + public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command ) throws IOException + { + int startLabelId = channel.getInt(); + int typeId = channel.getInt(); + int endLabelId = channel.getInt(); + long delta = channel.getLong(); + command.init( startLabelId, typeId, endLabelId, delta ); + return false; + } + + private Map readMap( ReadableLogChannel channel ) throws IOException + { + byte size = channel.get(); + Map result = new HashMap<>(); + for ( int i = 0; i < size; i++ ) + { + String key = read2bLengthAndString( channel ); + int id = getUnsignedShort( channel ); + if ( key == null ) + { + return null; + } + result.put( key, id ); + } + return result; + } + + private int getUnsignedShort( ReadableLogChannel channel ) throws IOException + { + return channel.getShort() & 0xFFFF; + } + + private IndexCommandHeader readIndexCommandHeader() throws ReadPastEndException, IOException + { + byte firstHeaderByte = channel.get(); + byte valueType = (byte) ((firstHeaderByte & 0x1C) >> 2); + byte entityType = (byte) ((firstHeaderByte & 0x2) >> 1); + boolean entityIdNeedsLong = (firstHeaderByte & 0x1) > 0; + + byte secondHeaderByte = channel.get(); + boolean startNodeNeedsLong = (secondHeaderByte & 0x80) > 0; + boolean endNodeNeedsLong = (secondHeaderByte & 0x40) > 0; + + int indexNameId = getUnsignedShort( channel ); + int keyId = getUnsignedShort( channel ); + if ( indexCommandHeader == null ) + { + indexCommandHeader = new IndexCommandHeader(); + } + return indexCommandHeader.set( valueType, entityType, entityIdNeedsLong, + indexNameId, startNodeNeedsLong, endNodeNeedsLong, keyId ); + } + + private Object readIndexValue( byte valueType ) throws IOException + { + switch ( valueType ) + { + case IndexCommand.VALUE_TYPE_NULL: + return null; + case IndexCommand.VALUE_TYPE_SHORT: + return channel.getShort(); + case IndexCommand.VALUE_TYPE_INT: + return channel.getInt(); + case IndexCommand.VALUE_TYPE_LONG: + return channel.getLong(); + case IndexCommand.VALUE_TYPE_FLOAT: + return channel.getFloat(); + case IndexCommand.VALUE_TYPE_DOUBLE: + return channel.getDouble(); + case IndexCommand.VALUE_TYPE_STRING: + return read3bLengthAndString( channel ); + default: + throw new RuntimeException( "Unknown value type " + valueType ); + } + } + + @Override + public void apply() + { // Nothing to apply + } + + @Override + public void close() + { // Nothing to close + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/CommandWriter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/CommandWriter.java index 947bcf1abb932..881c8434aceed 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/CommandWriter.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/CommandWriter.java @@ -291,13 +291,14 @@ public boolean visitRelationshipCountsCommand( RelationshipCountsCommand command return false; } - private void writeMap( Map map ) throws IOException + private void writeMap( Map map ) throws IOException { channel.put( (byte) map.size() ); - for ( Map.Entry entry : map.entrySet() ) + for ( Map.Entry entry : map.entrySet() ) { write2bLengthAndString( channel, entry.getKey() ); - channel.put( entry.getValue() ); + int id = entry.getValue(); + channel.putShort( (short) id ); } } @@ -360,12 +361,13 @@ protected void writeIndexCommandHeader( IndexCommand command ) throws IOExceptio } protected void writeIndexCommandHeader( byte valueType, byte entityType, byte entityIdNeedsLong, - byte startNodeNeedsLong, byte endNodeNeedsLong, byte indexNameId, - byte commandKeyId ) throws IOException + byte startNodeNeedsLong, byte endNodeNeedsLong, int indexNameId, + int keyId ) throws IOException { channel.put( (byte) ((valueType << 2) | (entityType << 1) | (entityIdNeedsLong)) ); - channel.put( (byte) ((startNodeNeedsLong << 7) | (endNodeNeedsLong << 6) | (indexNameId)) ); - channel.put( commandKeyId ); + channel.put( (byte) ((startNodeNeedsLong << 7) | (endNodeNeedsLong << 6)) ); + channel.putShort( (short) indexNameId ); + channel.putShort( (short) keyId ); } protected void putIntOrLong( long id ) throws IOException diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/DefaultLogEntryParserFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/DefaultLogEntryParserFactory.java index 5339d83d274b0..f1defbc59a202 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/DefaultLogEntryParserFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/DefaultLogEntryParserFactory.java @@ -50,6 +50,7 @@ private LogEntryParserDispatcher figureOutCorrectDispatcher( byte logVersion ) case LogVersions.LOG_VERSION_2_1: return new LogEntryParserDispatcher<>( LogEntryParsersV4.values() ); case LogVersions.LOG_VERSION_2_2: + case LogVersions.LOG_VERSION_2_2_4: return new LogEntryParserDispatcher<>( LogEntryParsersV5.values() ); case LogVersions.LOG_VERSION_2_3: return new LogEntryParserDispatcher<>( LogEntryParsersV6.values() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogVersions.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogVersions.java index e344a6b6712e0..d4294c4715c6f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogVersions.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogVersions.java @@ -32,12 +32,15 @@ private LogVersions() * version 4 as of 2014-02-06: neo4j 2.1 Dense nodes, split by type/direction into groups * version 5 as of 2014-05-23: neo4j 2.2 Removal of JTA / unified data source * version 6 as of 2015-05-26: neo4j 2.3 added checkpoint entry + * version 7 as of 2015-05-26: neo4j 2.2.4 legacy index command header has bigger id space */ public static final byte LOG_VERSION_1_9 = (byte) 2; public static final byte LOG_VERSION_2_0 = (byte) 3; public static final byte LOG_VERSION_2_1 = (byte) 4; public static final byte LOG_VERSION_2_2 = (byte) 5; - public static final byte LOG_VERSION_2_3 = (byte) 6; + + public static final byte LOG_VERSION_2_2_4 = (byte) 6; + public static final byte LOG_VERSION_2_3 = (byte) 7; public static final byte CURRENT_LOG_VERSION = LOG_VERSION_2_3; // on disk current format version diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexApplierTest.java index 88b6f609c657d..4d97e374d45ff 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexApplierTest.java @@ -60,8 +60,8 @@ public class LegacyIndexApplierTest public void shouldOnlyCreateOneApplierPerProvider() throws Exception { // GIVEN - Map names = MapUtil. genericMap( "first", (byte) 0, "second", (byte) 1 ); - Map keys = MapUtil. genericMap( "key", (byte) 0 ); + Map names = MapUtil. genericMap( "first", 0, "second", 1 ); + Map keys = MapUtil. genericMap( "key", 0 ); String applierName = "test-applier"; IndexConfigStore config = newIndexConfigStore( names, applierName ); LegacyIndexApplierLookup applierLookup = mock( LegacyIndexApplierLookup.class ); @@ -95,20 +95,20 @@ private static AddNodeCommand addNodeToIndex( IndexDefineCommand definitions, St return command; } - private static IndexDefineCommand definitions( Map names, Map keys ) + private static IndexDefineCommand definitions( Map names, Map keys ) { IndexDefineCommand definitions = new IndexDefineCommand(); definitions.init( names, keys ); return definitions; } - private IndexConfigStore newIndexConfigStore( Map names, String providerName ) + private IndexConfigStore newIndexConfigStore( Map names, String providerName ) { File dir = new File( "conf" ); EphemeralFileSystemAbstraction fileSystem = fs.get(); fileSystem.mkdirs( dir ); IndexConfigStore store = life.add( new IndexConfigStore( dir, fileSystem ) ); - for ( Map.Entry name : names.entrySet() ) + for ( Map.Entry name : names.entrySet() ) { store.set( Node.class, name.getKey(), stringMap( IndexManager.PROVIDER, providerName ) ); store.set( Relationship.class, name.getKey(), stringMap( IndexManager.PROVIDER, providerName ) ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/TransactionRepresentationStoreApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/TransactionRepresentationStoreApplierTest.java index de8e5e0794d25..c91fb5e003d34 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/TransactionRepresentationStoreApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/TransactionRepresentationStoreApplierTest.java @@ -23,6 +23,7 @@ import org.mockito.Matchers; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -171,9 +172,9 @@ private Collection indexTransaction() { IndexDefineCommand definitions = new IndexDefineCommand(); definitions.init( - MapUtil.genericMap( "one", (byte) 1 ), - MapUtil.genericMap( "two", (byte) 2 ) ); - return Collections.singletonList( definitions ); + MapUtil.genericMap( "one" ,1 ), + MapUtil.genericMap( "two", 2 ) ); + return Arrays.asList( definitions ); } private TransactionRepresentation createNodeTransaction( long nodeId ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/IndexDefineCommandTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/IndexDefineCommandTest.java index 397d6b70dce0f..f042eb3d4d7ec 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/IndexDefineCommandTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/IndexDefineCommandTest.java @@ -19,10 +19,10 @@ */ package org.neo4j.kernel.impl.index; -import static org.junit.Assert.fail; - import org.junit.Test; +import static org.junit.Assert.fail; + public class IndexDefineCommandTest { @Test @@ -30,10 +30,10 @@ public void testIndexCommandCreationEnforcesLimit() throws Exception { // Given IndexDefineCommand idc = new IndexDefineCommand(); + int max = (int) (Math.pow( 2, 16 ) - 1); // When - // 63 keys are used, it should be fine - for ( int i = 0; i < 63; i++ ) + for ( int i = 0; i < max; i++ ) { idc.getOrAssignKeyId( "index" + i ); } @@ -42,8 +42,8 @@ public void testIndexCommandCreationEnforcesLimit() throws Exception // it should break on the 64th try { - idc.getOrAssignKeyId( "index63" ); - fail("IndexDefineCommand should not allow more than 63 indexes per transaction"); + idc.getOrAssignKeyId( "dropThatOverflows" ); + fail( "IndexDefineCommand should not allow more than 63 indexes per transaction" ); } catch( IllegalStateException e ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1Test.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1Test.java index cf7047d4ec689..ad4b45bc75275 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1Test.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV1Test.java @@ -21,7 +21,6 @@ import org.junit.Test; -import org.neo4j.kernel.impl.index.IndexCommand.AddRelationshipCommand; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.PropertyType; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -39,43 +38,6 @@ public class PhysicalLogNeoCommandReaderV1Test { - @Test - public void shouldReadIndexCommandHeaderCorrectly() throws Exception - { - // This bug manifested in header byte[1] {0,1,2}, which contains: - // [x , ] start node needs long - // [ x , ] end node needs long - // [ xx,xxxx] index name id - // would have the mask for reading "start node needs long" to 0x8, where it should have been 0x80. - // So we need an index name id which has the 0x8 bit set to falsely read that value as "true". - // Number 12 will do just fine. - - // GIVEN - PhysicalLogNeoCommandReaderV1 reader = new PhysicalLogNeoCommandReaderV1(); - InMemoryLogChannel data = new InMemoryLogChannel(); - CommandWriter writer = new CommandWriter( data ); - AddRelationshipCommand command = new AddRelationshipCommand(); - byte indexNameId = (byte)12; - long entityId = 123; - byte keyId = (byte)1; - Object value = "test value"; - long startNode = 14; - long endNode = 15; - - // WHEN - command.init( indexNameId, entityId, keyId, value, startNode, endNode ); - writer.visitIndexAddRelationshipCommand( command ); - - // THEN - AddRelationshipCommand readCommand = (AddRelationshipCommand) reader.read( data ); - assertEquals( indexNameId, readCommand.getIndexNameId() ); - assertEquals( entityId, readCommand.getEntityId() ); - assertEquals( keyId, readCommand.getKeyId() ); - assertEquals( value, readCommand.getValue() ); - assertEquals( startNode, readCommand.getStartNode() ); - assertEquals( endNode, readCommand.getEndNode() ); - } - @Test public void shouldReadPropertyCommandWithDeletedDynamicRecords() throws Exception { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2Test.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2Test.java index 3e6350037c400..08e6f11e2a231 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2Test.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogNeoCommandReaderV2Test.java @@ -41,7 +41,7 @@ public void shouldReadIndexCommandHeaderCorrectly() throws Exception // Number 12 will do just fine. // GIVEN - PhysicalLogNeoCommandReaderV2 reader = new PhysicalLogNeoCommandReaderV2(); + PhysicalLogNeoCommandReaderV2_2_4 reader = new PhysicalLogNeoCommandReaderV2_2_4(); InMemoryLogChannel data = new InMemoryLogChannel(); CommandWriter writer = new CommandWriter( data ); AddRelationshipCommand command = new AddRelationshipCommand(); @@ -78,7 +78,7 @@ public void shouldProperlyMaskIndexIdFieldInIndexHeader() throws Exception */ // GIVEN - PhysicalLogNeoCommandReaderV2 reader = new PhysicalLogNeoCommandReaderV2(); + PhysicalLogNeoCommandReaderV2_2_4 reader = new PhysicalLogNeoCommandReaderV2_2_4(); InMemoryLogChannel data = new InMemoryLogChannel(); CommandWriter writer = new CommandWriter( data ); // Here we take advantage of the fact that all index commands have the same header written out diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/BatchingTransactionAppenderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/BatchingTransactionAppenderTest.java index b6760747b814a..019e6128c0a91 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/BatchingTransactionAppenderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/BatchingTransactionAppenderTest.java @@ -455,7 +455,7 @@ private TransactionRepresentation transactionWithLegacyIndexCommand() { Collection commands = new ArrayList<>(); IndexDefineCommand command = new IndexDefineCommand(); - command.init( new HashMap(), new HashMap() ); + command.init( new HashMap(), new HashMap() ); commands.add( command ); PhysicalTransactionRepresentation transaction = new PhysicalTransactionRepresentation( commands ); transaction.setHeader( new byte[0], 0, 0, 0, 0, 0, 0 ); @@ -523,8 +523,8 @@ private TransactionRepresentation createTransaction( boolean includeLegacyIndexC { IndexDefineCommand defineCommand = new IndexDefineCommand(); defineCommand.init( - MapUtil.genericMap( "one", (byte) 1 ), - MapUtil.genericMap( "two", (byte) 2 ) ); + MapUtil.genericMap( "one", 1 ), + MapUtil.genericMap( "two", 2 ) ); commands.add( defineCommand ); } else diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/LogTruncationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/LogTruncationTest.java index 08f1383e6ee89..a5f8abeb6d7b8 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/LogTruncationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/LogTruncationTest.java @@ -104,27 +104,28 @@ public class LogTruncationTest // Index commands AddRelationshipCommand addRelationshipCommand = new AddRelationshipCommand(); - addRelationshipCommand.init( (byte) 1, 1l, (byte) 1, "some value", 1, 1 ); + addRelationshipCommand.init( 1, 1l, 12345, "some value", 1, 1 ); permutations.put( AddRelationshipCommand.class, new Command[] { addRelationshipCommand } ); CreateCommand createCommand = new CreateCommand(); - createCommand.init( (byte) 1, IndexEntityType.Relationship.id(), MapUtil.stringMap( "string1", "string 2" ) ); + createCommand.init( 1, IndexEntityType.Relationship.id(), MapUtil.stringMap( "string1", "string 2" ) ); permutations.put( CreateCommand.class, new Command[] { createCommand } ); AddNodeCommand addCommand = new AddNodeCommand(); - addCommand.init( (byte) 1, 122l, (byte) 2, "value" ); + addCommand.init( 1234, 122l, 2, "value" ); permutations.put( AddNodeCommand.class, new Command[] { addCommand } ); DeleteCommand deleteCommand = new DeleteCommand(); - deleteCommand.init( (byte) 1, IndexEntityType.Relationship.id() ); + deleteCommand.init( 1, IndexEntityType.Relationship.id() ); permutations.put( DeleteCommand.class, new Command[] { deleteCommand } ); RemoveCommand removeCommand = new RemoveCommand(); - removeCommand.init( (byte) 1, IndexEntityType.Node.id(), 126, (byte) 3, "the value" ); + removeCommand.init( 1, IndexEntityType.Node.id(), 126, (byte) 3, "the value" ); permutations.put( RemoveCommand.class, new Command[] { removeCommand } ); IndexDefineCommand indexDefineCommand = new IndexDefineCommand(); - indexDefineCommand.init( MapUtil.genericMap( "string1", (byte) 45, "key1", (byte) 2 ), MapUtil.genericMap( "string", (byte) 2 ) ); + indexDefineCommand.init( MapUtil.genericMap( + "string1", 45, "key1", 2 ), MapUtil.genericMap( "string", 2 ) ); permutations.put( IndexDefineCommand.class, new Command[] { indexDefineCommand } ); // Counts commands diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/LuceneCommandApplierTest.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/LuceneCommandApplierTest.java index 0082a2de4c4ff..7714c139a6ef3 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/LuceneCommandApplierTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/LuceneCommandApplierTest.java @@ -65,15 +65,15 @@ public void shouldHandleMultipleIdSpaces() throws Exception { // WHEN issuing a command where the index name is mapped to a certain id IndexDefineCommand definitions = definitions( - MapUtil.genericMap( indexName, (byte) 0 ), - MapUtil.genericMap( key, (byte) 0 ) ); + MapUtil.genericMap( indexName, 0 ), + MapUtil.genericMap( key, 0 ) ); applier.visitIndexDefineCommand( definitions ); applier.visitIndexAddNodeCommand( addNodeToIndex( definitions, indexName, 0L ) ); // and then later issuing a command for that same index, but in another transaction where // the local index name id is a different one definitions = definitions( - MapUtil.genericMap( indexName, (byte) 1 ), - MapUtil.genericMap( key, (byte) 0 ) ); + MapUtil.genericMap( indexName, 1 ), + MapUtil.genericMap( key, 0 ) ); applier.visitIndexDefineCommand( definitions ); applier.visitIndexAddNodeCommand( addNodeToIndex( definitions, indexName, 1L ) ); applier.apply(); @@ -90,7 +90,7 @@ private static AddNodeCommand addNodeToIndex( IndexDefineCommand definitions, St return command; } - private static IndexDefineCommand definitions( Map names, Map keys ) + private static IndexDefineCommand definitions( Map names, Map keys ) { IndexDefineCommand definitions = new IndexDefineCommand(); definitions.init( names, keys );