From 5ef5a80e2616489edd26040297897d2648acdf8a Mon Sep 17 00:00:00 2001 From: Davide Grohmann Date: Mon, 20 Jun 2016 11:16:51 +0200 Subject: [PATCH] Remove the no longer used naive and durable raft log implementations --- .../org/neo4j/coreedge/raft/log/RaftLog.java | 2 + .../raft/log/RaftableContentAppender.java | 30 -- .../log/naive/DumpNaiveDurableRaftLog.java | 45 -- .../raft/log/naive/NaiveDurableRaftLog.java | 485 ----------------- .../log/physical/DumpPhysicalRaftLog.java | 219 -------- .../raft/log/physical/EntryReader.java | 29 - .../FilenameBasedLogVersionRepository.java | 47 -- .../coreedge/raft/log/physical/Header.java | 34 -- .../raft/log/physical/HeaderReader.java | 30 -- .../raft/log/physical/PhysicalRaftLog.java | 496 ------------------ .../log/physical/PhysicalRaftLogFile.java | 313 ----------- .../PhysicalRaftLogFileInformation.java | 102 ---- .../log/physical/PhysicalRaftLogFiles.java | 221 -------- .../log/physical/RaftAppendRecordCursor.java | 82 --- .../raft/log/physical/RaftEntryStore.java | 35 -- .../log/physical/RaftLogAppendRecord.java | 80 --- .../physical/RaftLogContinuationRecord.java | 84 --- .../raft/log/physical/RaftLogRecord.java | 37 -- .../raft/log/physical/RaftRecordCursor.java | 78 --- .../physical/ReaderRaftLogVersionBridge.java | 58 -- .../coreedge/raft/log/physical/Recovery.java | 98 ---- .../log/physical/SingleVersionReader.java | 71 --- .../VersionBridgingRaftEntryStore.java | 106 ---- .../raft/log/physical/VersionFiles.java | 30 -- .../raft/log/physical/VersionIndexRange.java | 83 --- .../raft/log/physical/VersionIndexRanges.java | 115 ---- .../pruning/RaftLogPruneStrategy.java | 78 --- .../pruning/RaftLogPruneStrategyFactory.java | 110 ---- .../core/EnterpriseCoreEditionModule.java | 66 +-- .../raft/log/MonitoredRaftLogTest.java | 15 +- .../log/NaiveDurableRaftLogContractTest.java | 44 -- .../raft/log/NaiveDurableRaftLogTest.java | 161 ------ .../raft/log/NaiveRaftLogVerificationIT.java | 48 -- .../raft/log/RaftLogDurabilityTest.java | 40 +- .../VersionBridgingRaftEntryStoreTest.java | 337 ------------ .../raft/log/debug/ReplayRaftLog.java | 16 +- ...FilenameBasedLogVersionRepositoryTest.java | 59 --- .../PhysicalRaftLogCacheConsistencyTest.java | 130 ----- .../physical/PhysicalRaftLogContractTest.java | 191 ------- .../log/physical/PhysicalRaftLogFileTest.java | 225 -------- .../physical/PhysicalRaftLogFilesTest.java | 174 ------ .../physical/PhysicalRaftLogRotationTest.java | 148 ------ .../PhysicalRaftLogVerificationIT.java | 65 --- .../raft/log/physical/RecoveryTest.java | 101 ---- ...hresholdBasedRaftLogPruneStrategyTest.java | 103 ---- .../log/physical/VersionIndexRangesTest.java | 217 -------- 46 files changed, 23 insertions(+), 5315 deletions(-) delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftableContentAppender.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/DumpNaiveDurableRaftLog.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/NaiveDurableRaftLog.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/DumpPhysicalRaftLog.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/EntryReader.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepository.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Header.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/HeaderReader.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLog.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFile.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileInformation.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFiles.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftAppendRecordCursor.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftEntryStore.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogAppendRecord.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogContinuationRecord.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogRecord.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftRecordCursor.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/ReaderRaftLogVersionBridge.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Recovery.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/SingleVersionReader.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionBridgingRaftEntryStore.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionFiles.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRange.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRanges.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategy.java delete mode 100644 enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategyFactory.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogContractTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveRaftLogVerificationIT.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/VersionBridgingRaftEntryStoreTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepositoryTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogCacheConsistencyTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogContractTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFilesTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogRotationTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogVerificationIT.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/RecoveryTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/ThresholdBasedRaftLogPruneStrategyTest.java delete mode 100644 enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRangesTest.java diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftLog.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftLog.java index a8ed89dc000f4..dce3cb6900dfe 100644 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftLog.java +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftLog.java @@ -30,6 +30,8 @@ */ public interface RaftLog extends ReadableRaftLog { + String PHYSICAL_LOG_DIRECTORY_NAME = "raft-log"; + /** * Appends entry to the end of the log. The first log index is 0. *

diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftableContentAppender.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftableContentAppender.java deleted file mode 100644 index 36d3b0e526acb..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/RaftableContentAppender.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log; - -import java.io.IOException; - -import org.neo4j.kernel.impl.transaction.log.Commitment; -import org.neo4j.kernel.impl.transaction.tracing.LogAppendEvent; - -public interface RaftableContentAppender -{ - Commitment append( RaftLogEntry transaction, LogAppendEvent logAppendEvent ) throws IOException; -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/DumpNaiveDurableRaftLog.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/DumpNaiveDurableRaftLog.java deleted file mode 100644 index afe714ef128f7..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/DumpNaiveDurableRaftLog.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.naive; - -import java.io.File; -import java.io.IOException; - -import org.neo4j.coreedge.raft.log.debug.LogPrinter; -import org.neo4j.coreedge.raft.net.CoreReplicatedContentMarshal; -import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.logging.NullLogProvider; - -public class DumpNaiveDurableRaftLog -{ - public static void main( String[] args ) throws IOException - { - for ( String arg : args ) - { - File logDirectory = new File( arg ); - System.out.println( "logDirectory = " + logDirectory ); - NaiveDurableRaftLog log = new NaiveDurableRaftLog( new DefaultFileSystemAbstraction(), - logDirectory, new CoreReplicatedContentMarshal(), NullLogProvider.getInstance() ); - - new LogPrinter( log ).print( System.out ); - System.out.println(); - } - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/NaiveDurableRaftLog.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/NaiveDurableRaftLog.java deleted file mode 100644 index 7da5726062b40..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/naive/NaiveDurableRaftLog.java +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.naive; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; - -import org.neo4j.com.NetworkFlushableChannel; -import org.neo4j.coreedge.raft.log.RaftLog; -import org.neo4j.coreedge.raft.log.RaftLogCursor; -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.net.NetworkFlushableChannelNetty4; -import org.neo4j.coreedge.raft.net.NetworkReadableClosableChannelNetty4; -import org.neo4j.coreedge.raft.replication.MarshallingException; -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.CursorValue; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.lifecycle.LifecycleAdapter; -import org.neo4j.logging.Log; -import org.neo4j.logging.LogProvider; -import org.neo4j.storageengine.api.WritableChannel; - -/** - * Writes a raft log to disk using 3 files: - *

- * 1. entries.log - * ┌─────────────────────────────┐ - * │term 8 bytes│ - * │contentPointer 8 bytes│ - * ├─────────────────────────────┤ - * │record length 16 bytes│ - * └─────────────────────────────┘ - *

- * 2. content.log - * ┌─────────────────────────────┐ - * │contentLength 4 bytes│ - * ├─────────────────────────────┤ - * │contentType 1 bytes│ - * │content variable│ - * ├─────────────────────────────┤ - * │record length variable│ - * └─────────────────────────────┘ - *

- * 3. meta.log - * ┌─────────────────────────────┐ - * │prevIndex 8 bytes│ - * │prevTerm 8 bytes│ - * ├─────────────────────────────┤ - * │record length 16 bytes│ - * └─────────────────────────────┘ - */ -public class NaiveDurableRaftLog extends LifecycleAdapter implements RaftLog -{ - public static final int ENTRY_RECORD_LENGTH = 16; - public static final int CONTENT_LENGTH_BYTES = 4; - public static final int META_BYTES = 8 * 2; - public static final String NAIVE_LOG_DIRECTORY_NAME = "raft-log"; - - private StoreChannel entriesChannel; - private StoreChannel contentChannel; - private final StoreChannel metaChannel; - - private final FileSystemAbstraction fileSystem; - private final File directory; - private final ChannelMarshal marshal; - private final Log log; - private long appendIndex = -1; - private long contentOffset; - private long term = -1; - private long prevIndex = -1; - private long prevTerm = -1; - - public NaiveDurableRaftLog( FileSystemAbstraction fileSystem, File directory, - ChannelMarshal marshal, LogProvider logProvider ) - { - this.fileSystem = fileSystem; - this.directory = directory; - this.marshal = marshal; - - directory.mkdirs(); - - try - { - entriesChannel = fileSystem.open( new File( directory, "entries.log" ), "rw" ); - contentChannel = fileSystem.open( new File( directory, "content.log" ), "rw" ); - metaChannel = fileSystem.open( new File( directory, "meta.log" ), "rw" ); - readMetadata(); - - appendIndex = prevIndex + entriesChannel.size() / ENTRY_RECORD_LENGTH; - contentOffset = contentChannel.size(); - log = logProvider.getLog( getClass() ); - - log.info( "Raft log created. appendIndex: %d", appendIndex ); - } - catch ( IOException e ) - { - throw new RuntimeException( e ); - } - } - - @Override - public void shutdown() throws Throwable - { - Exception container = new Exception( "Exception happened during shutdown of RaftLog. See suppressed " + - "exceptions for details" ); - boolean shouldThrow; - shouldThrow = forceAndCloseChannel( entriesChannel, container ); - shouldThrow = forceAndCloseChannel( contentChannel, container ) || shouldThrow; - shouldThrow = forceAndCloseChannel( metaChannel, container ) || shouldThrow; - if ( shouldThrow ) - { - throw container; - } - } - - /** - * This method will try to force and close a store channel. If any of these two operations fails, the exception - * will be added as suppressed in the provided container. In such a case, true will be returned. - * - * @param channel The channel to close - * @param container The container to add supressed exceptions in the case of failure - * @return True iff an exception was thrown by either force() or close() - */ - private boolean forceAndCloseChannel( StoreChannel channel, Exception container ) - { - boolean exceptionHappened = false; - try - { - channel.force( false ); - channel.close(); - } - catch ( Exception e ) - { - exceptionHappened = true; - container.addSuppressed( e ); - } - return exceptionHappened; - } - - @Override - public synchronized long append( RaftLogEntry... entries ) throws IOException - { - long newAppendIndex = appendIndex; - for ( RaftLogEntry entry : entries ) - { - newAppendIndex = appendSingle( entry ); - } - return newAppendIndex; - } - - private long appendSingle( RaftLogEntry logEntry ) throws IOException - { - if ( logEntry.term() >= term ) - { - term = logEntry.term(); - } - else - { - throw new IllegalStateException( String.format( "Non-monotonic term %d for in entry %s in term %d", - logEntry.term(), logEntry.toString(), term ) ); - } - - try - { - appendIndex++; - int length = writeContent( logEntry, contentChannel ); - writeEntry( appendIndex - (prevIndex + 1), new Entry( logEntry.term(), contentOffset ), entriesChannel ); - contentOffset += length; - return appendIndex; - } - catch ( MarshallingException | IOException e ) - { - throw new IOException( "Failed to append log entry", e ); - } - } - - @Override - public void truncate( long fromIndex ) throws IOException - { - if ( appendIndex >= fromIndex ) - { - Entry entry = readEntry( fromIndex ); - contentChannel.truncate( entry.contentPointer ); - contentOffset = entry.contentPointer; - - entriesChannel.truncate( ENTRY_RECORD_LENGTH * fromIndex ); - entriesChannel.force( false ); - - appendIndex = fromIndex - 1; - } - else - { - throw new IllegalArgumentException( "Cannot truncate at index " + fromIndex + " when append index is " + - appendIndex ); - } - term = readEntryTerm( appendIndex ); - } - - @Override - public long prune( long safeIndex ) throws IOException - { - try - { - if ( safeIndex > prevIndex ) - { - long safeIndexTerm = readEntryTerm( safeIndex ); - - StoreChannel tempEntriesChannel = fileSystem.open( new File( directory, "temp-entries.log" ), "rw" ); - StoreChannel tempContentChannel = fileSystem.open( new File( directory, "temp-content.log" ), "rw" ); - - contentOffset = 0; - for ( long i = safeIndex + 1; i <= appendIndex; i++ ) - { - RaftLogEntry logEntry = readLogEntry( i ); - int length = writeContent( logEntry, tempContentChannel ); - writeEntry( i - (safeIndex + 1), new Entry( logEntry.term(), contentOffset ), tempEntriesChannel ); - contentOffset += length; - } - tempEntriesChannel.close(); - tempContentChannel.close(); - entriesChannel.close(); - contentChannel.close(); - fileSystem.deleteFile( new File( directory, "entries.log" ) ); - fileSystem.deleteFile( new File( directory, "content.log" ) ); - fileSystem.renameFile( new File( directory, "temp-entries.log" ), new File( directory, "entries.log" - ) ); - fileSystem.renameFile( new File( directory, "temp-content.log" ), new File( directory, "content.log" - ) ); - entriesChannel = fileSystem.open( new File( directory, "entries.log" ), "rw" ); - contentChannel = fileSystem.open( new File( directory, "content.log" ), "rw" ); - - prevTerm = safeIndexTerm; - prevIndex = safeIndex; - - storeMetadata(); - } - - return prevIndex; - } - catch ( MarshallingException e ) - { - throw new IOException( e ); - } - } - - @Override - public long skip( long index, long term ) throws IOException - { - if ( index > appendIndex ) - { - entriesChannel.close(); - contentChannel.close(); - fileSystem.deleteFile( new File( directory, "entries.log" ) ); - fileSystem.deleteFile( new File( directory, "content.log" ) ); - entriesChannel = fileSystem.open( new File( directory, "entries.log" ), "rw" ); - contentChannel = fileSystem.open( new File( directory, "content.log" ), "rw" ); - - appendIndex = index; - prevIndex = index; - prevTerm = term; - - storeMetadata(); - } - return appendIndex; - } - - @Override - public long appendIndex() - { - return appendIndex; - } - - @Override - public long prevIndex() - { - return prevIndex; - } - - private RaftLogEntry readLogEntry( long logIndex ) throws IOException - { - Entry entry = readEntry( logIndex ); - ReplicatedContent content; - try - { - content = readContentFrom( entry.contentPointer ); - } - catch ( MarshallingException e ) - { - throw new IOException( e ); - } - - return new RaftLogEntry( entry.term, content ); - } - - @Override - public long readEntryTerm( long logIndex ) throws IOException - { - if ( logIndex == prevIndex ) - { - return prevTerm; - } - else if ( logIndex < prevIndex || logIndex > appendIndex ) - { - return -1; - } - - return readEntry( logIndex ).term; - } - - private static class Entry - { - private final long term; - - private final long contentPointer; - - public Entry( long term, long contentPointer ) - { - this.term = term; - this.contentPointer = contentPointer; - } - - @Override - public String toString() - { - return "Entry{" + - "term=" + term + - ", contentPointer=" + contentPointer + - '}'; - } - } - - @Override - public RaftLogCursor getEntryCursor( long fromIndex ) throws IOException - { - return new RaftLogCursor() - { - private long currentIndex = fromIndex - 1; // the cursor starts "before" the first entry - private CursorValue current = new CursorValue<>(); - - @Override - public boolean next() throws IOException - { - currentIndex++; - - try - { - current.set( readLogEntry( currentIndex ) ); - return true; - } - catch ( IllegalArgumentException e ) - { - current.invalidate(); - } - - return false; - } - - @Override - public void close() throws IOException - { - } - - @Override - public long index() - { - return currentIndex; - } - - @Override - public RaftLogEntry get() - { - return current.get(); - } - }; - } - - private void writeEntry( long index, Entry entry, StoreChannel entriesChannel ) throws IOException - { - ByteBuffer buffer = ByteBuffer.allocate( ENTRY_RECORD_LENGTH ); - buffer.putLong( entry.term ); - buffer.putLong( entry.contentPointer ); - buffer.flip(); - - entriesChannel.writeAll( buffer, index * ENTRY_RECORD_LENGTH ); - entriesChannel.force( false ); - } - - private Entry readEntry( long logIndex ) throws IOException - { - if ( logIndex <= prevIndex || logIndex > appendIndex ) - { - throw new IllegalArgumentException( "compaction exception" ); - } - - ByteBuffer buffer = ByteBuffer.allocate( ENTRY_RECORD_LENGTH ); - entriesChannel.read( buffer, (logIndex - (prevIndex + 1)) * ENTRY_RECORD_LENGTH ); - buffer.flip(); - long term = buffer.getLong(); - long contentPointer = buffer.getLong(); - return new Entry( term, contentPointer ); - } - - private int writeContent( RaftLogEntry logEntry, StoreChannel contentChannel ) throws MarshallingException, - IOException - { - ByteBuf buffer = Unpooled.buffer(); - WritableChannel channel = new NetworkFlushableChannelNetty4( buffer ); - marshal.marshal( logEntry.content(), channel ); - ByteBuffer contentBuffer = buffer.internalNioBuffer( 0, buffer.writerIndex() ); - int length = CONTENT_LENGTH_BYTES + contentBuffer.remaining(); - - ByteBuffer contentLengthBuffer = ByteBuffer.allocate( CONTENT_LENGTH_BYTES ); - contentLengthBuffer.putInt( length ); - contentLengthBuffer.flip(); - contentChannel.writeAll( contentLengthBuffer, contentOffset ); - contentChannel.writeAll( contentBuffer, contentOffset + CONTENT_LENGTH_BYTES ); - contentChannel.force( false ); - - return length; - } - - private ReplicatedContent readContentFrom( long contentPointer ) throws IOException, MarshallingException - { - ByteBuffer lengthBuffer = ByteBuffer.allocate( CONTENT_LENGTH_BYTES ); - contentChannel.read( lengthBuffer, contentPointer ); - lengthBuffer.flip(); - int contentLength = lengthBuffer.getInt(); - - ByteBuffer contentBuffer = ByteBuffer.allocate( contentLength - CONTENT_LENGTH_BYTES ); - contentChannel.read( contentBuffer, contentPointer + CONTENT_LENGTH_BYTES ); - contentBuffer.flip(); - ByteBuf byteBuf = Unpooled.wrappedBuffer( contentBuffer ); - return marshal.unmarshal( new NetworkReadableClosableChannelNetty4( byteBuf ) ); - } - - private void storeMetadata() throws IOException - { - ByteBuffer buffer = ByteBuffer.allocate( META_BYTES ); - buffer.putLong( prevIndex ); - buffer.putLong( prevTerm ); - buffer.flip(); - metaChannel.writeAll( buffer, 0 ); - metaChannel.force( false ); - } - - private void readMetadata() throws IOException - { - if ( metaChannel.size() < META_BYTES ) - { - prevIndex = -1; - prevTerm = -1; - } - else - { - ByteBuffer buffer = ByteBuffer.allocate( META_BYTES ); - metaChannel.read( buffer, 0 ); - buffer.flip(); - prevIndex = buffer.getLong(); - prevTerm = buffer.getLong(); - } - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/DumpPhysicalRaftLog.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/DumpPhysicalRaftLog.java deleted file mode 100644 index 39dced4e14279..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/DumpPhysicalRaftLog.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Comparator; -import java.util.TreeSet; - -import org.neo4j.coreedge.raft.net.CoreReplicatedContentMarshal; -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.IOCursor; -import org.neo4j.helpers.Args; -import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; - -import static org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles.getLogVersion; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeader.LOG_HEADER_SIZE; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; - -public class DumpPhysicalRaftLog -{ - private final FileSystemAbstraction fileSystem; - private static final String TO_FILE = "tofile"; - private ChannelMarshal marshal = new CoreReplicatedContentMarshal(); - - public DumpPhysicalRaftLog( FileSystemAbstraction fileSystem, ChannelMarshal marshal ) - { - this.fileSystem = fileSystem; - this.marshal = marshal; - } - - public int dump( String filenameOrDirectory, String logPrefix, PrintStream out ) throws IOException - { - int logsFound = 0; - PhysicalRaftLogFiles files = new PhysicalRaftLogFiles( new File( filenameOrDirectory ), fileSystem, marshal, new VersionIndexRanges() ); - SingleVersionReader reader = new SingleVersionReader( files, fileSystem, marshal ); - - for ( String fileName : filenamesOf( filenameOrDirectory, logPrefix ) ) - { - logsFound++; - out.println( "=== " + fileName + " ===" ); - StoreChannel fileChannel = fileSystem.open( new File( fileName ), "r" ); - - LogHeader logHeader; - try - { - logHeader = readLogHeader( ByteBuffer.allocateDirect( LOG_HEADER_SIZE ), fileChannel, false ); - } - catch ( IOException ex ) - { - out.println( "Unable to read timestamp information, no records in logical log." ); - out.println( ex.getMessage() ); - fileChannel.close(); - throw ex; - } - out.println( "Logical log format:" + logHeader.logFormatVersion + "version: " + logHeader.logVersion + - " with prev committed tx[" + logHeader.lastCommittedTxId + "]" ); - - try ( IOCursor cursor = - reader.readEntriesFrom( new LogPosition( logHeader.logVersion, LogHeader.LOG_HEADER_SIZE ) ) ) - { - while ( cursor.next() ) - { - out.println( cursor.get().toString() ); - } - } - } - return logsFound; - } - - protected String[] filenamesOf( String filenameOrDirectory, final String prefix ) - { - - File file = new File( filenameOrDirectory ); - if ( fileSystem.isDirectory( file ) ) - { - File[] files = fileSystem.listFiles( file, ( dir, name ) -> name.contains( prefix ) && !name.contains( "active" ) ); - Collection result = new TreeSet<>( sequentialComparator() ); - for ( File file1 : files ) - { - result.add( file1.getPath() ); - } - return result.toArray( new String[result.size()] ); - } - else - { - return new String[]{filenameOrDirectory}; - } - } - - private static Comparator sequentialComparator() - - { - return new Comparator() - { - @Override - public int compare( String o1, String o2 ) - { - return versionOf( o1 ).compareTo( versionOf( o2 ) ); - } - - private Long versionOf( String string ) - { - try - { - return getLogVersion( string ); - } - catch ( RuntimeException ignored ) - { - return Long.MAX_VALUE; - } - } - }; - } - - public static void main( String[] args ) throws IOException - { - Args arguments = Args.withFlags( TO_FILE ).parse( args ); - try ( Printer printer = getPrinter( arguments ) ) - { - for ( String fileAsString : arguments.orphans() ) - { - System.out.println("Reading file " + fileAsString ); - new DumpPhysicalRaftLog( new DefaultFileSystemAbstraction(), new CoreReplicatedContentMarshal() ) - .dump( fileAsString, PhysicalRaftLogFiles.BASE_FILE_NAME, printer.getFor( fileAsString ) ); - } - } - } - - public static Printer getPrinter( Args args ) - { - boolean toFile = args.getBoolean( TO_FILE, false, true ); - return toFile ? new FilePrinter() : SYSTEM_OUT_PRINTER; - } - - public interface Printer extends AutoCloseable - { - PrintStream getFor( String file ) throws FileNotFoundException; - - @Override - void close(); - } - - private static final Printer SYSTEM_OUT_PRINTER = new Printer() - { - @Override - public PrintStream getFor( String file ) - { - return System.out; - } - - @Override - public void close() - { // Don't close System.out - } - }; - - private static class FilePrinter implements Printer - { - private File directory; - private PrintStream out; - - @Override - public PrintStream getFor( String file ) throws FileNotFoundException - { - File absoluteFile = new File( file ).getAbsoluteFile(); - File dir = absoluteFile.isDirectory() ? absoluteFile : absoluteFile.getParentFile(); - if ( !dir.equals( directory ) ) - { - safeClose(); - File dumpFile = new File( dir, "dump-logical-log.txt" ); - System.out.println( "Redirecting the output to " + dumpFile.getPath() ); - out = new PrintStream( dumpFile ); - directory = dir; - } - return out; - } - - private void safeClose() - { - if ( out != null ) - { - out.close(); - } - } - - @Override - public void close() - { - safeClose(); - } - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/EntryReader.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/EntryReader.java deleted file mode 100644 index 32f74e02ec62c..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/EntryReader.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.cursor.IOCursor; - -public interface EntryReader -{ - IOCursor readEntriesInVersion( long version ) throws IOException; -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepository.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepository.java deleted file mode 100644 index 5b18b8c6abfb0..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepository.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; - -public class FilenameBasedLogVersionRepository implements LogVersionRepository -{ - private long version = 0; - - public FilenameBasedLogVersionRepository( PhysicalLogFiles logFiles) - { - version = Math.max(logFiles.getHighestLogVersion(), 0); - } - - @Override - public long getCurrentLogVersion() - { - return version; - } - - @Override - public long incrementAndGetVersion() throws IOException - { - return ++version; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Header.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Header.java deleted file mode 100644 index a25d2f3ba0596..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Header.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -public class Header -{ - public final long version; - public final long prevIndex; - public final long prevTerm; - - public Header( long version, long prevIndex, long prevTerm ) - { - this.version = version; - this.prevIndex = prevIndex; - this.prevTerm = prevTerm; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/HeaderReader.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/HeaderReader.java deleted file mode 100644 index 887068ccf1857..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/HeaderReader.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; - -public class HeaderReader -{ - public Header readHeader( File file ) - { - return null; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLog.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLog.java deleted file mode 100644 index d1848a1e3c119..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLog.java +++ /dev/null @@ -1,496 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; - -import org.neo4j.coreedge.raft.log.RaftLog; -import org.neo4j.coreedge.raft.log.RaftLogCursor; -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.CursorValue; -import org.neo4j.cursor.IOCursor; -import org.neo4j.helpers.Reference; -import org.neo4j.helpers.collection.LruCache; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.transaction.log.FlushablePositionAwareChannel; -import org.neo4j.kernel.impl.transaction.log.LogFileInformation; -import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.LogPositionMarker; -import org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; -import org.neo4j.kernel.impl.transaction.log.ReadableLogChannel; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; -import org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy; -import org.neo4j.kernel.impl.transaction.log.pruning.LogPruning; -import org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl; -import org.neo4j.kernel.impl.transaction.log.rotation.LogRotation; -import org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl; -import org.neo4j.kernel.impl.transaction.tracing.LogAppendEvent; -import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.kernel.lifecycle.Lifecycle; -import org.neo4j.logging.Log; -import org.neo4j.logging.LogProvider; - -import static java.lang.String.format; -import static org.neo4j.coreedge.raft.log.physical.pruning.RaftLogPruneStrategyFactory.fromConfigValue; - -// TODO: Handle recovery better; e.g add missing continuation records -// TODO: Better caching; e.g divide per log version, allow searching for non-exact start point, cache when closing cursor ... - -/** - * The physical RAFT log is an append only log supporting the operations required to support - * the RAFT consensus algorithm. The physical part relates to the fact that the implementation - * borrows from infrastructure around the already existing {@link PhysicalLogFile} and related. - * - * A RAFT log must be able to append new entries, but also truncate not yet committed entries, - * prune out old compacted entries and skip to a later starting point. - * - * The RAFT log consists of a sequence of individual log files with the following format: - * [HEADER] [CONTINUATION RECORD] [APPEND/COMMIT RECORDS]* - * - * Later log files are said to have a higher log version. This terminology is slightly confusing - * but inherited, so we stick with it. The version is not about the format itself, which is specifically - * referred to as a log format version. - * - * New log files are created when truncating, skipping or when appending and the file size has - * reached over the configured rotation threshold. This is called rotation. Each log file begins - * with a header and a continuation record and then follows a sequence of append and commit records. - */ -public class PhysicalRaftLog implements RaftLog, Lifecycle -{ - public static final String PHYSICAL_LOG_DIRECTORY_NAME = "raft-log"; - - private final PhysicalRaftLogFiles logFiles; - private final PhysicalRaftLogFile logFile; - private final ChannelMarshal marshal; - private final Supplier databaseHealthSupplier; - private final Log log; - private LogRotation logRotation; - private FlushablePositionAwareChannel writer; - private final RaftLogMetadataCache metadataCache; - private long prevIndex = -1; - private long prevTerm = -1; - private final AtomicLong appendIndex = new AtomicLong( -1 ); - private long commitIndex = -1; - private final LogPositionMarker positionMarker = new LogPositionMarker(); - private long term = -1; - - private final RaftEntryStore entryStore; - private final LruCache entryCache; - private final LogPruning logPruning; - - public PhysicalRaftLog( FileSystemAbstraction fileSystem, File directory, long rotateAtSize, String pruningConf, - int entryCacheSize, int headerCacheSize, - PhysicalRaftLogFile.Monitor monitor, ChannelMarshal marshal, - Supplier databaseHealthSupplier, LogProvider logProvider, - RaftLogMetadataCache raftLogMetadataCache ) - { - this.marshal = marshal; - this.databaseHealthSupplier = databaseHealthSupplier; - this.log = logProvider.getLog( getClass() ); - this.entryCache = new LruCache<>( "raft-log-entry-cache", entryCacheSize ); - - directory.mkdirs(); - - VersionIndexRanges ranges = new VersionIndexRanges(); - logFiles = new PhysicalRaftLogFiles( directory, fileSystem, marshal, ranges ); - - LogHeaderCache logHeaderCache = new LogHeaderCache( headerCacheSize ); - logFile = new PhysicalRaftLogFile( fileSystem, logFiles, rotateAtSize, - appendIndex::get, monitor, logHeaderCache ); - - LogFileInformation logFileInformation = new PhysicalRaftLogFileInformation( logFiles, logHeaderCache, - this::appendIndex, version -> 0L ); - LogPruneStrategy logPruneStrategy = fromConfigValue( fileSystem, logFileInformation, logFiles, pruningConf ) ; - this.logPruning = new LogPruningImpl( logPruneStrategy, logProvider ); - - this.metadataCache = raftLogMetadataCache; - SingleVersionReader reader = new SingleVersionReader( logFiles, fileSystem, marshal ); - entryStore = new VersionBridgingRaftEntryStore( ranges, reader, metadataCache ); - } - - @Override - public synchronized long append( RaftLogEntry... entries ) throws IOException - { - long newAppendIndex = appendIndex.get(); - for ( RaftLogEntry entry : entries ) - { - newAppendIndex = appendSingle( entry ); - } - return newAppendIndex; - } - - private long appendSingle( RaftLogEntry entry ) throws IOException - { - if ( entry.term() >= term ) - { - term = entry.term(); - } - else - { - throw new IllegalStateException( format( "Non-monotonic term %d for entry %s in term %d", - entry.term(), entry.toString(), term ) ); - } - - long newAppendIndex = appendIndex.incrementAndGet(); - - LogPositionMarker entryStartPosition = writer.getCurrentPosition( positionMarker ); - RaftLogAppendRecord.write( writer, marshal, newAppendIndex, term, entry.content() ); - writer.prepareForFlush().flush(); - entryCache.put( newAppendIndex, entry ); - metadataCache.cacheMetadata( newAppendIndex, entry.term(), entryStartPosition.newPosition() ); - - if( logRotation.rotateLogIfNeeded( LogAppendEvent.NULL ) ) - { - RaftLogContinuationRecord.write( writer, newAppendIndex, term ); - writer.prepareForFlush().flush(); - } - - return newAppendIndex; - } - - @Override - public void truncate( long fromIndex ) throws IOException - { - if ( fromIndex <= commitIndex ) - { - throw new IllegalArgumentException( format( "cannot truncate (%d) at or before the commit index (%d)", - fromIndex, commitIndex ) ); - } - - if ( appendIndex.get() < fromIndex ) - { - throw new IllegalArgumentException( "Cannot truncate at index " + fromIndex + " when append index is " + appendIndex ); - } - - long newAppendIndex = fromIndex - 1; - long newTerm = readEntryTerm( newAppendIndex ); - - entryCache.clear(); - metadataCache.removeUpwardsFrom( fromIndex ); - - appendIndex.set( newAppendIndex ); - logRotation.rotateLogFile(); - - term = newTerm; - RaftLogContinuationRecord.write( writer, newAppendIndex - 1, term ); - writer.prepareForFlush().flush(); - } - - @Override - public long appendIndex() - { - return appendIndex.get(); - } - - @Override - public long prevIndex() - { - return prevIndex; - } - - @Override - public RaftLogCursor getEntryCursor( long fromIndex ) throws IOException - { - final IOCursor inner = entryStore.getEntriesFrom( fromIndex ); - return new RaftLogCursor() - { - private CursorValue current = new CursorValue<>(); - private long index = fromIndex - 1; - - @Override - public boolean next() throws IOException - { - boolean hasNext = inner.next(); - if ( hasNext ) - { - current.set( inner.get().logEntry() ); - index++; - } - else - { - current.invalidate(); - } - return hasNext; - } - - @Override - public void close() throws IOException - { - inner.close(); - } - - @Override - public long index() - { - return index; - } - - @Override - public RaftLogEntry get() - { - return current.get(); - } - }; - } - - @Override - public long skip( long index, long term ) throws IOException - { - if( appendIndex.get() < index ) - { - logRotation.rotateLogFile(); - RaftLogContinuationRecord.write( writer, index, term ); - writer.prepareForFlush().flush(); - - prevIndex = index; - prevTerm = term; - appendIndex.set( index ); - - metadataCache.clear(); - } - - return appendIndex.get(); - } - - private RaftLogEntry readLogEntry( long logIndex ) throws IOException - { - RaftLogEntry entry = entryCache.get( logIndex ); - if( entry != null ) - { - return entry; - } - - try ( IOCursor entriesFrom = entryStore.getEntriesFrom( logIndex ) ) - { - while ( entriesFrom.next() ) - { - RaftLogAppendRecord raftLogAppendRecord = entriesFrom.get(); - if ( raftLogAppendRecord.logIndex() == logIndex ) - { - RaftLogEntry toReturn = raftLogAppendRecord.logEntry(); - entryCache.put( logIndex, toReturn ); - return toReturn; - } - else if ( raftLogAppendRecord.logIndex() > logIndex ) - { - throw new IllegalStateException( format( "Asked for index %d but got up to %d without " + - "finding it.", logIndex, raftLogAppendRecord.logIndex() ) ); - } - } - } - return null; - } - - @Override - public long readEntryTerm( long logIndex ) throws IOException - { - // Index -1 is not an existing log index, but represents the beginning of the log. - // It is a valid value to request the term for, and the term is -1. - if( logIndex == prevIndex ) - { - return prevTerm; - } - else if ( logIndex < prevIndex || logIndex > appendIndex.get() ) - { - return -1; - } - - RaftLogMetadataCache.RaftLogEntryMetadata metadata = metadataCache.getMetadata( logIndex ); - if( metadata != null ) - { - return metadata.getEntryTerm(); - } - - long resultTerm = -1; - - RaftLogEntry raftLogEntry = readLogEntry( logIndex ); - if ( raftLogEntry != null ) - { - resultTerm = raftLogEntry.term(); - } - - return resultTerm; - } - - @Override - public void init() throws IOException - { - logFiles.init(); - logFile.init(); - } - - @Override - public void start() throws IOException - { - this.logRotation = new LogRotationImpl( new LoggingLogFileMonitor( log ), logFile, databaseHealthSupplier.get() ); - - logFile.start(); - - recoverContinuationRecord(); - - restorePrevIndexAndTerm(); - restoreAppendIndex(); - - this.writer = logFile.getWriter(); - } - - /** This is just a very basic "recovery" making it so that a continuation record exists in the very first file right after creation. */ - private void recoverContinuationRecord() throws IOException - { - LogPositionMarker logPosition = new LogPositionMarker(); - - FlushablePositionAwareChannel writer = logFile.getWriter(); - writer.getCurrentPosition( logPosition ); - - if( logPosition.getLogVersion() == 0 && logPosition.getByteOffset() == LogHeader.LOG_HEADER_SIZE ) - { - RaftLogContinuationRecord.write( writer, -1, -1 ); - writer.prepareForFlush().flush(); - } - } - - @Override - public long prune( long safeIndex ) throws IOException - { - final long logVersionToPrune = findLogVersionToPrune( safeIndex ); - - if ( logVersionToPrune != -1 ) - { - logPruning.pruneLogs( logVersionToPrune ); - restorePrevIndexAndTerm(); - metadataCache.removeUpTo( prevIndex - 1 ); - } - - return prevIndex; - } - - /** - * Returns the log file version that contains entries which all have index less than the safeIndex argument. - * @param safeIndex The index value from which all entries should be less than in the returned log version - */ - public long findLogVersionToPrune( long safeIndex ) throws IOException - { - final LogPosition[] firstFileToPrune = {LogPosition.UNSPECIFIED}; - - logFile.accept( ( position, ignored, lastIndex ) -> { - if ( lastIndex < safeIndex ) - { - firstFileToPrune[0] = position; - return false; - } - return true; // keep going - } ); - - return firstFileToPrune[0].equals( LogPosition.UNSPECIFIED ) ? -1 : firstFileToPrune[0].getLogVersion(); - } - - private void restorePrevIndexAndTerm() throws IOException - { - final Reference firstFile = new Reference<>( null ); - - logFile.accept( ( position, ignored1, ignored2 ) -> { - firstFile.set( position ); - return true; - } ); - - if( firstFile.get() == null ) - { - throw new IOException( "No first log file found" ); - } - - ReadableLogChannel reader = logFile.getReader( firstFile.get() ); - - RaftRecordCursor recordCursor = new RaftRecordCursor<>( reader, marshal ); - recordCursor.next(); - RaftLogContinuationRecord cont = (RaftLogContinuationRecord) recordCursor.get(); - - prevIndex = cont.prevLogIndex(); - prevTerm = cont.prevLogTerm(); - - log.info( "Restored prev index at %d", prevIndex ); - log.info( "Restored prev term at %d", prevTerm ); - } - - private void restoreAppendIndex() throws IOException - { - long restoredAppendIndex = prevIndex; - try( IOCursor cursor = entryStore.getEntriesFrom( prevIndex + 1 ) ) - { - while( cursor.next() ) - { - restoredAppendIndex = cursor.get().logIndex(); - } - } - - appendIndex.set( restoredAppendIndex ); - log.info( "Restored append index at %d", restoredAppendIndex ); - } - - @Override - public void stop() throws Throwable - { - logFile.stop(); - this.writer = null; - } - - @Override - public void shutdown() throws Throwable - { - logFile.shutdown(); - } - - public enum RecordType - { - APPEND( (byte) 0 ), CONTINUATION( (byte) 1 ); - - private final byte value; - - RecordType( byte value ) - { - this.value = value; - } - - public byte value() - { - return value; - } - - public static RecordType forValue( byte value ) - { - switch ( value ) - { - case 0: - return APPEND; - case 1: - return CONTINUATION; - default: - throw new IllegalArgumentException( "Value " + value + " is not a known entry type" ); - } - } - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFile.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFile.java deleted file mode 100644 index f45b7fcc4b77e..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFile.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.function.Supplier; - -import org.neo4j.helpers.Exceptions; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.impl.transaction.log.FlushablePositionAwareChannel; -import org.neo4j.kernel.impl.transaction.log.LogFile; -import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; -import org.neo4j.kernel.impl.transaction.log.LogHeaderVisitor; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.LogVersionBridge; -import org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChannel; -import org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel; -import org.neo4j.kernel.impl.transaction.log.ReadableLogChannel; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; -import org.neo4j.kernel.lifecycle.Lifecycle; - -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeader.LOG_HEADER_SIZE; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader; -import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_LOG_VERSION; - -/** - * {@link LogFile} backed by one or more files in a {@link FileSystemAbstraction}. - */ -public class PhysicalRaftLogFile implements LogFile, Lifecycle -{ - public interface Monitor - { - void opened( File logFile, long logVersion, long lastTransactionId, boolean clean ); - - class Adapter implements Monitor - { - @Override - public void opened( File logFile, long logVersion, long lastTransactionId, boolean clean ) - { - } - } - } - - public static final String DEFAULT_VERSION_SUFFIX = "."; - private final long rotateAtSize; - private final FileSystemAbstraction fileSystem; - private final Supplier lastCommittedId; - private final PhysicalRaftLogFiles logFiles; - private final LogHeaderCache logHeaderCache; - private final Monitor monitor; - private final ByteBuffer headerBuffer = ByteBuffer.allocate( LOG_HEADER_SIZE ); - private PositionAwarePhysicalFlushableChannel writer; - private final LogVersionBridge readerLogVersionBridge; - - private volatile PhysicalLogVersionedStoreChannel channel; - - public PhysicalRaftLogFile( FileSystemAbstraction fileSystem, PhysicalRaftLogFiles logFiles, long rotateAtSize, - Supplier lastCommittedId, Monitor monitor, LogHeaderCache logHeaderCache - ) - { - this.fileSystem = fileSystem; - this.rotateAtSize = rotateAtSize; - this.lastCommittedId = lastCommittedId; - this.monitor = monitor; - this.logHeaderCache = logHeaderCache; - this.logFiles = logFiles; - this.readerLogVersionBridge = new ReaderRaftLogVersionBridge( fileSystem, logFiles ); - } - - @Override - public void init() throws IOException - { - // Make sure at least a bare bones log file is available before recovery - openLogChannelForVersion( logFiles.getHighestLogVersion() ).close(); - } - - @Override - public void start() throws IOException - { - // Recovery has taken place before this, so the log file has been truncated to last known good tx - // Just read header and move to the end - - channel = openLogChannelForVersion( logFiles.getHighestLogVersion() ); - // Move to the end - channel.position( channel.size() ); - - writer = new PositionAwarePhysicalFlushableChannel( channel ); - } - - @Override - public void stop() - { - // nothing to stop - } - - // In order to be able to write into a logfile after life.stop during shutdown sequence - // we will close channel and writer only during shutdown phase when all pending changes (like last - // checkpoint) are already in - @Override - public void shutdown() throws IOException - { - if ( writer != null ) - { - writer.close(); - } - if ( channel != null ) - { - channel.close(); - } - } - - @Override - public boolean rotationNeeded() throws IOException - { - /* - * Whereas channel.size() should be fine, we're safer calling position() due to possibility - * of this file being memory mapped or whatever. - */ - return channel.position() >= rotateAtSize; - } - - @Override - public synchronized void rotate() throws IOException - { - channel = rotate( channel ); - writer.setChannel( channel ); - } - - private PhysicalLogVersionedStoreChannel rotate( LogVersionedStoreChannel currentLog ) throws IOException - { - /* - * The store is now flushed. If we fail now the recovery code will open the - * current log file and replay everything. That's unnecessary but totally ok. - */ - currentLog.flush(); - /* - * The log version is now in the store, flushed and persistent. If we crash - * now, on recovery we'll attempt to open the version we're about to create - * (but haven't yet), discover it's not there. That will lead to creating - * the file, setting the header and continuing. We'll do just that now. - * Note that by this point, rotation is done. The next few lines are - * "simply overhead" for continuing to work with the new file. - */ - PhysicalLogVersionedStoreChannel newLog = openLogChannelForNextVersion(); - currentLog.close(); - return newLog; - } - - private PhysicalLogVersionedStoreChannel openLogChannelForNextVersion() throws IOException - { - long forVersion = logFiles.registerNextVersion( lastCommittedId.get() ); - - long lastTxId = lastCommittedId.get(); - writeLogHeader( headerBuffer, forVersion, lastTxId ); - logHeaderCache.putHeader( forVersion, lastTxId ); - - File toOpen = logFiles.getLogFileForVersion( forVersion ); - StoreChannel storeChannel = fileSystem.open( toOpen, "rw" ); - storeChannel.writeAll( headerBuffer ); - monitor.opened( toOpen, forVersion, lastTxId, true ); - - return new PhysicalLogVersionedStoreChannel( storeChannel, forVersion, CURRENT_LOG_VERSION ); - } - - private PhysicalLogVersionedStoreChannel openLogChannelForVersion( long forVersion ) throws IOException - { - File toOpen = logFiles.getLogFileForVersion( forVersion ); - StoreChannel storeChannel = fileSystem.open( toOpen, "rw" ); - LogHeader header = readLogHeader( headerBuffer, storeChannel, false ); - if ( header == null ) - { - // Either the header is not there in full or the file was new. Don't care - long lastTxId = lastCommittedId.get(); - writeLogHeader( headerBuffer, forVersion, lastTxId ); - logHeaderCache.putHeader( forVersion, lastTxId ); - logFiles.createdFile( forVersion, lastTxId ); - storeChannel.writeAll( headerBuffer ); - monitor.opened( toOpen, forVersion, lastTxId, true ); - } - byte formatVersion = header == null ? CURRENT_LOG_VERSION : header.logFormatVersion; - return new PhysicalLogVersionedStoreChannel( storeChannel, forVersion, formatVersion ); - } - - @Override - public FlushablePositionAwareChannel getWriter() - { - return writer; - } - - @Override - public ReadableLogChannel getReader( LogPosition position ) throws IOException - { - PhysicalLogVersionedStoreChannel logChannel = openForVersion( logFiles, fileSystem, position.getLogVersion() ); - logChannel.position( position.getByteOffset() ); - return new ReadAheadLogChannel( logChannel, readerLogVersionBridge ); - } - - public static PhysicalLogVersionedStoreChannel openForVersion( PhysicalRaftLogFiles logFiles, - FileSystemAbstraction fileSystem, - long version ) throws IOException - { - final File fileToOpen = logFiles.getLogFileForVersion( version ); - - if ( !fileSystem.fileExists( fileToOpen ) ) - { - throw new FileNotFoundException( String.format( "File does not exist [%s]", - fileToOpen.getCanonicalPath() ) ); - } - - StoreChannel rawChannel; - try - { - rawChannel = fileSystem.open( fileToOpen, "rw" ); - } - catch ( FileNotFoundException cause ) - { - throw Exceptions.withCause( new FileNotFoundException( String.format( "File could not be opened [%s]", - fileToOpen.getCanonicalPath() ) ), cause ); - } - - ByteBuffer buffer = ByteBuffer.allocate( LOG_HEADER_SIZE ); - LogHeader header = readLogHeader( buffer, rawChannel, true ); - assert header != null && header.logVersion == version : "went ham: " + header ; - - return new PhysicalLogVersionedStoreChannel( rawChannel, version, header.logFormatVersion ); - } - - public static PhysicalLogVersionedStoreChannel tryOpenForVersion( PhysicalRaftLogFiles logFiles, - FileSystemAbstraction fileSystem, long version ) - { - try - { - return openForVersion( logFiles, fileSystem, version ); - } - catch ( IOException ex ) - { - return null; - } - } - - @Override - public void accept( LogFileVisitor visitor, LogPosition startingFromPosition ) throws IOException - { - try ( ReadableLogChannel reader = getReader( startingFromPosition ) ) - { - visitor.visit( startingFromPosition, reader ); - } - } - - @Override - public void accept( LogHeaderVisitor visitor ) throws IOException - { - // Start from the where we're currently at and go backwards in time (versions) - long logVersion = logFiles.getHighestLogVersion(); - long highTransactionId = lastCommittedId.get(); - while ( logFiles.versionExists( logVersion ) ) - { - Long previousLogLastTxId = logHeaderCache.getLogHeader( logVersion ); - if ( previousLogLastTxId == null ) - { - LogHeader header = readLogHeader( fileSystem, logFiles.getLogFileForVersion( logVersion ) ); - assert logVersion == header.logVersion; - logHeaderCache.putHeader( header.logVersion, header.lastCommittedTxId ); - previousLogLastTxId = header.lastCommittedTxId; - } - - long lowTransactionId = previousLogLastTxId + 1; - LogPosition position = LogPosition.start( logVersion ); - if ( !visitor.visit( position, lowTransactionId, highTransactionId ) ) - { - break; - } - logVersion--; - highTransactionId = previousLogLastTxId; - } - } - - @Override - public File currentLogFile() - { - return logFiles.getLogFileForVersion( logFiles.getHighestLogVersion() ); - } - - @Override - public long currentLogVersion() - { - return logFiles.getHighestLogVersion(); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileInformation.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileInformation.java deleted file mode 100644 index eb6fc7a2defff..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileInformation.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.kernel.impl.transaction.log.LogFileInformation; -import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; - -public class PhysicalRaftLogFileInformation implements LogFileInformation -{ - public interface LogVersionToTimestamp - { - long getTimestampForVersion( long version ) throws IOException; - } - - public interface LastEntryInLog - { - long getLastEntryId(); - } - - private final PhysicalRaftLogFiles logFiles; - private final LogHeaderCache logHeaderCache; - private final LastEntryInLog lastEntryInLog; - private final LogVersionToTimestamp logVersionToTimestamp; - - public PhysicalRaftLogFileInformation( PhysicalRaftLogFiles logFiles, - LogHeaderCache logHeaderCache, - LastEntryInLog lastEntryInLog, - LogVersionToTimestamp logVersionToTimestamp ) - { - this.logFiles = logFiles; - this.logHeaderCache = logHeaderCache; - this.lastEntryInLog = lastEntryInLog; - this.logVersionToTimestamp = logVersionToTimestamp; - } - - @Override - public long getFirstExistingEntryId() throws IOException - { - long version = logFiles.getHighestLogVersion(); - long candidateFirstTx = -1; - while ( logFiles.versionExists( version ) ) - { - candidateFirstTx = getFirstEntryId( version ); - version--; - } - version++; // the loop above goes back one version too far. - - // OK, so we now have the oldest existing log version here. Open it and see if there's any transaction - // in there. If there is then that transaction is the first one that we have. - return logFiles.hasAnyEntries( version ) ? candidateFirstTx : -1; - } - - @Override - public long getFirstEntryId( long version ) throws IOException - { - Long logHeader = logHeaderCache.getLogHeader( version ); - if ( logHeader != null ) - { // It existed in cache - return logHeader + 1; - } - - // Wasn't cached, go look for it - if ( logFiles.versionExists( version ) ) - { - long previousVersionLastCommittedTx = logFiles.extractHeader( version ).lastCommittedTxId; - logHeaderCache.putHeader( version, previousVersionLastCommittedTx ); - return previousVersionLastCommittedTx + 1; - } - return -1; - } - - @Override - public long getLastEntryId() - { - return lastEntryInLog.getLastEntryId(); - } - - @Override - public long getFirstStartRecordTimestamp( long version ) throws IOException - { - return logVersionToTimestamp.getTimestampForVersion( version ); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFiles.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFiles.java deleted file mode 100644 index c87ab9a069dd3..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFiles.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.regex.Pattern; - -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.impl.transaction.log.LogVersionBridge; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; -import org.neo4j.kernel.lifecycle.LifecycleAdapter; - -import static java.lang.Math.max; -import static java.lang.Math.min; - -import static java.lang.String.format; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFile.openForVersion; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeader.LOG_HEADER_SIZE; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader; - -/** - * Used to figure out what logical log file to open when the database - * starts up. - */ -public class PhysicalRaftLogFiles extends LifecycleAdapter -{ - private final File logBaseName; - private final Pattern logFilePattern; - - private final FileSystemAbstraction fileSystem; - private final ChannelMarshal marshal; - private final VersionIndexRanges versionRanges; - private final ByteBuffer headerBuffer = ByteBuffer.allocate( LOG_HEADER_SIZE ); - - public static final String BASE_FILE_NAME = "raft.log"; - public static final String REGEX_DEFAULT_VERSION_SUFFIX = "\\."; - public static final String DEFAULT_VERSION_SUFFIX = "."; - - public PhysicalRaftLogFiles( - File directory, FileSystemAbstraction fileSystem, ChannelMarshal marshal, - VersionIndexRanges versionRanges ) - { - this.marshal = marshal; - this.logBaseName = new File( directory, BASE_FILE_NAME ); - this.logFilePattern = Pattern.compile( BASE_FILE_NAME + REGEX_DEFAULT_VERSION_SUFFIX + "\\d+" ); - this.fileSystem = fileSystem; - this.versionRanges = versionRanges; - } - - @Override - public void init() throws IOException - { - long lowest = -1; - long highest = -1; - - SortedSet versions = new TreeSet<>(); - - for ( File file : fileSystem.listFiles( logBaseName.getParentFile() ) ) - { - if ( logFilePattern.matcher( file.getName() ).matches() ) - { - // Get version based on the name - long logVersion = getLogVersion( file.getName() ); - versions.add( logVersion ); - highest = max( highest, logVersion ); - lowest = lowest == -1 ? logVersion : min( lowest, logVersion ); - } - } - long appendIndex = -1; - Long previousVersion = null; - for ( Long version : versions ) - { - File file = getLogFileForVersion( version ); - LogHeader logHeader = readLogHeader( fileSystem, file, false ); - if ( logHeader == null ) - { - if ( previousVersion != null ) - { - appendIndex = readLastIndex( previousVersion, appendIndex ); - } - writeHeader( file, version, appendIndex ); - } - else - { - appendIndex = logHeader.lastCommittedTxId; - } - versionRanges.add( version, appendIndex ); - previousVersion = version; - } - } - - private void writeHeader( File file, long version, long prevIndex ) throws IOException - { - // Either the header is not there in full or the file was new. Don't care - StoreChannel storeChannel = fileSystem.open( file, "rw" ); - writeLogHeader( headerBuffer, version, prevIndex ); - storeChannel.writeAll( headerBuffer ); - } - - private long readLastIndex( Long version, long previousAppendIndex ) throws IOException - { - long lastIndex = previousAppendIndex; - PhysicalLogVersionedStoreChannel logChannel = openForVersion( this, fileSystem, version ); - try ( RaftRecordCursor cursor = new RaftRecordCursor<>( new - ReadAheadLogChannel( logChannel, LogVersionBridge.NO_MORE_CHANNELS ), marshal )) - { - while ( cursor.next() ) - { - RaftLogRecord raftLogRecord = cursor.get(); - if ( raftLogRecord.getType() == PhysicalRaftLog.RecordType.APPEND ) - { - lastIndex = ((RaftLogAppendRecord) raftLogRecord).logIndex(); - } - } - } - - return lastIndex; - } - - public File getLogFileForVersion( long version ) - { - return new File( logBaseName.getPath() + DEFAULT_VERSION_SUFFIX + version ); - } - - public boolean versionExists( long version ) - { - return fileSystem.fileExists( getLogFileForVersion( version ) ); - } - - public LogHeader extractHeader( long version ) throws IOException - { - return readLogHeader( fileSystem, getLogFileForVersion( version ) ); - } - - public boolean hasAnyEntries( long version ) - { - return fileSystem.getFileSize( getLogFileForVersion( version ) ) > LOG_HEADER_SIZE; - } - - public long getHighestLogVersion() - { - return versionRanges.highestVersion(); - } - - public long getLowestLogVersion() - { - return versionRanges.lowestVersion(); - } - - static long getLogVersion( String historyLogFilename ) - { - int index = historyLogFilename.lastIndexOf( DEFAULT_VERSION_SUFFIX ); - if ( index == -1 ) - { - throw new RuntimeException( "Invalid log file '" + historyLogFilename + "'" ); - } - return Long.parseLong( historyLogFilename.substring( index + DEFAULT_VERSION_SUFFIX.length() ) ); - } - - public void createdFile( long forVersion, long prevEntryIndex ) - { - versionRanges.add( forVersion, prevEntryIndex ); - } - - public boolean containsEntries( long version ) - { - return fileSystem.getFileSize( getLogFileForVersion( version ) ) > LOG_HEADER_SIZE; - } - - public void pruneUpTo( long upper ) - { - // Find out which log is the earliest existing (lower bound to prune) - long lower = getLowestLogVersion(); - - // The reason we delete from lower to upper is that if it crashes in the middle - // we can be sure that no holes are created - for ( long version = lower; version <= upper; version++ ) - { - long lowestLogVersion = getLowestLogVersion(); - fileSystem.deleteFile( getLogFileForVersion( version ) ); - versionRanges.pruneVersion( version ); - assert version >= lowestLogVersion : - format( "Cannot prune to log version %d lower than the lowest existing %d", - version, lowestLogVersion ); - } - } - - public long registerNextVersion( Long prevLogIndex ) - { - long version = getHighestLogVersion() + 1; - versionRanges.add( version, prevLogIndex ); - return version; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftAppendRecordCursor.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftAppendRecordCursor.java deleted file mode 100644 index 371d3c504e4e5..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftAppendRecordCursor.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.CursorValue; -import org.neo4j.cursor.IOCursor; -import org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.ReadAheadChannel; -import org.neo4j.storageengine.api.ReadPastEndException; - -public class RaftAppendRecordCursor implements IOCursor -{ - private final ReadAheadChannel channel; - private final ChannelMarshal marshal; - private CursorValue currentRecord = new CursorValue<>(); - - public RaftAppendRecordCursor( ReadAheadChannel channel, ChannelMarshal marshal ) - { - this.channel = channel; - this.marshal = marshal; - } - - @Override - public boolean next() throws IOException - { - try - { - while ( true ) - { - byte type = channel.get(); - switch ( PhysicalRaftLog.RecordType.forValue( type ) ) - { - case APPEND: - currentRecord.set( RaftLogAppendRecord.read( channel, marshal ) ); - return true; - case CONTINUATION: - RaftLogContinuationRecord.read( channel ); - break; - default: - throw new IllegalStateException( "Not really sure how we got here. Read type value was " + type ); - } - } - } - catch ( ReadPastEndException notEnoughBytes ) - { - return false; - } - } - - @Override - public void close() throws IOException - { - channel.close(); - } - - @Override - public RaftLogAppendRecord get() - { - return currentRecord.get(); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftEntryStore.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftEntryStore.java deleted file mode 100644 index e117b6998d5f7..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftEntryStore.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.cursor.IOCursor; - -/** - * The only responsibility of a {@link RaftEntryStore} is to provide {@link IOCursor}s that run over a series of - * {@link RaftLogAppendRecord}s. The cursor is expected to start at the specified logIndex and run until the end of the - * raft log.Semantics as to whether truncated entries are returned, the behaviour when the end is reached and other - * details are purposefully left to the implementations. - */ -public interface RaftEntryStore -{ - IOCursor getEntriesFrom( long logIndex ) throws IOException; -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogAppendRecord.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogAppendRecord.java deleted file mode 100644 index 53d9db5b66090..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogAppendRecord.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.storageengine.api.ReadPastEndException; -import org.neo4j.storageengine.api.ReadableChannel; -import org.neo4j.storageengine.api.WritableChannel; - -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.RecordType.APPEND; - -public class RaftLogAppendRecord extends RaftLogRecord -{ - private final long appendIndex; - private final RaftLogEntry logEntry; - - public RaftLogAppendRecord( long appendIndex, RaftLogEntry logEntry ) - { - super( APPEND ); - this.appendIndex = appendIndex; - this.logEntry = logEntry; - } - - public RaftLogEntry logEntry() - { - return logEntry; - } - - public long logIndex() - { - return appendIndex; - } - - public static RaftLogAppendRecord read( ReadableChannel channel, ChannelMarshal marshal ) throws IOException - { - long appendIndex = channel.getLong(); - long term = channel.getLong(); - ReplicatedContent content = marshal.unmarshal( channel ); - if ( content == null ) - { - throw ReadPastEndException.INSTANCE; - } - return new RaftLogAppendRecord( appendIndex, new RaftLogEntry( term, content ) ); - } - - public static void write( WritableChannel channel, ChannelMarshal marshal, long appendIndex, long term, ReplicatedContent content ) throws IOException - { - channel.put( APPEND.value() ); - channel.putLong( appendIndex ); - channel.putLong( term ); - marshal.marshal( content, channel ); - } - - @Override - public String toString() - { - return String.format( "%d: %s", appendIndex, logEntry ); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogContinuationRecord.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogContinuationRecord.java deleted file mode 100644 index 0ec26d5d54fa4..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogContinuationRecord.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.storageengine.api.ReadableChannel; -import org.neo4j.storageengine.api.WritableChannel; - -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.RecordType.CONTINUATION; - -/** - * Continuation records are written at the beginning of a new log file - * and define the start position (prevLogIndex+1) for subsequently appended - * entries. At the same time prevLogIndex is thus the index of the last - * valid entry in the previous file, and prevLogTerm its term. These values - * are used for log matching. - * - * New log files are created when truncating, skipping or when the threshold - * size for a single log file has been exceeded. - */ -public class RaftLogContinuationRecord extends RaftLogRecord -{ - private final long prevLogIndex; - private final long prevLogTerm; - - RaftLogContinuationRecord( long prevLogIndex, long prevLogTerm ) - { - super( CONTINUATION ); - this.prevLogIndex = prevLogIndex; - this.prevLogTerm = prevLogTerm; - } - - public long prevLogIndex() - { - return prevLogIndex; - } - - public long prevLogTerm() - { - return prevLogTerm; - } - - public static RaftLogContinuationRecord read( ReadableChannel channel ) throws IOException - { - long prevLogIndex = channel.getLong(); - long prevLogTerm = channel.getLong(); - - return new RaftLogContinuationRecord( prevLogIndex, prevLogTerm ); - } - - public static void write( WritableChannel channel, long prevLogIndex, long prevLogTerm ) throws IOException - { - channel.put( CONTINUATION.value() ); - channel.putLong( prevLogIndex ); - channel.putLong( prevLogTerm ); - } - - @Override - public String toString() - { - return "RaftLogContinuationRecord{" + - "prevLogIndex=" + prevLogIndex + - ", prevLogTerm=" + prevLogTerm + - '}'; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogRecord.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogRecord.java deleted file mode 100644 index 347cc2589189f..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftLogRecord.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog; - -public abstract class RaftLogRecord -{ - protected final PhysicalRaftLog.RecordType type; - - RaftLogRecord( PhysicalRaftLog.RecordType type ) - { - this.type = type; - } - - public PhysicalRaftLog.RecordType getType() - { - return type; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftRecordCursor.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftRecordCursor.java deleted file mode 100644 index d01a32bde76e0..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/RaftRecordCursor.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.CursorValue; -import org.neo4j.cursor.IOCursor; -import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; -import org.neo4j.storageengine.api.ReadPastEndException; - -public class RaftRecordCursor implements IOCursor -{ - private final T channel; - private final ChannelMarshal marshal; - private CursorValue currentRecord = new CursorValue<>(); - - public RaftRecordCursor( T channel, ChannelMarshal marshal ) - { - this.channel = channel; - this.marshal = marshal; - } - - @Override - public boolean next() throws IOException - { - try - { - byte type = channel.get(); - switch ( PhysicalRaftLog.RecordType.forValue( type ) ) - { - case APPEND: - currentRecord.set( RaftLogAppendRecord.read( channel, marshal ) ); - return true; - case CONTINUATION: - currentRecord.set( RaftLogContinuationRecord.read( channel ) ); - return true; - default: - throw new IllegalStateException( "Not really sure how we got here. Read type value was " + type ); - } - } - catch ( ReadPastEndException notEnoughBytes ) - { - return false; - } - } - - @Override - public void close() throws IOException - { - channel.close(); - } - - @Override - public RaftLogRecord get() - { - return currentRecord.get(); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/ReaderRaftLogVersionBridge.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/ReaderRaftLogVersionBridge.java deleted file mode 100644 index 7adc617945c2c..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/ReaderRaftLogVersionBridge.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.FileNotFoundException; -import java.io.IOException; - -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.transaction.log.LogVersionBridge; -import org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; - -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFile.openForVersion; - -public class ReaderRaftLogVersionBridge implements LogVersionBridge -{ - private final FileSystemAbstraction fileSystem; - private final PhysicalRaftLogFiles logFiles; - - public ReaderRaftLogVersionBridge( FileSystemAbstraction fileSystem, PhysicalRaftLogFiles logFiles ) - { - this.fileSystem = fileSystem; - this.logFiles = logFiles; - } - - @Override - public LogVersionedStoreChannel next( LogVersionedStoreChannel channel ) throws IOException - { - PhysicalLogVersionedStoreChannel nextChannel; - try - { - nextChannel = openForVersion( logFiles, fileSystem, channel.getVersion() + 1 ); - } - catch ( FileNotFoundException e ) - { - return channel; - } - channel.close(); - return nextChannel; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Recovery.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Recovery.java deleted file mode 100644 index 4826667b5335b..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/Recovery.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; -import java.io.IOException; - -import org.neo4j.cursor.IOCursor; - -/** - * Determines which versions contain which entries. - */ -public class Recovery -{ - private final VersionFiles files; - private final HeaderReader headerReader; - private final EntryReader reader; - - public Recovery( VersionFiles files, HeaderReader headerReader, EntryReader reader ) - { - this.files = files; - this.headerReader = headerReader; - this.reader = reader; - } - - /** - * output: currentVersion, prevIndex, prevTerm, appendIndex - * effects: current version file starts with a valid header and contains no extra bytes beyond the last entry at appendIndex - */ - public LogState recover() throws IOException - { - long currentVersion = -1; - long prevIndex = -1; - long prevTerm = -1; - long appendIndex = -1; - VersionIndexRanges ranges = new VersionIndexRanges(); - - for ( File file : files.filesInVersionOrder() ) - { - Header header = headerReader.readHeader( file ); - if ( currentVersion < 0 ) - { - prevIndex = header.prevIndex; - prevTerm = header.prevTerm; - } - ranges.add( header.version, header.prevIndex ); - appendIndex = header.prevIndex; - currentVersion = header.version; - } - if ( currentVersion >= 0 ) - { - try ( IOCursor entryCursor = reader.readEntriesInVersion( currentVersion ) ) - { - while ( entryCursor.next() ) - { - appendIndex = entryCursor.get().logIndex(); - } - } - } - return new LogState( currentVersion, prevIndex, prevTerm, appendIndex, ranges ); - } - - static class LogState - { - public final long prevIndex; - public final long prevTerm; - public final long appendIndex; - public final long currentVersion; - public final VersionIndexRanges ranges; - - public LogState( - long currentVersion, long prevIndex, long prevTerm, long appendIndex, VersionIndexRanges ranges ) - { - this.currentVersion = currentVersion; - this.prevIndex = prevIndex; - this.prevTerm = prevTerm; - this.appendIndex = appendIndex; - this.ranges = ranges; - } - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/SingleVersionReader.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/SingleVersionReader.java deleted file mode 100644 index 79002da25b282..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/SingleVersionReader.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeader.LOG_HEADER_SIZE; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.neo4j.coreedge.raft.replication.ReplicatedContent; -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.cursor.IOCursor; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreChannel; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel; -import org.neo4j.kernel.impl.transaction.log.ReadAheadChannel; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; - -public class SingleVersionReader -{ - private final PhysicalRaftLogFiles files; - private final FileSystemAbstraction fileSystem; - private final ChannelMarshal marshal; - - public SingleVersionReader( PhysicalRaftLogFiles files, FileSystemAbstraction fileSystem, - ChannelMarshal marshal ) - { - this.files = files; - this.fileSystem = fileSystem; - this.marshal = marshal; - } - - public IOCursor readEntriesFrom( LogPosition position ) throws IOException - { - File file = files.getLogFileForVersion( position.getLogVersion() ); - - StoreChannel rawChannel = fileSystem.open( file, "rw" ); - ByteBuffer buffer = ByteBuffer.allocate( LOG_HEADER_SIZE ); - LogHeader header = readLogHeader( buffer, rawChannel, true ); - assert header != null && header.logVersion == position.getLogVersion(); - - PhysicalLogVersionedStoreChannel physicalLogVersionedStoreChannel = - new PhysicalLogVersionedStoreChannel( rawChannel, position.getLogVersion(), header.logFormatVersion ); - physicalLogVersionedStoreChannel.position( position.getByteOffset() ); - ReadAheadChannel readAheadChannel = new ReadAheadChannel<>( - physicalLogVersionedStoreChannel ); - - return new RaftAppendRecordCursor( readAheadChannel, marshal ); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionBridgingRaftEntryStore.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionBridgingRaftEntryStore.java deleted file mode 100644 index dd61b16bcd94f..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionBridgingRaftEntryStore.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.IOException; - -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.cursor.CursorValue; -import org.neo4j.cursor.IOCursor; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; - -/** - * A {@link RaftEntryStore} that uses a {@link VersionIndexRanges} to iterate over the - */ -public class VersionBridgingRaftEntryStore implements RaftEntryStore -{ - private final VersionIndexRanges ranges; - private final SingleVersionReader reader; - private final RaftLogMetadataCache raftLogMetadataCache; - - public VersionBridgingRaftEntryStore( VersionIndexRanges ranges, SingleVersionReader reader, RaftLogMetadataCache - raftLogMetadataCache ) - { - this.ranges = ranges; - this.reader = reader; - this.raftLogMetadataCache = raftLogMetadataCache; - } - - @Override - public IOCursor getEntriesFrom( long logIndex ) throws IOException - { - return new IOCursor() - { - private CursorValue cursorValue = new CursorValue<>(); - private long nextIndex = logIndex; - private VersionIndexRange currentRange = VersionIndexRange.OUT_OF_RANGE; - private IOCursor versionCursor; - - @Override - public boolean next() throws IOException - { - if ( !currentRange.includes( nextIndex ) ) - { - currentRange = ranges.versionForIndex( nextIndex ); - if ( !currentRange.includes( nextIndex ) ) - { - return false; - } - close(); - RaftLogMetadataCache.RaftLogEntryMetadata metadata = raftLogMetadataCache.getMetadata( nextIndex ); - LogPosition thePosition = new LogPosition( currentRange.version, LogHeader.LOG_HEADER_SIZE ); - if( metadata != null ) - { - thePosition = metadata.getStartPosition(); - } - versionCursor = reader.readEntriesFrom( thePosition ); - } - while ( versionCursor.next() ) - { - RaftLogAppendRecord record = versionCursor.get(); - if ( record.logIndex() == nextIndex ) - { - cursorValue.set( record ); - nextIndex++; - return true; - } - } - cursorValue.invalidate(); - return false; - } - - @Override - public void close() throws IOException - { - if ( versionCursor != null ) - { - versionCursor.close(); - } - } - - @Override - public RaftLogAppendRecord get() - { - return cursorValue.get(); - } - }; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionFiles.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionFiles.java deleted file mode 100644 index 2a033d26270c0..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionFiles.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; - -public class VersionFiles -{ - public Iterable filesInVersionOrder() - { - return null; - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRange.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRange.java deleted file mode 100644 index dbce7e93cf333..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRange.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import static java.lang.String.format; - -/** - * Instances of this class represent a pair of a log version and the range of index entries it contains. The starting - * index is set in creation and cannot be changed, but the end index is initially assumed to be unbounded (practically - * it is represented as {@link Long#MAX_VALUE} until it is capped by calling {@link #endAt(long)}. This corresponds - * to the way logs are structured, since until rotation happens the end index is unknown. - */ -public class VersionIndexRange -{ - public final long version; - public final long prevIndex; - private long lastIndex = Long.MAX_VALUE; - - public VersionIndexRange( long version, long prevIndex ) - { - this.version = version; - this.prevIndex = prevIndex; - } - - public boolean includes( long index ) - { - return index <= lastIndex && index > prevIndex; - } - - void endAt( long lastIndex ) - { - if ( lastIndex < prevIndex ) - { - throw new IllegalArgumentException( format( "A range cannot have the upper bound set to a value " + - "(%d) less than the lower bound (%d)", lastIndex, prevIndex ) ); - } - this.lastIndex = lastIndex; - } - - @Override - public String toString() - { - return format( "%d: %d < index <= %d", version, prevIndex, lastIndex ); - } - - public static final VersionIndexRange OUT_OF_RANGE = new VersionIndexRange( -1, -1 ) { - - @Override - public boolean includes( long index ) - { - return false; - } - - @Override - void endAt( long lastIndex ) - { - throw new UnsupportedOperationException( "Immutable" ); - } - - @Override - public String toString() - { - return "OUT_OF_RANGE"; - } - }; -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRanges.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRanges.java deleted file mode 100644 index 6bf3b356fa577..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRanges.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.util.LinkedList; - -import static java.lang.String.format; - -/** - * This data structure maintains a mapping from a log index value to the log version that contains it. This is - * maintained while entries are appended, log files are pruned and logs are truncated. To achieve this, there are - * two update methods - {@link #add(long, long)} for notifying of log creation and {@link #pruneVersion(long)} for - * notification of log pruning. - * - * The main query method is {@link #versionForIndex(long)} which returns the log version which contains the specified - * entry index. An abstraction to keep in mind is that the last log version is assumed to contain all entries - * greater than the prevIndex argument of the {@link #add(long, long)} call with the largest version argument so far. - * Practically, this means that asking for a log index that is beyond the current append index will return the current - * log version. - * - * A special word for truncations. They are communicated by calling {@link #add(long, long)} with the prevIndex at one - * less than the truncate index. This will update the previous version with the proper upper index. In the case where - * this process would leave one or more versions empty (for example, the prevIndex is less than the prevIndex with which - * a previous version was added) that version will be removed since logically is has been truncated away entirely. - */ -public class VersionIndexRanges -{ - /* - * This list is treated like a deque with the head (first entry, small list indexes) holding the lowest log - * index and the tail (last entry, highest list index) being the latest addition (i.e. largest log index). - */ - private final LinkedList ranges = new LinkedList<>(); - - public void add( long version, long prevIndex ) - { - // Ensure strictly monotonic additions - if ( !ranges.isEmpty() && ranges.peekLast().version >= version ) - { - throw new IllegalArgumentException( format( "Cannot accept range for version %d while having " + - "already accepted %d", version, ranges.peek().version ) ); - } - // Update the upper index of the stack's head, or completely remove it if it's been truncated away entirely - while ( !ranges.isEmpty() ) - { - VersionIndexRange range = ranges.peekLast(); - if ( range.prevIndex >= prevIndex ) - { - ranges.removeLast(); - } - else - { - range.endAt( prevIndex ); - break; - } - } - // Finally, add the new range at the stack's top - ranges.add( new VersionIndexRange( version, prevIndex ) ); - } - - public void pruneVersion( long version ) - { - // Keep removing at the queue's head until we reach the specified version - while( !ranges.isEmpty() && ranges.getFirst().version <= version ) - { - ranges.removeFirst(); - } - } - - public VersionIndexRange versionForIndex( long index ) - { - // Start at the stack's head, keep going backwards, since we assume most queries will be for recent entries - for ( int i = ranges.size() - 1; i >= 0; i-- ) - { - VersionIndexRange range = ranges.get( i ); - if ( range.includes( index ) ) - { - return range; - } - } - return VersionIndexRange.OUT_OF_RANGE; - } - - @Override - public String toString() - { - return format( "RaftLogVersionRanges{ranges=%s}", ranges ); - } - - public long highestVersion() - { - return ranges.isEmpty() ? 0 : Math.max(0, ranges.peekLast().version); - } - - public long lowestVersion() - { - return ranges.isEmpty() ? 0 : Math.max(0, ranges.peekFirst().version); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategy.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategy.java deleted file mode 100644 index 099ad480cc344..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategy.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical.pruning; - -import static org.neo4j.kernel.impl.transaction.log.LogVersionRepository.INITIAL_LOG_VERSION; - -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFiles; -import org.neo4j.kernel.impl.transaction.log.LogFileInformation; -import org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy; -import org.neo4j.kernel.impl.transaction.log.pruning.Threshold; - -public class RaftLogPruneStrategy implements LogPruneStrategy -{ - private final LogFileInformation logFileInformation; - private final PhysicalRaftLogFiles files; - private final Threshold threshold; - - public RaftLogPruneStrategy( LogFileInformation logFileInformation, PhysicalRaftLogFiles files, - Threshold threshold ) - { - this.logFileInformation = logFileInformation; - this.files = files; - this.threshold = threshold; - } - - @Override - public void prune( long upToLogVersion ) - { - if ( upToLogVersion == INITIAL_LOG_VERSION ) - { - return; - } - - threshold.init(); - long upper = upToLogVersion - 1; - boolean exceeded = false; - while ( upper >= 0 ) - { - if ( !files.versionExists( upper ) ) - { - // There aren't logs to prune anything. Just return - return; - } - - if ( files.containsEntries( upper ) && - threshold.reached( files.getLogFileForVersion( upper ), upper, logFileInformation ) ) - { - exceeded = true; - break; - } - upper--; - } - - if ( !exceeded ) - { - return; - } - - files.pruneUpTo( upper ); - } -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategyFactory.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategyFactory.java deleted file mode 100644 index df94d964fd151..0000000000000 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/raft/log/physical/pruning/RaftLogPruneStrategyFactory.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical.pruning; - -import java.util.concurrent.TimeUnit; - -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFiles; -import org.neo4j.helpers.Clock; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.transaction.log.LogFileInformation; -import org.neo4j.kernel.impl.transaction.log.pruning.EntryCountThreshold; -import org.neo4j.kernel.impl.transaction.log.pruning.EntryTimespanThreshold; -import org.neo4j.kernel.impl.transaction.log.pruning.FileCountThreshold; -import org.neo4j.kernel.impl.transaction.log.pruning.FileSizeThreshold; -import org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy; -import org.neo4j.kernel.impl.transaction.log.pruning.Threshold; -import org.neo4j.kernel.impl.transaction.log.pruning.ThresholdConfigParser; - -import static org.neo4j.kernel.impl.transaction.log.pruning.ThresholdConfigParser.parse; - -public class RaftLogPruneStrategyFactory -{ - public static final LogPruneStrategy NO_PRUNING = new LogPruneStrategy() - { - @Override - public void prune( long upToLogVersion ) - { - // do nothing - } - - @Override - public String toString() - { - return "NO_PRUNING"; - } - }; - - /** - * Parses a configuration value for log specifying log pruning. It has one of these forms: - *

- * For example: - * - */ - public static LogPruneStrategy fromConfigValue( FileSystemAbstraction fileSystem, - LogFileInformation logFileInformation, - PhysicalRaftLogFiles files, - String configValue ) - { - ThresholdConfigParser.ThresholdConfigValue value = parse( configValue ); - - if ( value == ThresholdConfigParser.ThresholdConfigValue.NO_PRUNING ) - { - return NO_PRUNING; - } - - Threshold thresholdToUse = getThresholdByType( fileSystem, value, configValue ); - return new RaftLogPruneStrategy( logFileInformation, files, thresholdToUse ); - } - - // visible for testing - private static Threshold getThresholdByType( FileSystemAbstraction fileSystem, - ThresholdConfigParser.ThresholdConfigValue value, String originalConfigValue ) - { - long thresholdValue = value.value; - - switch ( value.type ) - { - case "files": - return new FileCountThreshold( thresholdValue ); - case "size": - return new FileSizeThreshold( fileSystem, thresholdValue ); - case "txs": - case "entries": // txs and entries are synonyms - return new EntryCountThreshold( thresholdValue ); - case "hours": - return new EntryTimespanThreshold( Clock.SYSTEM_CLOCK, TimeUnit.HOURS, thresholdValue ); - case "days": - return new EntryTimespanThreshold( Clock.SYSTEM_CLOCK, TimeUnit.DAYS, thresholdValue ); - default: - throw new IllegalArgumentException( "Invalid log pruning configuration value '" + originalConfigValue + - "'. Invalid type '" + value.type + "', valid are files, size, txs, entries, hours, days." ); - } - } - -} diff --git a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java index aba3272d03278..2e955ee0f5033 100644 --- a/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java +++ b/enterprise/core-edge/src/main/java/org/neo4j/coreedge/server/core/EnterpriseCoreEditionModule.java @@ -57,10 +57,6 @@ import org.neo4j.coreedge.raft.log.MonitoredRaftLog; import org.neo4j.coreedge.raft.log.RaftLog; import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog; -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFile; import org.neo4j.coreedge.raft.log.pruning.PruningScheduler; import org.neo4j.coreedge.raft.log.segmented.InFlightMap; import org.neo4j.coreedge.raft.log.segmented.SegmentedRaftLog; @@ -163,7 +159,6 @@ import org.neo4j.udc.UsageData; import static java.util.concurrent.TimeUnit.SECONDS; - import static org.neo4j.kernel.impl.util.JobScheduler.SchedulingStrategy.NEW_THREAD; /** @@ -207,22 +202,7 @@ public void compact() throws IOException public enum RaftLogImplementation { - NAIVE, - IN_MEMORY, - PHYSICAL, - SEGMENTED; - - public static RaftLogImplementation fromString( String value ) - { - try - { - return RaftLogImplementation.valueOf( value ); - } - catch ( IllegalArgumentException ex ) - { - return NAIVE; - } - } + IN_MEMORY, SEGMENTED } @Override @@ -300,14 +280,13 @@ public EnterpriseCoreEditionModule( final PlatformModule platformModule, final DelayedRenewableTimeoutService raftTimeoutService = new DelayedRenewableTimeoutService( Clock.systemUTC(), logProvider ); - RaftLog underlyingLog = createRaftLog( - config, life, fileSystem, clusterStateDirectory, marshal, logProvider, databaseHealthSupplier ); + RaftLog underlyingLog = createRaftLog( config, life, fileSystem, clusterStateDirectory, marshal, logProvider ); MonitoredRaftLog raftLog = new MonitoredRaftLog( underlyingLog, platformModule.monitors ); - LocalDatabase localDatabase = new LocalDatabase( platformModule.storeDir, - new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), - platformModule.pageCache ), + CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, + platformModule.kernelExtensions.listFactories(), platformModule.pageCache ); + LocalDatabase localDatabase = new LocalDatabase( platformModule.storeDir, copiedStoreRecovery, new StoreFiles( new DefaultFileSystemAbstraction() ), dependencies.provideDependency( NeoStoreDataSource.class ), platformModule.dependencies.provideDependency( TransactionIdStore.class ), databaseHealthSupplier ); @@ -531,33 +510,15 @@ fileSystem, new File( clusterStateDirectory, "id-allocation-state" ), "id-alloca joinCatchupTimeout, logProvider ) ); } - private RaftLog createRaftLog( - Config config, LifeSupport life, FileSystemAbstraction fileSystem, File clusterStateDirectory, - CoreReplicatedContentMarshal marshal, LogProvider logProvider, - Supplier databaseHealthSupplier ) + private RaftLog createRaftLog( Config config, LifeSupport life, FileSystemAbstraction fileSystem, + File clusterStateDirectory, CoreReplicatedContentMarshal marshal, LogProvider logProvider ) { - RaftLogImplementation raftLogImplementation = RaftLogImplementation.fromString( - config.get( CoreEdgeClusterSettings.raft_log_implementation ) ); + RaftLogImplementation raftLogImplementation = + RaftLogImplementation.valueOf( config.get( CoreEdgeClusterSettings.raft_log_implementation ) ); switch ( raftLogImplementation ) { case IN_MEMORY: return new InMemoryRaftLog(); - case PHYSICAL: - { - long rotateAtSize = config.get( CoreEdgeClusterSettings.raft_log_rotation_size ); - String pruneConf = config.get( CoreEdgeClusterSettings.raft_log_pruning_strategy ); - int entryCacheSize = 32; - int metaDataCacheSize = 100_000; - int headerCacheSize = 10; - - return life.add( new PhysicalRaftLog( - fileSystem, - new File( clusterStateDirectory, PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME ), - rotateAtSize, pruneConf, entryCacheSize, headerCacheSize, - new PhysicalRaftLogFile.Monitor.Adapter(), marshal, databaseHealthSupplier, logProvider, - new RaftLogMetadataCache( metaDataCacheSize ) ) ); - } - case SEGMENTED: { long rotateAtSize = config.get( CoreEdgeClusterSettings.raft_log_rotation_size ); @@ -565,19 +526,14 @@ private RaftLog createRaftLog( return life.add( new SegmentedRaftLog( fileSystem, - new File( clusterStateDirectory, PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME ), + new File( clusterStateDirectory, RaftLog.PHYSICAL_LOG_DIRECTORY_NAME ), rotateAtSize, marshal, logProvider, pruningStrategyConfig ) ); } - - case NAIVE: default: - return life.add( new NaiveDurableRaftLog( - fileSystem, - new File( clusterStateDirectory, NaiveDurableRaftLog.NAIVE_LOG_DIRECTORY_NAME ), - marshal, logProvider ) ); + throw new IllegalStateException( "Unknown raft log implementation: " + raftLogImplementation ); } } diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/MonitoredRaftLogTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/MonitoredRaftLogTest.java index 8fe593aca7fe3..92af0a443a9be 100644 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/MonitoredRaftLogTest.java +++ b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/MonitoredRaftLogTest.java @@ -21,16 +21,10 @@ import org.junit.Test; -import java.io.File; - import org.neo4j.coreedge.raft.ReplicatedInteger; import org.neo4j.coreedge.raft.log.monitoring.RaftLogAppendIndexMonitor; import org.neo4j.coreedge.raft.log.monitoring.RaftLogCommitIndexMonitor; -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.monitoring.Monitors; -import org.neo4j.logging.NullLogProvider; import static org.junit.Assert.assertEquals; @@ -40,11 +34,6 @@ public class MonitoredRaftLogTest public void shouldMonitorAppendIndexAndCommitIndex() throws Exception { // Given - FileSystemAbstraction fsa = new EphemeralFileSystemAbstraction(); - - File directory = new File( "." ); - fsa.create( directory ); - Monitors monitors = new Monitors(); StubRaftLogAppendIndexMonitor appendMonitor = new StubRaftLogAppendIndexMonitor(); monitors.addMonitorListener( appendMonitor ); @@ -52,9 +41,7 @@ public void shouldMonitorAppendIndexAndCommitIndex() throws Exception StubRaftLogCommitIndexMonitor commitMonitor = new StubRaftLogCommitIndexMonitor(); monitors.addMonitorListener( commitMonitor ); - MonitoredRaftLog log = new MonitoredRaftLog( - new NaiveDurableRaftLog( fsa, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance() ), monitors ); + MonitoredRaftLog log = new MonitoredRaftLog( new InMemoryRaftLog(), monitors ); // When log.append( new RaftLogEntry( 0, ReplicatedInteger.valueOf( 1 ) ) ); diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogContractTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogContractTest.java deleted file mode 100644 index 2bbbffaa95909..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogContractTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log; - -import java.io.File; -import java.io.IOException; - -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.logging.NullLogProvider; - -import static org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog.NAIVE_LOG_DIRECTORY_NAME; - -public class NaiveDurableRaftLogContractTest extends RaftLogContractTest -{ - @Override - public RaftLog createRaftLog() throws IOException - { - FileSystemAbstraction fileSystem = new EphemeralFileSystemAbstraction(); - File directory = new File( NAIVE_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - - return new NaiveDurableRaftLog( fileSystem, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance() ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogTest.java deleted file mode 100644 index 8fdbe10035703..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveDurableRaftLogTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log; - -import org.junit.Test; -import org.mockito.Matchers; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.neo4j.coreedge.raft.ReplicatedInteger; -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.fs.StoreFileChannel; -import org.neo4j.logging.NullLogProvider; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class NaiveDurableRaftLogTest -{ - @Test - public void shouldCallWriteAllWhenStoringEntries() throws Exception - { - // Given - FileSystemAbstraction fsa = mock( FileSystemAbstraction.class ); - StoreFileChannel entriesChannel = mock( StoreFileChannel.class ); - StoreFileChannel contentChannel = mock( StoreFileChannel.class ); - StoreFileChannel commitChannel = mock( StoreFileChannel.class ); - - File directory = new File( "." ); - - File entriesFile = new File( directory, "entries.log" ); - File contentFile = new File( directory, "content.log" ); - File metaFile = new File( directory, "meta.log" ); - - when( fsa.open( Matchers.eq( entriesFile ), anyString() ) ).thenReturn( entriesChannel ); - when( fsa.open( Matchers.eq( contentFile ), anyString() ) ).thenReturn( contentChannel ); - when( fsa.open( Matchers.eq( metaFile ), anyString() ) ).thenReturn( commitChannel ); - - NaiveDurableRaftLog log = new NaiveDurableRaftLog( fsa, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance()); - - // When - log.append( new RaftLogEntry( 0, ReplicatedInteger.valueOf( 1 ) ) ); - - // Then - verify( entriesChannel ).writeAll( any( ByteBuffer.class ), anyInt() ); - verify( entriesChannel ).force( anyBoolean() ); - verify( contentChannel, times( 2 ) ).writeAll( any( ByteBuffer.class ), anyInt() ); - verify( contentChannel ).force( anyBoolean() ); - } - - @Test - public void shouldForceAndCloseFilesOnShutdown() throws Throwable - { - // Given - FileSystemAbstraction fsa = mock( FileSystemAbstraction.class ); - StoreFileChannel entriesChannel = mock( StoreFileChannel.class ); - StoreFileChannel contentChannel = mock( StoreFileChannel.class ); - StoreFileChannel commitChannel = mock( StoreFileChannel.class ); - - File directory = new File( "." ); - - File entriesFile = new File( directory, "entries.log" ); - File contentFile = new File( directory, "content.log" ); - File metaFile = new File( directory, "meta.log" ); - - when( fsa.open( Matchers.eq( entriesFile ), anyString() ) ).thenReturn( entriesChannel ); - when( fsa.open( Matchers.eq( contentFile ), anyString() ) ).thenReturn( contentChannel ); - when( fsa.open( Matchers.eq( metaFile ), anyString() ) ).thenReturn( commitChannel ); - - NaiveDurableRaftLog log = new NaiveDurableRaftLog( fsa, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance()); - - // When - log.shutdown(); - - // Then - verify( entriesChannel ).force( anyBoolean() ); - verify( entriesChannel ).close(); - - verify( contentChannel ).force( anyBoolean() ); - verify( contentChannel ).close(); - - verify( commitChannel ).force( anyBoolean() ); - verify( commitChannel ).close(); - } - - @Test - public void shouldForceAndCloseFilesOnShutdownEvenOnFailure() throws Throwable - { - // Given - FileSystemAbstraction fsa = mock( FileSystemAbstraction.class ); - StoreFileChannel entriesChannel = mock( StoreFileChannel.class ); - StoreFileChannel contentChannel = mock( StoreFileChannel.class ); - StoreFileChannel commitChannel = mock( StoreFileChannel.class ); - - doThrow( new IOException() ).when( entriesChannel ).force( anyBoolean() ); - doThrow( new IOException() ).when( contentChannel ).force( anyBoolean() ); - doThrow( new IOException() ).when( commitChannel ).force( anyBoolean() ); - - File directory = new File( "." ); - - File entriesFile = new File( directory, "entries.log" ); - File contentFile = new File( directory, "content.log" ); - File metaFile = new File( directory, "meta.log" ); - - when( fsa.open( Matchers.eq( entriesFile ), anyString() ) ).thenReturn( entriesChannel ); - when( fsa.open( Matchers.eq( contentFile ), anyString() ) ).thenReturn( contentChannel ); - when( fsa.open( Matchers.eq( metaFile ), anyString() ) ).thenReturn( commitChannel ); - - NaiveDurableRaftLog log = new NaiveDurableRaftLog( fsa, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance()); - - // When - try - { - log.shutdown(); - fail( "Should have thrown exception, by test design" ); - } - catch ( Exception e ) - { - // Then - // verify that all exceptions got added as suppressed - assertEquals( 3, e.getSuppressed().length ); // 3 exceptions, one for each file channel close() call - } - - // Then - verify( entriesChannel ).force( anyBoolean() ); - verify( contentChannel ).force( anyBoolean() ); - verify( commitChannel ).force( anyBoolean() ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveRaftLogVerificationIT.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveRaftLogVerificationIT.java deleted file mode 100644 index 74f4fb91a25ce..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/NaiveRaftLogVerificationIT.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log; - -import java.io.File; - -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.logging.NullLogProvider; - -import static org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog.NAIVE_LOG_DIRECTORY_NAME; - -public class NaiveRaftLogVerificationIT extends RaftLogVerificationIT -{ - @Override - protected RaftLog createRaftLog() throws Throwable - { - FileSystemAbstraction fsa = fsRule.get(); - File directory = new File( NAIVE_LOG_DIRECTORY_NAME ); - fsa.mkdir( directory ); - - return new NaiveDurableRaftLog( fsa, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance() ); - } - - @Override - protected long operations() - { - return 500; - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/RaftLogDurabilityTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/RaftLogDurabilityTest.java index cfad14f2ab0a1..13e17701a2ffb 100644 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/RaftLogDurabilityTest.java +++ b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/RaftLogDurabilityTest.java @@ -32,27 +32,20 @@ import org.neo4j.coreedge.raft.ReplicatedInteger; import org.neo4j.coreedge.raft.ReplicatedString; -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog; -import org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFile; import org.neo4j.coreedge.raft.log.segmented.SegmentedRaftLog; import org.neo4j.coreedge.server.core.EnterpriseCoreEditionModule.RaftLogImplementation; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.internal.DatabaseHealth; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.Mockito.mock; import static org.neo4j.coreedge.raft.ReplicatedInteger.valueOf; import static org.neo4j.coreedge.raft.log.RaftLogHelper.hasNoContent; import static org.neo4j.coreedge.raft.log.RaftLogHelper.readLogEntry; import static org.neo4j.coreedge.raft.log.segmented.SegmentedRaftLog.SEGMENTED_LOG_DIRECTORY_NAME; -import static org.neo4j.coreedge.server.core.EnterpriseCoreEditionModule.RaftLogImplementation.NAIVE; -import static org.neo4j.coreedge.server.core.EnterpriseCoreEditionModule.RaftLogImplementation.PHYSICAL; import static org.neo4j.coreedge.server.core.EnterpriseCoreEditionModule.RaftLogImplementation.SEGMENTED; @RunWith(Parameterized.class) @@ -71,38 +64,11 @@ public RaftLogDurabilityTest( RaftLogImplementation ignored, RaftLogFactory logF @Parameters(name = "log:{0}") public static Collection data() { - RaftLogFactory naive = ( fileSystem ) -> { - File directory = new File( SEGMENTED_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - return new NaiveDurableRaftLog( fileSystem, directory, new DummyRaftableContentSerializer(), - NullLogProvider.getInstance() ); - }; - - RaftLogFactory physical = ( fileSystem ) -> { - File directory = new File( SEGMENTED_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - - long rotateAtSizeBytes = 128; - int entryCacheSize = 4; - int metadataCacheSize = 8; - int fileHeaderCacheSize = 2; - - PhysicalRaftLog log = new PhysicalRaftLog( fileSystem, directory, rotateAtSizeBytes, "1 files", - entryCacheSize, fileHeaderCacheSize, - new PhysicalRaftLogFile.Monitor.Adapter(), new DummyRaftableContentSerializer(), () -> mock( - DatabaseHealth.class ), - NullLogProvider.getInstance(), new RaftLogMetadataCache( metadataCacheSize ) ); - log.init(); - log.start(); - return log; - }; - RaftLogFactory segmented = ( fileSystem ) -> { File directory = new File( SEGMENTED_LOG_DIRECTORY_NAME ); fileSystem.mkdir( directory ); long rotateAtSizeBytes = 128; - int entryCacheSize = 4; SegmentedRaftLog log = new SegmentedRaftLog( fileSystem, directory, rotateAtSizeBytes, new DummyRaftableContentSerializer(), NullLogProvider.getInstance(), "1 size" ); @@ -110,11 +76,7 @@ public static Collection data() return log; }; - return Arrays.asList( new Object[][]{ - {NAIVE, naive}, - {PHYSICAL, physical}, - {SEGMENTED, segmented}, - } ); + return Arrays.asList( new Object[][]{ {SEGMENTED, segmented} } ); } @Test diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/VersionBridgingRaftEntryStoreTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/VersionBridgingRaftEntryStoreTest.java deleted file mode 100644 index dbb58c3bb4103..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/VersionBridgingRaftEntryStoreTest.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log; - -import static java.util.Arrays.asList; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.hasItems; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.RETURNS_MOCKS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.neo4j.coreedge.raft.ReplicatedInteger.valueOf; -import static org.neo4j.kernel.impl.util.IOCursors.cursor; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.function.Consumer; - -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; -import org.junit.Test; -import org.neo4j.coreedge.raft.log.physical.RaftLogAppendRecord; -import org.neo4j.coreedge.raft.log.physical.SingleVersionReader; -import org.neo4j.coreedge.raft.log.physical.VersionBridgingRaftEntryStore; -import org.neo4j.coreedge.raft.log.physical.VersionIndexRanges; -import org.neo4j.cursor.IOCursor; -import org.neo4j.helpers.collection.Iterables; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; - -public class VersionBridgingRaftEntryStoreTest -{ - RaftLogAppendRecord entry0 = new RaftLogAppendRecord( 0, new RaftLogEntry( 10, valueOf( 100 ) ) ); - RaftLogAppendRecord entry1 = new RaftLogAppendRecord( 1, new RaftLogEntry( 10, valueOf( 101 ) ) ); - RaftLogAppendRecord entry2 = new RaftLogAppendRecord( 2, new RaftLogEntry( 10, valueOf( 102 ) ) ); - RaftLogAppendRecord entry3 = new RaftLogAppendRecord( 3, new RaftLogEntry( 10, valueOf( 103 ) ) ); - RaftLogAppendRecord entry4 = new RaftLogAppendRecord( 4, new RaftLogEntry( 11, valueOf( 104 ) ) ); - RaftLogAppendRecord entry5 = new RaftLogAppendRecord( 5, new RaftLogEntry( 11, valueOf( 105 ) ) ); - RaftLogAppendRecord entry6 = new RaftLogAppendRecord( 6, new RaftLogEntry( 12, valueOf( 106 ) ) ); - RaftLogAppendRecord entry7 = new RaftLogAppendRecord( 7, new RaftLogEntry( 12, valueOf( 107 ) ) ); - RaftLogAppendRecord entry8 = new RaftLogAppendRecord( 8, new RaftLogEntry( 12, valueOf( 108 ) ) ); - - private static LogPosition positionAtBeginningOfVersion( long version ) - { - return new LogPosition( version, LogHeader.LOG_HEADER_SIZE ); - } - - @Test - public void shouldReadEntriesFromSingleVersion() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class ); - - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( 0 ) ) ).thenReturn( cursor( entry0, entry1, entry2, entry3 ) ); - - VersionIndexRanges ranges = new VersionIndexRanges(); - ranges.add( 0, -1 ); - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, - mock( RaftLogMetadataCache.class ) ); - - // when - try ( IOCursor cursor = store.getEntriesFrom( 0 ) ) - { - // then - assertThat( allItems( cursor ), consistsOf( entry0, entry1, entry2, entry3 ) ); - } - } - - @Test - public void shouldReadEntriesFromMiddleOfFile() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class ); - - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( 0 ) ) ).thenReturn( cursor( entry0, entry1, entry2, entry3 ) ); - - VersionIndexRanges ranges = new VersionIndexRanges(); - ranges.add( 0, -1 ); - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, mock( RaftLogMetadataCache.class ) ); - - // when - try ( IOCursor cursor = store.getEntriesFrom( 2 ) ) - { - // then - assertThat( allItems( cursor ), consistsOf( entry2, entry3 ) ); - } - } - - @Test - public void shouldReadAcrossVersionBoundary() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class ); - - RaftLogAppendRecord truncatedEntry4 = new RaftLogAppendRecord( 3, new RaftLogEntry( 10, valueOf( -1 ) ) ); - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( 0 ) ) ).thenReturn( - cursor( entry0, entry1, entry2, entry3, truncatedEntry4 ) ); - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( 2 ) ) ).thenReturn( - cursor( entry4, entry5 ) ); - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( 3 ) ) ).thenReturn( - cursor( entry6, entry7, entry8 ) ); - - VersionIndexRanges ranges = new VersionIndexRanges(); - ranges.add( 0, -1 ); - ranges.add( 2, 3 ); - ranges.add( 3, 5 ); - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, mock( RaftLogMetadataCache.class ) ); - - // when - try ( IOCursor cursor = store.getEntriesFrom( 0 ) ) - { - // then - assertThat( allItems( cursor ), - consistsOf( entry0, entry1, entry2, entry3, entry4, entry5, entry6, entry7, entry8 ) ); - } - } - - @Test - public void shouldCloseUnderlyingCursors() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class ); - - VersionIndexRanges ranges = new VersionIndexRanges(); - List cursors = new ArrayList<>(); - for ( int i = 0; i < 10; i++ ) - { - ranges.add( i, i * 2 - 1 ); - StubCursor versionCursor = new StubCursor( cursor( - new RaftLogAppendRecord( i * 2, new RaftLogEntry( i, valueOf( i * 10 ) ) ), - new RaftLogAppendRecord( i * 2 + 1, new RaftLogEntry( i, valueOf( i * 10 ) ) ) - ) ); - cursors.add( versionCursor ); - when( versionReader.readEntriesFrom( positionAtBeginningOfVersion( i ) ) ).thenReturn( versionCursor ); - } - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, mock( RaftLogMetadataCache.class ) ); - - IOCursor cursor = store.getEntriesFrom( 0 ); - allItems( cursor ); - - // when - cursor.close(); - - // then - cursors.forEach( (versionCursor) -> assertTrue( versionCursor.isClosed() ) ); - } - - @Test - public void shouldReadNoEntriesFromEmptyLog() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class ); - - VersionIndexRanges ranges = new VersionIndexRanges(); // empty - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, mock( RaftLogMetadataCache.class ) ); - - // when - try ( IOCursor cursor = store.getEntriesFrom( 0 ) ) - { - // then - assertThat( allItems( cursor ), empty() ); - } - } - - @Test - public void shouldUsePositionCacheIfItContainsTargetEntry() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class, RETURNS_MOCKS ); - VersionIndexRanges ranges = new VersionIndexRanges(); // empty - int currentVersion = 4; - ranges.add( currentVersion, 9 ); // log version 4 contains everything from 9 forward - RaftLogMetadataCache raftLogMetadataCache = new RaftLogMetadataCache( 3 ); - // and a cache that has metadata for the entry we are after - LogPosition thePosition = new LogPosition( currentVersion, 128 ); - int cachedIndex = 10; - raftLogMetadataCache.cacheMetadata( cachedIndex, 30, thePosition ); - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, raftLogMetadataCache ); - - // when - // we ask the store for an entry at an index that has its position cached - IOCursor entriesFrom = store.getEntriesFrom( cachedIndex ); - // and we ask the cursor to actually read the thing - entriesFrom.next(); - - // then - verify( versionReader, times( 1 ) ).readEntriesFrom( thePosition ); - } - - @Test - public void shouldReturnEmtpyCursorIfRequestedIndexIsNotInRange() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class, RETURNS_MOCKS ); - VersionIndexRanges ranges = new VersionIndexRanges(); // empty - ranges.add( 4, 11 ); // log version 4 contains everything from 11 forward - RaftLogMetadataCache raftLogMetadataCache = new RaftLogMetadataCache( 3 ); - // and a cache that has metadata for the entry we are after - long requestedIndex = 10; - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, raftLogMetadataCache ); - - // when - // we ask the store for an entry at an index that is not present in the ranges - IOCursor entriesFrom = store.getEntriesFrom( requestedIndex ); - - // then - // the returned cursor should be empty - assertFalse( entriesFrom.next() ); - } - - @Test - public void shouldCreateCursorPositionedAfterHeaderIfTheEntryHasNoMetadataCached() throws Exception - { - // given - SingleVersionReader versionReader = mock( SingleVersionReader.class, RETURNS_MOCKS ); - VersionIndexRanges ranges = new VersionIndexRanges(); // empty - int currentVersion = 0; - ranges.add( currentVersion, 1 ); // log version 0 contains everything from 1 forward - RaftLogMetadataCache raftLogMetadataCache = new RaftLogMetadataCache( 3 ); - long requestedIndex = 10; - - VersionBridgingRaftEntryStore store = new VersionBridgingRaftEntryStore( ranges, versionReader, - raftLogMetadataCache ); - - // when - // we ask the store for an entry at an index that has its position not present in the cache - IOCursor entriesFrom = store.getEntriesFrom( requestedIndex ); - // and we ask the cursor to actually read the thing - entriesFrom.next(); - - // then - verify( versionReader, times( 1 ) ).readEntriesFrom( new LogPosition( currentVersion, LogHeader.LOG_HEADER_SIZE ) ); - } - - @SafeVarargs - public static Matcher> consistsOf(T... expected) - { - Matcher> hasItems = hasItems( expected ); - return new TypeSafeMatcher>() - { - @Override - protected boolean matchesSafely( Iterable actual ) - { - return expected.length == Iterables.count( actual ) && hasItems.matches( actual ); - } - - @Override - public void describeTo( Description description ) - { - description.appendText( "A collection of " ); - description.appendValue( expected.length ); - description.appendText( " items, specifically: " ); - description.appendValue( asList( expected ) ); - } - }; - } - - private List allItems( IOCursor cursor ) throws IOException - { - LinkedList list = new LinkedList<>(); - while ( cursor.next() ) - { - list.add( cursor.get() ); - } - return list; - } - - private class StubCursor implements IOCursor - { - private final IOCursor inner; - private boolean closed = false; - - private StubCursor( IOCursor inner ) - { - this.inner = inner; - } - - @Override - public boolean next() throws IOException - { - return inner.next(); - } - - @Override - public void close() throws IOException - { - closed = true; - } - - @Override - public void forAll( Consumer consumer ) throws IOException - { - inner.forAll( consumer ); - } - - @Override - public RaftLogAppendRecord get() - { - return inner.get(); - } - - public boolean isClosed() - { - return closed; - } - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/debug/ReplayRaftLog.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/debug/ReplayRaftLog.java index a8329fe266c0e..2e02c9c2f1326 100644 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/debug/ReplayRaftLog.java +++ b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/debug/ReplayRaftLog.java @@ -22,16 +22,19 @@ import java.io.File; import java.io.IOException; -import org.neo4j.coreedge.raft.log.naive.NaiveDurableRaftLog; +import org.neo4j.coreedge.raft.log.segmented.SegmentedRaftLog; import org.neo4j.coreedge.raft.net.CoreReplicatedContentMarshal; import org.neo4j.coreedge.raft.replication.ReplicatedContent; import org.neo4j.coreedge.raft.replication.tx.ReplicatedTransaction; import org.neo4j.coreedge.raft.replication.tx.ReplicatedTransactionFactory; +import org.neo4j.coreedge.server.CoreEdgeClusterSettings; import org.neo4j.helpers.Args; import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.NullLogProvider; import static org.neo4j.coreedge.raft.log.RaftLogHelper.readLogEntry; +import static org.neo4j.helpers.collection.MapUtil.stringMap; public class ReplayRaftLog { @@ -46,14 +49,12 @@ public static void main( String[] args ) throws IOException File logDirectory = new File( from ); System.out.println( "logDirectory = " + logDirectory ); - NaiveDurableRaftLog log = new NaiveDurableRaftLog( new DefaultFileSystemAbstraction(), - logDirectory, new CoreReplicatedContentMarshal(), NullLogProvider.getInstance() ); + Config config = new Config( stringMap() ); + SegmentedRaftLog log = new SegmentedRaftLog( new DefaultFileSystemAbstraction(), logDirectory, + config.get( CoreEdgeClusterSettings.raft_log_rotation_size ), new CoreReplicatedContentMarshal(), + NullLogProvider.getInstance(), config.get( CoreEdgeClusterSettings.raft_log_pruning_strategy ) ); long totalCommittedEntries = log.appendIndex(); // Not really, but we need to have a way to pass in the commit index - -// File target = new File( to ); -// RandomAccessFile newLog = new RandomAccessFile( target, "rw" ); - for ( int i = 0; i <= totalCommittedEntries; i++ ) { ReplicatedContent content = readLogEntry( log, i ).content(); @@ -67,6 +68,5 @@ public static void main( String[] args ) throws IOException } ); } } -// newLog.close(); } } diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepositoryTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepositoryTest.java deleted file mode 100644 index ecc713601a415..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/FilenameBasedLogVersionRepositoryTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.junit.Rule; -import org.junit.Test; - -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; -import org.neo4j.test.rule.TargetDirectory; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FilenameBasedLogVersionRepositoryTest -{ - @Rule - public TargetDirectory.TestDirectory testDir; - - @Test - public void shouldStartAtVersion0() throws Exception - { - PhysicalLogFiles logFiles = mock( PhysicalLogFiles.class ); - when( logFiles.getHighestLogVersion() ).thenReturn( -1L ); - - FilenameBasedLogVersionRepository repository = new FilenameBasedLogVersionRepository( logFiles ); - - assertEquals( 0L, repository.getCurrentLogVersion() ); - } - - @Test - public void shouldPickHighestVersionAvailable() throws Exception - { - PhysicalLogFiles logFiles = mock( PhysicalLogFiles.class ); - when( logFiles.getHighestLogVersion() ).thenReturn( 2L ); - - FilenameBasedLogVersionRepository repository = new FilenameBasedLogVersionRepository( logFiles ); - - assertEquals( 2L, repository.getCurrentLogVersion() ); - - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogCacheConsistencyTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogCacheConsistencyTest.java deleted file mode 100644 index f49ccadbe2503..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogCacheConsistencyTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.junit.After; -import org.junit.Test; - -import java.io.File; - -import org.neo4j.coreedge.raft.ReplicatedInteger; -import org.neo4j.coreedge.raft.log.DummyRaftableContentSerializer; -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.kernel.lifecycle.LifeSupport; -import org.neo4j.logging.NullLogProvider; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.mockito.Mockito.mock; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME; - -public class PhysicalRaftLogCacheConsistencyTest -{ - private LifeSupport life = new LifeSupport(); - private FileSystemAbstraction fileSystem; - - @After - public void tearDown() throws Throwable - { - life.stop(); - life.shutdown(); - } - - private PhysicalRaftLog createRaftLog( long rotateAtSize, PhysicalRaftLogFile.Monitor logFileMonitor, - RaftLogMetadataCache raftLogMetadataCache ) - { - if ( fileSystem == null ) - { - fileSystem = new EphemeralFileSystemAbstraction(); - } - File directory = new File( PHYSICAL_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - - PhysicalRaftLog newRaftLog = new PhysicalRaftLog( fileSystem, directory, rotateAtSize, "1 files", 100, 10, - logFileMonitor, new DummyRaftableContentSerializer(), () -> mock( DatabaseHealth.class ), - NullLogProvider.getInstance(), raftLogMetadataCache ); - life.add( newRaftLog ); - life.init(); - life.start(); - return newRaftLog; - } - - @Test - public void shouldUpdateMetaDataCacheWhenLogsArePruned() throws Exception - { - // Given - int entryCount = 100; - RaftLogMetadataCache metadataCache = new RaftLogMetadataCache( entryCount * 2 ); - - PhysicalRaftLog log = createRaftLog( 100, mock( PhysicalRaftLogFile.Monitor.class ), metadataCache ); - - // When - for ( int i = 0; i < entryCount; i++ ) - { - log.append( new RaftLogEntry( i, ReplicatedInteger.valueOf(i) ) ); - } - - long newPrevIndex = log.prune( 50L ); - - // Then - for ( int i = 0; i < newPrevIndex; i++ ) - { - assertNull( metadataCache.getMetadata( i ) ); - } - for ( long i = newPrevIndex; i < entryCount; i++ ) - { - assertNotNull( metadataCache.getMetadata( i ) ); - } - } - - @Test - public void shouldUpdateMetaDataCacheWhenLogIsTruncated() throws Exception - { - // Given - int entryCount = 100; - RaftLogMetadataCache metadataCache = new RaftLogMetadataCache( entryCount * 2 ); - - PhysicalRaftLog log = createRaftLog( 100, mock( PhysicalRaftLogFile.Monitor.class ), metadataCache ); - - // When - for ( long i = 0; i < entryCount; i++ ) - { - log.append( new RaftLogEntry( i, ReplicatedInteger.valueOf( (int) i ) ) ); - } - - long truncateIndex = 50L; - log.truncate( truncateIndex ); - - // Then - for ( long i = 0; i < truncateIndex; i++ ) - { - assertNotNull( metadataCache.getMetadata( i ) ); - } - - for ( long i = truncateIndex; i < entryCount; i++ ) - { - assertNull( metadataCache.getMetadata( i ) ); - } - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogContractTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogContractTest.java deleted file mode 100644 index 81b43877d6344..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogContractTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.junit.After; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; - -import org.neo4j.coreedge.raft.ReplicatedInteger; -import org.neo4j.coreedge.raft.log.DummyRaftableContentSerializer; -import org.neo4j.coreedge.raft.log.RaftLog; -import org.neo4j.coreedge.raft.log.RaftLogContractTest; -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.kernel.lifecycle.LifeSupport; -import org.neo4j.logging.NullLogProvider; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.neo4j.coreedge.raft.log.RaftLogHelper.readLogEntry; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME; - -public class PhysicalRaftLogContractTest extends RaftLogContractTest -{ - private PhysicalRaftLog raftLog; - private LifeSupport life = new LifeSupport(); - private FileSystemAbstraction fileSystem; - - @Override - public RaftLog createRaftLog() throws IOException - { - this.raftLog = createRaftLog( 100 ); - return raftLog; - } - - @After - public void tearDown() throws Throwable - { - life.stop(); - life.shutdown(); - } - - private PhysicalRaftLog createRaftLog( int cacheSize ) - { - if ( fileSystem == null ) - { - fileSystem = new EphemeralFileSystemAbstraction(); - } - File directory = new File( PHYSICAL_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - - PhysicalRaftLog newRaftLog = new PhysicalRaftLog( fileSystem, directory, 1024, "1 files", cacheSize, 10, - new PhysicalRaftLogFile.Monitor.Adapter(), new DummyRaftableContentSerializer(), - () -> mock( DatabaseHealth.class ), NullLogProvider.getInstance(), new RaftLogMetadataCache( 10 ) ); - life.add( newRaftLog ); - life.init(); - life.start(); - return newRaftLog; - } - - @Test - public void shouldReadBackInCachedEntry() throws Throwable - { - // Given - PhysicalRaftLog raftLog = (PhysicalRaftLog) createRaftLog(); - int term = 0; - ReplicatedInteger content = ReplicatedInteger.valueOf( 4 ); - - // When - long entryIndex = raftLog.append( new RaftLogEntry( term, content ) ); - - // Then - assertEquals( entryIndex, raftLog.appendIndex() ); - assertEquals( content, readLogEntry( raftLog, entryIndex ).content() ); - assertEquals( term, raftLog.readEntryTerm( entryIndex ) ); - } - - @Test - public void shouldReadBackNonCachedEntry() throws Exception - { - // Given - int cacheSize = 1; - PhysicalRaftLog raftLog = createRaftLog( cacheSize ); - int term = 0; - ReplicatedInteger content1 = ReplicatedInteger.valueOf( 4 ); - ReplicatedInteger content2 = ReplicatedInteger.valueOf( 5 ); - - // When - long entryIndex1 = raftLog.append( new RaftLogEntry( term, content1 ) ); - long entryIndex2 = raftLog.append( new RaftLogEntry( term, content2 ) ); // this will push the first entry out of cache - - // Then - // entry 1 should be there - assertEquals( content1, readLogEntry( raftLog, entryIndex1 ).content() ); - assertEquals( term, raftLog.readEntryTerm( entryIndex1 ) ); - - // entry 2 should be there also - assertEquals( content2, readLogEntry( raftLog, entryIndex2 ).content() ); - assertEquals( term, raftLog.readEntryTerm( entryIndex2 ) ); - } - - @Test - public void shouldRestoreCommitIndexOnStartup() throws Throwable - { - // Given - PhysicalRaftLog raftLog = createRaftLog( 100 /* cache size */ ); - int term = 0; - ReplicatedInteger content1 = ReplicatedInteger.valueOf( 4 ); - ReplicatedInteger content2 = ReplicatedInteger.valueOf( 5 ); - raftLog.append( new RaftLogEntry( term, content1 ) ); - long entryIndex2 = raftLog.append( new RaftLogEntry( term, content2 ) ); - - // When - // we restart the raft log - life.remove( raftLog ); // stops the removed instance - raftLog = createRaftLog( 100 ); - - // Then - assertEquals( entryIndex2, raftLog.appendIndex() ); - } - - @Test - public void shouldRestoreCorrectCommitAndAppendIndexOnStartupAfterTruncation() throws Exception - { - // Given - PhysicalRaftLog raftLog = createRaftLog( 100 /* cache size */ ); - int term = 0; - ReplicatedInteger content = ReplicatedInteger.valueOf( 4 ); - raftLog.append( new RaftLogEntry( term, content ) ); - raftLog.append( new RaftLogEntry( term, content ) ); - long entryIndex3 = raftLog.append( new RaftLogEntry( term, content ) ); - long entryIndex4 = raftLog.append( new RaftLogEntry( term, content ) ); - - raftLog.truncate( entryIndex4 ); - - // When - // we restart the raft log - life.remove( raftLog ); // stops the removed instance - raftLog = createRaftLog( 100 ); - - // Then - assertEquals( entryIndex3, raftLog.appendIndex() ); - } - - @Test - public void shouldRestoreCorrectCommitAndAppendIndexWithTruncationRecordsAndAppendedRecordsAfterThat() throws Exception - { - // Given - PhysicalRaftLog raftLog = createRaftLog( 100 /* cache size */ ); - int term = 0; - ReplicatedInteger content = ReplicatedInteger.valueOf( 4 ); - raftLog.append( new RaftLogEntry( term, content ) ); - raftLog.append( new RaftLogEntry( term, content ) ); - raftLog.append( new RaftLogEntry( term, content ) ); - long entryIndex4 = raftLog.append( new RaftLogEntry( term, content ) ); - - raftLog.truncate( entryIndex4 ); - - long entryIndex5 = raftLog.append( new RaftLogEntry( term, content ) ); - - // When - // we restart the raft log - life.remove( raftLog ); // stops the removed instance - raftLog = createRaftLog( 100 ); - - // Then - assertEquals( entryIndex5, raftLog.appendIndex() ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileTest.java deleted file mode 100644 index 2801b5f36585c..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFileTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.transaction.DeadSimpleLogVersionRepository; -import org.neo4j.kernel.impl.transaction.DeadSimpleTransactionIdStore; -import org.neo4j.kernel.impl.transaction.log.FlushablePositionAwareChannel; -import org.neo4j.kernel.impl.transaction.log.LogFile; -import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.kernel.impl.transaction.log.LogPositionMarker; -import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile.Monitor; -import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; -import org.neo4j.kernel.impl.transaction.log.ReadableClosableChannel; -import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; -import org.neo4j.kernel.lifecycle.LifeSupport; -import org.neo4j.test.rule.TargetDirectory; -import org.neo4j.test.rule.TargetDirectory.TestDirectory; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; - -public class PhysicalRaftLogFileTest -{ - @Rule - public final TestDirectory directory = TargetDirectory.testDirForTest( getClass() ); - private final FileSystemAbstraction fs = new DefaultFileSystemAbstraction(); - private final LogVersionRepository logVersionRepository = new DeadSimpleLogVersionRepository( 1L ); - private final TransactionIdStore transactionIdStore = new DeadSimpleTransactionIdStore( 5L, 0, 0, 0 ); - - @Test - public void shouldOpenInFreshDirectoryAndFinallyAddHeader() throws Exception - { - // GIVEN - String name = "log"; - LifeSupport life = new LifeSupport(); - PhysicalLogFiles logFiles = new PhysicalLogFiles( directory.directory(), name, fs ); - life.add( new PhysicalLogFile( fs, logFiles, 1000, transactionIdStore::getLastCommittedTransactionId, - logVersionRepository, mock( Monitor.class ), new LogHeaderCache( 10 ) ) ); - - // WHEN - life.start(); - life.shutdown(); - - // THEN - File file = new PhysicalLogFiles( directory.directory(), name, fs ).getLogFileForVersion( 1L ); - LogHeader header = readLogHeader( fs, file ); - assertEquals( 1L, header.logVersion ); - assertEquals( 5L, header.lastCommittedTxId ); - } - - @Test - public void shouldWriteSomeDataIntoTheLog() throws Exception - { - // GIVEN - String name = "log"; - LifeSupport life = new LifeSupport(); - PhysicalLogFiles logFiles = new PhysicalLogFiles( directory.directory(), name, fs ); - Monitor monitor = mock( Monitor.class ); - LogFile logFile = life.add( new PhysicalLogFile( fs, logFiles, 1000, - transactionIdStore::getLastCommittedTransactionId, logVersionRepository, monitor, - new LogHeaderCache( 10 ) ) ); - - // WHEN - try - { - life.start(); - - FlushablePositionAwareChannel writer = logFile.getWriter(); - LogPositionMarker positionMarker = new LogPositionMarker(); - writer.getCurrentPosition( positionMarker ); - int intValue = 45; - long longValue = 4854587; - writer.putInt( intValue ); - writer.putLong( longValue ); - writer.prepareForFlush().flush(); - - // THEN - try ( ReadableClosableChannel reader = logFile.getReader( positionMarker.newPosition() ) ) - { - assertEquals( intValue, reader.getInt() ); - assertEquals( longValue, reader.getLong() ); - } - } - finally - { - life.shutdown(); - } - } - - @Test - public void shouldReadOlderLogs() throws Exception - { - // GIVEN - String name = "log"; - LifeSupport life = new LifeSupport(); - PhysicalLogFiles logFiles = new PhysicalLogFiles( directory.directory(), name, fs ); - LogFile logFile = life.add( new PhysicalLogFile( fs, logFiles, 50, - transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( Monitor.class ), - new LogHeaderCache( 10 ) ) ); - - // WHEN - life.start(); - try - { - FlushablePositionAwareChannel writer = logFile.getWriter(); - LogPositionMarker positionMarker = new LogPositionMarker(); - writer.getCurrentPosition( positionMarker ); - LogPosition position1 = positionMarker.newPosition(); - int intValue = 45; - long longValue = 4854587; - byte[] someBytes = someBytes( 40 ); - writer.putInt( intValue ); - writer.putLong( longValue ); - writer.put( someBytes, someBytes.length ); - writer.prepareForFlush().flush(); - writer.getCurrentPosition( positionMarker ); - LogPosition position2 = positionMarker.newPosition(); - long longValue2 = 123456789L; - writer.putLong( longValue2 ); - writer.put( someBytes, someBytes.length ); - writer.prepareForFlush().flush(); - - // THEN - try ( ReadableClosableChannel reader = logFile.getReader( position1 ) ) - { - assertEquals( intValue, reader.getInt() ); - assertEquals( longValue, reader.getLong() ); - assertArrayEquals( someBytes, readBytes( reader, 40 ) ); - } - try ( ReadableClosableChannel reader = logFile.getReader( position2 ) ) - { - assertEquals( longValue2, reader.getLong() ); - assertArrayEquals( someBytes, readBytes( reader, 40 ) ); - } - } - finally - { - life.shutdown(); - } - } - - @Test - public void shouldVisitLogFile() throws Exception - { - // GIVEN - String name = "log"; - LifeSupport life = new LifeSupport(); - PhysicalLogFiles logFiles = new PhysicalLogFiles( directory.directory(), name, fs ); - LogFile logFile = life.add( new PhysicalLogFile( fs, logFiles, 50, - transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( Monitor.class ), - new LogHeaderCache( 10 ) ) ); - life.start(); - FlushablePositionAwareChannel writer = logFile.getWriter(); - LogPositionMarker mark = new LogPositionMarker(); - writer.getCurrentPosition( mark ); - for ( int i = 0; i < 5; i++ ) - { - writer.put( (byte)i ); - } - writer.prepareForFlush(); - - // WHEN/THEN - final AtomicBoolean called = new AtomicBoolean(); - logFile.accept( ( position, channel ) -> { - for ( int i = 0; i < 5; i++ ) - { - assertEquals( (byte)i, channel.get() ); - } - called.set( true ); - return true; - }, mark.newPosition() ); - assertTrue( called.get() ); - life.shutdown(); - } - - private byte[] readBytes( ReadableClosableChannel reader, int length ) throws IOException - { - byte[] result = new byte[length]; - reader.get( result, length ); - return result; - } - - private byte[] someBytes( int length ) - { - byte[] result = new byte[length]; - for ( int i = 0; i < length; i++ ) - { - result[i] = (byte) (i%5); - } - return result; - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFilesTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFilesTest.java deleted file mode 100644 index 037f2ddbe05df..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogFilesTest.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; - -import org.junit.Test; - -import org.neo4j.coreedge.raft.state.ChannelMarshal; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.impl.transaction.log.entry.LogHeader; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFile.DEFAULT_VERSION_SUFFIX; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLogFiles.BASE_FILE_NAME; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader; -import static org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader; - -public class PhysicalRaftLogFilesTest -{ - private final FileSystemAbstraction fs = new EphemeralFileSystemAbstraction(); - private final File tmpDirectory = new File( "." ); - - @Test - public void shouldGetTheFileNameForAGivenVersion() - { - // given - final PhysicalRaftLogFiles files = new PhysicalRaftLogFiles( tmpDirectory, fs, mock( ChannelMarshal.class ), - mock( VersionIndexRanges.class ) ); - final int version = 12; - - // when - final File versionFileName = files.getLogFileForVersion( version ); - - // then - final File expected = new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + version ); - assertEquals( expected, versionFileName ); - } - - @Test - public void shouldBeAbleToRetrieveTheHighestLogVersionWithOtherFilesPresent() throws Exception - { - // given - fs.mkdir( tmpDirectory ); - PhysicalRaftLogFiles files = new PhysicalRaftLogFiles( tmpDirectory, fs, mock( ChannelMarshal.class ), - new VersionIndexRanges() ); - - writeLogHeader( fs, new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + "1" ), 1, -1 ); - writeLogHeader( fs, new File( tmpDirectory, "unknown" + DEFAULT_VERSION_SUFFIX + "4" ), 1, -1 ); - writeLogHeader( fs, new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + "3" ), 3, -1 ); - writeLogHeader( fs, new File( tmpDirectory, BASE_FILE_NAME ), 1, -1 ); - - files.init(); - - // when - final long highestLogVersion = files.getHighestLogVersion(); - - // then - assertEquals( 3, highestLogVersion ); - } - - @Test - public void shouldBeAbleToRetrieveTheHighestLogVersionWithEmptyFilePresent() throws Exception - { - // given - fs.mkdir( tmpDirectory ); - PhysicalRaftLogFiles files = new PhysicalRaftLogFiles( tmpDirectory, fs, mock( ChannelMarshal.class ), - new VersionIndexRanges() ); - - writeLogHeader( fs, new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + "1" ), 1, -1 ); - writeLogHeader( fs, new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + "3" ), 3, 42 ); - File emptyFile = new File( tmpDirectory, BASE_FILE_NAME + DEFAULT_VERSION_SUFFIX + "4" ); - fs.create( emptyFile ); - - files.init(); - - // when - final long highestLogVersion = files.getHighestLogVersion(); - - // then - assertEquals( 4, highestLogVersion ); - LogHeader logHeader = readLogHeader( fs, emptyFile ); - assertEquals( 4, logHeader.logVersion ); - assertEquals( 42, logHeader.lastCommittedTxId ); - } - - @Test - public void shouldFindTheVersionBasedOnTheFilename() - { - // given - final File file = - new File( "v" + DEFAULT_VERSION_SUFFIX + DEFAULT_VERSION_SUFFIX + DEFAULT_VERSION_SUFFIX + "2" ); - - // when - // Get version based on the name - long logVersion = PhysicalRaftLogFiles.getLogVersion( file.getName() ); - - // then - assertEquals( 2, logVersion ); - } - - @Test - public void shouldThrowIfThereIsNoVersionInTheFileName() - { - // given - final File file = new File( "wrong" ); - - // when - try - { - // Get version based on the name - PhysicalRaftLogFiles.getLogVersion( file.getName() ); - fail( "should have thrown" ); - } - catch ( RuntimeException ex ) - { - assertEquals( "Invalid log file '" + file.getName() + "'", ex.getMessage() ); - } - } - - @Test(expected = NumberFormatException.class) - public void shouldThrowIfVersionIsNotANumber() - { - // given - final File file = new File( "aa" + DEFAULT_VERSION_SUFFIX + "A" ); - - // when - // Get version based on the name - PhysicalRaftLogFiles.getLogVersion( file.getName() ); - } - - @Test - public void shouldIncrementVersionOnRegisterNewVersion() throws Exception - - { - // given - long prevLogIndex = 123L; - PhysicalRaftLogFiles files = new PhysicalRaftLogFiles( tmpDirectory, fs, mock( ChannelMarshal.class ), - new VersionIndexRanges() ); - - // when - long initialVersion = files.getHighestLogVersion(); - // then - assertEquals( 0, initialVersion ); - - // when - long newVersion = files.registerNextVersion( prevLogIndex ); - - // then - assertEquals( initialVersion + 1, newVersion ); - assertEquals( initialVersion + 1, files.getHighestLogVersion() ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogRotationTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogRotationTest.java deleted file mode 100644 index 40f3c97073451..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogRotationTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.junit.After; -import org.junit.Test; - -import java.io.File; -import java.util.concurrent.atomic.AtomicLong; - -import org.neo4j.coreedge.raft.ReplicatedInteger; -import org.neo4j.coreedge.raft.ReplicatedString; -import org.neo4j.coreedge.raft.log.DummyRaftableContentSerializer; -import org.neo4j.coreedge.raft.log.RaftLogEntry; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.kernel.lifecycle.LifeSupport; -import org.neo4j.logging.NullLogProvider; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME; - -public class PhysicalRaftLogRotationTest -{ - private LifeSupport life = new LifeSupport(); - private FileSystemAbstraction fileSystem; - - @After - public void tearDown() throws Throwable - { - life.stop(); - life.shutdown(); - } - - private PhysicalRaftLog createRaftLog( long rotateAtSize, PhysicalRaftLogFile.Monitor logFileMonitor ) - { - if ( fileSystem == null ) - { - fileSystem = new EphemeralFileSystemAbstraction(); - } - File directory = new File( PHYSICAL_LOG_DIRECTORY_NAME ); - fileSystem.mkdir( directory ); - - PhysicalRaftLog newRaftLog = new PhysicalRaftLog( fileSystem, directory, rotateAtSize, "1 files", 100, 10, - logFileMonitor, new DummyRaftableContentSerializer(), () -> mock( DatabaseHealth.class ), - NullLogProvider.getInstance(), new RaftLogMetadataCache( 10 ) ); - life.add( newRaftLog ); - life.init(); - life.start(); - return newRaftLog; - } - - @Test - public void shouldRotateOnAppendWhenRotateSizeIsReached() throws Exception - { - // Given - AtomicLong currentVersion = new AtomicLong(); - PhysicalRaftLogFile.Monitor logFileMonitor = - ( logFile, logVersion, lastTransactionId, clean ) -> currentVersion.set( logVersion ); - int rotateAtSize = 100; - PhysicalRaftLog log = createRaftLog( rotateAtSize, logFileMonitor ); - - StringBuilder builder = new StringBuilder(); - for ( int i = 0; i < rotateAtSize; i++ ) - { - builder.append( "i" ); - } - - // When - ReplicatedString stringThatIsMoreThan100Bytes = new ReplicatedString( builder.toString() ); - log.append( new RaftLogEntry( 0, stringThatIsMoreThan100Bytes ) ); - - // Then - assertEquals( 1, currentVersion.get() ); - } - - @Test - public void shouldRotateOnTruncate() throws Exception - { - // Given - AtomicLong currentVersion = new AtomicLong(); - PhysicalRaftLogFile.Monitor logFileMonitor = - ( logFile, logVersion, lastTransactionId, clean ) -> currentVersion.set( logVersion ); - int rotateAtSize = 100; - PhysicalRaftLog log = createRaftLog( rotateAtSize, logFileMonitor ); - - StringBuilder builder = new StringBuilder(); - for ( int i = 0; i < rotateAtSize - 60; i++ ) - { - builder.append( "i" ); - } - - // When - ReplicatedString stringThatGetsTheSizeToAlmost100Bytes = new ReplicatedString( builder.toString() ); - long indexToTruncate = log.append( new RaftLogEntry( 0, stringThatGetsTheSizeToAlmost100Bytes ) ); - assertEquals( 0, currentVersion.get() ); - log.truncate( indexToTruncate ); - - // Then - assertEquals( 1, currentVersion.get() ); - } - - @Test - public void shouldBeAbleToRecoverToLatestStateAfterRotation() throws Throwable - { - int rotateAtSize = 100; - PhysicalRaftLog log = createRaftLog( rotateAtSize, new PhysicalRaftLogFile.Monitor.Adapter() ); - - StringBuilder builder = new StringBuilder(); - for ( int i = 0; i < rotateAtSize - 40; i++ ) - { - builder.append( "i" ); - } - - ReplicatedString stringThatGetsTheSizeToAlmost100Bytes = new ReplicatedString( builder.toString() ); - int term = 0; - log.append( new RaftLogEntry( term, stringThatGetsTheSizeToAlmost100Bytes ) ); - long indexToRestoreTo = log.append( new RaftLogEntry( term, ReplicatedInteger.valueOf( 1 ) ) ); - - // When - life.remove( log ); - log = createRaftLog( rotateAtSize, new PhysicalRaftLogFile.Monitor.Adapter() ); - - // Then - assertEquals( indexToRestoreTo, log.appendIndex() ); - assertEquals( term, log.readEntryTerm( indexToRestoreTo ) ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogVerificationIT.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogVerificationIT.java deleted file mode 100644 index 93b327ae566a8..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/PhysicalRaftLogVerificationIT.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; - -import org.neo4j.coreedge.raft.log.DummyRaftableContentSerializer; -import org.neo4j.coreedge.raft.log.RaftLog; -import org.neo4j.coreedge.raft.log.RaftLogMetadataCache; -import org.neo4j.coreedge.raft.log.RaftLogVerificationIT; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.logging.NullLogProvider; - -import static org.mockito.Mockito.mock; -import static org.neo4j.coreedge.raft.log.physical.PhysicalRaftLog.PHYSICAL_LOG_DIRECTORY_NAME; - -public class PhysicalRaftLogVerificationIT extends RaftLogVerificationIT -{ - @Override - protected RaftLog createRaftLog() throws Throwable - { - FileSystemAbstraction fsa = fsRule.get(); - - File directory = new File( PHYSICAL_LOG_DIRECTORY_NAME ); - fsa.mkdir( directory ); - - long rotateAtSizeBytes = 128; - int entryCacheSize = 4; - int metadataCacheSize = 8; - int fileHeaderCacheSize = 2; - - PhysicalRaftLog newRaftLog = new PhysicalRaftLog( fsa, directory, rotateAtSizeBytes, "1 files", entryCacheSize, - fileHeaderCacheSize, new PhysicalRaftLogFile.Monitor.Adapter(), - new DummyRaftableContentSerializer(), () -> mock( DatabaseHealth.class ), NullLogProvider.getInstance(), new RaftLogMetadataCache( metadataCacheSize ) ); - - newRaftLog.init(); - newRaftLog.start(); - - return newRaftLog; - } - - @Override - protected long operations() - { - return 500; - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/RecoveryTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/RecoveryTest.java deleted file mode 100644 index a018092ed4850..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/RecoveryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import java.io.File; - -import org.junit.Test; - -import org.neo4j.coreedge.raft.log.RaftLogEntry; - -import static java.util.Arrays.asList; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import static org.neo4j.coreedge.raft.ReplicatedInteger.valueOf; -import static org.neo4j.kernel.impl.util.IOCursors.cursor; - -public class RecoveryTest -{ - @Test - public void shouldRecoverStateFromFiles() throws Exception - { - // given - VersionFiles versionFiles = mock( VersionFiles.class ); - when( versionFiles.filesInVersionOrder() ) - .thenReturn( asList( new File( "v1 " ), new File( "v2 " ), new File( "v3 " ) ) ); - - HeaderReader headerReader = mock( HeaderReader.class ); - when( headerReader.readHeader( new File( "v1 " ) ) ).thenReturn( new Header( 1, -1, -1 ) ); - when( headerReader.readHeader( new File( "v2 " ) ) ).thenReturn( new Header( 2, 9, 0 ) ); - when( headerReader.readHeader( new File( "v3 " ) ) ).thenReturn( new Header( 3, 19, 0 ) ); - - EntryReader entryReader = mock( EntryReader.class ); - when( entryReader.readEntriesInVersion( 3 ) ).thenReturn( cursor( - new RaftLogAppendRecord( 20, new RaftLogEntry( 0, valueOf( 120 ) ) ), - new RaftLogAppendRecord( 21, new RaftLogEntry( 0, valueOf( 121 ) ) ), - new RaftLogAppendRecord( 22, new RaftLogEntry( 0, valueOf( 122 ) ) ) - ) ); - - Recovery recovery = new Recovery( versionFiles, headerReader, entryReader ); - - // when - Recovery.LogState state = recovery.recover(); - - // then - assertEquals( -1, state.prevIndex ); - assertEquals( -1, state.prevTerm ); - assertEquals( 22, state.appendIndex ); - assertEquals( 3, state.currentVersion ); - assertEquals( 1, state.ranges.lowestVersion() ); - assertEquals( 3, state.ranges.highestVersion() ); - } - - @Test - public void shouldRecoverStateFromPrunedFiles() throws Exception - { - // given - VersionFiles versionFiles = mock( VersionFiles.class ); - when( versionFiles.filesInVersionOrder() ) - .thenReturn( asList( new File( "v2 " ), new File( "v3 " ) ) ); - - HeaderReader headerReader = mock( HeaderReader.class ); - when( headerReader.readHeader( new File( "v2 " ) ) ).thenReturn( new Header( 2, 9, 0 ) ); - when( headerReader.readHeader( new File( "v3 " ) ) ).thenReturn( new Header( 3, 19, 0 ) ); - - EntryReader entryReader = mock( EntryReader.class ); - when( entryReader.readEntriesInVersion( 3 ) ).thenReturn( cursor() ); - - Recovery recovery = new Recovery( versionFiles, headerReader, entryReader ); - - // when - Recovery.LogState state = recovery.recover(); - - // then - assertEquals( 9, state.prevIndex ); - assertEquals( 0, state.prevTerm ); - assertEquals( 19, state.appendIndex ); - assertEquals( 3, state.currentVersion ); - assertEquals( 2, state.ranges.lowestVersion() ); - assertEquals( 3, state.ranges.highestVersion() ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/ThresholdBasedRaftLogPruneStrategyTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/ThresholdBasedRaftLogPruneStrategyTest.java deleted file mode 100644 index 6a93c8cfed468..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/ThresholdBasedRaftLogPruneStrategyTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.junit.Test; -import org.mockito.Matchers; -import org.neo4j.coreedge.raft.log.physical.pruning.RaftLogPruneStrategy; -import org.neo4j.kernel.impl.transaction.log.LogFileInformation; -import org.neo4j.kernel.impl.transaction.log.pruning.Threshold; - -public class ThresholdBasedRaftLogPruneStrategyTest -{ - private final LogFileInformation logFileInfo = mock( PhysicalRaftLogFileInformation.class ); - private final PhysicalRaftLogFiles files = mock( PhysicalRaftLogFiles.class ); - private final Threshold threshold = mock( Threshold.class ); - - @Test - public void shouldNotDeleteAnythingIfThresholdDoesNotAllow() throws Exception - { - // Given - when( threshold.reached( any(), anyLong(), any() ) ).thenReturn( false ); - - when( files.containsEntries( anyLong() ) ).thenReturn( true ); - when( files.versionExists( anyLong() ) ).thenReturn( true ); - - final RaftLogPruneStrategy strategy = new RaftLogPruneStrategy( logFileInfo, files, threshold ); - - // When - strategy.prune( 7L ); - - // Then - verify( threshold, times( 1 ) ).init(); - verify( files, times( 0 ) ).pruneUpTo( anyLong() ); - } - - @Test - public void shouldDeleteJustWhatTheThresholdSays() throws Exception - { - // Given - when( threshold.reached( any(), Matchers.eq( 6L ), any() ) ).thenReturn( false ); - when( threshold.reached( any(), Matchers.eq( 5L ), any() ) ).thenReturn( false ); - when( threshold.reached( any(), Matchers.eq( 4L ), any() ) ).thenReturn( false ); - when( threshold.reached( any(), Matchers.eq( 3L ), any() ) ).thenReturn( true ); - - when( files.containsEntries( anyLong() ) ).thenReturn( true ); - when( files.versionExists( anyLong() ) ).thenReturn( true ); - - final RaftLogPruneStrategy strategy = new RaftLogPruneStrategy( logFileInfo, files, threshold ); - - // When - strategy.prune( 7L ); - - // Then - verify( threshold, times( 1 ) ).init(); - verify( files, times( 1 ) ).pruneUpTo( 3 ); - } - - @Test - public void shouldAlwaysKeepOneFileAround() throws Exception - { - // Given - when( threshold.reached( any(), Matchers.eq( 3L ), any() ) ).thenReturn( true ); - when( threshold.reached( any(), Matchers.eq( 2L ), any() ) ).thenReturn( true ); - when( threshold.reached( any(), Matchers.eq( 1L ), any() ) ).thenReturn( true ); - when( threshold.reached( any(), Matchers.eq( 0L ), any() ) ).thenReturn( true ); - - when( files.containsEntries( anyLong() ) ).thenReturn( true ); - when( files.versionExists( anyLong() ) ).thenReturn( true ); - - final RaftLogPruneStrategy strategy = new RaftLogPruneStrategy( logFileInfo, files, threshold ); - - // When - strategy.prune( 3L ); - - // Then - verify( threshold, times( 1 ) ).init(); - verify( files, times( 1 ) ).pruneUpTo( 2 ); - } -} diff --git a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRangesTest.java b/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRangesTest.java deleted file mode 100644 index df5a6a7550ef6..0000000000000 --- a/enterprise/core-edge/src/test/java/org/neo4j/coreedge/raft/log/physical/VersionIndexRangesTest.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.coreedge.raft.log.physical; - -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -import static org.neo4j.coreedge.raft.log.physical.VersionIndexRange.OUT_OF_RANGE; - -public class VersionIndexRangesTest -{ - @Test - public void shouldBuildListOfFiles() throws Exception - { - VersionIndexRanges ranges = new VersionIndexRanges(); - assertEquals( OUT_OF_RANGE, ranges.versionForIndex( 0 ) ); - assertEquals( OUT_OF_RANGE, ranges.versionForIndex( 1 ) ); - - ranges.add( 0, -1 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 0, ranges.versionForIndex( Long.MAX_VALUE ).version ); - - ranges.add( 1, 24 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 0, ranges.versionForIndex( 24 ).version ); - assertEquals( 1, ranges.versionForIndex( 25 ).version ); - assertEquals( 1, ranges.versionForIndex( Long.MAX_VALUE ).version ); - - ranges.add( 2, 68 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 0, ranges.versionForIndex( 24 ).version ); - assertEquals( 1, ranges.versionForIndex( 25 ).version ); - assertEquals( 1, ranges.versionForIndex( 68 ).version ); - assertEquals( 2, ranges.versionForIndex( 69 ).version ); - assertEquals( 2, ranges.versionForIndex( Long.MAX_VALUE ).version ); - - ranges.add( 3, 48 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 0, ranges.versionForIndex( 24 ).version ); - assertEquals( 1, ranges.versionForIndex( 25 ).version ); - assertEquals( 1, ranges.versionForIndex( 48 ).version ); - assertEquals( 3, ranges.versionForIndex( 49 ).version ); - assertEquals( 3, ranges.versionForIndex( 69 ).version ); - assertEquals( 3, ranges.versionForIndex( Long.MAX_VALUE ).version ); - - ranges.add( 4, 62 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 0, ranges.versionForIndex( 24 ).version ); - assertEquals( 1, ranges.versionForIndex( 25 ).version ); - assertEquals( 1, ranges.versionForIndex( 48 ).version ); - assertEquals( 3, ranges.versionForIndex( 49 ).version ); - assertEquals( 3, ranges.versionForIndex( 62 ).version ); - assertEquals( 4, ranges.versionForIndex( 63 ).version ); - assertEquals( 4, ranges.versionForIndex( 69 ).version ); - assertEquals( 4, ranges.versionForIndex( Long.MAX_VALUE ).version ); - - ranges.add( 5, 2 ); - assertEquals( 0, ranges.versionForIndex( 0 ).version ); - assertEquals( 5, ranges.versionForIndex( 24 ).version ); - assertEquals( 5, ranges.versionForIndex( 25 ).version ); - assertEquals( 5, ranges.versionForIndex( 48 ).version ); - assertEquals( 5, ranges.versionForIndex( 49 ).version ); - assertEquals( 5, ranges.versionForIndex( 62 ).version ); - assertEquals( 5, ranges.versionForIndex( 63 ).version ); - assertEquals( 5, ranges.versionForIndex( 69 ).version ); - assertEquals( 5, ranges.versionForIndex( Long.MAX_VALUE ).version ); - } - - @Test - public void shouldPruneEarlyRanges() throws Exception - { - VersionIndexRanges ranges = new VersionIndexRanges(); - - ranges.add( 0, 2 ); - ranges.add( 1, 24 ); - ranges.add( 2, 68 ); - - ranges.pruneVersion( 1 ); - - assertEquals( OUT_OF_RANGE, ranges.versionForIndex( 0 ) ); - assertEquals( OUT_OF_RANGE, ranges.versionForIndex( 68 ) ); - assertEquals( 2, ranges.versionForIndex( 69 ).version ); - } - - @Test - public void shouldPruneAllRanges() throws Exception - { - // given - VersionIndexRanges ranges = new VersionIndexRanges(); - ranges.add( 0, 2 ); - - // when - ranges.pruneVersion( 0 ); - - // then - assertEquals( OUT_OF_RANGE, ranges.versionForIndex( 0 ) ); - } - - @Test - public void shouldRejectAlreadyContainedVersion() throws Exception - { - VersionIndexRanges ranges = new VersionIndexRanges(); - - ranges.add( 0, 2 ); - ranges.add( 1, 24 ); - ranges.add( 2, 68 ); - - assertThat( ranges, rejectsVersion( 1, 2 ) ); - assertThat( ranges, rejectsVersion( 2, 2 ) ); - } - - @Test - public void shouldMaintainHighestVersion() throws Exception - { - // Given - VersionIndexRanges ranges = new VersionIndexRanges(); - - // Then - assertEquals( 0, ranges.highestVersion() ); - - // When - ranges.add( 1, 12 ); - - // Then - assertEquals( 1, ranges.highestVersion() ); - - // When - ranges.add( 2, 12 ); - - // Then - assertEquals( 2, ranges.highestVersion() ); - } - - @Test - public void shouldMaintainLowestVersion() throws Exception - { - // Given - VersionIndexRanges ranges = new VersionIndexRanges(); - - // Then - assertEquals( 0, ranges.lowestVersion() ); - - // When - ranges.add( 1, 12 ); - - // Then - assertEquals( 1, ranges.lowestVersion() ); - - // When - // we truncate the previous version away - ranges.add( 2, 12 ); - - // Then - // Version 1 is gone, 2 is the new lowest one - assertEquals( 2, ranges.lowestVersion() ); - - // When - ranges.add( 3, 15 ); - - // Then - assertEquals( 2, ranges.lowestVersion() ); - - // When - ranges.pruneVersion( 2 ); - - // Then - assertEquals( 3, ranges.lowestVersion() ); - } - - private Matcher rejectsVersion( long version, long prevIndex ) - { - return new TypeSafeMatcher() - { - @Override - protected boolean matchesSafely( VersionIndexRanges ranges ) - { - try - { - ranges.add( version, prevIndex ); - return false; - } - catch ( Exception e ) - { - return true; - } - } - - @Override - public void describeTo( Description description ) - { - description.appendText( "Ranges that rejects version: " ).appendValue( version ); - } - }; - } -}