diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index 4b171b1a3b7b2..babee24fd0cbb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -131,6 +131,8 @@ import org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy; import org.neo4j.kernel.impl.transaction.log.pruning.LogPruning; import org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl; +import org.neo4j.kernel.impl.transaction.log.reverse.ReverseTransactionCursorLoggingMonitor; +import org.neo4j.kernel.impl.transaction.log.reverse.ReversedSingleFileTransactionCursor; import org.neo4j.kernel.impl.transaction.log.rotation.LogRotation; import org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl; import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; @@ -437,6 +439,7 @@ public void start() throws IOException LogTailScanner tailScanner = new LogTailScanner( logFiles, fs, logEntryReader, monitors, failOnCorruptedLogFiles ); monitors.addMonitorListener( new LoggingLogTailScannerMonitor( logService.getInternalLog( LogTailScanner.class ) ) ); + monitors.addMonitorListener( new ReverseTransactionCursorLoggingMonitor( logService.getInternalLog( ReversedSingleFileTransactionCursor.class ) ) ); LogVersionUpgradeChecker.check( tailScanner, config ); // Upgrade the store before we begin @@ -659,7 +662,7 @@ private NeoStoreTransactionLogModule buildTransactionLogs( logFile, logRotation, transactionMetadataCache, transactionIdStore, explicitIndexTransactionOrdering, databaseHealth ) ); final LogicalTransactionStore logicalTransactionStore = - new PhysicalLogicalTransactionStore( logFile, transactionMetadataCache, logEntryReader, logService ); + new PhysicalLogicalTransactionStore( logFile, transactionMetadataCache, logEntryReader, monitors ); int txThreshold = config.get( GraphDatabaseSettings.check_point_interval_tx ); final CountCommittedTransactionThreshold countCommittedTransactionThreshold = diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStore.java index dbf7ac6c98066..4e98a12ecb784 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStore.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.neo4j.cursor.IOCursor; -import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache.TransactionMetadata; import org.neo4j.kernel.impl.transaction.log.entry.LogEntry; @@ -31,6 +30,9 @@ import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; +import org.neo4j.kernel.impl.transaction.log.reverse.ReversedMultiFileTransactionCursor; +import org.neo4j.kernel.impl.transaction.log.reverse.ReversedTransactionCursorMonitor; +import org.neo4j.kernel.monitoring.Monitors; import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_CHECKSUM; import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP; @@ -47,15 +49,15 @@ public class PhysicalLogicalTransactionStore implements LogicalTransactionStore private final LogFile logFile; private final TransactionMetadataCache transactionMetadataCache; private final LogEntryReader logEntryReader; - private final LogService logService; + private final Monitors monitors; public PhysicalLogicalTransactionStore( LogFile logFile, TransactionMetadataCache transactionMetadataCache, - LogEntryReader logEntryReader, LogService logService ) + LogEntryReader logEntryReader, Monitors monitors ) { this.logFile = logFile; this.transactionMetadataCache = transactionMetadataCache; this.logEntryReader = logEntryReader; - this.logService = logService; + this.monitors = monitors; } @Override @@ -68,7 +70,8 @@ public TransactionCursor getTransactions( LogPosition position ) throws IOExcept public TransactionCursor getTransactionsInReverseOrder( LogPosition backToPosition ) throws IOException { - return ReversedMultiFileTransactionCursor.fromLogFile( logFile, backToPosition, logService ); + return ReversedMultiFileTransactionCursor + .fromLogFile( logFile, backToPosition, monitors.newMonitor( ReversedTransactionCursorMonitor.class ) ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java index 6e85e3d7f2576..b4c7825839edc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReadOnlyTransactionStore.java @@ -24,7 +24,6 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache.TransactionMetadata; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; @@ -40,8 +39,8 @@ public class ReadOnlyTransactionStore extends LifecycleAdapter implements Logica private final LifeSupport life = new LifeSupport(); private final LogicalTransactionStore physicalStore; - public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Monitors monitors, - LogService logService ) throws IOException + public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, File fromPath, Monitors monitors ) + throws IOException { PhysicalLogFiles logFiles = new PhysicalLogFiles( fromPath, fs ); TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache( 100 ); @@ -53,7 +52,7 @@ public ReadOnlyTransactionStore( PageCache pageCache, FileSystemAbstraction fs, monitors.newMonitor( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); physicalStore = new PhysicalLogicalTransactionStore( logFile, transactionMetadataCache, logEntryReader, - logService ); + monitors ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/EagerlyReversedTransactionCursor.java similarity index 87% rename from community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursor.java rename to community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/EagerlyReversedTransactionCursor.java index 31e10a6c01f4f..850ff9eb6b07d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/EagerlyReversedTransactionCursor.java @@ -17,13 +17,15 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.kernel.impl.transaction.log; +package org.neo4j.kernel.impl.transaction.log.reverse; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.kernel.impl.transaction.log.TransactionCursor; /** * Eagerly exhausts a {@link TransactionCursor} and allows moving through it in reverse order. @@ -35,13 +37,13 @@ * * @see ReversedMultiFileTransactionCursor */ -class EagerlyReversedTransactionCursor implements TransactionCursor +public class EagerlyReversedTransactionCursor implements TransactionCursor { private final List txs = new ArrayList<>(); private final TransactionCursor cursor; private int indexToReturn; - EagerlyReversedTransactionCursor( TransactionCursor cursor ) throws IOException + public EagerlyReversedTransactionCursor( TransactionCursor cursor ) throws IOException { this.cursor = cursor; while ( cursor.next() ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReverseTransactionCursorLoggingMonitor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReverseTransactionCursorLoggingMonitor.java new file mode 100644 index 0000000000000..1aedd08bb24aa --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReverseTransactionCursorLoggingMonitor.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.log.reverse; + +import org.neo4j.logging.Log; + +import static java.lang.String.format; + +public class ReverseTransactionCursorLoggingMonitor implements ReversedTransactionCursorMonitor +{ + private final Log log; + + public ReverseTransactionCursorLoggingMonitor( Log log ) + { + this.log = log; + } + + @Override + public void transactionalLogRecordReadFailure( Throwable t, long[] transactionOffsets, int transactionIndex, long logVersion ) + { + log.warn( transactionIndex > 0 ? + format( "Fail to read transaction log version %d. Last valid transaction start offset is: %d.", + logVersion, transactionOffsets[transactionIndex - 1] ) : + format( "Fail to read first transaction of log version %d.", logVersion) ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursor.java similarity index 85% rename from community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursor.java rename to community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursor.java index 1d90c300ac1eb..f4dfa3089724c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursor.java @@ -17,19 +17,25 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.kernel.impl.transaction.log; +package org.neo4j.kernel.impl.transaction.log.reverse; import java.io.IOException; import org.neo4j.function.ThrowingFunction; -import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.LogFile; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.kernel.impl.transaction.log.PhysicalTransactionCursor; +import org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel; +import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; +import org.neo4j.kernel.impl.transaction.log.ReadableLogChannel; +import org.neo4j.kernel.impl.transaction.log.TransactionCursor; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; -import static org.neo4j.kernel.impl.transaction.log.EagerlyReversedTransactionCursor.eagerlyReverse; import static org.neo4j.kernel.impl.transaction.log.LogPosition.start; import static org.neo4j.kernel.impl.transaction.log.LogVersionBridge.NO_MORE_CHANNELS; +import static org.neo4j.kernel.impl.transaction.log.reverse.EagerlyReversedTransactionCursor.eagerlyReverse; /** * Similar to {@link PhysicalTransactionCursor} and actually uses it internally. This main difference is that transactions @@ -43,7 +49,7 @@ * * @see ReversedSingleFileTransactionCursor */ -class ReversedMultiFileTransactionCursor implements TransactionCursor +public class ReversedMultiFileTransactionCursor implements TransactionCursor { private final LogPosition backToPosition; private final ThrowingFunction cursorFactory; @@ -57,13 +63,13 @@ class ReversedMultiFileTransactionCursor implements TransactionCursor * * @param logFile {@link LogFile} to supply log entries forming transactions. * @param backToPosition {@link LogPosition} to read backwards to. - * @param logService logging service used to to create log. + * @param monitor reverse transaction cursor monitor * @return a {@link TransactionCursor} which returns transactions from the end of the log stream and backwards to * and including transaction starting at {@link LogPosition}. * @throws IOException on I/O error. */ - static TransactionCursor fromLogFile( LogFile logFile, LogPosition backToPosition, LogService logService ) throws - IOException + public static TransactionCursor fromLogFile( LogFile logFile, LogPosition backToPosition, + ReversedTransactionCursorMonitor monitor ) throws IOException { long highestVersion = logFile.currentLogVersion(); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>(); @@ -74,7 +80,7 @@ static TransactionCursor fromLogFile( LogFile logFile, LogPosition backToPositio { // This is a channel which can be positioned explicitly and is the typical case for such channels // Let's take advantage of this fact and use a bit smarter reverse implementation - return new ReversedSingleFileTransactionCursor( (ReadAheadLogChannel) channel, logEntryReader, logService ); + return new ReversedSingleFileTransactionCursor( (ReadAheadLogChannel) channel, logEntryReader, monitor ); } // Fall back to simply eagerly reading each single log file and reversing in memory diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursor.java similarity index 90% rename from community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursor.java rename to community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursor.java index 61f248297a289..9d12b894a133f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursor.java @@ -17,17 +17,22 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.kernel.impl.transaction.log; +package org.neo4j.kernel.impl.transaction.log.reverse; import java.io.IOException; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; -import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.kernel.impl.transaction.log.LogVersionBridge; +import org.neo4j.kernel.impl.transaction.log.PhysicalTransactionCursor; +import org.neo4j.kernel.impl.transaction.log.ReadAheadChannel; +import org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel; +import org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel; +import org.neo4j.kernel.impl.transaction.log.TransactionCursor; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader; -import org.neo4j.logging.Log; import static java.lang.String.format; @@ -57,16 +62,16 @@ * * @see ReversedMultiFileTransactionCursor */ -class ReversedSingleFileTransactionCursor implements TransactionCursor +public class ReversedSingleFileTransactionCursor implements TransactionCursor { // Should this be passed in or extracted from the read-ahead channel instead? private static final int CHUNK_SIZE = ReadAheadChannel.DEFAULT_READ_AHEAD_SIZE; private final ReadAheadLogChannel channel; + private final ReversedTransactionCursorMonitor monitor; private final TransactionCursor transactionCursor; // Should be generally large enough to hold transactions in a chunk, where one chunk is the read-ahead size of ReadAheadLogChannel private final Deque chunkTransactions = new ArrayDeque<>( 20 ); - private final Log log; private CommittedTransactionRepresentation currentChunkTransaction; // May be longer than required, offsetLength holds the actual length. private final long[] offsets; @@ -75,19 +80,18 @@ class ReversedSingleFileTransactionCursor implements TransactionCursor private long totalSize; ReversedSingleFileTransactionCursor( ReadAheadLogChannel channel, - LogEntryReader logEntryReader, LogService logService ) - throws IOException + LogEntryReader logEntryReader, + ReversedTransactionCursorMonitor monitor ) throws IOException { this.channel = channel; + this.monitor = monitor; // There's an assumption here: that the underlying channel can move in between calls and that the // transaction cursor will just happily read from the new position. this.transactionCursor = new PhysicalTransactionCursor<>( channel, logEntryReader ); - this.log = logService.getInternalLog( ReversedSingleFileTransactionCursor.class ); this.offsets = sketchOutTransactionStartOffsets(); } // Also initializes offset indexes - // This method could use some way of reading log entries w/o creating objects. Would be great private long[] sketchOutTransactionStartOffsets() throws IOException { // Grows on demand. Initially sized to be able to hold all transaction start offsets for a single log file @@ -110,7 +114,7 @@ private long[] sketchOutTransactionStartOffsets() throws IOException } catch ( Throwable t ) { - log.warn( buildReadErrorMessage( offsets, offsetCursor, logVersion ), t ); + monitor.transactionalLogRecordReadFailure( t, offsets, offsetCursor, logVersion ); } if ( channel.getVersion() != logVersion ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedTransactionCursorMonitor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedTransactionCursorMonitor.java new file mode 100644 index 0000000000000..17ea21387f54b --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedTransactionCursorMonitor.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.transaction.log.reverse; + +public interface ReversedTransactionCursorMonitor +{ + void transactionalLogRecordReadFailure( Throwable t, long[] offsets, int offsetCursor, long logVersion ); +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java index 2a4c9f7275601..44f2fd0542662 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryTest.java @@ -156,7 +156,8 @@ public void shouldRecoverExistingData() throws Exception LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); - LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, logService ); + LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, + monitors ); TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, txStore, NO_MONITOR ) { @@ -259,7 +260,7 @@ public void shouldSeeThatACleanDatabaseShouldNotRequireRecovery() throws Excepti LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); - LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, logService ); + LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors ); TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, txStore, NO_MONITOR ) { @@ -399,7 +400,7 @@ private boolean recover( File storeDir, PhysicalLogFiles logFiles ) LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 50, transactionIdStore::getLastCommittedTransactionId, logVersionRepository, mock( PhysicalLogFile.Monitor.class ), logHeaderCache ) ); - LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, logService ); + LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, metadataCache, reader, monitors ); TransactionLogPruner logPruner = new TransactionLogPruner( storeDir, logFiles, fileSystemRule.get() ); life.add( new Recovery( new DefaultRecoveryService( storageEngine, tailScanner, transactionIdStore, txStore, NO_MONITOR ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursorTest.java index 3a6dc0909941e..64c6a035015c2 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/EagerlyReversedTransactionCursorTest.java @@ -22,11 +22,11 @@ import org.junit.Test; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.reverse.EagerlyReversedTransactionCursor; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; - import static org.neo4j.helpers.collection.Iterators.array; import static org.neo4j.kernel.impl.transaction.log.GivenTransactionCursor.exhaust; import static org.neo4j.kernel.impl.transaction.log.GivenTransactionCursor.given; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStoreTest.java index 0c2b828e808af..07ec7108a11c0 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/PhysicalLogicalTransactionStoreTest.java @@ -32,7 +32,6 @@ import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.core.StartupStatisticsProvider; -import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.DeadSimpleTransactionIdStore; @@ -75,6 +74,7 @@ public class PhysicalLogicalTransactionStoreTest @Rule public TestDirectory dir = TestDirectory.testDirectory(); private File testDir; + private Monitors monitors = new Monitors(); @Before public void setup() @@ -96,7 +96,7 @@ public void extractTransactionFromLogFilesSkippingLastLogFileWithoutHeader() thr long timeCommitted = timeStarted + 10; LifeSupport life = new LifeSupport(); final PhysicalLogFiles logFiles = new PhysicalLogFiles( testDir, DEFAULT_NAME, fileSystemRule.get() ); - Monitor monitor = new Monitors().newMonitor( PhysicalLogFile.Monitor.class ); + Monitor monitor = monitors.newMonitor( PhysicalLogFile.Monitor.class ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 1000, transactionIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); @@ -120,7 +120,7 @@ public void extractTransactionFromLogFilesSkippingLastLogFileWithoutHeader() thr positionCache.clear(); final LogicalTransactionStore store = new PhysicalLogicalTransactionStore( emptyLogFile, positionCache, - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitors ); verifyTransaction( transactionIdStore, positionCache, additionalHeader, masterId, authorId, timeStarted, latestCommittedTxWhenStarted, timeCommitted, store ); } @@ -135,7 +135,7 @@ public void shouldOpenCleanStore() throws Exception LifeSupport life = new LifeSupport(); PhysicalLogFiles logFiles = new PhysicalLogFiles( testDir, DEFAULT_NAME, fileSystemRule.get() ); - Monitor monitor = new Monitors().newMonitor( PhysicalLogFile.Monitor.class ); + Monitor monitor = monitors.newMonitor( PhysicalLogFile.Monitor.class ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 1000, transactionIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); @@ -169,7 +169,7 @@ public void shouldOpenAndRecoverExistingData() throws Exception long timeCommitted = timeStarted + 10; LifeSupport life = new LifeSupport(); final PhysicalLogFiles logFiles = new PhysicalLogFiles( testDir, DEFAULT_NAME, fileSystemRule.get() ); - Monitor monitor = new Monitors().newMonitor( PhysicalLogFile.Monitor.class ); + Monitor monitor = monitors.newMonitor( PhysicalLogFile.Monitor.class ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 1000, transactionIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); @@ -193,7 +193,7 @@ public void shouldOpenAndRecoverExistingData() throws Exception transactionIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, positionCache, - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitors ); life.add( new BatchingTransactionAppender( logFile, NO_ROTATION, positionCache, transactionIdStore, BYPASS, DATABASE_HEALTH ) ); @@ -267,7 +267,7 @@ public void shouldExtractMetadataFromExistingTransaction() throws Exception long timeCommitted = timeStarted + 10; LifeSupport life = new LifeSupport(); PhysicalLogFiles logFiles = new PhysicalLogFiles( testDir, DEFAULT_NAME, fileSystemRule.get() ); - Monitor monitor = new Monitors().newMonitor( PhysicalLogFile.Monitor.class ); + Monitor monitor = monitors.newMonitor( PhysicalLogFile.Monitor.class ); LogFile logFile = life.add( new PhysicalLogFile( fileSystemRule.get(), logFiles, 1000, txIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); @@ -288,7 +288,7 @@ public void shouldExtractMetadataFromExistingTransaction() throws Exception txIdStore::getLastCommittedTransactionId, mock( LogVersionRepository.class ), monitor, logHeaderCache ) ); final LogicalTransactionStore store = new PhysicalLogicalTransactionStore( logFile, positionCache, - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitors ); // WHEN life.start(); @@ -313,7 +313,7 @@ public void shouldThrowNoSuchTransactionExceptionIfMetadataNotFound() throws Exc LifeSupport life = new LifeSupport(); final LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, cache, - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitors ); try { @@ -348,7 +348,7 @@ public void shouldThrowNoSuchTransactionExceptionIfLogFileIsMissing() throws Exc LifeSupport life = new LifeSupport(); final LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore( logFile, cache, - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitors ); try { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursorTest.java similarity index 97% rename from community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursorTest.java rename to community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursorTest.java index 75bf6a1f6e4d2..48c1717af1736 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedMultiFileTransactionCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedMultiFileTransactionCursorTest.java @@ -17,7 +17,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.kernel.impl.transaction.log; +package org.neo4j.kernel.impl.transaction.log.reverse; import org.junit.Test; @@ -27,6 +27,8 @@ import org.neo4j.function.ThrowingFunction; import org.neo4j.helpers.ArrayUtil; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.kernel.impl.transaction.log.TransactionCursor; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit; import static java.lang.Math.toIntExact; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursorTest.java similarity index 86% rename from community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursorTest.java rename to community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursorTest.java index 7597eb15788f3..68bd514d06e00 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/ReversedSingleFileTransactionCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/reverse/ReversedSingleFileTransactionCursorTest.java @@ -17,7 +17,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.kernel.impl.transaction.log; +package org.neo4j.kernel.impl.transaction.log.reverse; import org.junit.Before; import org.junit.Rule; @@ -28,15 +28,25 @@ import java.util.ArrayList; import java.util.Collection; -import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.DeadSimpleLogVersionRepository; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.command.Command; +import org.neo4j.kernel.impl.transaction.log.FlushablePositionAwareChannel; +import org.neo4j.kernel.impl.transaction.log.LogHeaderCache; +import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile; +import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles; +import org.neo4j.kernel.impl.transaction.log.PhysicalTransactionRepresentation; +import org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.TransactionLogWriter; import org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter; import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader; import org.neo4j.kernel.lifecycle.LifeRule; +import org.neo4j.logging.AssertableLogProvider; +import org.neo4j.logging.LogProvider; import org.neo4j.storageengine.api.StorageCommand; import org.neo4j.test.rule.RandomRule; import org.neo4j.test.rule.TestDirectory; @@ -64,6 +74,9 @@ public class ReversedSingleFileTransactionCursorTest public final RuleChain rules = RuleChain.outerRule( random ).around( fs ).around( directory ).around( life ); private long txId = TransactionIdStore.BASE_TX_ID; + private LogProvider logProvider = new AssertableLogProvider( true ); + private ReverseTransactionCursorLoggingMonitor monitor = new ReverseTransactionCursorLoggingMonitor( + logProvider.getLog( ReversedSingleFileTransactionCursor.class ) ); private PhysicalLogFile logFile; @Before @@ -136,7 +149,7 @@ public void shouldDetectAndPreventChannelReadingMultipleLogVersions() throws Exc // when try ( ReadAheadLogChannel channel = (ReadAheadLogChannel) logFile.getReader( start( 0 ) ) ) { - new ReversedSingleFileTransactionCursor( channel, new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new ReversedSingleFileTransactionCursor( channel, new VersionAwareLogEntryReader<>(), monitor ); fail( "Should've failed" ); } catch ( IllegalArgumentException e ) @@ -170,7 +183,7 @@ private ReversedSingleFileTransactionCursor txCursor() throws IOException { return new ReversedSingleFileTransactionCursor( (ReadAheadLogChannel) logFile.getReader( start( 0 ), NO_MORE_CHANNELS ), - new VersionAwareLogEntryReader<>(), NullLogService.getInstance() ); + new VersionAwareLogEntryReader<>(), monitor ); } private void writeTransactions( int transactionCount, int minTransactionSize, int maxTransactionSize ) throws IOException diff --git a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java index 53b94a442d4d3..06bb7ad984ad2 100644 --- a/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java +++ b/tools/src/main/java/org/neo4j/tools/applytx/ApplyTransactionsCommand.java @@ -35,7 +35,6 @@ import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; -import org.neo4j.kernel.impl.logging.SimpleLogService; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore; @@ -45,7 +44,6 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.monitoring.Monitors; -import org.neo4j.logging.FormattedLogProvider; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.tools.console.input.ArgsCommand; @@ -108,9 +106,8 @@ private long applyTransactions( File fromPath, GraphDatabaseAPI toDb, long fromT try ( DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); PageCache pageCache = StandalonePageCacheFactory.createPageCache( fileSystem ) ) { - SimpleLogService logService = new SimpleLogService( FormattedLogProvider.toOutputStream( System.out ) ); LogicalTransactionStore source = life.add( new ReadOnlyTransactionStore( pageCache, fileSystem, fromPath, - new Monitors(), logService ) ); + new Monitors() ) ); life.start(); long lastAppliedTx = fromTxExclusive; // Some progress if there are more than a couple of transactions to apply