From bcdec9400d57cfa2b6ce7b2f2aa5308c93e7b9ff Mon Sep 17 00:00:00 2001 From: Martin Furmanski Date: Fri, 2 Feb 2018 16:54:34 +0100 Subject: [PATCH] Fix cluster seeding --- .../catchup/storecopy/CommitState.java | 58 +++++++++ .../catchup/storecopy/LocalDatabase.java | 12 ++ .../catchup/storecopy/RemoteStore.java | 111 +++++++++++------- .../catchup/storecopy/StoreCopyClient.java | 8 +- .../core/server/CoreServerModule.java | 2 +- .../state/snapshot/CoreStateDownloader.java | 13 +- .../EnterpriseReadReplicaEditionModule.java | 2 +- .../backup/ClusterSeedingIT.java | 28 ++--- .../backup/FileCopyDetector.java | 2 +- .../backup/NewMemberSeedingIT.java | 6 +- .../backup_stores/AbstractStoreGenerator.java | 2 +- .../backup/backup_stores/BackupStore.java | 2 +- .../BackupStoreWithSomeData.java | 2 +- ...StoreWithSomeDataButNoTransactionLogs.java | 2 +- .../backup_stores/EmptyBackupStore.java | 2 +- .../backup/backup_stores/NoStore.java | 2 +- .../backup/cluster_load/ClusterLoad.java | 2 +- .../backup/cluster_load/NoLoad.java | 2 +- .../backup/cluster_load/SmallBurst.java | 2 +- .../catchup/storecopy/RemoteStoreTest.java | 9 +- .../snapshot/CoreStateDownloaderTest.java | 16 +-- .../causalclustering/helpers/BackupUtil.java | 2 +- 22 files changed, 193 insertions(+), 94 deletions(-) create mode 100644 enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CommitState.java diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CommitState.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CommitState.java new file mode 100644 index 000000000000..3557dc537b68 --- /dev/null +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/CommitState.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.causalclustering.catchup.storecopy; + +import java.util.Optional; + +class CommitState +{ + private final long metaDataStoreIndex; + private final Long transactionLogIndex; + + CommitState( long metaDataStoreIndex ) + { + this.metaDataStoreIndex = metaDataStoreIndex; + this.transactionLogIndex = null; + } + + CommitState( long metaDataStoreIndex, long transactionLogIndex ) + { + assert transactionLogIndex >= metaDataStoreIndex; + + this.metaDataStoreIndex = metaDataStoreIndex; + this.transactionLogIndex = transactionLogIndex; + } + + long metaDataStoreIndex() + { + return metaDataStoreIndex; + } + + Optional transactionLogIndex() + { + return Optional.ofNullable( transactionLogIndex ); + } + + @Override + public String toString() + { + return "CommitState{" + "metaDataStoreIndex=" + metaDataStoreIndex + ", transactionLogIndex=" + transactionLogIndex + '}'; + } +} diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java index 65e07bd26f64..4c2d02a13912 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java @@ -34,6 +34,7 @@ import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.storemigration.LogFiles; import org.neo4j.kernel.impl.storemigration.StoreFile; import org.neo4j.kernel.impl.transaction.log.TransactionAppender; import org.neo4j.kernel.impl.transaction.state.DataSourceManager; @@ -252,4 +253,15 @@ private void dropAvailabilityGuard() availabilityGuard.fulfill( currentRequirement ); currentRequirement = null; } + + public boolean hasTxLogs() + { + File[] files = storeDir.listFiles( LogFiles.FILENAME_FILTER ); + if ( files == null ) + { + throw new RuntimeException( "Files was null. Incorrect directory or I/O error?" ); + } + + return files.length > 0; + } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java index ae8b7579839e..d2447432456a 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStore.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.util.Optional; import org.neo4j.causalclustering.catchup.CatchUpClientException; import org.neo4j.causalclustering.catchup.CatchupResult; @@ -28,7 +29,6 @@ import org.neo4j.causalclustering.catchup.tx.TransactionLogCatchUpFactory; import org.neo4j.causalclustering.catchup.tx.TransactionLogCatchUpWriter; import org.neo4j.causalclustering.catchup.tx.TxPullClient; -import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.io.fs.FileSystemAbstraction; @@ -38,12 +38,12 @@ import org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore; import org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionStore; import org.neo4j.kernel.impl.transaction.log.TransactionCursor; -import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; +import static org.neo4j.causalclustering.catchup.CatchupResult.E_TRANSACTION_PRUNED; import static org.neo4j.causalclustering.catchup.CatchupResult.SUCCESS_END_OF_BATCH; import static org.neo4j.causalclustering.catchup.CatchupResult.SUCCESS_END_OF_STREAM; import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID; @@ -54,6 +54,7 @@ public class RemoteStore { private final Log log; + private final LocalDatabase localDatabase; private final Monitors monitors; private final FileSystemAbstraction fs; private final PageCache pageCache; @@ -62,11 +63,8 @@ public class RemoteStore private final TxPullClient txPullClient; private final TransactionLogCatchUpFactory transactionLogFactory; - public RemoteStore( LogProvider logProvider, - FileSystemAbstraction fs, PageCache pageCache, - StoreCopyClient storeCopyClient, TxPullClient txPullClient, - TransactionLogCatchUpFactory transactionLogFactory, - Monitors monitors ) + public RemoteStore( LogProvider logProvider, FileSystemAbstraction fs, PageCache pageCache, StoreCopyClient storeCopyClient, TxPullClient txPullClient, + TransactionLogCatchUpFactory transactionLogFactory, Monitors monitors, LocalDatabase localDatabase ) { this.logProvider = logProvider; this.storeCopyClient = storeCopyClient; @@ -76,32 +74,36 @@ public RemoteStore( LogProvider logProvider, this.transactionLogFactory = transactionLogFactory; this.monitors = monitors; this.log = logProvider.getLog( getClass() ); + this.localDatabase = localDatabase; } - /** - * Later stages of the startup process require at least one transaction to - * figure out the mapping between the transaction log and the consensus log. - * - * If there are no transaction logs then we can pull from and including - * the index which the metadata store points to. This would be the case - * for example with a backup taken during an idle period of the system. - * - * However, if there are transaction logs then we want to find out where - * they end and pull from there, excluding the last one so that we do not - * get duplicate entries. - */ - private long getPullIndex( File storeDir ) throws IOException + private CommitState getStoreState() throws IOException { - /* this is the metadata store */ - ReadOnlyTransactionIdStore txIdStore = new ReadOnlyTransactionIdStore( pageCache, storeDir ); + ReadOnlyTransactionIdStore metaDataStore = new ReadOnlyTransactionIdStore( pageCache, localDatabase.storeDir() ); + long metaDataStoreTxId = metaDataStore.getLastCommittedTransactionId(); - /* Clean as in clean shutdown. Without transaction logs this should be the truth, - * but otherwise it can be used as a starting point for scanning the logs. */ - long lastCleanTxId = txIdStore.getLastCommittedTransactionId(); - log.info( "Last Clean Tx Id: %d", lastCleanTxId ); + Optional latestTransactionLogIndex = getLatestTransactionLogIndex( metaDataStoreTxId ); - /* these are the transaction logs */ - ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, storeDir, new Monitors() ); + //noinspection OptionalIsPresent + if ( latestTransactionLogIndex.isPresent() ) + { + return new CommitState( metaDataStoreTxId, latestTransactionLogIndex.get() ); + } + else + { + return new CommitState( metaDataStoreTxId ); + } + } + + private Optional getLatestTransactionLogIndex( long startTxId ) throws IOException + { + if ( !localDatabase.hasTxLogs() ) + { + return Optional.empty(); + } + + // this is not really a read-only store, because it will create an empty transaction log if there is none + ReadOnlyTransactionStore txStore = new ReadOnlyTransactionStore( pageCache, fs, localDatabase.storeDir(), new Monitors() ); long lastTxId = BASE_TX_ID; try ( Lifespan ignored = new Lifespan( txStore ) ) @@ -109,12 +111,11 @@ private long getPullIndex( File storeDir ) throws IOException TransactionCursor cursor; try { - cursor = txStore.getTransactions( lastCleanTxId ); + cursor = txStore.getTransactions( startTxId ); } catch ( NoSuchTransactionException e ) { - log.info( "No transaction logs found. Will use metadata store as base for pull request." ); - return Math.max( TransactionIdStore.BASE_TX_ID + 1, lastCleanTxId ); + return Optional.empty(); } while ( cursor.next() ) @@ -123,20 +124,48 @@ private long getPullIndex( File storeDir ) throws IOException lastTxId = tx.getCommitEntry().getTxId(); } - if ( lastTxId < lastCleanTxId ) - { - throw new IllegalStateException( "Metadata index was higher than transaction log index." ); - } - - // we don't want to pull a transaction we already have in the log, hence +1 - return lastTxId + 1; + return Optional.of( lastTxId ); } } - public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId, File storeDir ) throws StoreCopyFailedException, IOException + /** + * Later stages of the startup process require at least one transaction to + * figure out the mapping between the transaction log and the consensus log. + * + * If there are no transaction logs then we can pull from and including + * the index which the metadata store points to. This would be the case + * for example with a backup taken during an idle period of the system. + * + * However, if there are transaction logs then we want to find out where + * they end and pull from there, excluding the last one so that we do not + * get duplicate entries. + */ + public CatchupResult tryCatchingUp( AdvertisedSocketAddress from, StoreId expectedStoreId ) throws StoreCopyFailedException, IOException { - long pullIndex = getPullIndex( storeDir ); - return pullTransactions( from, expectedStoreId, storeDir, pullIndex, false ); + CommitState commitState = getStoreState(); + log.info( "Store commit state: " + commitState ); + + if ( commitState.transactionLogIndex().isPresent() ) + { + return pullTransactions( from, expectedStoreId, localDatabase.storeDir(), commitState.transactionLogIndex().get() + 1, false ); + } + else + { + CatchupResult catchupResult; + if ( commitState.metaDataStoreIndex() == BASE_TX_ID ) + { + return pullTransactions( from, expectedStoreId, localDatabase.storeDir(), commitState.metaDataStoreIndex() + 1, false ); + } + else + { + catchupResult = pullTransactions( from, expectedStoreId, localDatabase.storeDir(), commitState.metaDataStoreIndex(), false ); + if ( catchupResult == E_TRANSACTION_PRUNED ) + { + return pullTransactions( from, expectedStoreId, localDatabase.storeDir(), commitState.metaDataStoreIndex() + 1, false ); + } + } + return catchupResult; + } } public void copy( AdvertisedSocketAddress from, StoreId expectedStoreId, File destDir ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClient.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClient.java index 637c0d88cacf..c21b14680716 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClient.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/StoreCopyClient.java @@ -20,22 +20,16 @@ package org.neo4j.causalclustering.catchup.storecopy; import java.io.IOException; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.neo4j.causalclustering.catchup.CatchUpClient; import org.neo4j.causalclustering.catchup.CatchUpClientException; import org.neo4j.causalclustering.catchup.CatchUpResponseAdaptor; -import org.neo4j.causalclustering.core.state.snapshot.TopologyLookupException; -import org.neo4j.causalclustering.discovery.TopologyService; -import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.causalclustering.identity.StoreId; import org.neo4j.helpers.AdvertisedSocketAddress; import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; -import static java.lang.String.format; - public class StoreCopyClient { private final CatchUpClient catchUpClient; @@ -77,7 +71,7 @@ public boolean onFileContent( CompletableFuture signal, FileChunk fileChun public void onFileStreamingComplete( CompletableFuture signal, StoreCopyFinishedResponse response ) { - log.info( "Finished streaming %s", destination ); + log.info( "Finished streaming" ); signal.complete( response.lastCommittedTxBeforeStoreCopy() ); } } ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java index 770ce674270f..5e7144a1d9a1 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/server/CoreServerModule.java @@ -204,7 +204,7 @@ private CoreStateDownloader createCoreStateDownloader( LifeSupport servicesToSto RemoteStore remoteStore = new RemoteStore( logProvider, platformModule.fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), - new TxPullClient( catchUpClient, platformModule.monitors ), new TransactionLogCatchUpFactory(), platformModule.monitors ); + new TxPullClient( catchUpClient, platformModule.monitors ), new TransactionLogCatchUpFactory(), platformModule.monitors, localDatabase ); CopiedStoreRecovery copiedStoreRecovery = platformModule.life.add( new CopiedStoreRecovery( platformModule.config, platformModule.kernelExtensions.listFactories(), platformModule.pageCache ) ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java index a62970d9bcc1..ad628533fae9 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java @@ -77,9 +77,16 @@ void downloadSnapshot( MemberId source ) throws StoreCopyFailedException /* Extract some key properties before shutting it down. */ boolean isEmptyStore = localDatabase.isEmpty(); - if ( !isEmptyStore ) + /* + * There is no reason to try to recover if there are no transaction logs and in fact it is + * also problematic for the initial transaction pull during the snapshot download because the + * kernel will create a transaction log with a header where previous index points to the same + * index as that written down into the metadata store. This is problematic because we have no + * guarantee that there are later transactions and we need at least one transaction in + * the log to figure out the Raft log index (see {@link RecoverConsensusLogIndex}). + */ + if ( localDatabase.hasTxLogs() ) { - /* make sure it's recovered before we start messing with catchup */ localDatabase.start(); localDatabase.stop(); } @@ -120,7 +127,7 @@ public void onCoreSnapshot( CompletableFuture signal, CoreSnapshot else { StoreId localStoreId = localDatabase.storeId(); - CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId, localDatabase.storeDir() ); + CatchupResult catchupResult = remoteStore.tryCatchingUp( fromAddress, localStoreId ); if ( catchupResult == E_TRANSACTION_PRUNED ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index fc602fa16767..2450551f375e 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -227,7 +227,7 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule new RemoteStore( platformModule.logging.getInternalLogProvider(), fileSystem, platformModule.pageCache, new StoreCopyClient( catchUpClient, logProvider ), new TxPullClient( catchUpClient, platformModule.monitors ), new TransactionLogCatchUpFactory(), - platformModule.monitors ); + platformModule.monitors, localDatabase ); CopiedStoreRecovery copiedStoreRecovery = new CopiedStoreRecovery( config, platformModule.kernelExtensions.listFactories(), diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/ClusterSeedingIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/ClusterSeedingIT.java index c9009950ab36..7b4a26c4a16b 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/ClusterSeedingIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/ClusterSeedingIT.java @@ -27,7 +27,6 @@ import org.junit.runners.Parameterized; import java.io.File; -import java.util.Arrays; import java.util.Map; import java.util.Optional; import java.util.function.IntFunction; @@ -50,7 +49,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually; import static org.neo4j.causalclustering.helpers.BackupUtil.restoreFromBackup; @@ -63,23 +62,22 @@ public class ClusterSeedingIT private FileCopyDetector fileCopyDetector; @Parameterized.Parameters( name = "{0}" ) - public static Iterable data() throws Exception + public static Object[][] data() throws Exception { - return stores(); - } - - private static Iterable stores() - { - return Arrays.asList( - new NoStore(), - new EmptyBackupStore(), - new BackupStoreWithSomeData(), - new BackupStoreWithSomeDataButNoTransactionLogs() ); + return new Object[][]{ + {new NoStore(), true }, + {new EmptyBackupStore(), false }, + {new BackupStoreWithSomeData(), false }, + {new BackupStoreWithSomeDataButNoTransactionLogs(), false } + }; } @Parameterized.Parameter() public BackupStore initialStore; + @Parameterized.Parameter( 1 ) + public boolean shouldStoreCopy; + @Rule public TestDirectory testDir = TestDirectory.testDirectory(); public DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); @@ -143,11 +141,11 @@ public void shouldSeedNewCluster() throws Exception // when cluster.start(); - //then + // then if ( backup.isPresent() ) { dataMatchesEventually( DbRepresentation.of( backup.get() ), cluster.coreMembers() ); } - assertFalse( fileCopyDetector.hasDetectedAnyFileCopied() ); + assertEquals( shouldStoreCopy, fileCopyDetector.hasDetectedAnyFileCopied() ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/FileCopyDetector.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/FileCopyDetector.java index 77dfb0728c67..f6106934791a 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/FileCopyDetector.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/FileCopyDetector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/NewMemberSeedingIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/NewMemberSeedingIT.java index 20fc85ecab29..c62b971c785c 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/NewMemberSeedingIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/NewMemberSeedingIT.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. @@ -91,8 +91,7 @@ private static Iterable loads() private static Iterable stores() { - return Arrays - .asList( new EmptyBackupStore(), new BackupStoreWithSomeData(), + return Arrays.asList( new EmptyBackupStore(), new BackupStoreWithSomeData(), new BackupStoreWithSomeDataButNoTransactionLogs() ); } @@ -151,6 +150,7 @@ public void shouldSeedNewMemberToCluster() throws Exception { restoreFromBackup( backup.get(), fsa, newCoreClusterMember ); } + // we want the new instance to seed from backup and not delete and re-download the store newCoreClusterMember.monitors().addMonitorListener( fileCopyDetector ); newCoreClusterMember.start(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/AbstractStoreGenerator.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/AbstractStoreGenerator.java index 484383751589..bb111db73971 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/AbstractStoreGenerator.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/AbstractStoreGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStore.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStore.java index 1515d2e7cfb0..4c37905a7e11 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStore.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStore.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeData.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeData.java index 1d7abe9b6100..cf269512a9d7 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeData.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeData.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeDataButNoTransactionLogs.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeDataButNoTransactionLogs.java index 753d823e3363..70c69f530559 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeDataButNoTransactionLogs.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/BackupStoreWithSomeDataButNoTransactionLogs.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/EmptyBackupStore.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/EmptyBackupStore.java index 21669e510241..d25682683e6f 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/EmptyBackupStore.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/EmptyBackupStore.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/NoStore.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/NoStore.java index c305824bfd6e..70c4710eb27b 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/NoStore.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/backup_stores/NoStore.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/ClusterLoad.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/ClusterLoad.java index ea3a0d06b189..3001e9d3871a 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/ClusterLoad.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/ClusterLoad.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/NoLoad.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/NoLoad.java index 2ad80b98f43d..2600f106ccd3 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/NoLoad.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/NoLoad.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/SmallBurst.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/SmallBurst.java index 68424eb707f8..f3e57aec91c5 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/SmallBurst.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/backup/cluster_load/SmallBurst.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java index af068487cd99..f498796b5a63 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/RemoteStoreTest.java @@ -49,6 +49,8 @@ public class RemoteStoreTest { + private LocalDatabase localDatabase = mock( LocalDatabase.class ); + @Test public void shouldCopyStoreFilesAndPullTransactions() throws Exception { @@ -60,7 +62,7 @@ public void shouldCopyStoreFilesAndPullTransactions() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), new Monitors(), localDatabase ); // when AdvertisedSocketAddress localhost = new AdvertisedSocketAddress( "127.0.0.1", 1234 ); @@ -90,7 +92,7 @@ public void shouldSetLastPulledTransactionId() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), new Monitors(), localDatabase ); // when remoteStore.copy( localhost, wantedStoreId, new File( "destination" ) ); @@ -110,8 +112,7 @@ public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception TransactionLogCatchUpWriter writer = mock( TransactionLogCatchUpWriter.class ); RemoteStore remoteStore = new RemoteStore( NullLogProvider.getInstance(), mock( FileSystemAbstraction.class ), - null, - storeCopyClient, txPullClient, factory( writer ), new Monitors() ); + null, storeCopyClient, txPullClient, factory( writer ), new Monitors(), localDatabase ); doThrow( StoreCopyFailedException.class ).when( txPullClient ) .pullTransactions( any( AdvertisedSocketAddress.class ), eq( storeId ), anyLong(), any( TransactionLogCatchUpWriter.class ) ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java index 9bfccb04f2e8..fd639ae871a6 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java @@ -57,7 +57,7 @@ public class CoreStateDownloaderTest private final RemoteStore remoteStore = mock( RemoteStore.class ); private final CatchUpClient catchUpClient = mock( CatchUpClient.class ); private final StoreCopyProcess storeCopyProcess = mock( StoreCopyProcess.class ); - private CoreSnapshotService snaptshotService = mock( CoreSnapshotService.class ); + private CoreSnapshotService snapshotService = mock( CoreSnapshotService.class ); private TopologyService topologyService = mock( TopologyService.class ); private final CoreStateMachines coreStateMachines = mock( CoreStateMachines.class ); @@ -71,7 +71,7 @@ public class CoreStateDownloaderTest private final CoreStateDownloader downloader = new CoreStateDownloader( localDatabase, startStopLife, remoteStore, catchUpClient, logProvider, - storeCopyProcess, coreStateMachines, snaptshotService, topologyService ); + storeCopyProcess, coreStateMachines, snapshotService, topologyService ); @Before public void commonMocking() throws IOException @@ -93,7 +93,7 @@ public void shouldDownloadCompleteStoreWhenEmpty() throws Throwable downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any() ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, remoteStoreId ); } @@ -134,7 +134,7 @@ public void shouldNotOverwriteNonEmptyMismatchingStore() throws Exception // then verify( remoteStore, never() ).copy( any(), any(), any() ); - verify( remoteStore, never() ).tryCatchingUp( any(), any(), any() ); + verify( remoteStore, never() ).tryCatchingUp( any(), any() ); } @Test @@ -143,13 +143,13 @@ public void shouldCatchupIfPossible() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( SUCCESS_END_OF_STREAM ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId ) ).thenReturn( SUCCESS_END_OF_STREAM ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId ); verify( remoteStore, never() ).copy( any(), any(), any() ); } @@ -159,13 +159,13 @@ public void shouldDownloadWholeStoreIfCannotCatchUp() throws Exception // given when( localDatabase.isEmpty() ).thenReturn( false ); when( remoteStore.getStoreId( remoteAddress ) ).thenReturn( storeId ); - when( remoteStore.tryCatchingUp( remoteAddress, storeId, storeDir ) ).thenReturn( E_TRANSACTION_PRUNED ); + when( remoteStore.tryCatchingUp( remoteAddress, storeId ) ).thenReturn( E_TRANSACTION_PRUNED ); // when downloader.downloadSnapshot( remoteMember ); // then - verify( remoteStore ).tryCatchingUp( remoteAddress, storeId, storeDir ); + verify( remoteStore ).tryCatchingUp( remoteAddress, storeId ); verify( storeCopyProcess ).replaceWithStoreFrom( remoteAddress, storeId ); } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/helpers/BackupUtil.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/helpers/BackupUtil.java index 32b0bf829283..0fef68b80fc3 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/helpers/BackupUtil.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/helpers/BackupUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2017 "Neo Technology," + * Copyright (c) 2002-2018 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j.