diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java index 5a99d498aa1bf..02940ce357c7f 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabase.java @@ -46,7 +46,10 @@ public class LocalDatabase implements Lifecycle { - private static final AvailabilityRequirement NOT_STOPPED = availabilityRequirement( "Database is stopped" ); + private static final AvailabilityRequirement NOT_STOPPED = + availabilityRequirement( "Database is stopped" ); + private static final AvailabilityRequirement NOT_COPYING_STORE = + availabilityRequirement( "Database is stopped to copy store from another cluster member" ); private final File storeDir; @@ -60,7 +63,7 @@ public class LocalDatabase implements Lifecycle private volatile StoreId storeId; private volatile DatabaseHealth databaseHealth; - private boolean started = false; + private AvailabilityRequirement currentRequirement; private volatile TransactionCommitProcess localCommit; @@ -79,7 +82,7 @@ public LocalDatabase( File storeDir, StoreFiles storeFiles, this.availabilityGuard = availabilityGuard; this.log = logProvider.getLog( getClass() ); - raiseAvailabilityGuard(); + raiseAvailabilityGuard( NOT_STOPPED ); } @Override @@ -97,24 +100,28 @@ public synchronized void start() throws Throwable dataSourceManager.start(); dropAvailabilityGuard(); - started = true; } @Override - public synchronized void stop() throws Throwable + public void stop() throws Throwable { - log.info( "Stopping" ); - databaseHealth = null; - localCommit = null; - dataSourceManager.stop(); + stopWithRequirement( NOT_STOPPED ); + } - raiseAvailabilityGuard(); - started = false; + /** + * Stop database to perform a store copy. This will raise {@link AvailabilityGuard} with + * a more friendly blocking requirement. + * + * @throws Throwable if any of the components are unable to stop. + */ + public void stopForStoreCopy() throws Throwable + { + stopWithRequirement( NOT_COPYING_STORE ); } public boolean isAvailable() { - return started; + return currentRequirement == null; } @Override @@ -125,7 +132,7 @@ public void shutdown() throws Throwable public synchronized StoreId storeId() { - if ( started ) + if ( isAvailable() ) { return storeId; } @@ -226,13 +233,31 @@ public TransactionCommitProcess getCommitProcess() return localCommit; } - private void raiseAvailabilityGuard() + private synchronized void stopWithRequirement( AvailabilityRequirement requirement ) throws Throwable + { + log.info( "Stopping, reason: " + requirement.description() ); + databaseHealth = null; + localCommit = null; + dataSourceManager.stop(); + + raiseAvailabilityGuard( requirement ); + } + + private void raiseAvailabilityGuard( AvailabilityRequirement requirement ) { - availabilityGuard.require( NOT_STOPPED ); + // it is possible for the local database to be created and stopped right after that to perform a store copy + // in this case we need to impose new requirement and drop the old one + availabilityGuard.require( requirement ); + if ( currentRequirement != null ) + { + dropAvailabilityGuard(); + } + currentRequirement = requirement; } private void dropAvailabilityGuard() { - availabilityGuard.fulfill( NOT_STOPPED ); + availabilityGuard.fulfill( currentRequirement ); + currentRequirement = null; } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcess.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcess.java index cd2707aa9d037..9507417218270 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcess.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcess.java @@ -305,7 +305,7 @@ private void downloadDatabase( MemberId core, StoreId localStoreId ) { try { - localDatabase.stop(); + localDatabase.stopForStoreCopy(); startStopOnStoreCopy.stop(); } catch ( Throwable throwable ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java index 1a22d0ae78dd1..4432202679980 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloader.java @@ -83,7 +83,7 @@ public synchronized void downloadSnapshot( MemberId source, CoreState coreState } startStopOnStoreCopy.stop(); - localDatabase.stop(); + localDatabase.stopForStoreCopy(); log.info( "Downloading snapshot from core server at %s", source ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java index 9e3bf892c0518..9d61d2c7bd7e1 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/storecopy/LocalDatabaseTest.java @@ -81,6 +81,22 @@ public void availabilityGuardRaisedOnStop() throws Throwable assertDatabaseIsStoppedAndUnavailable( guard ); } + @Test + public void availabilityGuardRaisedOnStopForStoreCopy() throws Throwable + { + AvailabilityGuard guard = newAvailabilityGuard(); + assertTrue( guard.isAvailable() ); + + LocalDatabase localDatabase = newLocalDatabase( guard ); + assertFalse( guard.isAvailable() ); + + localDatabase.start(); + assertTrue( guard.isAvailable() ); + + localDatabase.stopForStoreCopy(); + assertDatabaseIsStoppedForStoreCopyAndUnavailable( guard ); + } + private static LocalDatabase newLocalDatabase( AvailabilityGuard availabilityGuard ) { return new LocalDatabase( mock( File.class ), mock( StoreFiles.class ), mock( DataSourceManager.class ), @@ -98,4 +114,10 @@ private static void assertDatabaseIsStoppedAndUnavailable( AvailabilityGuard gua assertFalse( guard.isAvailable() ); assertThat( guard.describeWhoIsBlocking(), containsString( "Database is stopped" ) ); } + + private static void assertDatabaseIsStoppedForStoreCopyAndUnavailable( AvailabilityGuard guard ) + { + assertFalse( guard.isAvailable() ); + assertThat( guard.describeWhoIsBlocking(), containsString( "Database is stopped to copy store" ) ); + } } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcessTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcessTest.java index 4d07fa685faf3..cde5aded5b986 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcessTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/CatchupPollingProcessTest.java @@ -170,7 +170,7 @@ public void nextStateShouldBeTxPullingAfterASuccessfulStoreCopy() throws Throwab timeoutService.invokeTimeout( TX_PULLER_TIMEOUT ); // then - verify( localDatabase ).stop(); + verify( localDatabase ).stopForStoreCopy(); verify( startStopOnStoreCopy ).stop(); verify( storeCopyProcess ).replaceWithStoreFrom( any( MemberId.class ), eq( storeId ) ); verify( localDatabase ).start(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java index 86a90531b8b71..6d60aeb7beba4 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/snapshot/CoreStateDownloaderTest.java @@ -27,9 +27,9 @@ import org.neo4j.causalclustering.catchup.CatchUpClient; import org.neo4j.causalclustering.catchup.storecopy.LocalDatabase; +import org.neo4j.causalclustering.catchup.storecopy.RemoteStore; import org.neo4j.causalclustering.catchup.storecopy.StoreCopyFailedException; import org.neo4j.causalclustering.catchup.storecopy.StoreCopyProcess; -import org.neo4j.causalclustering.catchup.storecopy.RemoteStore; import org.neo4j.causalclustering.core.state.CoreState; import org.neo4j.causalclustering.core.state.machines.CoreStateMachines; import org.neo4j.causalclustering.identity.MemberId; @@ -100,7 +100,7 @@ public void shouldStopDatabaseDuringDownload() throws Throwable // then verify( startStopLife ).stop(); - verify( localDatabase ).stop(); + verify( localDatabase ).stopForStoreCopy(); verify( localDatabase ).start(); verify( startStopLife ).start(); } diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaStoreCopyIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaStoreCopyIT.java index 256b0ef91259b..792342dea2cf8 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaStoreCopyIT.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/scenarios/ReadReplicaStoreCopyIT.java @@ -41,6 +41,7 @@ import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.test.causalclustering.ClusterRule; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; @@ -81,6 +82,7 @@ public void shouldNotBePossibleToStartTransactionsWhenReadReplicaCopiesStore() t catch ( Exception e ) { assertThat( e, instanceOf( TransactionFailureException.class ) ); + assertThat( e.getMessage(), containsString( "Database is stopped to copy store" ) ); } } finally