diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index 4ac1619ea2d4..971b280f8db2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -248,40 +248,40 @@ public Collection createCommands( // Create objects to be populated with command-friendly changes Collection commands = new ArrayList<>(); - if ( txState == null ) + if ( txState != null ) { - // Bypass creating commands for record transaction state if we have no transaction state - legacyIndexTransactionState.extractCommands( commands ); - return commands; + NeoStoreTransactionContext context = new NeoStoreTransactionContext( neoStores, locks ); + TransactionRecordState recordState = new TransactionRecordState( neoStores, integrityValidator, context ); + recordState.initialize( lastTransactionIdWhenStarted ); + + // Visit transaction state and populate these record state objects + TxStateVisitor txStateVisitor = new TransactionToRecordStateVisitor( recordState, + schemaStateChangeCallback, schemaStorage, constraintSemantics, providerMap, + legacyIndexTransactionState, procedureCache ); + CountsRecordState countsRecordState = new CountsRecordState(); + txStateVisitor = constraintSemantics.decorateTxStateVisitor( + operations, + storeStatement, + storeLayer, + new DirectTxStateHolder( txState, legacyIndexTransactionState ), + txStateVisitor ); + txStateVisitor = new TransactionCountingStateVisitor( + txStateVisitor, storeLayer, txState, countsRecordState ); + try ( TxStateVisitor visitor = txStateVisitor ) + { + txState.accept( txStateVisitor ); + } + + // Convert record state into commands + recordState.extractCommands( commands ); + countsRecordState.extractCommands( commands ); } - NeoStoreTransactionContext context = new NeoStoreTransactionContext( neoStores, locks ); - TransactionRecordState recordState = new TransactionRecordState( neoStores, integrityValidator, context ); - recordState.initialize( lastTransactionIdWhenStarted ); - - // Visit transaction state and populate these record state objects - TxStateVisitor txStateVisitor = new TransactionToRecordStateVisitor( recordState, - schemaStateChangeCallback, schemaStorage, constraintSemantics, providerMap, - legacyIndexTransactionState, procedureCache ); - CountsRecordState countsRecordState = new CountsRecordState(); - txStateVisitor = constraintSemantics.decorateTxStateVisitor( - operations, - storeStatement, - storeLayer, - new DirectTxStateHolder( txState, legacyIndexTransactionState ), - txStateVisitor ); - txStateVisitor = new TransactionCountingStateVisitor( - txStateVisitor, storeLayer, txState, countsRecordState ); - try ( TxStateVisitor visitor = txStateVisitor ) + if ( legacyIndexTransactionState != null ) { - txState.accept( txStateVisitor ); + legacyIndexTransactionState.extractCommands( commands ); } - // Convert record state into commands - recordState.extractCommands( commands ); - legacyIndexTransactionState.extractCommands( commands ); - countsRecordState.extractCommands( commands ); - return commands; } @@ -472,5 +472,6 @@ public void shutdown() throws Throwable { labelScanStore.shutdown(); indexingService.shutdown(); + neoStores.close(); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineRule.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineRule.java new file mode 100644 index 000000000000..fa27e7f347f3 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineRule.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.storageengine.impl.recordstorage; + +import java.io.File; + +import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.IdGeneratorFactory; +import org.neo4j.kernel.KernelEventHandlers; +import org.neo4j.kernel.api.TokenNameLookup; +import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.api.labelscan.LabelScanStore; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.LegacyIndexProviderLookup; +import org.neo4j.kernel.impl.api.index.IndexingService; +import org.neo4j.kernel.impl.api.scan.InMemoryLabelScanStore; +import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; +import org.neo4j.kernel.impl.constraints.StandardConstraintSemantics; +import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator; +import org.neo4j.kernel.impl.core.LabelTokenHolder; +import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder; +import org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder; +import org.neo4j.kernel.impl.index.IndexConfigStore; +import org.neo4j.kernel.impl.locking.ReentrantLockService; +import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.kernel.impl.util.Neo4jJobScheduler; +import org.neo4j.kernel.internal.DatabaseHealth; +import org.neo4j.kernel.lifecycle.LifeSupport; +import org.neo4j.logging.NullLog; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.test.ExternalResource; +import org.neo4j.test.PageCacheRule; +import org.neo4j.test.impl.EphemeralIdGenerator; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Conveniently manages a {@link RecordStorageEngine} in a test. Needs {@link FileSystemAbstraction} and + * {@link PageCache}, which usually are managed by test rules themselves. That's why they are passed in + * when {@link #getWith(FileSystemAbstraction, PageCache) getting (constructing)} the engine. Further + * dependencies can be overridden in that returned builder as well. + * + * Keep in mind that this rule must be created BEFORE {@link PageCacheRule} and any file system rule so that + * shutdown order gets correct. + */ +public class RecordStorageEngineRule extends ExternalResource +{ + private final LifeSupport life = new LifeSupport(); + + @Override + protected void before() throws Throwable + { + super.before(); + life.start(); + } + + public Builder getWith( FileSystemAbstraction fs, PageCache pageCache ) + { + return new Builder( fs, pageCache ); + } + + private RecordStorageEngine get( FileSystemAbstraction fs, PageCache pageCache, LabelScanStore labelScanStore, + SchemaIndexProvider schemaIndexProvider, DatabaseHealth databaseHealth, File storeDirectory ) + { + if ( !fs.fileExists( storeDirectory ) && !fs.mkdir( storeDirectory ) ) + { + throw new IllegalStateException(); + } + IdGeneratorFactory idGeneratorFactory = new EphemeralIdGenerator.Factory(); + LabelScanStoreProvider labelScanStoreProvider = new LabelScanStoreProvider( labelScanStore, 42 ); + LegacyIndexProviderLookup legacyIndexProviderLookup = mock( LegacyIndexProviderLookup.class ); + when( legacyIndexProviderLookup.all() ).thenReturn( Iterables.empty() ); + IndexConfigStore indexConfigStore = new IndexConfigStore( storeDirectory, fs ); + JobScheduler scheduler = life.add( new Neo4jJobScheduler() ); + + return life.add( new RecordStorageEngine( storeDirectory, new Config(), idGeneratorFactory, pageCache, fs, + NullLogProvider.getInstance(), mock( PropertyKeyTokenHolder.class ), mock( LabelTokenHolder.class ), + mock( RelationshipTypeTokenHolder.class ), () -> {}, new StandardConstraintSemantics(), + scheduler, mock( TokenNameLookup.class ), new ReentrantLockService(), + schemaIndexProvider, IndexingService.NO_MONITOR, databaseHealth, + labelScanStoreProvider, legacyIndexProviderLookup, indexConfigStore ) ); + } + + @Override + protected void after( boolean successful ) throws Throwable + { + life.shutdown(); + super.after( successful ); + } + + public class Builder + { + private final FileSystemAbstraction fs; + private final PageCache pageCache; + private LabelScanStore labelScanStore = new InMemoryLabelScanStore(); + private DatabaseHealth databaseHealth = new DatabaseHealth( + new DatabasePanicEventGenerator( new KernelEventHandlers( NullLog.getInstance() ) ), + NullLog.getInstance() ); + private File storeDirectory = new File( "graph.db" ); + private SchemaIndexProvider schemaIndexProvider = SchemaIndexProvider.NO_INDEX_PROVIDER; + + public Builder( FileSystemAbstraction fs, PageCache pageCache ) + { + this.fs = fs; + this.pageCache = pageCache; + } + + public Builder labelScanStore( LabelScanStore labelScanStore ) + { + this.labelScanStore = labelScanStore; + return this; + } + + public Builder indexProvider( SchemaIndexProvider schemaIndexProvider ) + { + this.schemaIndexProvider = schemaIndexProvider; + return this; + } + + public Builder databaseHealth( DatabaseHealth databaseHealth ) + { + this.databaseHealth = databaseHealth; + return this; + } + + public Builder storeDirectory( File storeDirectory ) + { + this.storeDirectory = storeDirectory; + return this; + } + + // Add more here + + public RecordStorageEngine build() + { + return get( fs, pageCache, labelScanStore, schemaIndexProvider, databaseHealth, storeDirectory ); + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java index 721490ce8a12..df53ab324516 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngineTest.java @@ -22,45 +22,21 @@ import org.junit.Rule; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.concurrent.ThreadLocalRandom; -import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.helpers.Exceptions; -import org.neo4j.helpers.collection.Iterables; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.IdGeneratorFactory; -import org.neo4j.kernel.api.TokenNameLookup; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; -import org.neo4j.kernel.impl.api.LegacyIndexProviderLookup; import org.neo4j.kernel.impl.api.TransactionApplicationMode; import org.neo4j.kernel.impl.api.TransactionToApply; -import org.neo4j.kernel.impl.api.index.IndexingService; -import org.neo4j.kernel.impl.api.scan.InMemoryLabelScanStore; -import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; -import org.neo4j.kernel.impl.constraints.StandardConstraintSemantics; -import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator; -import org.neo4j.kernel.impl.core.LabelTokenHolder; -import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder; -import org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder; -import org.neo4j.kernel.impl.index.IndexConfigStore; -import org.neo4j.kernel.impl.locking.ReentrantLockService; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.FakeCommitment; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.internal.DatabaseHealth; -import org.neo4j.logging.NullLog; -import org.neo4j.logging.NullLogProvider; import org.neo4j.test.EphemeralFileSystemRule; -import org.neo4j.test.OnDemandJobScheduler; import org.neo4j.test.PageCacheRule; -import org.neo4j.test.impl.EphemeralIdGenerator; - import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; @@ -72,6 +48,8 @@ public class RecordStorageEngineTest { + @Rule + public final RecordStorageEngineRule storageEngineRule = new RecordStorageEngineRule(); @Rule public final EphemeralFileSystemRule fsRule = new EphemeralFileSystemRule(); @Rule @@ -80,73 +58,40 @@ public class RecordStorageEngineTest @Test( timeout = 30_000 ) public void shutdownRecordStorageEngineAfterFailedTransaction() throws Throwable { - RecordStorageEngine engine = start( recordStorageEngine() ); - try - { - Exception applicationError = executeFailingTransaction( engine ); - assertNotNull( applicationError ); - } - finally - { - // shutdown should not hang after a failed transaction - shutdown( engine ); - } + RecordStorageEngine engine = + storageEngineRule.getWith( fsRule.get(), pageCacheRule.getPageCache( fsRule.get() ) ).build(); + Exception applicationError = executeFailingTransaction( engine ); + assertNotNull( applicationError ); } @Test public void databasePanicIsRaisedWhenTxApplicationFails() throws Throwable { DatabaseHealth databaseHealth = mock( DatabaseHealth.class ); - RecordStorageEngine engine = start( recordStorageEngine( databaseHealth ) ); - try - { - Exception applicationError = executeFailingTransaction( engine ); - verify( databaseHealth ).panic( applicationError ); - } - finally - { - shutdown( engine ); - } + RecordStorageEngine engine = + storageEngineRule.getWith( fsRule.get(), pageCacheRule.getPageCache( fsRule.get() ) ) + .databaseHealth( databaseHealth ) + .build(); + Exception applicationError = executeFailingTransaction( engine ); + verify( databaseHealth ).panic( applicationError ); } @Test( timeout = 30_000 ) public void obtainCountsStoreResetterAfterFailedTransaction() throws Throwable { - RecordStorageEngine engine = start( recordStorageEngine() ); - try - { - Exception applicationError = executeFailingTransaction( engine ); - assertNotNull( applicationError ); - - CountsTracker countsStore = engine.neoStores().getCounts(); - // possible to obtain a resetting updater that internally has a write lock on the counts store - try ( CountsAccessor.Updater updater = countsStore.reset( 0 ) ) - { - assertNotNull( updater ); - } - } - finally + RecordStorageEngine engine = + storageEngineRule.getWith( fsRule.get(), pageCacheRule.getPageCache( fsRule.get() ) ).build(); + Exception applicationError = executeFailingTransaction( engine ); + assertNotNull( applicationError ); + + CountsTracker countsStore = engine.neoStores().getCounts(); + // possible to obtain a resetting updater that internally has a write lock on the counts store + try ( CountsAccessor.Updater updater = countsStore.reset( 0 ) ) { - shutdown( engine ); + assertNotNull( updater ); } } - private static RecordStorageEngine start( RecordStorageEngine engine ) throws Throwable - { - engine.init(); - engine.start(); - return engine; - } - - private static void shutdown( RecordStorageEngine engine ) throws Throwable - { - // this have to be done here because RecordStorageEngine creates NeoStores but does not manage their lifecycle... - engine.neoStores().close(); - - engine.stop(); - engine.shutdown(); - } - private Exception executeFailingTransaction( RecordStorageEngine engine ) throws IOException { Exception applicationError = new UnderlyingStorageException( "No space left on device" ); @@ -177,35 +122,4 @@ private static TransactionToApply newTransactionThatFailsWith( Exception error ) txToApply.commitment( commitment, txId ); return txToApply; } - - private RecordStorageEngine recordStorageEngine() - { - DatabasePanicEventGenerator panicEventGenerator = mock( DatabasePanicEventGenerator.class ); - DatabaseHealth databaseHealth = new DatabaseHealth( panicEventGenerator, NullLog.getInstance() ); - return recordStorageEngine( databaseHealth ); - } - - private RecordStorageEngine recordStorageEngine( DatabaseHealth databaseHealth ) - { - EphemeralFileSystemAbstraction fs = fsRule.get(); - File storeDir = new File( "graph.db" ); - if ( !fs.mkdir( storeDir ) ) - { - throw new IllegalStateException(); - } - IdGeneratorFactory idGeneratorFactory = new EphemeralIdGenerator.Factory(); - PageCache pageCache = pageCacheRule.getPageCache( fs ); - InMemoryLabelScanStore labelScanStore = new InMemoryLabelScanStore(); - LabelScanStoreProvider labelScanStoreProvider = new LabelScanStoreProvider( labelScanStore, 42 ); - LegacyIndexProviderLookup legacyIndexProviderLookup = mock( LegacyIndexProviderLookup.class ); - when( legacyIndexProviderLookup.all() ).thenReturn( Iterables.empty() ); - IndexConfigStore indexConfigStore = new IndexConfigStore( storeDir, fs ); - - return new RecordStorageEngine( storeDir, new Config(), idGeneratorFactory, pageCache, fs, - NullLogProvider.getInstance(), mock( PropertyKeyTokenHolder.class ), mock( LabelTokenHolder.class ), - mock( RelationshipTypeTokenHolder.class ), () -> {}, new StandardConstraintSemantics(), - new OnDemandJobScheduler(), mock( TokenNameLookup.class ), new ReentrantLockService(), - SchemaIndexProvider.NO_INDEX_PROVIDER, IndexingService.NO_MONITOR, databaseHealth, - labelScanStoreProvider, legacyIndexProviderLookup, indexConfigStore ); - } }