diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/ConsistencyCheckService.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/ConsistencyCheckService.java index 75c887e01f0f6..e11b42448edd6 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/ConsistencyCheckService.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/ConsistencyCheckService.java @@ -147,7 +147,7 @@ public PrintWriter get() .build(); SchemaIndexProvider indexes = new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, - storeDir, tuningConfiguration, operationalMode ); + storeDir, logProvider, tuningConfiguration, operationalMode ); DirectStoreAccess stores = new DirectStoreAccess( store, labelScanStore, indexes ); FullCheck check = new FullCheck( tuningConfiguration, progressFactory ); summary = check.execute( stores, new DuplicatingLog( log, reportLog ) ); diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/GraphStoreFixture.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/GraphStoreFixture.java index 846d38abb2565..a7dd678801091 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/GraphStoreFixture.java @@ -62,6 +62,7 @@ import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.impl.transaction.tracing.CommitEvent; import org.neo4j.logging.FormattedLogProvider; +import org.neo4j.logging.NullLogProvider; import org.neo4j.test.PageCacheRule; import org.neo4j.test.TargetDirectory; import org.neo4j.test.TestGraphDatabaseFactory; @@ -106,8 +107,8 @@ public DirectStoreAccess directStoreAccess() private SchemaIndexProvider createIndexes( FileSystemAbstraction fileSystem, Config config, OperationalMode operationalMode ) { - return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, config, - operationalMode ); + return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, + NullLogProvider.getInstance(), config, operationalMode ); } public File directory() diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java index 164e0f6824a44..e7e0948c66739 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java @@ -152,7 +152,7 @@ public PrintWriter get() SchemaIndexProvider indexes = new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, - storeDir, consistencyCheckerConfig, operationalMode ); + storeDir, logProvider, consistencyCheckerConfig, operationalMode ); int numberOfThreads = defaultConsistencyCheckThreadsNumber(); Statistics statistics; diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java index f49df2ff41878..7b2d60e543c1f 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java @@ -142,7 +142,8 @@ public DirectStoreAccess directStoreAccess() private SchemaIndexProvider createIndexes( FileSystemAbstraction fileSystem, Config config, OperationalMode operationalMode ) { - return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, config, operationalMode ); + return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, + FormattedLogProvider.toOutputStream( System.out ), config, operationalMode ); } public File directory() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyFactory.java index 2bc8ee494107a..84b819d60f38a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyFactory.java @@ -21,5 +21,5 @@ public interface IndexProxyFactory { - IndexProxy create(); + IndexProxy create() throws Exception; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxySetup.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxySetup.java index b67ad4410ac3a..3bd9be5b9678b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxySetup.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxySetup.java @@ -31,10 +31,10 @@ import org.neo4j.kernel.impl.api.UpdateableSchemaState; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; import static java.lang.String.format; - import static org.neo4j.kernel.impl.api.index.IndexPopulationFailure.failure; public class IndexProxySetup @@ -75,7 +75,7 @@ public IndexProxy createPopulatingIndexProxy( final long ruleId, // TODO: This is here because there is a circular dependency from PopulatingIndexProxy to FlippableIndexProxy final String indexUserDescription = indexUserDescription( descriptor, providerDescriptor ); final IndexConfiguration config = new IndexConfiguration( constraint ); - IndexPopulator populator = populatorFromProvider( providerDescriptor, ruleId, descriptor, config, + final IndexPopulator populator = populatorFromProvider( providerDescriptor, ruleId, descriptor, config, samplingConfig ); FailedIndexProxyFactory failureDelegateFactory = new FailedPopulatingIndexProxyFactory( @@ -97,25 +97,17 @@ public IndexProxy createPopulatingIndexProxy( final long ruleId, flipper.setFlipTarget( new IndexProxyFactory() { @Override - public IndexProxy create() + public IndexProxy create() throws Exception { - try - { - monitor.populationCompleteOn( descriptor ); - OnlineIndexProxy onlineProxy = new OnlineIndexProxy( - descriptor, config, onlineAccessorFromProvider( providerDescriptor, ruleId, - config, samplingConfig ), storeView, providerDescriptor - ); - if ( constraint ) - { - return new TentativeConstraintIndexProxy( flipper, onlineProxy ); - } - return onlineProxy; - } - catch ( IOException e ) + monitor.populationCompleteOn( descriptor ); + OnlineIndexProxy onlineProxy = new OnlineIndexProxy( + descriptor, config, onlineAccessorFromProvider( providerDescriptor, ruleId, + config, samplingConfig ), storeView, providerDescriptor ); + if ( constraint ) { - return createFailedIndexProxy( ruleId, descriptor, providerDescriptor, constraint, failure( e ) ); + return new TentativeConstraintIndexProxy( flipper, onlineProxy ); } + return onlineProxy; } } ); @@ -151,7 +143,10 @@ public IndexProxy createOnlineIndexProxy( long ruleId, } catch ( IOException e ) { - return createFailedIndexProxy( ruleId, descriptor, providerDescriptor, unique, failure( e ) ); + logProvider.getLog( getClass() ).error( "Failed to open index: " + ruleId + + " (" + descriptor.userDescription( tokenNameLookup ) + + "), requesting re-population.", e ); + return createRecoveringIndexProxy( descriptor, providerDescriptor, unique ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java index e25d4385e3cdb..82a0281ac5ce5 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java @@ -234,7 +234,6 @@ public void init() break; case POPULATING: // The database was shut down during population, or a crash has occurred, or some other sad thing. - indexProxy = proxySetup.createRecoveringIndexProxy( descriptor, providerDescriptor, constraint ); break; case FAILED: diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java index a139620d0ea8d..f3fd6a6359999 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java @@ -19,12 +19,6 @@ */ package org.neo4j.kernel.impl.api.index; -import org.junit.Rule; -import org.junit.Test; -import org.mockito.InOrder; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -34,6 +28,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.helpers.collection.ArrayIterator; @@ -71,6 +75,7 @@ import org.neo4j.register.Register.DoubleLongRegister; import org.neo4j.test.DoubleLatch; +import static java.util.Arrays.asList; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.Matchers.equalTo; @@ -94,15 +99,13 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; - -import static java.util.Arrays.asList; - import static org.neo4j.collection.primitive.PrimitiveLongCollections.setOf; import static org.neo4j.helpers.collection.IteratorUtil.asCollection; import static org.neo4j.helpers.collection.IteratorUtil.asResourceIterator; import static org.neo4j.helpers.collection.IteratorUtil.asSet; import static org.neo4j.helpers.collection.IteratorUtil.iterator; import static org.neo4j.helpers.collection.IteratorUtil.loop; +import static org.neo4j.kernel.api.index.InternalIndexState.FAILED; import static org.neo4j.kernel.api.index.InternalIndexState.ONLINE; import static org.neo4j.kernel.api.index.InternalIndexState.POPULATING; import static org.neo4j.kernel.impl.api.index.IndexUpdateMode.BATCHED; @@ -801,6 +804,108 @@ public void awaitingPopulationOfRecoveredIndex( long index, IndexDescriptor desc assertEquals( ONLINE, indexing.getIndexProxy( indexId.get() ).getState() ); } + @Test + public void shouldStoreIndexFailureWhenFailingToCreateOnlineAccessorAfterPopulating() throws Exception + { + // given + long indexId = 1; + IndexingService indexing = newIndexingServiceWithMockedDependencies( populator, accessor, withData() ); + + IOException exception = new IOException( "Expected failure" ); + when( nameLookup.labelGetName( labelId ) ).thenReturn( "TheLabel" ); + when( nameLookup.propertyKeyGetName( propertyKeyId ) ).thenReturn( "propertyKey" ); + + when( indexProvider.getOnlineAccessor( + eq( indexId ), any( IndexConfiguration.class ), any( IndexSamplingConfig.class ) ) ) + .thenThrow( exception ); + + life.start(); + ArgumentCaptor closeArgs = ArgumentCaptor.forClass( Boolean.class ); + + // when + indexing.createIndex( indexRule( indexId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR ) ); + verify( populator, timeout( 1000 ).times( 2 ) ).close( closeArgs.capture() ); + + // then + assertEquals( FAILED, indexing.getIndexProxy( 1 ).getState() ); + assertEquals( asList( true, false ), closeArgs.getAllValues() ); + assertThat( storedFailure(), containsString( "java.io.IOException: Expected failure\n\tat " ) ); + logProvider.assertAtLeastOnce( inLog( IndexPopulationJob.class ).error( equalTo( + "Failed to populate index: [:TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]]" ), + causedBy( exception ) ) ); + logProvider.assertNone( inLog( IndexPopulationJob.class ).info( + "Index population completed. Index is now online: [%s]", + ":TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]" ) ); + } + + @Test + public void shouldStoreIndexFailureWhenFailingToCreateOnlineAccessorAfterRecoveringPopulatingIndex() throws Exception + { + // given + long indexId = 1; + IndexingService indexing = newIndexingServiceWithMockedDependencies( populator, accessor, withData(), + indexRule( indexId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR ) ); + + IOException exception = new IOException( "Expected failure" ); + when( nameLookup.labelGetName( labelId ) ).thenReturn( "TheLabel" ); + when( nameLookup.propertyKeyGetName( propertyKeyId ) ).thenReturn( "propertyKey" ); + + when( indexProvider.getInitialState( indexId ) ).thenReturn( POPULATING ); + when( indexProvider.getOnlineAccessor( + eq( indexId ), any( IndexConfiguration.class ), any( IndexSamplingConfig.class ) ) ) + .thenThrow( exception ); + + life.start(); + ArgumentCaptor closeArgs = ArgumentCaptor.forClass( Boolean.class ); + + // when + verify( populator, timeout( 1000 ).times( 2 ) ).close( closeArgs.capture() ); + + // then + assertEquals( FAILED, indexing.getIndexProxy( 1 ).getState() ); + assertEquals( asList( true, false ), closeArgs.getAllValues() ); + assertThat( storedFailure(), containsString( "java.io.IOException: Expected failure\n\tat " ) ); + logProvider.assertAtLeastOnce( inLog( IndexPopulationJob.class ).error( equalTo( + "Failed to populate index: [:TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]]" ), + causedBy( exception ) ) ); + logProvider.assertNone( inLog( IndexPopulationJob.class ).info( + "Index population completed. Index is now online: [%s]", + ":TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]" ) ); + } + + static Matcher causedBy( final Throwable exception ) + { + return new TypeSafeMatcher() + { + @Override + protected boolean matchesSafely( Throwable item ) + { + while ( item != null ) + { + if ( item == exception ) + { + return true; + } + item = item.getCause(); + } + return false; + } + + @Override + public void describeTo( Description description ) + { + description.appendText( "exception caused by " ).appendValue( exception ); + } + }; + } + + private String storedFailure() throws IOException + { + ArgumentCaptor reason = ArgumentCaptor.forClass( String.class ); + verify( populator ).markAsFailed( reason.capture() ); + return reason.getValue(); + } + private static class ControlledIndexPopulator extends IndexPopulator.Adapter { private final DoubleLatch latch; diff --git a/community/lucene-index/pom.xml b/community/lucene-index/pom.xml index 990cf618b6b25..ef28a3e5877e6 100644 --- a/community/lucene-index/pom.xml +++ b/community/lucene-index/pom.xml @@ -64,6 +64,13 @@ the relevant Commercial Agreement. test-jar test + + org.neo4j + neo4j-logging + ${project.version} + test-jar + test + org.neo4j neo4j-io diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java index 73f00e02c0f4a..0f56f9cc07227 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java @@ -19,15 +19,10 @@ */ package org.neo4j.kernel.api.impl.index; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.Directory; -import java.io.EOFException; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; @@ -45,6 +40,8 @@ import org.neo4j.kernel.impl.store.StoreFactory; import org.neo4j.kernel.impl.storemigration.SchemaIndexMigrator; import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; import org.neo4j.udc.UsageDataKeys.OperationalMode; public class LuceneSchemaIndexProvider extends SchemaIndexProvider @@ -54,10 +51,11 @@ public class LuceneSchemaIndexProvider extends SchemaIndexProvider private final FailureStorage failureStorage; private final FolderLayout folderLayout; private final boolean readOnly; - private final Map failures = new HashMap<>(); + private final Log log; - public LuceneSchemaIndexProvider( FileSystemAbstraction fileSystem, DirectoryFactory directoryFactory, - File storeDir, Config config, OperationalMode operationalMode ) + public LuceneSchemaIndexProvider( + FileSystemAbstraction fileSystem, DirectoryFactory directoryFactory, + File storeDir, LogProvider logging, Config config, OperationalMode operationalMode ) { super( LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR, 1 ); this.directoryFactory = directoryFactory; @@ -65,6 +63,7 @@ public LuceneSchemaIndexProvider( FileSystemAbstraction fileSystem, DirectoryFac this.folderLayout = new FolderLayout( rootDirectory ); this.failureStorage = new FailureStorage( fileSystem, folderLayout ); this.readOnly = isReadOnly( config, operationalMode ); + this.log = logging.getLog( getClass() ); } @Override @@ -114,38 +113,24 @@ public void shutdown() throws Throwable @Override public InternalIndexState getInitialState( long indexId ) { + String failure = failureStorage.loadIndexFailure( indexId ); + if ( failure != null ) + { + return InternalIndexState.FAILED; + } + File indexLocation = folderLayout.getFolder( indexId ); try { - String failure = failureStorage.loadIndexFailure( indexId ); - if ( failure != null ) - { - failures.put( indexId, failure ); - return InternalIndexState.FAILED; - } - - try ( Directory directory = directoryFactory.open( folderLayout.getFolder( indexId ) ) ) + try ( Directory directory = directoryFactory.open( indexLocation ) ) { boolean status = LuceneIndexWriter.isOnline( directory ); return status ? InternalIndexState.ONLINE : InternalIndexState.POPULATING; } } - catch( CorruptIndexException e ) + catch( IOException e ) { - return InternalIndexState.FAILED; - } - catch( FileNotFoundException e ) - { - failures.put( indexId, "File not found: " + e.getMessage() ); - return InternalIndexState.FAILED; - } - catch( EOFException e ) - { - failures.put( indexId, "EOF encountered: " + e.getMessage() ); - return InternalIndexState.FAILED; - } - catch ( IOException e ) - { - throw new RuntimeException( e ); + log.error( "Failed to open index:" + indexId + ", requesting re-population.", e ); + return InternalIndexState.POPULATING; } } @@ -160,10 +145,6 @@ public String getPopulationFailure( long indexId ) throws IllegalStateException { String failure = failureStorage.loadIndexFailure( indexId ); if ( failure == null ) - { - failure = failures.get( indexId ); - } - if ( failure == null ) { throw new IllegalStateException( "Index " + indexId + " isn't failed" ); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderFactory.java index cbf6e880a7f95..aafc03b196c63 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderFactory.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderFactory.java @@ -25,7 +25,9 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.spi.KernelContext; +import org.neo4j.logging.LogProvider; import static org.neo4j.kernel.api.impl.index.LuceneKernelExtensions.directoryFactory; @@ -41,6 +43,8 @@ public class LuceneSchemaIndexProviderFactory extends public interface Dependencies { Config getConfig(); + + LogService getLogging(); } public LuceneSchemaIndexProviderFactory() @@ -52,12 +56,13 @@ public LuceneSchemaIndexProviderFactory() public LuceneSchemaIndexProvider newInstance( KernelContext context, Dependencies dependencies ) throws Throwable { Config config = dependencies.getConfig(); + LogProvider logging = dependencies.getLogging().getInternalLogProvider(); boolean ephemeral = config.get( GraphDatabaseFacadeFactory.Configuration.ephemeral ); FileSystemAbstraction fileSystem = context.fileSystem(); DirectoryFactory directoryFactory = directoryFactory( ephemeral, fileSystem ); - return new LuceneSchemaIndexProvider( fileSystem, directoryFactory, context.storeDir(), config, + return new LuceneSchemaIndexProvider( fileSystem, directoryFactory, context.storeDir(), logging, config, context.operationalMode() ); } } diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/NonUniqueIndexTests.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/NonUniqueIndexTests.java index aa5007ef59125..cf1fedcbcd9ed 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/NonUniqueIndexTests.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/NonUniqueIndexTests.java @@ -49,6 +49,7 @@ import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.util.Neo4jJobScheduler; import org.neo4j.logging.LogProvider; +import org.neo4j.logging.NullLogProvider; import org.neo4j.test.TargetDirectory; import org.neo4j.udc.UsageDataKeys.OperationalMode; @@ -153,7 +154,8 @@ private List nodeIdsInIndex( int indexId, String value ) throws IOExceptio { Config config = new Config(); SchemaIndexProvider indexProvider = new LuceneSchemaIndexProvider( new DefaultFileSystemAbstraction(), - DirectoryFactory.PERSISTENT, directory.graphDbDir(), new Config(), OperationalMode.single ); + DirectoryFactory.PERSISTENT, directory.graphDbDir(), NullLogProvider.getInstance(), + new Config(), OperationalMode.single ); IndexConfiguration indexConfig = new IndexConfiguration( false ); IndexSamplingConfig samplingConfig = new IndexSamplingConfig( config ); try ( IndexAccessor accessor = indexProvider.getOnlineAccessor( indexId, indexConfig, samplingConfig ); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexRecoveryIT.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexRecoveryIT.java index 86fce5d5a980d..129d67a5779d7 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexRecoveryIT.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexRecoveryIT.java @@ -46,6 +46,7 @@ import org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo; import org.neo4j.kernel.impl.transaction.log.rotation.LogRotation; import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.logging.NullLogProvider; import org.neo4j.test.EphemeralFileSystemRule; import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.udc.UsageDataKeys.OperationalMode; @@ -343,7 +344,7 @@ public Lifecycle newInstance( KernelContext context, LuceneSchemaIndexProviderFa throws Throwable { return new LuceneSchemaIndexProvider( fs.get(), ignoreCloseDirectoryFactory, context.storeDir(), - dependencies.getConfig(), OperationalMode.single ) + dependencies.getLogging().getInternalLogProvider(), dependencies.getConfig(), OperationalMode.single ) { @Override public InternalIndexState getInitialState( long indexId ) @@ -366,7 +367,7 @@ public Lifecycle newInstance( KernelContext context, LuceneSchemaIndexProviderFa throws Throwable { return new LuceneSchemaIndexProvider( fs.get(), ignoreCloseDirectoryFactory, context.storeDir(), - dependencies.getConfig(), OperationalMode.single ) + dependencies.getLogging().getInternalLogProvider(), dependencies.getConfig(), OperationalMode.single ) { @Override public int compareTo( SchemaIndexProvider o ) diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexCorruptionTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexCorruptionTest.java index 3d77783babbc0..0c0dc682ba9af 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexCorruptionTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexCorruptionTest.java @@ -20,6 +20,7 @@ package org.neo4j.kernel.api.impl.index; import org.apache.lucene.index.CorruptIndexException; +import org.hamcrest.CoreMatchers; import org.junit.Rule; import org.junit.Test; @@ -29,31 +30,36 @@ import org.neo4j.kernel.api.index.InternalIndexState; import org.neo4j.kernel.configuration.Config; +import org.neo4j.logging.AssertableLogProvider; import org.neo4j.test.EphemeralFileSystemRule; import org.neo4j.test.TargetDirectory; import org.neo4j.udc.UsageDataKeys.OperationalMode; +import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.neo4j.logging.AssertableLogProvider.inLog; public class LuceneSchemaIndexCorruptionTest { @Rule public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); + @Rule public final EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); + private final AssertableLogProvider logProvider = new AssertableLogProvider(); @Test - public void shouldMarkIndexAsFailedIfIndexIsCorrupt() throws Exception + public void shouldRequestIndexPopulationIfTheIndexIsCorrupt() throws Exception { // Given DirectoryFactory dirFactory = mock(DirectoryFactory.class); // This isn't quite correct, but it will trigger the correct code paths in our code - when(dirFactory.open( any(File.class) )).thenThrow(new CorruptIndexException( "It's borken." )); + CorruptIndexException toThrow = new CorruptIndexException( "It's borken." ); + when(dirFactory.open( any(File.class) )).thenThrow( toThrow ); LuceneSchemaIndexProvider p = getLuceneSchemaIndexProvider( dirFactory ); @@ -61,11 +67,12 @@ public void shouldMarkIndexAsFailedIfIndexIsCorrupt() throws Exception InternalIndexState initialState = p.getInitialState( 1l ); // Then - assertThat( initialState, equalTo(InternalIndexState.FAILED) ); + assertThat( initialState, equalTo(InternalIndexState.POPULATING) ); + logProvider.assertAtLeastOnce( loggedException( toThrow ) ); } @Test - public void shouldMarkAsFailedAndReturnCorrectFailureMessageWhenFailingWithFileNotFoundException() throws Exception + public void shouldRequestIndexPopulationFailingWithFileNotFoundException() throws Exception { // Given DirectoryFactory dirFactory = mock(DirectoryFactory.class); @@ -80,12 +87,12 @@ public void shouldMarkAsFailedAndReturnCorrectFailureMessageWhenFailingWithFileN InternalIndexState initialState = p.getInitialState( 1l ); // Then - assertThat( initialState, equalTo(InternalIndexState.FAILED) ); - assertThat( p.getPopulationFailure( 1l ), equalTo( "File not found: " + toThrow.getMessage() ) ); + assertThat( initialState, equalTo(InternalIndexState.POPULATING) ); + logProvider.assertAtLeastOnce( loggedException( toThrow ) ); } @Test - public void shouldMarkAsFailedAndReturnCorrectFailureMessageWhenFailingWithEOFException() throws Exception + public void shouldRequestIndexPopulationWhenFailingWithEOFException() throws Exception { // Given DirectoryFactory dirFactory = mock(DirectoryFactory.class); @@ -100,43 +107,19 @@ public void shouldMarkAsFailedAndReturnCorrectFailureMessageWhenFailingWithEOFEx InternalIndexState initialState = p.getInitialState( 1l ); // Then - assertThat( initialState, equalTo(InternalIndexState.FAILED) ); - assertThat( p.getPopulationFailure( 1l ), equalTo( "EOF encountered: " + toThrow.getMessage() ) ); + assertThat( initialState, equalTo(InternalIndexState.POPULATING) ); + logProvider.assertAtLeastOnce( loggedException( toThrow ) ); } - @Test - public void shouldDenyFailureForNonFailedIndex() throws Exception + private LuceneSchemaIndexProvider getLuceneSchemaIndexProvider( DirectoryFactory dirFactory ) { - - // Given - DirectoryFactory dirFactory = mock(DirectoryFactory.class); - - // This isn't quite correct, but it will trigger the correct code paths in our code - EOFException toThrow = new EOFException( "/some/path/somewhere" ); - when(dirFactory.open( any(File.class) )).thenThrow( toThrow ); - - LuceneSchemaIndexProvider p = getLuceneSchemaIndexProvider( dirFactory ); - - // When - InternalIndexState initialState = p.getInitialState( 1l ); - - // Then - assertThat( initialState, equalTo( InternalIndexState.FAILED ) ); - boolean exceptionOnOtherIndexThrown = false; - try - { - p.getPopulationFailure( 2l ); - } - catch( IllegalStateException e ) - { - exceptionOnOtherIndexThrown = true; - } - assertTrue( exceptionOnOtherIndexThrown ); + return new LuceneSchemaIndexProvider( fs.get(), dirFactory, testDirectory.graphDbDir(), + logProvider, new Config(), OperationalMode.single ); } - private LuceneSchemaIndexProvider getLuceneSchemaIndexProvider( DirectoryFactory dirFactory ) + private static AssertableLogProvider.LogMatcher loggedException( Throwable exception ) { - return new LuceneSchemaIndexProvider( fs.get(), dirFactory, testDirectory.graphDbDir(), - new Config(), OperationalMode.single ); + return inLog( CoreMatchers.any( String.class ) ) + .error( CoreMatchers.any( String.class ), sameInstance( exception ) ); } } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexPopulatorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexPopulatorTest.java index ec13b8af76d8e..429f33b226bb0 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexPopulatorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexPopulatorTest.java @@ -47,6 +47,7 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.IndexStoreView; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.logging.NullLogProvider; import org.neo4j.test.EphemeralFileSystemRule; import org.neo4j.udc.UsageDataKeys.OperationalMode; @@ -244,7 +245,7 @@ public void before() throws Exception DirectoryFactory directoryFactory = new DirectoryFactory.Single( new DirectoryFactory.UncloseableDirectory( directory ) ); provider = new LuceneSchemaIndexProvider( fs.get(), directoryFactory, new File( "target/whatever" ), - new Config(), OperationalMode.single ); + NullLogProvider.getInstance(), new Config(), OperationalMode.single ); indexDescriptor = new IndexDescriptor( 42, propertyKeyId ); indexStoreView = mock( IndexStoreView.class ); IndexConfiguration indexConfig = new IndexConfiguration( false ); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderTest.java index 19715f282fc58..b580c2d8f564c 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProviderTest.java @@ -33,6 +33,7 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.logging.NullLogProvider; import org.neo4j.udc.UsageDataKeys.OperationalMode; import static org.neo4j.helpers.collection.MapUtil.stringMap; @@ -105,6 +106,6 @@ private IndexAccessor getIndexAccessor( Config readOnlyConfig, LuceneSchemaIndex private LuceneSchemaIndexProvider getLuceneSchemaIndexProvider( Config config, DirectoryFactory directoryFactory ) { - return new LuceneSchemaIndexProvider( fs, directoryFactory, graphDbDir, config, OperationalMode.single ); + return new LuceneSchemaIndexProvider( fs, directoryFactory, graphDbDir, NullLogProvider.getInstance(), config, OperationalMode.single ); } } diff --git a/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java b/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java new file mode 100644 index 0000000000000..20c49a92d2ed2 --- /dev/null +++ b/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.index; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.hamcrest.Matcher; +import org.junit.Rule; +import org.junit.Test; + +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.schema.IndexDefinition; +import org.neo4j.graphdb.schema.Schema; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.test.DatabaseRule; +import org.neo4j.test.EmbeddedDatabaseRule; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.DynamicLabel.label; +import static org.neo4j.graphdb.schema.Schema.IndexState.ONLINE; + +public class IndexFailureOnStartupTest +{ + private static final Label PERSON = label( "Person" ); + @Rule + public final DatabaseRule db = new EmbeddedDatabaseRule(); + + @Test + public void failedIndexShouldRepairAutomatically() throws Exception + { + // given + try ( Transaction tx = db.beginTx() ) + { + db.schema().indexFor( PERSON ).on( "name" ).create(); + tx.success(); + } + awaitIndexesOnline( 5, SECONDS ); + createNamed( PERSON, "Johan" ); + // when - we restart the database in a state where the index is not operational + db.restartDatabase( new DeleteIndexFile( "_0.cfs" ) ); + // then - the database should still be operational + createNamed( PERSON, "Lars" ); + indexStateShouldBe( equalTo( ONLINE ) ); + assertFindsNamed( PERSON, "Lars" ); + } + + @Test + public void shouldNotBeAbleToViolateConstraintWhenBackingIndexFailsToOpen() throws Exception + { + // given + try ( Transaction tx = db.beginTx() ) + { + db.schema().constraintFor( PERSON ).assertPropertyIsUnique( "name" ).create(); + tx.success(); + } + createNamed( PERSON, "Lars" ); + // when - we restart the database in a state where the index is not operational + db.restartDatabase( new DeleteIndexFile( "_0.cfs" ) ); + // then - we must not be able to violate the constraint + createNamed( PERSON, "Johan" ); + Throwable failure = null; + try + { + createNamed( PERSON, "Lars" ); + } + catch ( Throwable e ) + { + // this must fail, otherwise we have violated the constraint + failure = e; + } + assertNotNull( failure ); + indexStateShouldBe( equalTo( ONLINE ) ); + } + + private void awaitIndexesOnline( int timeout, TimeUnit unit ) + { + try ( Transaction tx = db.beginTx() ) + { + db.schema().awaitIndexesOnline( timeout, unit ); + tx.success(); + } + } + + private void assertFindsNamed( Label label, String name ) + { + try ( Transaction tx = db.beginTx() ) + { + assertNotNull( "Must be able to find node created while index was offline", + db.findNode( label, "name", name ) ); + tx.success(); + } + } + + private void indexStateShouldBe( Matcher matchesExpectation ) + { + try ( Transaction tx = db.beginTx() ) + { + for ( IndexDefinition index : db.schema().getIndexes() ) + { + assertThat( db.schema().getIndexState( index ), matchesExpectation ); + } + tx.success(); + } + } + + private void createNamed( Label label, String name ) + { + try ( Transaction tx = db.beginTx() ) + { + db.createNode( label ).setProperty( "name", name ); + tx.success(); + } + } + + private static class DeleteIndexFile implements DatabaseRule.RestartAction + { + private final String source; + + DeleteIndexFile( String source ) + { + this.source = source; + } + + @Override + public void run( FileSystemAbstraction fs, File base ) throws IOException + { + fs.deleteFile( new File( soleIndexDir( fs, base ), source ) ); + } + } + + private static File soleIndexDir( FileSystemAbstraction fs, File base ) + { + File[] indexes = fs.listFiles( new File( base, "schema/index/lucene" ) ); + assert indexes.length == 1 : "expecting only a single index directory"; + return indexes[0]; + } +} diff --git a/enterprise/enterprise-performance-tests/src/main/java/org/neo4j/perftest/enterprise/ccheck/ConsistencyPerformanceCheck.java b/enterprise/enterprise-performance-tests/src/main/java/org/neo4j/perftest/enterprise/ccheck/ConsistencyPerformanceCheck.java index df68a0d129305..0041945ad4dce 100644 --- a/enterprise/enterprise-performance-tests/src/main/java/org/neo4j/perftest/enterprise/ccheck/ConsistencyPerformanceCheck.java +++ b/enterprise/enterprise-performance-tests/src/main/java/org/neo4j/perftest/enterprise/ccheck/ConsistencyPerformanceCheck.java @@ -148,7 +148,7 @@ private static DirectStoreAccess createScannableStores( File storeDir, Config tu SchemaIndexProvider indexes = new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, - storeDir, tuningConfiguration, operationalMode ); + storeDir, NullLogProvider.getInstance(), tuningConfiguration, operationalMode ); LuceneLabelScanStoreBuilder labelScanStoreBuilder = new LuceneLabelScanStoreBuilder( storeDir, neoStores, fileSystem, tuningConfiguration, operationalMode, NullLogProvider.getInstance() ); LabelScanStore labelScanStore = labelScanStoreBuilder.build(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java index bcba20297b9df..48608bc9129bb 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java @@ -68,6 +68,7 @@ import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant; import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.logging.NullLogProvider; import org.neo4j.register.Register.DoubleLong; import org.neo4j.test.DoubleLatch; import org.neo4j.test.ha.ClusterRule; @@ -556,14 +557,16 @@ public Lifecycle newInstance( KernelContext context, SchemaIndexHaIT.IndexProvid { ControlledSchemaIndexProvider provider = new ControlledSchemaIndexProvider( new LuceneSchemaIndexProvider( new DefaultFileSystemAbstraction(), - DirectoryFactory.PERSISTENT, context.storeDir(), deps.config(), context.operationalMode() ) ); + DirectoryFactory.PERSISTENT, context.storeDir(), NullLogProvider.getInstance(), + deps.config(), context.operationalMode() ) ); perDbIndexProvider.put( deps.db(), provider ); return provider; } else { return new LuceneSchemaIndexProvider( new DefaultFileSystemAbstraction(), - DirectoryFactory.PERSISTENT, context.storeDir(), deps.config(), context.operationalMode() ); + DirectoryFactory.PERSISTENT, context.storeDir(), NullLogProvider.getInstance(), + deps.config(), context.operationalMode() ); } } }