Skip to content

Commit

Permalink
Merge branch '3.0' into 3.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrisvest
chrisvest committed Aug 19, 2016
2 parents f23dfd4 + b4c5f93 commit 72f261b
Show file tree
Hide file tree
Showing 34 changed files with 617 additions and 171 deletions.
18 changes: 0 additions & 18 deletions community/bolt/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,22 +123,4 @@
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
2 changes: 1 addition & 1 deletion community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ storeDir, fullStoreLabelUpdateStream( () -> indexStoreView ),
SchemaIndexProvider indexes = new LuceneSchemaIndexProvider(
fileSystem,
DirectoryFactory.PERSISTENT,
storeDir, consistencyCheckerConfig, operationalMode );
storeDir, logProvider, consistencyCheckerConfig, operationalMode );

int numberOfThreads = defaultConsistencyCheckThreadsNumber();
Statistics statistics;
Expand Down
4 changes: 2 additions & 2 deletions ...ity/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,8 @@ public DirectStoreAccess directStoreAccess()

private SchemaIndexProvider createIndexes( FileSystemAbstraction fileSystem, Config config, OperationalMode operationalMode )
{
return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, config,
operationalMode );
return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory,
FormattedLogProvider.toOutputStream( System.out ), config, operationalMode );
}

public File directory()
Expand Down
5 changes: 5 additions & 0 deletions community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,4 +581,9 @@ public static BoltConnector boltConnector( String key )
{
return new BoltConnector( key );
}

@Description( "Create an archive of an index before re-creating it if failing to load on startup." )
@Internal
public static final Setting<Boolean> archive_failed_index = setting(
"unsupported.dbms.index.archive_failed", BOOLEAN, "false" );
}
33 changes: 14 additions & 19 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.neo4j.logging.LogProvider;

import static java.lang.String.format;
import static org.neo4j.kernel.impl.api.index.IndexPopulationFailure.failure;

/**
* Helper class of {@link IndexingService}. Used mainly as factory of index proxies.
Expand Down Expand Up @@ -60,15 +59,15 @@ public IndexProxyCreator( IndexSamplingConfig samplingConfig,
public IndexProxy createPopulatingIndexProxy( final long ruleId,
final IndexDescriptor descriptor,
final SchemaIndexProvider.Descriptor providerDescriptor,
final boolean constraint,
IndexConfiguration config,
final boolean flipToTentative,
final IndexingService.Monitor monitor,
final IndexPopulationJob populationJob ) throws IOException
{
final FlippableIndexProxy flipper = new FlippableIndexProxy();

// TODO: This is here because there is a circular dependency from PopulatingIndexProxy to FlippableIndexProxy
final String indexUserDescription = indexUserDescription( descriptor, providerDescriptor );
final IndexConfiguration config = IndexConfiguration.of( constraint );
IndexPopulator populator = populatorFromProvider( providerDescriptor, ruleId, descriptor, config,
samplingConfig );

Expand All @@ -92,23 +91,16 @@ public IndexProxy createPopulatingIndexProxy( final long ruleId,

// Prepare for flipping to online mode
flipper.setFlipTarget( () -> {
try
monitor.populationCompleteOn( descriptor );
OnlineIndexProxy onlineProxy = new OnlineIndexProxy(
descriptor, config, onlineAccessorFromProvider( providerDescriptor, ruleId,
config, samplingConfig ), storeView, providerDescriptor, true
);
if ( flipToTentative )
{
monitor.populationCompleteOn( descriptor );
OnlineIndexProxy onlineProxy = new OnlineIndexProxy(
descriptor, config, onlineAccessorFromProvider( providerDescriptor, ruleId,
config, samplingConfig ), storeView, providerDescriptor, true
);
if ( constraint )
{
return new TentativeConstraintIndexProxy( flipper, onlineProxy );
}
return onlineProxy;
}
catch ( IOException e )
{
return createFailedIndexProxy( ruleId, descriptor, providerDescriptor, constraint, failure( e ) );
return new TentativeConstraintIndexProxy( flipper, onlineProxy );
}
return onlineProxy;
} );

return new ContractCheckingIndexProxy( flipper, false );
Expand Down Expand Up @@ -141,7 +133,10 @@ public IndexProxy createOnlineIndexProxy( long ruleId,
}
catch ( IOException e )
{
return createFailedIndexProxy( ruleId, descriptor, providerDescriptor, unique, failure( e ) );
logProvider.getLog( getClass() ).error( "Failed to open index: " + ruleId +
" (" + descriptor.userDescription( tokenNameLookup ) +
"), requesting re-population.", e );
return createRecoveringIndexProxy( descriptor, providerDescriptor, unique );
}
}

Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxyFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@

public interface IndexProxyFactory
{
IndexProxy create();
IndexProxy create() throws Exception;
}
38 changes: 24 additions & 14 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException;
import org.neo4j.kernel.api.exceptions.schema.ConstraintVerificationFailedKernelException;
import org.neo4j.kernel.api.index.IndexConfiguration;
import org.neo4j.kernel.api.index.IndexDescriptor;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.InternalIndexState;
Expand Down Expand Up @@ -209,9 +210,18 @@ public void init()
break;
case POPULATING:
// The database was shut down during population, or a crash has occurred, or some other sad thing.

indexProxy = indexProxyCreator
.createRecoveringIndexProxy( descriptor, providerDescriptor, constraint );
if ( constraint && indexRule.getOwningConstraint() == null )
{
// don't bother rebuilding if we are going to throw the index away anyhow
indexProxy = indexProxyCreator.createFailedIndexProxy(
indexId, descriptor, providerDescriptor, constraint,
failure( "Constraint for index was not committed." ) );
}
else
{
indexProxy = indexProxyCreator
.createRecoveringIndexProxy( descriptor, providerDescriptor, constraint );
}
break;
case FAILED:
IndexPopulationFailure failure = failure( provider.getPopulationFailure( indexId ) );
Expand Down Expand Up @@ -271,17 +281,16 @@ public void start() throws Exception
for ( Map.Entry<Long,RebuildingIndexDescriptor> entry : rebuildingDescriptors.entrySet() )
{
long indexId = entry.getKey();
RebuildingIndexDescriptor descriptors = entry.getValue();

/*
* Passing in "false" for unique here may seem surprising, and.. well, yes, it is, I was surprised too.
* However, it is actually perfectly safe, because whenever we have constraint indexes here, they will
* be in a state where they didn't finish populating, and despite the fact that we re-create them here,
* they will get dropped as soon as recovery is completed by the constraint system.
*/
RebuildingIndexDescriptor descriptor = entry.getValue();

IndexProxy proxy = indexProxyCreator.createPopulatingIndexProxy(
indexId, descriptors.getIndexDescriptor(), descriptors.getProviderDescriptor(), false,
monitor, populationJob );
indexId,
descriptor.getIndexDescriptor(),
descriptor.getProviderDescriptor(),
descriptor.getConfiguration(),
false, // never pass through a tentative online state during recovery
monitor,
populationJob );
proxy.start();
indexMap.putIndexProxy( indexId, proxy );
}
Expand Down Expand Up @@ -529,7 +538,8 @@ public void createIndexes( IndexRule... rules ) throws IOException
{
populationJob = populationJob == null ? newIndexPopulationJob() : populationJob;
index = indexProxyCreator.createPopulatingIndexProxy(
ruleId, descriptor, providerDescriptor, constraint, monitor, populationJob );
ruleId, descriptor, providerDescriptor, IndexConfiguration.of( constraint ), constraint,
monitor, populationJob );
index.start();
}
else
Expand Down
1 change: 1 addition & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/IndexRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ public Long getOwningConstraint()
{
throw new IllegalStateException( "Can only get owner from constraint indexes." );
}
long owningConstraint = this.owningConstraint;
if ( owningConstraint == NO_OWNING_CONSTRAINT )
{
return null;
Expand Down
109 changes: 108 additions & 1 deletion community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@
*/
package org.neo4j.kernel.impl.api.index;

import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
Expand Down Expand Up @@ -120,6 +124,7 @@
import static org.neo4j.helpers.collection.Iterators.iterator;
import static org.neo4j.helpers.collection.Iterators.loop;
import static org.neo4j.helpers.collection.MapUtil.stringMap;
import static org.neo4j.kernel.api.index.InternalIndexState.FAILED;
import static org.neo4j.kernel.api.index.InternalIndexState.ONLINE;
import static org.neo4j.kernel.api.index.InternalIndexState.POPULATING;
import static org.neo4j.kernel.impl.api.index.IndexUpdateMode.RECOVERY;
Expand Down Expand Up @@ -817,7 +822,7 @@ public void shouldCreateMultipleIndexesInOneCall() throws Exception
// GIVEN
IndexingService.Monitor monitor = IndexingService.NO_MONITOR;
IndexingService indexing = newIndexingServiceWithMockedDependencies( populator, accessor,
withData( NodePropertyUpdate.add( 0, 0, "value", new long[] {1} ) ), monitor );
withData( NodePropertyUpdate.add( 0, 0, "value", new long[]{1} ) ), monitor );
life.start();

// WHEN
Expand All @@ -837,6 +842,108 @@ public void shouldCreateMultipleIndexesInOneCall() throws Exception
waitForIndexesToComeOnline( indexing, 0, 1, 2 );
}

@Test
public void shouldStoreIndexFailureWhenFailingToCreateOnlineAccessorAfterPopulating() throws Exception
{
// given
long indexId = 1;
IndexingService indexing = newIndexingServiceWithMockedDependencies( populator, accessor, withData() );

IOException exception = new IOException( "Expected failure" );
when( nameLookup.labelGetName( labelId ) ).thenReturn( "TheLabel" );
when( nameLookup.propertyKeyGetName( propertyKeyId ) ).thenReturn( "propertyKey" );

when( indexProvider.getOnlineAccessor(
eq( indexId ), any( IndexConfiguration.class ), any( IndexSamplingConfig.class ) ) )
.thenThrow( exception );

life.start();
ArgumentCaptor<Boolean> closeArgs = ArgumentCaptor.forClass( Boolean.class );

// when
indexing.createIndexes( indexRule( indexId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR ) );
verify( populator, timeout( 1000 ).times( 2 ) ).close( closeArgs.capture() );

// then
assertEquals( FAILED, indexing.getIndexProxy( 1 ).getState() );
assertEquals( asList( true, false ), closeArgs.getAllValues() );
assertThat( storedFailure(), containsString( "java.io.IOException: Expected failure\n\tat " ) );
logProvider.assertAtLeastOnce( inLog( IndexPopulationJob.class ).error( equalTo(
"Failed to populate index: [:TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]]" ),
causedBy( exception ) ) );
logProvider.assertNone( inLog( IndexPopulationJob.class ).info(
"Index population completed. Index is now online: [%s]",
":TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]" ) );
}

@Test
public void shouldStoreIndexFailureWhenFailingToCreateOnlineAccessorAfterRecoveringPopulatingIndex() throws Exception
{
// given
long indexId = 1;
IndexingService indexing = newIndexingServiceWithMockedDependencies( populator, accessor, withData(),
indexRule( indexId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR ) );

IOException exception = new IOException( "Expected failure" );
when( nameLookup.labelGetName( labelId ) ).thenReturn( "TheLabel" );
when( nameLookup.propertyKeyGetName( propertyKeyId ) ).thenReturn( "propertyKey" );

when( indexProvider.getInitialState( indexId ) ).thenReturn( POPULATING );
when( indexProvider.getOnlineAccessor(
eq( indexId ), any( IndexConfiguration.class ), any( IndexSamplingConfig.class ) ) )
.thenThrow( exception );

life.start();
ArgumentCaptor<Boolean> closeArgs = ArgumentCaptor.forClass( Boolean.class );

// when
verify( populator, timeout( 1000 ).times( 2 ) ).close( closeArgs.capture() );

// then
assertEquals( FAILED, indexing.getIndexProxy( 1 ).getState() );
assertEquals( asList( true, false ), closeArgs.getAllValues() );
assertThat( storedFailure(), containsString( "java.io.IOException: Expected failure\n\tat " ) );
logProvider.assertAtLeastOnce( inLog( IndexPopulationJob.class ).error( equalTo(
"Failed to populate index: [:TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]]" ),
causedBy( exception ) ) );
logProvider.assertNone( inLog( IndexPopulationJob.class ).info(
"Index population completed. Index is now online: [%s]",
":TheLabel(propertyKey) [provider: {key=quantum-dex, version=25.0}]" ) );
}

static Matcher<? extends Throwable> causedBy( final Throwable exception )
{
return new TypeSafeMatcher<Throwable>()
{
@Override
protected boolean matchesSafely( Throwable item )
{
while ( item != null )
{
if ( item == exception )
{
return true;
}
item = item.getCause();
}
return false;
}

@Override
public void describeTo( Description description )
{
description.appendText( "exception caused by " ).appendValue( exception );
}
};
}

private String storedFailure() throws IOException
{
ArgumentCaptor<String> reason = ArgumentCaptor.forClass( String.class );
verify( populator ).markAsFailed( reason.capture() );
return reason.getValue();
}

private static class ControlledIndexPopulator extends IndexPopulator.Adapter
{
private final DoubleLatch latch;
Expand Down

0 comments on commit 72f261b

Please sign in to comment.