Skip to content

Commit

Permalink
IndexingService log when deprecated IndexProviders are used
Browse files Browse the repository at this point in the history 
Log a warning if an index backed by a deprecated index provider is loaded during
IndexingService.init or IndexingService.start.

Deprecated index providers are:
- lucene-1.0
- lucene+native-1.0
- lucene+native-2.0

Also improve AssertableLogProvider to be able to verify multiple matchers
in any order on a single log call.
  • Loading branch information
@burqen
burqen committed Oct 11, 2018
1 parent 7798054 commit 20b0f1e
Show file tree
Hide file tree
Showing 7 changed files with 287 additions and 37 deletions.
2 changes: 1 addition & 1 deletion community/community-it/kernel-it/src/test/java/recovery/RecoveryCleanupIT.java
View file
Expand Up @@ -188,7 +188,7 @@ private void nativeIndexMustLogCrashPointerCleanupDuringRecovery( GraphDatabaseS
matchers.add( indexRecoveryFinishedLogMatcher( subType ) );
matchers.add( indexRecoveryLogMatcher( "Schema index cleanup job closed", subType ) );
}
matchers.forEach( logProvider::assertContainsExactlyOneMessageMatching );
matchers.forEach( logProvider::assertContainsExactlyOneMessageMatchingInAnyOrder );
}

private Matcher<String> indexRecoveryLogMatcher( String logMessage, String subIndexProviderKey )
Expand Down
17 changes: 12 additions & 5 deletions community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java
View file
Expand Up @@ -613,19 +613,21 @@ public class GraphDatabaseSettings implements LoadableConfig

public enum SchemaIndex
{
NATIVE_BTREE10( "native-btree", "1.0" ),
NATIVE20( "lucene+native", "2.0" ),
NATIVE10( "lucene+native", "1.0" ),
LUCENE10( "lucene", "1.0" );
NATIVE_BTREE10( "native-btree", "1.0", false ),
NATIVE20( "lucene+native", "2.0", true ),
NATIVE10( "lucene+native", "1.0", true ),
LUCENE10( "lucene", "1.0", true );

private final String providerKey;
private final String providerVersion;
private final boolean deprecated;
private final String providerName;

SchemaIndex( String providerKey, String providerVersion )
SchemaIndex( String providerKey, String providerVersion, boolean deprecated )
{
this.providerKey = providerKey;
this.providerVersion = providerVersion;
this.deprecated = deprecated;
this.providerName = toProviderName( providerKey, providerVersion );
}

Expand All @@ -644,6 +646,11 @@ public String providerVersion()
return providerVersion;
}

public boolean deprecated()
{
return deprecated;
}

private static String toProviderName( String providerName, String providerVersion )
{
return providerName + "-" + providerVersion;
Expand Down
45 changes: 43 additions & 2 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java
View file
Expand Up @@ -19,6 +19,7 @@
*/
package org.neo4j.kernel.impl.api.index;

import org.apache.commons.lang3.mutable.MutableBoolean;
import org.eclipse.collections.api.LongIterable;
import org.eclipse.collections.api.block.procedure.primitive.LongObjectProcedure;
import org.eclipse.collections.api.map.primitive.MutableLongObjectMap;
Expand All @@ -31,16 +32,21 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.neo4j.function.ThrowingConsumer;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.internal.kernel.api.InternalIndexState;
import org.neo4j.internal.kernel.api.TokenNameLookup;
Expand Down Expand Up @@ -204,15 +210,19 @@ public void init()
indexMapRef.modify( indexMap ->
{
Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>( InternalIndexState.class );
Map<IndexProviderDescriptor,List<IndexLogRecord>> indexProviders = new HashMap<>();
for ( StoreIndexDescriptor indexDescriptor : indexDescriptors )
{
IndexProxy indexProxy;

IndexProviderDescriptor providerDescriptor = indexDescriptor.providerDescriptor();
IndexProvider provider = providerMap.lookup( providerDescriptor );
InternalIndexState initialState = provider.getInitialState( indexDescriptor );
IndexLogRecord indexLogRecord = new IndexLogRecord( indexDescriptor );
indexStates.computeIfAbsent( initialState, internalIndexState -> new ArrayList<>() )
.add( new IndexLogRecord( indexDescriptor ) );
.add( indexLogRecord );
indexProviders.computeIfAbsent( providerDescriptor, indexProviderDescriptor -> new ArrayList<>() )
.add( indexLogRecord );

log.debug( indexStateInfo( "init", initialState, indexDescriptor ) );
switch ( initialState )
Expand All @@ -237,6 +247,7 @@ public void init()
indexMap.putIndexProxy( indexProxy );
}
logIndexStateSummary( "init", indexStates );
logIndexProviderSummary( "init", indexProviders );
return indexMap;
} );
}
Expand Down Expand Up @@ -269,14 +280,19 @@ public void start()
indexMapRef.modify( indexMap ->
{
Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>( InternalIndexState.class );
Map<IndexProviderDescriptor,List<IndexLogRecord>> indexProviders = new HashMap<>();

// Find all indexes that are not already online, do not require rebuilding, and create them
indexMap.forEachIndexProxy( ( indexId, proxy ) ->
{
InternalIndexState state = proxy.getState();
StoreIndexDescriptor descriptor = proxy.getDescriptor();
IndexProviderDescriptor providerDescriptor = descriptor.providerDescriptor();
IndexLogRecord indexLogRecord = new IndexLogRecord( descriptor );
indexStates.computeIfAbsent( state, internalIndexState -> new ArrayList<>() )
.add( new IndexLogRecord( descriptor ) );
.add( indexLogRecord );
indexProviders.computeIfAbsent( providerDescriptor, indexProviderDescriptor -> new ArrayList<>() )
.add( indexLogRecord );
log.debug( indexStateInfo( "start", state, descriptor ) );
switch ( state )
{
Expand All @@ -295,6 +311,7 @@ public void start()
}
} );
logIndexStateSummary( "start", indexStates );
logIndexProviderSummary( "start", indexProviders );

// Drop placeholder proxies for indexes that need to be rebuilt
dropRecoveringIndexes( indexMap, rebuildingDescriptors.keySet() );
Expand Down Expand Up @@ -774,6 +791,30 @@ private void logIndexStateSummary( String method, Map<InternalIndexState,List<In
log.info( format( "IndexingService.%s: indexes not specifically mentioned above are %s", method, mostPopularState ) );
}

private void logIndexProviderSummary( String method, Map<IndexProviderDescriptor,List<IndexLogRecord>> indexProviders )
{
Set<String> deprecatedIndexProviders = Arrays.stream( GraphDatabaseSettings.SchemaIndex.values() )
.filter( GraphDatabaseSettings.SchemaIndex::deprecated )
.map( GraphDatabaseSettings.SchemaIndex::providerName )
.collect( Collectors.toSet() );
StringJoiner joiner = new StringJoiner( ", ", format( "IndexingService.%s: Deprecated index providers in use: ", method ), "" );
MutableBoolean anyDeprecated = new MutableBoolean();
indexProviders.forEach( ( indexProviderDescriptor, indexLogRecords ) ->
{
if ( deprecatedIndexProviders.contains( indexProviderDescriptor.name() ) )
{
anyDeprecated.setTrue();
int numberOfIndexes = indexLogRecords.size();
joiner.add( indexProviderDescriptor.name() + " (" + numberOfIndexes + (numberOfIndexes == 1 ? " index" : " indexes") + ")" );
}
} );
if ( anyDeprecated.getValue() )
{
// todo use userLog here
log.info( joiner.toString() );
}
}

private final class IndexPopulationStarter implements Function<IndexMap,IndexMap>
{
private final boolean verifyBeforeFlipping;
Expand Down

0 comments on commit 20b0f1e

Please sign in to comment.