Skip to content

Commit

Permalink
Merge pull request #9804 from tinwelint/3.3-nsni-combined-index-provi…
Browse files Browse the repository at this point in the history
…der--migration

Native + Lucene Fusion index as default
  • Loading branch information
tinwelint committed Aug 24, 2017
2 parents 16b8e4a + 49ab091 commit 7d0e62c
Show file tree
Hide file tree
Showing 88 changed files with 1,600 additions and 917 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,13 @@
import org.neo4j.io.pagecache.tracing.PageCacheTracer;
import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier;
import org.neo4j.kernel.api.direct.DirectStoreAccess;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensions;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
import org.neo4j.kernel.impl.factory.OperationalMode;
import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
import org.neo4j.kernel.impl.logging.SimpleLogService;
import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.store.StoreAccess;
Expand All @@ -64,8 +63,12 @@
import org.neo4j.logging.LogProvider;

import static java.lang.String.format;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.RECOVERY_PREVENTING_COLLECTOR;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.instantiateKernelExtensions;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.loadSchemaIndexProviders;
import static org.neo4j.io.file.Files.createOrOpenAsOuputStream;
import static org.neo4j.kernel.configuration.Settings.TRUE;
import static org.neo4j.kernel.impl.factory.DatabaseInfo.COMMUNITY;

public class ConsistencyCheckService
{
Expand Down Expand Up @@ -204,7 +207,6 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr
{
Log log = logProvider.getLog( getClass() );
config.augment( GraphDatabaseSettings.read_only, TRUE );
OperationalMode operationalMode = OperationalMode.single;

StoreFactory factory = new StoreFactory( storeDir, config,
new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider );
Expand All @@ -225,11 +227,17 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr

// Bootstrap kernel extensions
LifeSupport life = new LifeSupport();
KernelExtensions extensions = life.add( instantiateKernelExtensions( storeDir,
fileSystem, config, new SimpleLogService( logProvider, logProvider ), pageCache,
RECOVERY_PREVENTING_COLLECTOR,
// May be enterprise edition, but in consistency checker we only care about the operational mode
COMMUNITY ) );

try ( NeoStores neoStores = factory.openAllNeoStores() )
{
life.start();
SchemaIndexProvider indexes = life.add( new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT,
storeDir, logProvider, config, operationalMode ) );

SchemaIndexProviderMap indexes = loadSchemaIndexProviders( extensions );

LabelScanStore labelScanStore =
new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, true, new Monitors(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ private PrimitiveLongIterator queryIndexOrEmpty( IndexReader reader, IndexQuery[
Arrays.toString( query ) ), e );
}

return reader.hasFullNumberPrecision()
return reader.hasFullNumberPrecision( query )
? indexedNodeIds : LookupFilter.exactIndexMatches( propertyReader, indexedNodeIds, query );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.InternalIndexState;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.store.RecordStore;
import org.neo4j.kernel.impl.store.SchemaStorage;
Expand All @@ -43,7 +44,7 @@ public class IndexAccessors implements Closeable
private final List<IndexRule> onlineIndexRules = new ArrayList<>();
private final List<IndexRule> notOnlineIndexRules = new ArrayList<>();

public IndexAccessors( SchemaIndexProvider provider,
public IndexAccessors( SchemaIndexProviderMap providers,
RecordStore<DynamicRecord> schemaStore,
IndexSamplingConfig samplingConfig ) throws IOException
{
Expand All @@ -58,7 +59,8 @@ public IndexAccessors( SchemaIndexProvider provider,
// - populating indexes will be rebuilt on next startup
// - failed indexes have to be dropped by the user anyways
IndexRule indexRule = rules.next();
if ( InternalIndexState.ONLINE == provider.getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) )
if ( InternalIndexState.ONLINE == provider( providers, indexRule )
.getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) )
{
onlineIndexRules.add( indexRule );
}
Expand All @@ -81,10 +83,16 @@ public IndexAccessors( SchemaIndexProvider provider,
for ( IndexRule indexRule : onlineIndexRules )
{
long indexId = indexRule.getId();
accessors.put( indexId, provider.getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) );
accessors.put( indexId, provider( providers, indexRule )
.getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) );
}
}

private SchemaIndexProvider provider( SchemaIndexProviderMap providers, IndexRule indexRule )
{
return providers.apply( indexRule.getProviderDescriptor() );
}

public Collection<IndexRule> notOnlineRules()
{
return notOnlineIndexRules;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.consistency.internal;

import java.io.File;

import org.neo4j.helpers.Service;
import org.neo4j.index.internal.gbptree.CleanupJob;
import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.extension.KernelExtensions;
import org.neo4j.kernel.extension.UnsatisfiedDependencyStrategies;
import org.neo4j.kernel.extension.dependency.AllByPrioritySelectionStrategy;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.factory.DatabaseInfo;
import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.spi.KernelContext;
import org.neo4j.kernel.impl.spi.SimpleKernelContext;
import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap;
import org.neo4j.kernel.impl.util.Dependencies;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;

/**
* Utility for loading {@link SchemaIndexProvider} instances from {@link KernelExtensions}.
*/
public class SchemaIndexExtensionLoader
{
/**
* Used in scenarios where recovery isn't allowed.
*/
public static final RecoveryCleanupWorkCollector RECOVERY_PREVENTING_COLLECTOR = new RecoveryPreventingCollector();

public static SchemaIndexProviderMap loadSchemaIndexProviders( KernelExtensions extensions )
{
AllByPrioritySelectionStrategy<SchemaIndexProvider> indexProviderSelection = new AllByPrioritySelectionStrategy<>();
SchemaIndexProvider defaultIndexProvider =
extensions.resolveDependency( SchemaIndexProvider.class, indexProviderSelection );
return new DefaultSchemaIndexProviderMap( defaultIndexProvider,
indexProviderSelection.lowerPrioritizedCandidates() );
}

@SuppressWarnings( "unchecked" )
public static KernelExtensions instantiateKernelExtensions( File storeDir, FileSystemAbstraction fileSystem,
Config config, LogService logService, PageCache pageCache,
RecoveryCleanupWorkCollector recoveryCollector, DatabaseInfo databaseInfo )
{
Dependencies deps = new Dependencies();
deps.satisfyDependencies( fileSystem, config, logService, pageCache, recoveryCollector );
@SuppressWarnings( "rawtypes" )
Iterable kernelExtensions = Service.load( KernelExtensionFactory.class );
KernelContext kernelContext = new SimpleKernelContext( storeDir, databaseInfo, deps );
return new KernelExtensions( kernelContext, kernelExtensions, deps, UnsatisfiedDependencyStrategies.ignore() );
}

private static class RecoveryPreventingCollector extends LifecycleAdapter implements RecoveryCleanupWorkCollector
{
@Override
public void add( CleanupJob job )
{
if ( job.needed() )
{
throw new IllegalStateException( "Consistency checker should not do recovery" );
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -216,19 +216,9 @@ public void shouldReportMissingSchemaIndex() throws Exception
File storeDir = testDirectory.absolutePath();
GraphDatabaseService gds = getGraphDatabaseService( storeDir );

IndexDefinition indexDefinition;

try ( Transaction tx = gds.beginTx() )
{
indexDefinition = gds.schema().indexFor( Label.label( "label" ) ).on( "propKey" ).create();
tx.success();
}

try ( Transaction tx = gds.beginTx() )
{
gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES );
tx.success();
}
Label label = Label.label( "label" );
String propKey = "propKey";
createIndex( gds, label, propKey );

gds.shutdown();

Expand All @@ -251,6 +241,49 @@ public void shouldReportMissingSchemaIndex() throws Exception
) );
}

@Test
public void oldLuceneSchemaIndexShouldBeConsideredConsistentWithFusionProvider() throws Exception
{
File storeDir = testDirectory.graphDbDir();
String enableNativeIndex = GraphDatabaseSettings.enable_native_schema_index.name();
Label label = Label.label( "label" );
String propKey = "propKey";

// Given a lucene index
GraphDatabaseService db = getGraphDatabaseService( storeDir, enableNativeIndex, Settings.FALSE );
createIndex( db, label, propKey );
try ( Transaction tx = db.beginTx() )
{
db.createNode( label ).setProperty( propKey, 1 );
db.createNode( label ).setProperty( propKey, "string" );
tx.success();
}
db.shutdown();

ConsistencyCheckService service = new ConsistencyCheckService();
Config configuration =
Config.defaults( settings( enableNativeIndex, Settings.TRUE ) );
Result result = runFullConsistencyCheck( service, configuration, storeDir );
assertTrue( result.isSuccessful() );
}

private void createIndex( GraphDatabaseService gds, Label label, String propKey )
{
IndexDefinition indexDefinition;

try ( Transaction tx = gds.beginTx() )
{
indexDefinition = gds.schema().indexFor( label ).on( propKey ).create();
tx.success();
}

try ( Transaction tx = gds.beginTx() )
{
gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES );
tx.success();
}
}

private File findFile( String targetFile, File directory )
{
File file = new File( directory, targetFile );
Expand All @@ -267,9 +300,14 @@ private GraphDatabaseService getGraphDatabaseService()
}

private GraphDatabaseService getGraphDatabaseService( File storeDir )
{
return getGraphDatabaseService( storeDir, new String[0] );
}

private GraphDatabaseService getGraphDatabaseService( File storeDir, String... settings )
{
GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir );
builder.setConfig( settings() );
builder.setConfig( settings( settings ) );

return builder.newGraphDatabase();
}
Expand Down

0 comments on commit 7d0e62c

Please sign in to comment.