Skip to content

Commit

Permalink
Revert "Merge pull request #9910 from burqen/3.3-nsni-revert"
Browse files Browse the repository at this point in the history
This reverts commit df2f827, reversing
changes made to d301cc4.
  • Loading branch information
tinwelint committed Aug 29, 2017
1 parent 896f2f8 commit 20987c7
Show file tree
Hide file tree
Showing 88 changed files with 1,600 additions and 917 deletions.
Expand Up @@ -44,14 +44,13 @@
import org.neo4j.io.pagecache.tracing.PageCacheTracer;
import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier;
import org.neo4j.kernel.api.direct.DirectStoreAccess;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensions;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
import org.neo4j.kernel.impl.factory.OperationalMode;
import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
import org.neo4j.kernel.impl.logging.SimpleLogService;
import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.store.StoreAccess;
Expand All @@ -64,8 +63,12 @@
import org.neo4j.logging.LogProvider;

import static java.lang.String.format;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.RECOVERY_PREVENTING_COLLECTOR;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.instantiateKernelExtensions;
import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.loadSchemaIndexProviders;
import static org.neo4j.io.file.Files.createOrOpenAsOuputStream;
import static org.neo4j.kernel.configuration.Settings.TRUE;
import static org.neo4j.kernel.impl.factory.DatabaseInfo.COMMUNITY;

public class ConsistencyCheckService
{
Expand Down Expand Up @@ -204,7 +207,6 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr
{
Log log = logProvider.getLog( getClass() );
config.augment( GraphDatabaseSettings.read_only, TRUE );
OperationalMode operationalMode = OperationalMode.single;

StoreFactory factory = new StoreFactory( storeDir, config,
new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider );
Expand All @@ -225,11 +227,17 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr

// Bootstrap kernel extensions
LifeSupport life = new LifeSupport();
KernelExtensions extensions = life.add( instantiateKernelExtensions( storeDir,
fileSystem, config, new SimpleLogService( logProvider, logProvider ), pageCache,
RECOVERY_PREVENTING_COLLECTOR,
// May be enterprise edition, but in consistency checker we only care about the operational mode
COMMUNITY ) );

try ( NeoStores neoStores = factory.openAllNeoStores() )
{
life.start();
SchemaIndexProvider indexes = life.add( new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT,
storeDir, logProvider, config, operationalMode ) );

SchemaIndexProviderMap indexes = loadSchemaIndexProviders( extensions );

LabelScanStore labelScanStore =
new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, true, new Monitors(),
Expand Down
Expand Up @@ -244,7 +244,7 @@ private PrimitiveLongIterator queryIndexOrEmpty( IndexReader reader, IndexQuery[
Arrays.toString( query ) ), e );
}

return reader.hasFullNumberPrecision()
return reader.hasFullNumberPrecision( query )
? indexedNodeIds : LookupFilter.exactIndexMatches( propertyReader, indexedNodeIds, query );
}

Expand Down
Expand Up @@ -31,6 +31,7 @@
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.InternalIndexState;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.store.RecordStore;
import org.neo4j.kernel.impl.store.SchemaStorage;
Expand All @@ -43,7 +44,7 @@ public class IndexAccessors implements Closeable
private final List<IndexRule> onlineIndexRules = new ArrayList<>();
private final List<IndexRule> notOnlineIndexRules = new ArrayList<>();

public IndexAccessors( SchemaIndexProvider provider,
public IndexAccessors( SchemaIndexProviderMap providers,
RecordStore<DynamicRecord> schemaStore,
IndexSamplingConfig samplingConfig ) throws IOException
{
Expand All @@ -58,7 +59,8 @@ public IndexAccessors( SchemaIndexProvider provider,
// - populating indexes will be rebuilt on next startup
// - failed indexes have to be dropped by the user anyways
IndexRule indexRule = rules.next();
if ( InternalIndexState.ONLINE == provider.getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) )
if ( InternalIndexState.ONLINE == provider( providers, indexRule )
.getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) )
{
onlineIndexRules.add( indexRule );
}
Expand All @@ -81,10 +83,16 @@ public IndexAccessors( SchemaIndexProvider provider,
for ( IndexRule indexRule : onlineIndexRules )
{
long indexId = indexRule.getId();
accessors.put( indexId, provider.getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) );
accessors.put( indexId, provider( providers, indexRule )
.getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) );
}
}

private SchemaIndexProvider provider( SchemaIndexProviderMap providers, IndexRule indexRule )
{
return providers.apply( indexRule.getProviderDescriptor() );
}

public Collection<IndexRule> notOnlineRules()
{
return notOnlineIndexRules;
Expand Down
@@ -0,0 +1,87 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.consistency.internal;

import java.io.File;

import org.neo4j.helpers.Service;
import org.neo4j.index.internal.gbptree.CleanupJob;
import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.extension.KernelExtensions;
import org.neo4j.kernel.extension.UnsatisfiedDependencyStrategies;
import org.neo4j.kernel.extension.dependency.AllByPrioritySelectionStrategy;
import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap;
import org.neo4j.kernel.impl.factory.DatabaseInfo;
import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.spi.KernelContext;
import org.neo4j.kernel.impl.spi.SimpleKernelContext;
import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap;
import org.neo4j.kernel.impl.util.Dependencies;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;

/**
* Utility for loading {@link SchemaIndexProvider} instances from {@link KernelExtensions}.
*/
public class SchemaIndexExtensionLoader
{
/**
* Used in scenarios where recovery isn't allowed.
*/
public static final RecoveryCleanupWorkCollector RECOVERY_PREVENTING_COLLECTOR = new RecoveryPreventingCollector();

public static SchemaIndexProviderMap loadSchemaIndexProviders( KernelExtensions extensions )
{
AllByPrioritySelectionStrategy<SchemaIndexProvider> indexProviderSelection = new AllByPrioritySelectionStrategy<>();
SchemaIndexProvider defaultIndexProvider =
extensions.resolveDependency( SchemaIndexProvider.class, indexProviderSelection );
return new DefaultSchemaIndexProviderMap( defaultIndexProvider,
indexProviderSelection.lowerPrioritizedCandidates() );
}

@SuppressWarnings( "unchecked" )
public static KernelExtensions instantiateKernelExtensions( File storeDir, FileSystemAbstraction fileSystem,
Config config, LogService logService, PageCache pageCache,
RecoveryCleanupWorkCollector recoveryCollector, DatabaseInfo databaseInfo )
{
Dependencies deps = new Dependencies();
deps.satisfyDependencies( fileSystem, config, logService, pageCache, recoveryCollector );
@SuppressWarnings( "rawtypes" )
Iterable kernelExtensions = Service.load( KernelExtensionFactory.class );
KernelContext kernelContext = new SimpleKernelContext( storeDir, databaseInfo, deps );
return new KernelExtensions( kernelContext, kernelExtensions, deps, UnsatisfiedDependencyStrategies.ignore() );
}

private static class RecoveryPreventingCollector extends LifecycleAdapter implements RecoveryCleanupWorkCollector
{
@Override
public void add( CleanupJob job )
{
if ( job.needed() )
{
throw new IllegalStateException( "Consistency checker should not do recovery" );
}
}
}
}
Expand Up @@ -216,19 +216,9 @@ public void shouldReportMissingSchemaIndex() throws Exception
File storeDir = testDirectory.absolutePath();
GraphDatabaseService gds = getGraphDatabaseService( storeDir );

IndexDefinition indexDefinition;

try ( Transaction tx = gds.beginTx() )
{
indexDefinition = gds.schema().indexFor( Label.label( "label" ) ).on( "propKey" ).create();
tx.success();
}

try ( Transaction tx = gds.beginTx() )
{
gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES );
tx.success();
}
Label label = Label.label( "label" );
String propKey = "propKey";
createIndex( gds, label, propKey );

gds.shutdown();

Expand All @@ -251,6 +241,49 @@ public void shouldReportMissingSchemaIndex() throws Exception
) );
}

@Test
public void oldLuceneSchemaIndexShouldBeConsideredConsistentWithFusionProvider() throws Exception
{
File storeDir = testDirectory.graphDbDir();
String enableNativeIndex = GraphDatabaseSettings.enable_native_schema_index.name();
Label label = Label.label( "label" );
String propKey = "propKey";

// Given a lucene index
GraphDatabaseService db = getGraphDatabaseService( storeDir, enableNativeIndex, Settings.FALSE );
createIndex( db, label, propKey );
try ( Transaction tx = db.beginTx() )
{
db.createNode( label ).setProperty( propKey, 1 );
db.createNode( label ).setProperty( propKey, "string" );
tx.success();
}
db.shutdown();

ConsistencyCheckService service = new ConsistencyCheckService();
Config configuration =
Config.defaults( settings( enableNativeIndex, Settings.TRUE ) );
Result result = runFullConsistencyCheck( service, configuration, storeDir );
assertTrue( result.isSuccessful() );
}

private void createIndex( GraphDatabaseService gds, Label label, String propKey )
{
IndexDefinition indexDefinition;

try ( Transaction tx = gds.beginTx() )
{
indexDefinition = gds.schema().indexFor( label ).on( propKey ).create();
tx.success();
}

try ( Transaction tx = gds.beginTx() )
{
gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES );
tx.success();
}
}

private File findFile( String targetFile, File directory )
{
File file = new File( directory, targetFile );
Expand All @@ -267,9 +300,14 @@ private GraphDatabaseService getGraphDatabaseService()
}

private GraphDatabaseService getGraphDatabaseService( File storeDir )
{
return getGraphDatabaseService( storeDir, new String[0] );
}

private GraphDatabaseService getGraphDatabaseService( File storeDir, String... settings )
{
GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir );
builder.setConfig( settings() );
builder.setConfig( settings( settings ) );

return builder.newGraphDatabase();
}
Expand Down

0 comments on commit 20987c7

Please sign in to comment.