diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java index 1e737d797850..a4d909d7ec5d 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java @@ -44,13 +44,14 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; import org.neo4j.kernel.api.direct.DirectStoreAccess; +import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.extension.KernelExtensions; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream; +import org.neo4j.kernel.impl.factory.OperationalMode; import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; -import org.neo4j.kernel.impl.logging.SimpleLogService; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.StoreAccess; @@ -63,12 +64,8 @@ import org.neo4j.logging.LogProvider; import static java.lang.String.format; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.RECOVERY_PREVENTING_COLLECTOR; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.instantiateKernelExtensions; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.loadSchemaIndexProviders; import static org.neo4j.io.file.Files.createOrOpenAsOuputStream; import static org.neo4j.kernel.configuration.Settings.TRUE; -import static org.neo4j.kernel.impl.factory.DatabaseInfo.COMMUNITY; public class ConsistencyCheckService { @@ -207,6 +204,7 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr { Log log = logProvider.getLog( getClass() ); config.augment( GraphDatabaseSettings.read_only, TRUE ); + OperationalMode operationalMode = OperationalMode.single; StoreFactory factory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider ); @@ -227,17 +225,11 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr // Bootstrap kernel extensions LifeSupport life = new LifeSupport(); - KernelExtensions extensions = life.add( instantiateKernelExtensions( storeDir, - fileSystem, config, new SimpleLogService( logProvider, logProvider ), pageCache, - RECOVERY_PREVENTING_COLLECTOR, - // May be enterprise edition, but in consistency checker we only care about the operational mode - COMMUNITY ) ); - try ( NeoStores neoStores = factory.openAllNeoStores() ) { life.start(); - - SchemaIndexProviderMap indexes = loadSchemaIndexProviders( extensions ); + SchemaIndexProvider indexes = life.add( new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, + storeDir, logProvider, config, operationalMode ) ); LabelScanStore labelScanStore = new NativeLabelScanStore( pageCache, storeDir, FullStoreChangeStream.EMPTY, true, new Monitors(), diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/PropertyAndNodeIndexedCheck.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/PropertyAndNodeIndexedCheck.java index 56b05c3af3fb..3243dbeee6e4 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/PropertyAndNodeIndexedCheck.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/PropertyAndNodeIndexedCheck.java @@ -244,7 +244,7 @@ private PrimitiveLongIterator queryIndexOrEmpty( IndexReader reader, IndexQuery[ Arrays.toString( query ) ), e ); } - return reader.hasFullNumberPrecision( query ) + return reader.hasFullNumberPrecision() ? indexedNodeIds : LookupFilter.exactIndexMatches( propertyReader, indexedNodeIds, query ); } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/index/IndexAccessors.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/index/IndexAccessors.java index e4e8cc5ec021..592a989eefd3 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/index/IndexAccessors.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/index/IndexAccessors.java @@ -31,7 +31,6 @@ import org.neo4j.kernel.api.index.IndexAccessor; import org.neo4j.kernel.api.index.InternalIndexState; import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.SchemaStorage; @@ -44,7 +43,7 @@ public class IndexAccessors implements Closeable private final List onlineIndexRules = new ArrayList<>(); private final List notOnlineIndexRules = new ArrayList<>(); - public IndexAccessors( SchemaIndexProviderMap providers, + public IndexAccessors( SchemaIndexProvider provider, RecordStore schemaStore, IndexSamplingConfig samplingConfig ) throws IOException { @@ -59,8 +58,7 @@ public IndexAccessors( SchemaIndexProviderMap providers, // - populating indexes will be rebuilt on next startup // - failed indexes have to be dropped by the user anyways IndexRule indexRule = rules.next(); - if ( InternalIndexState.ONLINE == provider( providers, indexRule ) - .getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) ) + if ( InternalIndexState.ONLINE == provider.getInitialState( indexRule.getId(), indexRule.getIndexDescriptor() ) ) { onlineIndexRules.add( indexRule ); } @@ -83,16 +81,10 @@ public IndexAccessors( SchemaIndexProviderMap providers, for ( IndexRule indexRule : onlineIndexRules ) { long indexId = indexRule.getId(); - accessors.put( indexId, provider( providers, indexRule ) - .getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) ); + accessors.put( indexId, provider.getOnlineAccessor( indexId, indexRule.getIndexDescriptor(), samplingConfig ) ); } } - private SchemaIndexProvider provider( SchemaIndexProviderMap providers, IndexRule indexRule ) - { - return providers.apply( indexRule.getProviderDescriptor() ); - } - public Collection notOnlineRules() { return notOnlineIndexRules; diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/internal/SchemaIndexExtensionLoader.java b/community/consistency-check/src/main/java/org/neo4j/consistency/internal/SchemaIndexExtensionLoader.java deleted file mode 100644 index 2a240c1d345a..000000000000 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/internal/SchemaIndexExtensionLoader.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.consistency.internal; - -import java.io.File; - -import org.neo4j.helpers.Service; -import org.neo4j.index.internal.gbptree.CleanupJob; -import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.extension.KernelExtensionFactory; -import org.neo4j.kernel.extension.KernelExtensions; -import org.neo4j.kernel.extension.UnsatisfiedDependencyStrategies; -import org.neo4j.kernel.extension.dependency.AllByPrioritySelectionStrategy; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; -import org.neo4j.kernel.impl.factory.DatabaseInfo; -import org.neo4j.kernel.impl.logging.LogService; -import org.neo4j.kernel.impl.spi.KernelContext; -import org.neo4j.kernel.impl.spi.SimpleKernelContext; -import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; -import org.neo4j.kernel.impl.util.Dependencies; -import org.neo4j.kernel.lifecycle.LifecycleAdapter; - -/** - * Utility for loading {@link SchemaIndexProvider} instances from {@link KernelExtensions}. - */ -public class SchemaIndexExtensionLoader -{ - /** - * Used in scenarios where recovery isn't allowed. - */ - public static final RecoveryCleanupWorkCollector RECOVERY_PREVENTING_COLLECTOR = new RecoveryPreventingCollector(); - - public static SchemaIndexProviderMap loadSchemaIndexProviders( KernelExtensions extensions ) - { - AllByPrioritySelectionStrategy indexProviderSelection = new AllByPrioritySelectionStrategy<>(); - SchemaIndexProvider defaultIndexProvider = - extensions.resolveDependency( SchemaIndexProvider.class, indexProviderSelection ); - return new DefaultSchemaIndexProviderMap( defaultIndexProvider, - indexProviderSelection.lowerPrioritizedCandidates() ); - } - - @SuppressWarnings( "unchecked" ) - public static KernelExtensions instantiateKernelExtensions( File storeDir, FileSystemAbstraction fileSystem, - Config config, LogService logService, PageCache pageCache, - RecoveryCleanupWorkCollector recoveryCollector, DatabaseInfo databaseInfo ) - { - Dependencies deps = new Dependencies(); - deps.satisfyDependencies( fileSystem, config, logService, pageCache, recoveryCollector ); - @SuppressWarnings( "rawtypes" ) - Iterable kernelExtensions = Service.load( KernelExtensionFactory.class ); - KernelContext kernelContext = new SimpleKernelContext( storeDir, databaseInfo, deps ); - return new KernelExtensions( kernelContext, kernelExtensions, deps, UnsatisfiedDependencyStrategies.ignore() ); - } - - private static class RecoveryPreventingCollector extends LifecycleAdapter implements RecoveryCleanupWorkCollector - { - @Override - public void add( CleanupJob job ) - { - if ( job.needed() ) - { - throw new IllegalStateException( "Consistency checker should not do recovery" ); - } - } - } -} diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckServiceIntegrationTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckServiceIntegrationTest.java index 7d3218ba0af7..24193e323218 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckServiceIntegrationTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/ConsistencyCheckServiceIntegrationTest.java @@ -216,9 +216,19 @@ public void shouldReportMissingSchemaIndex() throws Exception File storeDir = testDirectory.absolutePath(); GraphDatabaseService gds = getGraphDatabaseService( storeDir ); - Label label = Label.label( "label" ); - String propKey = "propKey"; - createIndex( gds, label, propKey ); + IndexDefinition indexDefinition; + + try ( Transaction tx = gds.beginTx() ) + { + indexDefinition = gds.schema().indexFor( Label.label( "label" ) ).on( "propKey" ).create(); + tx.success(); + } + + try ( Transaction tx = gds.beginTx() ) + { + gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES ); + tx.success(); + } gds.shutdown(); @@ -241,49 +251,6 @@ public void shouldReportMissingSchemaIndex() throws Exception ) ); } - @Test - public void oldLuceneSchemaIndexShouldBeConsideredConsistentWithFusionProvider() throws Exception - { - File storeDir = testDirectory.graphDbDir(); - String enableNativeIndex = GraphDatabaseSettings.enable_native_schema_index.name(); - Label label = Label.label( "label" ); - String propKey = "propKey"; - - // Given a lucene index - GraphDatabaseService db = getGraphDatabaseService( storeDir, enableNativeIndex, Settings.FALSE ); - createIndex( db, label, propKey ); - try ( Transaction tx = db.beginTx() ) - { - db.createNode( label ).setProperty( propKey, 1 ); - db.createNode( label ).setProperty( propKey, "string" ); - tx.success(); - } - db.shutdown(); - - ConsistencyCheckService service = new ConsistencyCheckService(); - Config configuration = - Config.defaults( settings( enableNativeIndex, Settings.TRUE ) ); - Result result = runFullConsistencyCheck( service, configuration, storeDir ); - assertTrue( result.isSuccessful() ); - } - - private void createIndex( GraphDatabaseService gds, Label label, String propKey ) - { - IndexDefinition indexDefinition; - - try ( Transaction tx = gds.beginTx() ) - { - indexDefinition = gds.schema().indexFor( label ).on( propKey ).create(); - tx.success(); - } - - try ( Transaction tx = gds.beginTx() ) - { - gds.schema().awaitIndexOnline( indexDefinition, 1, TimeUnit.MINUTES ); - tx.success(); - } - } - private File findFile( String targetFile, File directory ) { File file = new File( directory, targetFile ); @@ -300,14 +267,9 @@ private GraphDatabaseService getGraphDatabaseService() } private GraphDatabaseService getGraphDatabaseService( File storeDir ) - { - return getGraphDatabaseService( storeDir, new String[0] ); - } - - private GraphDatabaseService getGraphDatabaseService( File storeDir, String... settings ) { GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ); - builder.setConfig( settings( settings ) ); + builder.setConfig( settings() ); return builder.newGraphDatabase(); } diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java index 23c7d9fa392e..fde994039881 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java @@ -45,19 +45,18 @@ import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.exceptions.TransactionFailureException; +import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.extension.KernelExtensions; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.IndexStoreView; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.scan.FullLabelStream; -import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.OperationalMode; import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; import org.neo4j.kernel.impl.locking.LockService; -import org.neo4j.kernel.impl.logging.LogService; -import org.neo4j.kernel.impl.logging.SimpleLogService; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeLabelsField; @@ -76,8 +75,8 @@ import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView; import org.neo4j.kernel.impl.transaction.tracing.CommitEvent; import org.neo4j.kernel.internal.GraphDatabaseAPI; -import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.monitoring.Monitors; +import org.neo4j.logging.FormattedLogProvider; import org.neo4j.logging.LogProvider; import org.neo4j.logging.NullLog; import org.neo4j.logging.NullLogProvider; @@ -90,9 +89,6 @@ import static java.lang.System.currentTimeMillis; import static org.neo4j.consistency.ConsistencyCheckService.defaultConsistencyCheckThreadsNumber; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.RECOVERY_PREVENTING_COLLECTOR; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.instantiateKernelExtensions; -import static org.neo4j.consistency.internal.SchemaIndexExtensionLoader.loadSchemaIndexProviders; public abstract class GraphStoreFixture extends ConfigurablePageCacheRule implements TestRule { @@ -113,20 +109,19 @@ public abstract class GraphStoreFixture extends ConfigurablePageCacheRule implem private int relTypeId; private int propKeyId; private DefaultFileSystemAbstraction fileSystem; - private final LifeSupport life = new LifeSupport(); /** * Record format used to generate initial database. */ private String formatName = StringUtils.EMPTY; - private GraphStoreFixture( boolean keepStatistics, String formatName ) + public GraphStoreFixture( boolean keepStatistics, String formatName ) { this.keepStatistics = keepStatistics; this.formatName = formatName; } - protected GraphStoreFixture( String formatName ) + public GraphStoreFixture( String formatName ) { this( false, formatName ); } @@ -135,7 +130,6 @@ protected GraphStoreFixture( String formatName ) protected void after( boolean success ) { super.after( success ); - life.shutdown(); if ( fileSystem != null ) { try @@ -158,7 +152,6 @@ public DirectStoreAccess directStoreAccess() { if ( directStoreAccess == null ) { - life.start(); fileSystem = new DefaultFileSystemAbstraction(); PageCache pageCache = getPageCache( fileSystem ); LogProvider logProvider = NullLogProvider.getInstance(); @@ -180,12 +173,14 @@ public DirectStoreAccess directStoreAccess() nativeStores.initialize(); Config config = Config.defaults(); + OperationalMode operationalMode = OperationalMode.single; IndexStoreView indexStoreView = new NeoStoreIndexStoreView( LockService.NO_LOCK_SERVICE, nativeStores.getRawNeoStores() ); + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector = RecoveryCleanupWorkCollector.IMMEDIATE; LabelScanStore labelScanStore = startLabelScanStore( pageCache, indexStoreView ); - SchemaIndexProviderMap indexes = createIndexes( pageCache, fileSystem, directory, config, logProvider ); - directStoreAccess = new DirectStoreAccess( nativeStores, labelScanStore, indexes ); + directStoreAccess = new DirectStoreAccess( nativeStores, labelScanStore, createIndexes( fileSystem, + config, operationalMode ) ); } return directStoreAccess; } @@ -207,13 +202,10 @@ private LabelScanStore startLabelScanStore( PageCache pageCache, IndexStoreView return labelScanStore; } - private SchemaIndexProviderMap createIndexes( PageCache pageCache, FileSystemAbstraction fileSystem, File storeDir, - Config config, LogProvider logProvider ) + private SchemaIndexProvider createIndexes( FileSystemAbstraction fileSystem, Config config, OperationalMode operationalMode ) { - LogService logService = new SimpleLogService( logProvider, logProvider ); - KernelExtensions extensions = life.add( instantiateKernelExtensions( storeDir, fileSystem, config, logService, - pageCache, RECOVERY_PREVENTING_COLLECTOR, DatabaseInfo.COMMUNITY ) ); - return loadSchemaIndexProviders( extensions ); + return new LuceneSchemaIndexProvider( fileSystem, DirectoryFactory.PERSISTENT, directory, + FormattedLogProvider.toOutputStream( System.out ), config, operationalMode ); } public File directory() @@ -228,7 +220,7 @@ public Statistics getAccessStatistics() public abstract static class Transaction { - final long startTimestamp = currentTimeMillis(); + public final long startTimestamp = currentTimeMillis(); protected abstract void transactionData( TransactionDataBuilder tx, IdGenerator next ); @@ -305,7 +297,7 @@ public int propertyKey() return propKeyId++; } - void updateCorrespondingIdGenerators( NeoStores neoStores ) + public void updateCorrespondingIdGenerators( NeoStores neoStores ) { neoStores.getNodeStore().setHighestPossibleIdInUse( nodeId ); neoStores.getRelationshipStore().setHighestPossibleIdInUse( relId ); @@ -318,7 +310,7 @@ public static final class TransactionDataBuilder private final TransactionWriter writer; private final NodeStore nodes; - TransactionDataBuilder( TransactionWriter writer, NodeStore nodes ) + public TransactionDataBuilder( TransactionWriter writer, NodeStore nodes ) { this.writer = writer; this.nodes = nodes; @@ -454,12 +446,12 @@ protected void stop() throws Throwable } } - private int myId() + protected int myId() { return 1; } - private int masterId() + protected int masterId() { return -1; } @@ -471,7 +463,7 @@ public class Applier implements AutoCloseable private final TransactionIdStore transactionIdStore; private final NeoStores neoStores; - Applier() + public Applier() { database = (GraphDatabaseAPI) new TestGraphDatabaseFactory().newEmbeddedDatabase( directory ); DependencyResolver dependencyResolver = database.getDependencyResolver(); @@ -506,7 +498,7 @@ public Applier createApplier() return new Applier(); } - private void applyTransaction( Transaction transaction ) throws TransactionFailureException + protected void applyTransaction( Transaction transaction ) throws TransactionFailureException { // TODO you know... we could have just appended the transaction representation to the log // and the next startup of the store would do recovery where the transaction would have been diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java index d2383ff896ee..f47e67336923 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java @@ -38,10 +38,10 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.direct.DirectStoreAccess; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.MyRelTypes; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.StoreAccess; @@ -125,7 +125,7 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception StoreAccess storeAccess = new StoreAccess( neoStores ).initialize(); DirectStoreAccess directStoreAccess = new DirectStoreAccess( storeAccess, db.getDependencyResolver().resolveDependency( LabelScanStore.class ), - db.getDependencyResolver().resolveDependency( SchemaIndexProviderMap.class ) ); + db.getDependencyResolver().resolveDependency( SchemaIndexProvider.class ) ); int threads = random.intBetween( 2, 10 ); FullCheck checker = new FullCheck( getTuningConfiguration(), ProgressMonitorFactory.NONE, diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java index d40be988b48d..4848990a7e3b 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java @@ -444,8 +444,8 @@ public void shouldNotReportIndexInconsistenciesIfIndexIsFailed() throws Exceptio { IndexRule rule = rules.next(); IndexSamplingConfig samplingConfig = new IndexSamplingConfig( Config.defaults() ); - IndexPopulator populator = storeAccess.indexes().apply( rule.getProviderDescriptor() ) - .getPopulator( rule.getId(), rule.getIndexDescriptor(), samplingConfig ); + IndexPopulator populator = + storeAccess.indexes().getPopulator( rule.getId(), rule.getIndexDescriptor(), samplingConfig ); populator.markAsFailed( "Oh noes! I was a shiny index and then I was failed" ); populator.close( false ); @@ -556,13 +556,11 @@ public void shouldReportNodesThatAreNotIndexed() throws Exception while ( indexRuleIterator.hasNext() ) { IndexRule indexRule = indexRuleIterator.next(); - IndexAccessor accessor = fixture.directStoreAccess().indexes(). - apply( indexRule.getProviderDescriptor() ).getOnlineAccessor( - indexRule.getId(), indexRule.getIndexDescriptor(), samplingConfig ); + IndexAccessor accessor = fixture.directStoreAccess().indexes().getOnlineAccessor( + indexRule.getId(), indexRule.getIndexDescriptor(), samplingConfig ); IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE ); updater.remove( asPrimitiveLongSet( indexedNodes ) ); updater.close(); - accessor.force(); accessor.close(); } @@ -584,12 +582,11 @@ public void shouldReportNodesWithDuplicatePropertyValueInUniqueIndex() throws Ex while ( indexRuleIterator.hasNext() ) { IndexRule indexRule = indexRuleIterator.next(); - IndexAccessor accessor = fixture.directStoreAccess().indexes().apply( indexRule.getProviderDescriptor() ) + IndexAccessor accessor = fixture.directStoreAccess().indexes() .getOnlineAccessor( indexRule.getId(), indexRule.getIndexDescriptor(), samplingConfig ); IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE ); updater.process( IndexEntryUpdate.add( 42, indexRule.getIndexDescriptor().schema(), values( indexRule ) ) ); updater.close(); - accessor.force(); accessor.close(); } @@ -1036,8 +1033,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, DynamicRecord record1Before = record1.clone(); DynamicRecord record2Before = record2.clone(); - IndexRule rule1 = constraintIndexRule( ruleId1, labelId, propertyKeyId, DESCRIPTOR, ruleId1 ); - IndexRule rule2 = constraintIndexRule( ruleId2, labelId, propertyKeyId, DESCRIPTOR, ruleId1 ); + IndexRule rule1 = constraintIndexRule( ruleId1, labelId, propertyKeyId, DESCRIPTOR, (long) ruleId1 ); + IndexRule rule2 = constraintIndexRule( ruleId2, labelId, propertyKeyId, DESCRIPTOR, (long) ruleId1 ); Collection records1 = serializeRule( rule1, record1 ); Collection records2 = serializeRule( rule2, record2 ); @@ -1081,7 +1078,7 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, DynamicRecord record1Before = record1.clone(); DynamicRecord record2Before = record2.clone(); - IndexRule rule1 = constraintIndexRule( ruleId1, labelId, propertyKeyId, DESCRIPTOR, ruleId2 ); + IndexRule rule1 = constraintIndexRule( ruleId1, labelId, propertyKeyId, DESCRIPTOR, (long) ruleId2 ); ConstraintRule rule2 = uniquenessConstraintRule( ruleId2, labelId, propertyKeyId, ruleId2 ); Collection records1 = serializeRule( rule1, record1 ); diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/InternalTreeLogic.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/InternalTreeLogic.java index 4ff22803cd0f..a96bdd77de90 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/InternalTreeLogic.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/InternalTreeLogic.java @@ -387,11 +387,9 @@ private int search( PageCursor cursor, KEY key, KEY readKey, int keyCount ) private boolean cursorIsAtExpectedLocation( PageCursor cursor ) { assert currentLevel >= 0 : "Uninitialized tree logic, currentLevel:" + currentLevel; - long currentPageId = cursor.getCurrentPageId(); - long expectedPageId = levels[currentLevel].treeNodeId; - assert currentPageId == expectedPageId : "Expected cursor to be at page:" + - expectedPageId + " at level:" + currentLevel + ", but was at page:" + - currentPageId; + assert cursor.getCurrentPageId() == levels[currentLevel].treeNodeId : "Expected cursor to be at page:" + + levels[currentLevel].treeNodeId + " at level:" + currentLevel + ", but was at page:" + + cursor.getCurrentPageId(); return true; } diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index f1297b34bcb2..b878c9643665 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -409,10 +409,6 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting multi_threaded_schema_index_population_enabled = setting( "unsupported.dbms.multi_threaded_schema_index_population_enabled", BOOLEAN, TRUE ); - @Internal - public static final Setting enable_native_schema_index = - setting( "unsupported.dbms.enable_native_schema_index", BOOLEAN, TRUE ); - // Store settings @Description( "Make Neo4j keep the logical transaction logs for being able to backup the database. " + "Can be used for specifying the threshold to prune logical logs after. For example \"10 days\" will " + diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index 003ea1b39ed2..7704fb9d359e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -45,7 +45,7 @@ import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.legacyindex.AutoIndexing; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.extension.dependency.AllByPrioritySelectionStrategy; +import org.neo4j.kernel.extension.dependency.HighestSelectionStrategy; import org.neo4j.kernel.guard.Guard; import org.neo4j.kernel.impl.api.CommitProcessFactory; import org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations; @@ -69,7 +69,6 @@ import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionHooks; import org.neo4j.kernel.impl.api.index.IndexingService; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.operations.QueryRegistrationOperations; import org.neo4j.kernel.impl.api.state.ConstraintIndexCreator; import org.neo4j.kernel.impl.constraints.ConstraintSemantics; @@ -134,7 +133,6 @@ import org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl; import org.neo4j.kernel.impl.transaction.log.rotation.LogRotation; import org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl; -import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; import org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue; @@ -271,7 +269,7 @@ boolean applicable( DiagnosticsPhase phase ) private Dependencies dependencies; private LifeSupport life; - private SchemaIndexProviderMap schemaIndexProviderMap; + private SchemaIndexProvider schemaIndexProvider; private File storeDir; private boolean readOnly; private final IdController idController; @@ -411,16 +409,9 @@ public void start() throws IOException life.add( recoveryCleanupWorkCollector ); - AllByPrioritySelectionStrategy indexProviderSelection = new AllByPrioritySelectionStrategy<>(); - SchemaIndexProvider defaultIndexProvider = - dependencyResolver.resolveDependency( SchemaIndexProvider.class, indexProviderSelection ); - - // todo do we really need to have this dependency? - dependencies.satisfyDependency( defaultIndexProvider ); - - schemaIndexProviderMap = - new DefaultSchemaIndexProviderMap( defaultIndexProvider, indexProviderSelection.lowerPrioritizedCandidates() ); - dependencies.satisfyDependency( schemaIndexProviderMap ); + schemaIndexProvider = dependencyResolver.resolveDependency( SchemaIndexProvider.class, + HighestSelectionStrategy.getInstance() ); + dependencies.satisfyDependency( schemaIndexProvider ); IndexConfigStore indexConfigStore = new IndexConfigStore( storeDir, fs ); dependencies.satisfyDependency( lockService ); @@ -569,7 +560,7 @@ private void upgradeStore( RecordFormats format ) fs, config, logService, - schemaIndexProviderMap, + schemaIndexProvider, indexProviders, pageCache, format ).migrate( storeDir ); @@ -584,7 +575,7 @@ private StorageEngine buildStorageEngine( RecordStorageEngine storageEngine = new RecordStorageEngine( storeDir, config, pageCache, fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaState, constraintSemantics, scheduler, - tokenNameLookup, lockService, schemaIndexProviderMap, indexingServiceMonitor, databaseHealth, + tokenNameLookup, lockService, schemaIndexProvider, indexingServiceMonitor, databaseHealth, legacyIndexProviderLookup, indexConfigStore, legacyIndexTransactionOrdering, idGeneratorFactory, idController, monitors, recoveryCleanupWorkCollector, operationalMode ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/direct/DirectStoreAccess.java b/community/kernel/src/main/java/org/neo4j/kernel/api/direct/DirectStoreAccess.java index 348860fac86b..7a4c41460aef 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/direct/DirectStoreAccess.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/direct/DirectStoreAccess.java @@ -22,22 +22,24 @@ import java.io.Closeable; import java.io.IOException; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.store.StoreAccess; +import org.neo4j.kernel.lifecycle.LifeSupport; public class DirectStoreAccess implements Closeable { + private final LifeSupport life = new LifeSupport(); private final StoreAccess nativeStores; private final LabelScanStore labelScanStore; - private final SchemaIndexProviderMap indexes; + private final SchemaIndexProvider indexes; public DirectStoreAccess( - StoreAccess nativeStores, LabelScanStore labelScanStore, SchemaIndexProviderMap indexes ) + StoreAccess nativeStores, LabelScanStore labelScanStore, SchemaIndexProvider indexes ) { this.nativeStores = nativeStores; this.labelScanStore = labelScanStore; - this.indexes = indexes; + this.indexes = life.add( indexes ); } public StoreAccess nativeStores() @@ -50,7 +52,7 @@ public LabelScanStore labelScanStore() return labelScanStore; } - public SchemaIndexProviderMap indexes() + public SchemaIndexProvider indexes() { return indexes; } @@ -60,5 +62,6 @@ public void close() throws IOException { nativeStores.close(); labelScanStore.shutdown(); + life.shutdown(); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java index 446a00147100..c3a9628d9fbc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java @@ -60,9 +60,9 @@ public void close() } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { - return delegate.hasFullNumberPrecision( predicates ); + return delegate.hasFullNumberPrecision(); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexPopulator.java index a7b4fbd381d6..048674a34026 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexPopulator.java @@ -57,6 +57,13 @@ public interface IndexPopulator void add( Collection> updates ) throws IndexEntryConflictException, IOException; + /** + * Variant of {@link #add(Collection) + * @param update to be inserted + */ + void add( IndexEntryUpdate update ) + throws IndexEntryConflictException, IOException; + /** * Verifies that each value in this index is unique. * This method is called after the index has been fully populated and is guaranteed to not have @@ -144,6 +151,11 @@ public void add( Collection> updates ) throws Inde { } + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + } + public IndexUpdater newPopulatingUpdater( PropertyAccessor accessor ) { return SwallowingIndexUpdater.INSTANCE; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java index f65ffdaaecb0..a3091aecc84a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java @@ -205,31 +205,25 @@ public boolean equals( Object o ) public int hashCode() { int result = priority; - result = 31 * result + providerDescriptor.hashCode(); + result = 31 * result + (providerDescriptor != null ? providerDescriptor.hashCode() : 0); return result; } /** * Get schema index store root directory in specified store. * @param storeDir store root directory - * @return schema index store root directory + * @return shema index store root directory */ public File getSchemaIndexStoreDirectory( File storeDir ) { - return getSchemaIndexStoreDirectory( storeDir, getProviderDescriptor() ); - } - - public static File getSchemaIndexStoreDirectory( File storeDir, Descriptor descriptor ) - { - return new File( new File( new File( storeDir, "schema" ), "index" ), descriptor.getKey() ); + return new File( new File( new File( storeDir, "schema" ), "index" ), getProviderDescriptor().getKey() ); } public abstract StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache ); /** * Provides a snapshot of meta files about this index provider, not the indexes themselves. - * - * @return {@link ResourceIterator} over all meta files for this index provider. + * @return */ public ResourceIterator snapshotMetaFiles() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/schema/IndexQuery.java b/community/kernel/src/main/java/org/neo4j/kernel/api/schema/IndexQuery.java index 6bd00f5286ef..9994a04e4f40 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/schema/IndexQuery.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/schema/IndexQuery.java @@ -302,16 +302,6 @@ public Number to() return (Number)to.asObject(); } - public Value fromAsValue() - { - return from; - } - - public Value toAsValue() - { - return to; - } - public boolean fromInclusive() { return fromInclusive; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/extension/UnsatisfiedDependencyStrategies.java b/community/kernel/src/main/java/org/neo4j/kernel/extension/UnsatisfiedDependencyStrategies.java index 627b901442e9..80aea49f3d58 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/extension/UnsatisfiedDependencyStrategies.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/extension/UnsatisfiedDependencyStrategies.java @@ -19,8 +19,6 @@ */ package org.neo4j.kernel.extension; -import java.io.PrintStream; - public class UnsatisfiedDependencyStrategies { private UnsatisfiedDependencyStrategies() @@ -36,18 +34,9 @@ public static UnsatisfiedDependencyStrategy fail() } public static UnsatisfiedDependencyStrategy ignore() - { - return ( kernelExtensionFactory, e ) -> - { // just ignore - }; - } - - // Perhaps not used, but very useful for debugging kernel extension loading problems - public static UnsatisfiedDependencyStrategy print( PrintStream out ) { return ( kernelExtensionFactory, e ) -> { - out.println( kernelExtensionFactory + " missing dep " + e ); }; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/extension/dependency/AllByPrioritySelectionStrategy.java b/community/kernel/src/main/java/org/neo4j/kernel/extension/dependency/AllByPrioritySelectionStrategy.java deleted file mode 100644 index 42dc2694aca0..000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/extension/dependency/AllByPrioritySelectionStrategy.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.extension.dependency; - -import java.util.Collections; -import java.util.List; - -import org.neo4j.graphdb.DependencyResolver; -import org.neo4j.helpers.collection.Iterables; - -import static org.neo4j.kernel.extension.KernelExtensionUtil.servicesClassPathEntryInformation; - -/** - * Selects the candidate with highest priority (assumed to implement {@link Comparable}) and returns - * in {@link #select(Class, Iterable)}, but keeps the others for access too. - * - * @param type of items expected to be provided into {@link #select(Class, Iterable)}. Due to signature of the - * {@link #select(Class, Iterable) select method} where an explicit and local {@code R} is defined a cast from - * {@code R} to {@code T} is required and so will fail if {@code T} isn't matching {@code R}. - */ -public class AllByPrioritySelectionStrategy> implements DependencyResolver.SelectionStrategy -{ - private List lowerPrioritizedCandidates = Collections.emptyList(); - - @SuppressWarnings( "unchecked" ) - @Override - public R select( Class type, Iterable candidates ) throws IllegalArgumentException - { - List all = (List) Iterables.asList( candidates ); - if ( all.isEmpty() ) - { - throw new IllegalArgumentException( "Could not resolve dependency of type: " + - type.getName() + ". " + servicesClassPathEntryInformation() ); - } - Collections.sort( all, Collections.reverseOrder() ); - R highest = (R) all.remove( 0 ); - lowerPrioritizedCandidates = all; - return highest; - } - - public Iterable lowerPrioritizedCandidates() - { - return lowerPrioritizedCandidates; - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java index 5dc3db602807..8f00b82dc78d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java @@ -827,7 +827,7 @@ public long nodeGetFromUniqueIndexSeek( * a fresh reader that isn't associated with the current transaction and hence will not be * automatically closed. */ PrimitiveLongResourceIterator committed = resourceIterator( reader.query( query ), reader ); - PrimitiveLongIterator exactMatches = reader.hasFullNumberPrecision( query ) + PrimitiveLongIterator exactMatches = reader.hasFullNumberPrecision() ? committed : LookupFilter.exactIndexMatches( this, state, committed, query ); PrimitiveLongIterator changesFiltered = filterIndexStateChangesForSeek( state, exactMatches, index, IndexQuery.asValueTuple( query ) ); @@ -841,7 +841,7 @@ public PrimitiveLongIterator indexQuery( KernelStatement state, IndexDescriptor StorageStatement storeStatement = state.getStoreStatement(); IndexReader reader = storeStatement.getIndexReader( index ); PrimitiveLongIterator committed = reader.query( predicates ); - PrimitiveLongIterator exactMatches = reader.hasFullNumberPrecision( predicates ) + PrimitiveLongIterator exactMatches = reader.hasFullNumberPrecision() ? committed : LookupFilter.exactIndexMatches( this, state, committed, predicates ); IndexQuery firstPredicate = predicates[0]; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java index 8e8bbcf9bdd4..1d0eac89d09d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java @@ -19,7 +19,13 @@ */ package org.neo4j.kernel.impl.api.index; +import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -33,8 +39,12 @@ import org.neo4j.function.Predicates; import org.neo4j.helpers.Exceptions; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException; import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.api.schema.index.IndexDescriptor; import org.neo4j.logging.LogProvider; import org.neo4j.storageengine.api.schema.PopulationProgress; import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles; @@ -58,6 +68,7 @@ public class BatchingMultipleIndexPopulator extends MultipleIndexPopulator { static final String TASK_QUEUE_SIZE_NAME = "task_queue_size"; static final String AWAIT_TIMEOUT_MINUTES_NAME = "await_timeout_minutes"; + static final String BATCH_SIZE_NAME = "batch_size"; static final String MAXIMUM_NUMBER_OF_WORKERS_NAME = "population_workers_maximum"; private static final String EOL = System.lineSeparator(); @@ -68,9 +79,11 @@ public class BatchingMultipleIndexPopulator extends MultipleIndexPopulator private final int TASK_QUEUE_SIZE = FeatureToggles.getInteger( getClass(), TASK_QUEUE_SIZE_NAME, getNumberOfPopulationWorkers() * 2 ); private final int AWAIT_TIMEOUT_MINUTES = FeatureToggles.getInteger( getClass(), AWAIT_TIMEOUT_MINUTES_NAME, 30 ); + private final int BATCH_SIZE = FeatureToggles.getInteger( getClass(), BATCH_SIZE_NAME, 10_000 ); private final AtomicLong activeTasks = new AtomicLong(); private final ExecutorService executor; + private final Map>> batchedUpdates = new HashMap<>(); /** * Creates a new multi-threaded populator for the given store view. @@ -78,7 +91,7 @@ public class BatchingMultipleIndexPopulator extends MultipleIndexPopulator * @param storeView the view of the store as a visitable of nodes * @param logProvider the log provider */ - BatchingMultipleIndexPopulator( IndexStoreView storeView, LogProvider logProvider ) + public BatchingMultipleIndexPopulator( IndexStoreView storeView, LogProvider logProvider ) { super( storeView, logProvider ); this.executor = createThreadPool(); @@ -106,6 +119,15 @@ public StoreScan indexAllNodes() return new BatchingStoreScan<>( storeScan ); } + @Override + protected IndexPopulation createPopulation( IndexPopulator populator, long indexId, + IndexDescriptor descriptor, SchemaIndexProvider.Descriptor providerDescriptor, + FlippableIndexProxy flipper, FailedIndexProxyFactory failedIndexProxyFactory, String indexUserDescription ) + { + return new BatchingIndexPopulation( populator, indexId, descriptor, providerDescriptor, flipper, + failedIndexProxyFactory, indexUserDescription ); + } + @Override protected void populateFromQueue( long currentlyIndexedNodeId ) { @@ -119,9 +141,9 @@ protected void populateFromQueue( long currentlyIndexedNodeId ) @Override public String toString() { - String updatesString = populations + String updatesString = batchedUpdates.values() .stream() - .map( population -> population.batchedUpdates.size() + " updates" ) + .map( entry -> entry.size() + " updates" ) .collect( joining( ", ", "[", "]" ) ); return "BatchingMultipleIndexPopulator{activeTasks=" + activeTasks + ", executor=" + executor + ", " + @@ -150,23 +172,63 @@ private void awaitCompletion() } } + /** + * Add given {@link IndexEntryUpdate update} to the list of updates already present for the given + * {@link IndexPopulation population}. Flushes all updates if {@link #BATCH_SIZE} is reached. + * + * @param population the index population. + * @param update updates to add to the batch. + */ + private void batchUpdate( IndexPopulation population, IndexEntryUpdate update ) + { + List> batch = batchedUpdates.computeIfAbsent( population, key -> newBatch() ); + batch.add( update ); + flushIfNeeded( population, batch ); + } + + /** + * Insert given list of updates to the index if {@link #BATCH_SIZE} is reached. + * + * @param population the index population. + * @param batch the list of updates for the index. + */ + private void flushIfNeeded( IndexPopulation population, List> batch ) + { + if ( batch.size() >= BATCH_SIZE ) + { + batchedUpdates.remove( population ); + flush( population, batch ); + } + } + /** * Insert all batched updates into corresponding indexes. */ private void flushAll() { - populations.forEach( population -> flush( population ) ); + Iterator>>> entries = batchedUpdates.entrySet().iterator(); + while ( entries.hasNext() ) + { + Map.Entry>> entry = entries.next(); + IndexPopulation population = entry.getKey(); + List> updates = entry.getValue(); + entries.remove(); + if ( updates != null && !updates.isEmpty() ) + { + flush( population, updates ); + } + } } /** * Insert the given batch of updates into the index defined by the given {@link IndexPopulation}. * * @param population the index population. + * @param batch the list of updates to insert. */ - private void flush( IndexPopulation population ) + private void flush( IndexPopulation population, List> batch ) { activeTasks.incrementAndGet(); - Collection> batch = population.takeCurrentBatch(); executor.execute( () -> { @@ -231,6 +293,11 @@ private void handleInterrupt() log.warn( "Interrupted while waiting for index population tasks to complete." + EOL + this ); } + private List> newBatch() + { + return new ArrayList<>( BATCH_SIZE ); + } + private ExecutorService createThreadPool() { int threads = getNumberOfPopulationWorkers(); @@ -266,6 +333,28 @@ private int getNumberOfPopulationWorkers() return Math.max( 2, MAXIMUM_NUMBER_OF_WORKERS ); } + /** + * An {@link IndexPopulation} that does not insert updates one by one into the index but instead adds them to the + * map containing batches of updates for each index. + */ + private class BatchingIndexPopulation extends IndexPopulation + { + BatchingIndexPopulation( IndexPopulator populator, long indexId, IndexDescriptor descriptor, + SchemaIndexProvider.Descriptor providerDescriptor, FlippableIndexProxy flipper, + FailedIndexProxyFactory failedIndexProxyFactory, String indexUserDescription ) + { + super( populator, indexId, descriptor, providerDescriptor, flipper, failedIndexProxyFactory, + indexUserDescription ); + } + + @Override + protected void add( IndexEntryUpdate updates ) throws IOException, + IndexEntryConflictException + { + batchUpdate( this, updates ); + } + } + /** * A delegating {@link StoreScan} implementation that flushes all pending updates and terminates the executor after * the delegate store scan completes. diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java index 0acf4d8586b5..b1ded4696e17 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java @@ -20,9 +20,7 @@ package org.neo4j.kernel.impl.api.index; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -85,11 +83,9 @@ */ public class MultipleIndexPopulator implements IndexPopulator { - public static final String QUEUE_THRESHOLD_NAME = "queue_threshold"; - static final String BATCH_SIZE_NAME = "batch_size"; + public static final String QUEUE_THRESHOLD_NAME = "queue_threshold"; private final int QUEUE_THRESHOLD = FeatureToggles.getInteger( getClass(), QUEUE_THRESHOLD_NAME, 20_000 ); - private final int BATCH_SIZE = FeatureToggles.getInteger( BatchingMultipleIndexPopulator.class, BATCH_SIZE_NAME, 10_000 ); // Concurrency queue since multiple concurrent threads may enqueue updates into it. It is important for this queue // to have fast #size() method since it might be drained in batches @@ -98,7 +94,7 @@ public class MultipleIndexPopulator implements IndexPopulator // Populators are added into this list. The same thread adding populators will later call #indexAllNodes. // Multiple concurrent threads might fail individual populations. // Failed populations are removed from this list while iterating over it. - final List populations = new CopyOnWriteArrayList<>(); + private final List populations = new CopyOnWriteArrayList<>(); private final IndexStoreView storeView; private final LogProvider logProvider; @@ -162,6 +158,12 @@ public void add( Collection> updates ) throw new UnsupportedOperationException( "Can't populate directly using this populator implementation. " ); } + @Override + public void add( IndexEntryUpdate update ) + { + throw new UnsupportedOperationException( "Can't populate directly using this populator implementation. " ); + } + public StoreScan indexAllNodes() { int[] labelIds = labelIds(); @@ -462,8 +464,6 @@ public class IndexPopulation implements LabelSchemaSupplier final FailedIndexProxyFactory failedIndexProxyFactory; final String indexUserDescription; - List> batchedUpdates; - IndexPopulation( IndexPopulator populator, long indexId, @@ -481,7 +481,6 @@ public class IndexPopulation implements LabelSchemaSupplier this.failedIndexProxyFactory = failedIndexProxyFactory; this.indexUserDescription = indexUserDescription; this.indexCountsRemover = new IndexCountsRemover( storeView, indexId ); - this.batchedUpdates = new ArrayList<>( BATCH_SIZE ); } private void flipToFailed( Throwable t ) @@ -495,17 +494,19 @@ private void onUpdate( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException { populator.includeSample( update ); - if ( batch( update ) ) - { - populator.add( takeCurrentBatch() ); - } + add( update ); + } + + void add( IndexEntryUpdate update ) + throws IOException, IndexEntryConflictException + { + populator.add( update ); } private void flip() throws FlipFailedKernelException { flipper.flip( () -> { - populator.add( takeCurrentBatch() ); populateFromQueueIfAvailable( Long.MAX_VALUE ); IndexSample sample = populator.sampleResult(); storeView.replaceIndexCounts( indexId, sample.uniqueValues(), sample.sampleSize(), @@ -521,23 +522,6 @@ public LabelSchemaDescriptor schema() { return descriptor.schema(); } - - public boolean batch( IndexEntryUpdate update ) - { - batchedUpdates.add( update ); - return batchedUpdates.size() >= BATCH_SIZE; - } - - Collection> takeCurrentBatch() - { - if ( batchedUpdates.isEmpty() ) - { - return Collections.emptyList(); - } - Collection> batch = batchedUpdates; - batchedUpdates = new ArrayList<>( BATCH_SIZE ); - return batch; - } } private class NodePopulationVisitor implements Visitor visitor ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndex.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndex.java index 496d429b16ac..9368334440a1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndex.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndex.java @@ -51,16 +51,10 @@ class NativeSchemaNumberIndex headerWriter ) throws IOException { - ensureDirectoryExist(); tree = new GBPTree<>( pageCache, storeFile, layout, 0, NO_MONITOR, NO_HEADER_READER, headerWriter, recoveryCleanupWorkCollector ); } - private void ensureDirectoryExist() throws IOException - { - pageCache.getCachedFileSystem().mkdirs( storeFile.getParentFile() ); - } - void closeTree() throws IOException { tree = closeIfPresent( tree ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexAccessor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexAccessor.java index 71fbecdad0a6..ae7a16dc5b20 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexAccessor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexAccessor.java @@ -110,6 +110,7 @@ public ResourceIterator snapshotFiles() throws IOException @Override public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) throws IndexEntryConflictException, IOException - { // Not needed since uniqueness is verified automatically w/o cost for every update. + { + throw new UnsupportedOperationException( "Implement me" ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulator.java index 725d8c52d06c..9175b7c7acbe 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulator.java @@ -22,14 +22,8 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collection; -import java.util.concurrent.ExecutionException; -import org.neo4j.collection.primitive.PrimitiveLongSet; -import org.neo4j.concurrent.Work; -import org.neo4j.concurrent.WorkSync; -import org.neo4j.helpers.Exceptions; import org.neo4j.index.internal.gbptree.GBPTree; import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; @@ -62,7 +56,6 @@ public abstract class NativeSchemaNumberIndexPopulator conflictDetectingValueMerger; private final NativeSchemaNumberIndexUpdater singleUpdater; - private WorkSync workSync; private Writer singleTreeWriter; private byte[] failureBytes; @@ -81,10 +74,8 @@ public abstract class NativeSchemaNumberIndexPopulator( new IndexUpdateApply<>( treeKey, treeValue, singleTreeWriter, conflictDetectingValueMerger ) ); } void instantiateWriter() throws IOException @@ -111,7 +102,17 @@ public synchronized void drop() throws IOException @Override public void add( Collection> updates ) throws IndexEntryConflictException, IOException { - applyWithWorkSync( updates ); + for ( IndexEntryUpdate update : updates ) + { + add( update ); + } + } + + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + NativeSchemaNumberIndexUpdater + .processAdd( treeKey, treeValue, update, singleTreeWriter, conflictDetectingValueMerger ); } @Override @@ -124,38 +125,7 @@ public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) @Override public IndexUpdater newPopulatingUpdater( PropertyAccessor accessor ) throws IOException { - return new IndexUpdater() - { - private boolean closed; - private Collection> updates = new ArrayList<>(); - - @Override - public void process( IndexEntryUpdate update ) throws IOException, IndexEntryConflictException - { - assertOpen(); - updates.add( update ); - } - - @Override - public void close() throws IOException, IndexEntryConflictException - { - applyWithWorkSync( updates ); - closed = true; - } - - @Override - public void remove( PrimitiveLongSet nodeIds ) throws IOException - { // no-op - } - - private void assertOpen() - { - if ( closed ) - { - throw new IllegalStateException( "Updater has been closed" ); - } - } - }; + return singleUpdater.initialize( singleTreeWriter, false ); } @Override @@ -187,18 +157,6 @@ public synchronized void close( boolean populationCompletedSuccessfully ) throws } } - private void applyWithWorkSync( Collection> updates ) throws IOException - { - try - { - workSync.apply( new IndexUpdateWork( updates ) ); - } - catch ( ExecutionException e ) - { - throw Exceptions.launderedException( IOException.class, e ); - } - } - private void assertNotDropped() { if ( dropped ) @@ -247,53 +205,4 @@ void closeWriter() throws IOException { singleTreeWriter = closeIfPresent( singleTreeWriter ); } - - private static class IndexUpdateApply - { - private final KEY treeKey; - private final VALUE treeValue; - private final Writer writer; - private final ConflictDetectingValueMerger conflictDetectingValueMerger; - - IndexUpdateApply( KEY treeKey, VALUE treeValue, Writer writer, - ConflictDetectingValueMerger conflictDetectingValueMerger ) - { - this.treeKey = treeKey; - this.treeValue = treeValue; - this.writer = writer; - this.conflictDetectingValueMerger = conflictDetectingValueMerger; - } - - public void process( IndexEntryUpdate indexEntryUpdate ) throws Exception - { - NativeSchemaNumberIndexUpdater.processUpdate( treeKey, treeValue, indexEntryUpdate, writer, conflictDetectingValueMerger ); - } - } - - private static class IndexUpdateWork implements Work - { - private final Collection> updates; - - IndexUpdateWork( Collection> updates ) - { - this.updates = updates; - } - - @Override - public IndexUpdateWork combine( IndexUpdateWork work ) - { - ArrayList> combined = new ArrayList<>( updates ); - combined.addAll( work.updates ); - return new IndexUpdateWork( combined ); - } - - @Override - public void apply( IndexUpdateApply indexUpdateApply ) throws Exception - { - for ( IndexEntryUpdate indexEntryUpdate : updates ) - { - indexUpdateApply.process( indexEntryUpdate ); - } - } - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexProvider.java index a5d909a54e36..1822253578f1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexProvider.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexProvider.java @@ -21,7 +21,6 @@ import java.io.File; import java.io.IOException; - import org.neo4j.index.internal.gbptree.GBPTree; import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; @@ -47,19 +46,19 @@ public class NativeSchemaNumberIndexProvider extends SchemaIndexProvider { public static final String KEY = "native"; - public static final Descriptor NATIVE_PROVIDER_DESCRIPTOR = new Descriptor( KEY, "1.0" ); + private static final Descriptor NATIVE_PROVIDER_DESCRIPTOR = new Descriptor( KEY, "1.0" ); private final PageCache pageCache; private final File nativeSchemaIndexBaseDir; private final Log log; private final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector; private final boolean readOnly; - public NativeSchemaNumberIndexProvider( PageCache pageCache, File storeDir, LogProvider logging, + NativeSchemaNumberIndexProvider( PageCache pageCache, File nativeSchemaIndexBaseDir, LogProvider logging, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, boolean readOnly ) { super( NATIVE_PROVIDER_DESCRIPTOR, 0 ); this.pageCache = pageCache; - this.nativeSchemaIndexBaseDir = getSchemaIndexStoreDirectory( storeDir ); + this.nativeSchemaIndexBaseDir = getSchemaIndexStoreDirectory( nativeSchemaIndexBaseDir ); this.log = logging.getLog( getClass() ); this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector; this.readOnly = readOnly; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexReader.java index dc2940810fe1..fc0c04cb79bd 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexReader.java @@ -24,7 +24,6 @@ import java.util.HashSet; import java.util.Set; -import org.neo4j.collection.primitive.PrimitiveLongCollections; import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.cursor.RawCursor; import org.neo4j.index.internal.gbptree.GBPTree; @@ -40,7 +39,7 @@ import org.neo4j.storageengine.api.schema.IndexReader; import org.neo4j.storageengine.api.schema.IndexSampler; import org.neo4j.values.storable.Value; -import org.neo4j.values.storable.ValueGroup; +import org.neo4j.values.storable.Values; class NativeSchemaNumberIndexReader implements IndexReader @@ -49,7 +48,7 @@ class NativeSchemaNumberIndexReader layout; private final KEY treeKeyFrom; private final KEY treeKeyTo; - private final Set,IOException>> openSeekers; + private Set,IOException>> openSeekers; NativeSchemaNumberIndexReader( GBPTree tree, Layout layout ) { @@ -122,60 +121,33 @@ public PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotAp return startSeekForInitializedRange(); case exact: ExactPredicate exactPredicate = (ExactPredicate) predicate; - treeKeyFrom.from( Long.MIN_VALUE, exactPredicate.value() ); - treeKeyTo.from( Long.MAX_VALUE, exactPredicate.value() ); + Value[] values = new Value[] {exactPredicate.value()}; + treeKeyFrom.from( Long.MIN_VALUE, values ); + treeKeyTo.from( Long.MAX_VALUE, values ); return startSeekForInitializedRange(); case rangeNumeric: // todo: NumberRangePredicate should return NumberValue instead of Number NumberRangePredicate rangePredicate = (NumberRangePredicate) predicate; - initFromForRange( rangePredicate ); - initToForRange( rangePredicate ); + treeKeyFrom.from( rangePredicate.fromInclusive() ? Long.MIN_VALUE : Long.MAX_VALUE, + new Value[] {Values.of( rangePredicate.from() )} ); + treeKeyFrom.entityIdIsSpecialTieBreaker = true; + treeKeyTo.from( rangePredicate.toInclusive() ? Long.MAX_VALUE : Long.MIN_VALUE, + new Value[] {Values.of( rangePredicate.to() )} ); + treeKeyTo.entityIdIsSpecialTieBreaker = true; return startSeekForInitializedRange(); default: throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." ); } } - private void initToForRange( NumberRangePredicate rangePredicate ) - { - Value toValue = rangePredicate.toAsValue(); - if ( toValue.valueGroup() == ValueGroup.NO_VALUE ) - { - treeKeyTo.initAsHighest(); - } - else - { - treeKeyTo.from( rangePredicate.toInclusive() ? Long.MAX_VALUE : Long.MIN_VALUE, toValue ); - treeKeyTo.entityIdIsSpecialTieBreaker = true; - } - } - - private void initFromForRange( NumberRangePredicate rangePredicate ) - { - Value fromValue = rangePredicate.fromAsValue(); - if ( fromValue.valueGroup() == ValueGroup.NO_VALUE ) - { - treeKeyFrom.initAsLowest(); - } - else - { - treeKeyFrom.from( rangePredicate.fromInclusive() ? Long.MIN_VALUE : Long.MAX_VALUE, fromValue ); - treeKeyFrom.entityIdIsSpecialTieBreaker = true; - } - } - @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { return true; } private PrimitiveLongIterator startSeekForInitializedRange() { - if ( layout.compare( treeKeyFrom, treeKeyTo ) > 0 ) - { - return PrimitiveLongCollections.emptyIterator(); - } try { RawCursor,IOException> seeker = tree.seek( treeKeyFrom, treeKeyTo ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexUpdater.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexUpdater.java index 68e41752abba..338a2427ea56 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexUpdater.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexUpdater.java @@ -63,7 +63,20 @@ NativeSchemaNumberIndexUpdater initialize( Writer writer, public void process( IndexEntryUpdate update ) throws IOException, IndexEntryConflictException { assertOpen(); - processUpdate( treeKey, treeValue, update, writer, conflictDetectingValueMerger ); + switch ( update.updateMode() ) + { + case ADDED: + processAdd( treeKey, treeValue, update, writer, conflictDetectingValueMerger ); + break; + case CHANGED: + processChange( treeKey, treeValue, update, writer, conflictDetectingValueMerger ); + break; + case REMOVED: + processRemove( treeKey, update, writer ); + break; + default: + throw new IllegalArgumentException(); + } } @Override @@ -90,26 +103,6 @@ private void assertOpen() } } - static void processUpdate( KEY treeKey, VALUE treeValue, - IndexEntryUpdate update, Writer writer, ConflictDetectingValueMerger conflictDetectingValueMerger ) - throws IOException, IndexEntryConflictException - { - switch ( update.updateMode() ) - { - case ADDED: - processAdd( treeKey, treeValue, update, writer, conflictDetectingValueMerger ); - break; - case CHANGED: - processChange( treeKey, treeValue, update, writer, conflictDetectingValueMerger ); - break; - case REMOVED: - processRemove( treeKey, update, writer ); - break; - default: - throw new IllegalArgumentException(); - } - } - private static void processRemove( KEY treeKey, IndexEntryUpdate update, Writer writer ) throws IOException { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java index 137ae89ea6bc..823ac146540f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java @@ -20,7 +20,6 @@ package org.neo4j.kernel.impl.index.schema.fusion; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; @@ -69,14 +68,16 @@ public void drop() throws IOException @Override public void add( Collection> updates ) throws IndexEntryConflictException, IOException { - Collection> luceneBatch = new ArrayList<>(); - Collection> nativeBatch = new ArrayList<>(); for ( IndexEntryUpdate update : updates ) { - selector.select( nativeBatch, luceneBatch, update.values() ).add( update ); + add( update ); } - lucenePopulator.add( luceneBatch ); - nativePopulator.add( nativeBatch ); + } + + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + selector.select( nativePopulator, lucenePopulator, update.values() ).add( update ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java index d3cfefebe5a2..8c945397c474 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java @@ -100,25 +100,9 @@ public PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotAp } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { - if ( predicates.length > 1 ) - { - return false; - } - - IndexQuery predicate = predicates[0]; - if ( predicate instanceof ExactPredicate ) - { - Value value = ((ExactPredicate) predicate).value(); - return selector.select( - nativeReader.hasFullNumberPrecision( predicates ), - luceneReader.hasFullNumberPrecision( predicates ), value ); - } - if ( predicates[0] instanceof NumberRangePredicate ) - { - return nativeReader.hasFullNumberPrecision( predicates ); - } - return false; + // Since we know that native reader can do this we return true + return true; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProvider.java index 27e872938cce..7b99c01420b9 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProvider.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProvider.java @@ -32,6 +32,7 @@ import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant; import org.neo4j.storageengine.api.schema.IndexSample; import org.neo4j.values.storable.Value; +import static java.lang.String.format; /** * This {@link SchemaIndexProvider index provider} act as one logical index but is backed by two physical @@ -49,10 +50,11 @@ public interface Selector private final SchemaIndexProvider luceneProvider; private final Selector selector; - public FusionSchemaIndexProvider( SchemaIndexProvider nativeProvider, SchemaIndexProvider luceneProvider, Selector selector, - SchemaIndexProvider.Descriptor descriptor, int priority ) + public FusionSchemaIndexProvider( SchemaIndexProvider nativeProvider, SchemaIndexProvider luceneProvider, Selector selector ) { - super( descriptor, priority ); + super( new Descriptor( + nativeProvider.getProviderDescriptor().getKey() + "+" + luceneProvider.getProviderDescriptor().getKey(), + nativeProvider.getProviderDescriptor().getVersion() + "+" + luceneProvider.getProviderDescriptor().getVersion() ), 0 ); this.nativeProvider = nativeProvider; this.luceneProvider = luceneProvider; this.selector = selector; @@ -78,28 +80,16 @@ public IndexAccessor getOnlineAccessor( long indexId, IndexDescriptor descriptor @Override public String getPopulationFailure( long indexId ) throws IllegalStateException { - String nativeFailure = null; - try + String nativeFailure = nativeProvider.getPopulationFailure( indexId ); + String luceneFailure = luceneProvider.getPopulationFailure( indexId ); + if ( nativeFailure != null ) { - nativeFailure = nativeProvider.getPopulationFailure( indexId ); + return luceneFailure == null ? nativeFailure : nativeFailure + " and " + luceneFailure; } - catch ( IllegalStateException e ) - { // Just catch - } - String luceneFailure = null; - try + else { - luceneFailure = luceneProvider.getPopulationFailure( indexId ); - } - catch ( IllegalStateException e ) - { // Just catch + return luceneFailure; } - - if ( nativeFailure != null || luceneFailure != null ) - { - return "native: " + nativeFailure + " lucene: " + luceneFailure; - } - throw new IllegalStateException( "None of the indexes were in a failed state" ); } @Override @@ -107,17 +97,12 @@ public InternalIndexState getInitialState( long indexId, IndexDescriptor descrip { InternalIndexState nativeState = nativeProvider.getInitialState( indexId, descriptor ); InternalIndexState luceneState = luceneProvider.getInitialState( indexId, descriptor ); - if ( nativeState == InternalIndexState.FAILED || luceneState == InternalIndexState.FAILED ) - { - // One of the state is FAILED, the whole state must be considered FAILED - return InternalIndexState.FAILED; - } - if ( nativeState == InternalIndexState.POPULATING || luceneState == InternalIndexState.POPULATING ) + if ( nativeState != luceneState ) { - // No state is FAILED and one of the state is POPULATING, the whole state must be considered POPULATING - return InternalIndexState.POPULATING; + throw new IllegalStateException( + format( "Internal providers answer with different state native:%s, lucene:%s", + nativeState, luceneState ) ); } - // This means that both states are ONLINE return nativeState; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index cff98dd99b7a..b0b85b0fc8c6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -38,6 +38,7 @@ import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.exceptions.schema.ConstraintValidationException; import org.neo4j.kernel.api.exceptions.schema.CreateConstraintFailureException; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.api.txstate.TransactionCountingStateVisitor; @@ -58,7 +59,6 @@ import org.neo4j.kernel.impl.api.index.IndexingServiceFactory; import org.neo4j.kernel.impl.api.index.IndexingUpdateService; import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.scan.FullLabelStream; import org.neo4j.kernel.impl.api.store.SchemaCache; import org.neo4j.kernel.impl.api.store.StorageLayer; @@ -89,6 +89,7 @@ import org.neo4j.kernel.impl.transaction.command.IndexUpdatesWork; import org.neo4j.kernel.impl.transaction.command.LabelUpdateWork; import org.neo4j.kernel.impl.transaction.command.NeoStoreBatchTransactionApplier; +import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; import org.neo4j.kernel.impl.transaction.state.IntegrityValidator; import org.neo4j.kernel.impl.transaction.state.Loaders; import org.neo4j.kernel.impl.transaction.state.PropertyCreator; @@ -143,7 +144,7 @@ public class RecordStorageEngine implements StorageEngine, Lifecycle private final IntegrityValidator integrityValidator; private final CacheAccessBackDoor cacheAccess; private final LabelScanStore labelScanStore; - private final SchemaIndexProviderMap schemaIndexProviderMap; + private final DefaultSchemaIndexProviderMap schemaIndexProviderMap; private final LegacyIndexApplierLookup legacyIndexApplierLookup; private final SchemaState schemaState; private final SchemaStorage schemaStorage; @@ -180,7 +181,7 @@ public RecordStorageEngine( JobScheduler scheduler, TokenNameLookup tokenNameLookup, LockService lockService, - SchemaIndexProviderMap indexProviderMap, + SchemaIndexProvider indexProvider, IndexingService.Monitor indexingServiceMonitor, DatabaseHealth databaseHealth, LegacyIndexProviderLookup legacyIndexProviderLookup, @@ -219,7 +220,7 @@ public RecordStorageEngine( readOnly, monitors, recoveryCleanupWorkCollector ); indexStoreView = new DynamicIndexStoreView( neoStoreIndexStoreView, labelScanStore, lockService, neoStores, logProvider ); - schemaIndexProviderMap = indexProviderMap; + schemaIndexProviderMap = new DefaultSchemaIndexProviderMap( indexProvider ); indexingService = IndexingServiceFactory.createIndexingService( config, scheduler, schemaIndexProviderMap, indexStoreView, tokenNameLookup, Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).indexesGetAll() ), logProvider, diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java index 56ddd26161a2..d1bca2fddfd3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DatabaseMigrator.java @@ -24,8 +24,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; @@ -49,14 +49,14 @@ public class DatabaseMigrator private final FileSystemAbstraction fs; private final Config config; private final LogService logService; - private final SchemaIndexProviderMap schemaIndexProviderMap; + private final SchemaIndexProvider schemaIndexProvider; private final Map indexProviders; private final PageCache pageCache; private final RecordFormats format; public DatabaseMigrator( MigrationProgressMonitor progressMonitor, FileSystemAbstraction fs, - Config config, LogService logService, SchemaIndexProviderMap schemaIndexProviderMap, + Config config, LogService logService, SchemaIndexProvider schemaIndexProvider, Map indexProviders, PageCache pageCache, RecordFormats format ) { @@ -64,7 +64,7 @@ public DatabaseMigrator( this.fs = fs; this.config = config; this.logService = logService; - this.schemaIndexProviderMap = schemaIndexProviderMap; + this.schemaIndexProvider = schemaIndexProvider; this.indexProviders = indexProviders; this.pageCache = pageCache; this.format = format; @@ -84,13 +84,13 @@ public void migrate( File storeDir ) StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); + StoreMigrationParticipant schemaMigrator = schemaIndexProvider.storeMigrationParticipant( fs, pageCache ); LegacyIndexMigrator legacyIndexMigrator = new LegacyIndexMigrator( fs, indexProviders, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); - schemaIndexProviderMap.accept( - provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); + storeUpgrader.addParticipant( schemaMigrator ); storeUpgrader.addParticipant( legacyIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMap.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMap.java index 6ee6d548abc8..ea1c5770ed9e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMap.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMap.java @@ -19,65 +19,34 @@ */ package org.neo4j.kernel.impl.transaction.state; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.api.index.SchemaIndexProvider.Descriptor; import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; public class DefaultSchemaIndexProviderMap implements SchemaIndexProviderMap { - private final SchemaIndexProvider defaultIndexProvider; - private final Map indexProviders = new HashMap<>(); + private final SchemaIndexProvider indexProvider; - public DefaultSchemaIndexProviderMap( SchemaIndexProvider defaultIndexProvider ) + public DefaultSchemaIndexProviderMap( SchemaIndexProvider indexProvider ) { - this( defaultIndexProvider, Collections.emptyList() ); - } - - public DefaultSchemaIndexProviderMap( SchemaIndexProvider defaultIndexProvider, - Iterable additionalIndexProviders ) - { - this.defaultIndexProvider = defaultIndexProvider; - indexProviders.put( defaultIndexProvider.getProviderDescriptor(), defaultIndexProvider ); - for ( SchemaIndexProvider provider : additionalIndexProviders ) - { - Descriptor providerDescriptor = provider.getProviderDescriptor(); - SchemaIndexProvider existing = indexProviders.putIfAbsent( providerDescriptor, provider ); - if ( existing != null ) - { - throw new IllegalArgumentException( "Tried to load multiple schema index providers with the same provider descriptor " + - providerDescriptor + ". First loaded " + existing + " then " + provider ); - } - } + this.indexProvider = indexProvider; } @Override public SchemaIndexProvider getDefaultProvider() { - return defaultIndexProvider; + return indexProvider; } @Override public SchemaIndexProvider apply( SchemaIndexProvider.Descriptor descriptor ) { - SchemaIndexProvider provider = indexProviders.get( descriptor ); - if ( provider != null ) + if ( indexProvider.getProviderDescriptor().getKey().equals( descriptor.getKey() ) ) { - return provider; + return indexProvider; } throw new IllegalArgumentException( "Tried to get index provider for an existing index with provider " + - descriptor + " whereas available providers in this session being " + indexProviders + - ", and default being " + defaultIndexProvider ); - } - - @Override - public void accept( Consumer visitor ) - { - indexProviders.values().forEach( visitor ); + descriptor + " whereas the default and only supported provider in this session is " + + indexProvider.getProviderDescriptor() ); } } diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java index e02eb361ad86..7caa36fe0c29 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java @@ -51,14 +51,13 @@ public interface IndexReader extends Resource PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotApplicableKernelException; /** - * @param predicates query to determine whether or not index has full number precision for. * @return whether or not this reader will only return 100% matching results from {@link #query(IndexQuery...)} * when calling with predicates involving numbers, such as {@link IndexQuery#exact(int, Object)} * w/ a {@link Number} or {@link IndexQuery#range(int, Number, boolean, Number, boolean)}. * If {@code false} is returned this means that the caller of {@link #query(IndexQuery...)} will have to * do additional filtering, double-checking of actual property values, externally. */ - boolean hasFullNumberPrecision( IndexQuery... predicates ); + boolean hasFullNumberPrecision(); IndexReader EMPTY = new IndexReader() { @@ -87,7 +86,7 @@ public void close() } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { return true; } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java index a8a1d54d3152..eb431bd68328 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java @@ -1303,10 +1303,8 @@ public void forceFlush() private static class IndexPopulatorWithSchema extends IndexPopulator.Adapter implements LabelSchemaSupplier { - private final int batchSize = 1_000; private final IndexPopulator populator; private final IndexDescriptor index; - private Collection> batchedUpdates = new ArrayList<>( batchSize ); IndexPopulatorWithSchema( IndexPopulator populator, IndexDescriptor index ) { @@ -1325,21 +1323,16 @@ public IndexDescriptor index() return index; } + @Override public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException { - batchedUpdates.add( update ); - if ( batchedUpdates.size() > batchSize ) - { - populator.add( batchedUpdates ); - batchedUpdates = new ArrayList<>( batchSize ); - } + populator.add( update ); } @Override public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) throws IndexEntryConflictException, IOException { - populator.add( batchedUpdates ); populator.verifyDeferredConstraints( propertyAccessor ); } diff --git a/community/kernel/src/test/java/org/neo4j/graphdb/SchemaIndexWaitingAcceptanceTest.java b/community/kernel/src/test/java/org/neo4j/graphdb/SchemaIndexWaitingAcceptanceTest.java index 86fe4b6014ad..20b0c7fbb493 100644 --- a/community/kernel/src/test/java/org/neo4j/graphdb/SchemaIndexWaitingAcceptanceTest.java +++ b/community/kernel/src/test/java/org/neo4j/graphdb/SchemaIndexWaitingAcceptanceTest.java @@ -51,7 +51,7 @@ protected void configure( GraphDatabaseFactory databaseFactory ) { List> extensions; extensions = Collections.singletonList( singleInstanceSchemaIndexProviderFactory( "test", provider ) ); - ((TestGraphDatabaseFactory) databaseFactory).setKernelExtensions( extensions ); + ((TestGraphDatabaseFactory) databaseFactory).addKernelExtensions( extensions ); } }; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/index/CompositeIndexPopulatorCompatibility.java b/community/kernel/src/test/java/org/neo4j/kernel/api/index/CompositeIndexPopulatorCompatibility.java index c7a6892ea7e8..e63339ff5609 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/index/CompositeIndexPopulatorCompatibility.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/index/CompositeIndexPopulatorCompatibility.java @@ -68,24 +68,21 @@ public void shouldProvidePopulatorThatAcceptsDuplicateEntries() throws Exception { // when IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - p.create(); - p.add( Arrays.asList( - add( 1, descriptor.schema(), "v1", "v2" ), - add( 2, descriptor.schema(), "v1", "v2" ) ) ); - p.close( true ); - } ); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + populator.create(); + populator.add( Arrays.asList( + add( 1, descriptor.schema(), "v1", "v2" ), + add( 2, descriptor.schema(), "v1", "v2" ) ) ); + populator.close( true ); // then - try ( IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ) ) + IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ); + try ( IndexReader reader = accessor.newReader() ) { - try ( IndexReader reader = accessor.newReader() ) - { - PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( 1, "v1" ), IndexQuery.exact( 1, "v2" ) ); - assertEquals( asSet( 1L, 2L ), PrimitiveLongCollections.toSet( nodes ) ); - } + PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( 1, "v1" ), IndexQuery.exact( 1, "v2" ) ); + assertEquals( asSet( 1L, 2L ), PrimitiveLongCollections.toSet( nodes ) ); } + accessor.close(); } } @@ -108,29 +105,28 @@ public void shouldEnforceUniqueConstraintsDirectly() throws Exception { // when IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + + populator.create(); + populator.add( Arrays.asList( + IndexEntryUpdate.add( nodeId1, descriptor.schema(), value1, value2 ), + IndexEntryUpdate.add( nodeId2, descriptor.schema(), value1, value2 ) ) ); + try + { + NodePropertyAccessor propertyAccessor = + new NodePropertyAccessor( nodeId1, descriptor.schema(), value1, value2 ); + propertyAccessor.addNode( nodeId2, descriptor.schema(), value1, value2 ); + populator.verifyDeferredConstraints( propertyAccessor ); + + fail( "expected exception" ); + } + // then + catch ( IndexEntryConflictException conflict ) { - p.create(); - p.add( Arrays.asList( - IndexEntryUpdate.add( nodeId1, descriptor.schema(), value1, value2 ), - IndexEntryUpdate.add( nodeId2, descriptor.schema(), value1, value2 ) ) ); - try - { - NodePropertyAccessor propertyAccessor = - new NodePropertyAccessor( nodeId1, descriptor.schema(), value1, value2 ); - propertyAccessor.addNode( nodeId2, descriptor.schema(), value1, value2 ); - p.verifyDeferredConstraints( propertyAccessor ); - - fail( "expected exception" ); - } - // then - catch ( IndexEntryConflictException conflict ) - { - assertEquals( nodeId1, conflict.getExistingNodeId() ); - assertEquals( ValueTuple.of( value1, value2 ), conflict.getPropertyValues() ); - assertEquals( nodeId2, conflict.getAddedNodeId() ); - } - } ); + assertEquals( nodeId1, conflict.getExistingNodeId() ); + assertEquals( ValueTuple.of( value1, value2 ), conflict.getPropertyValues() ); + assertEquals( nodeId2, conflict.getAddedNodeId() ); + } } @Test @@ -138,22 +134,21 @@ public void shouldNotRestrictUpdatesDifferingOnSecondProperty() throws Exception { // given IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - p.create(); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); - // when - p.add( Arrays.asList( - IndexEntryUpdate.add( nodeId1, descriptor.schema(), value1, value2 ), - IndexEntryUpdate.add( nodeId2, descriptor.schema(), value1, value3 ) ) ); + populator.create(); - NodePropertyAccessor propertyAccessor = - new NodePropertyAccessor( nodeId1, descriptor.schema(), value1, value2 ); - propertyAccessor.addNode( nodeId2, descriptor.schema(), value1, value3 ); + // when + populator.add( Arrays.asList( + IndexEntryUpdate.add( nodeId1, descriptor.schema(), value1, value2 ), + IndexEntryUpdate.add( nodeId2, descriptor.schema(), value1, value3 ) ) ); + + NodePropertyAccessor propertyAccessor = + new NodePropertyAccessor( nodeId1, descriptor.schema(), value1, value2 ); + propertyAccessor.addNode( nodeId2, descriptor.schema(), value1, value3 ); - // then this should pass fine - p.verifyDeferredConstraints( propertyAccessor ); - } ); + // then this should pass fine + populator.verifyDeferredConstraints( propertyAccessor ); } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexProviderCompatibilityTestSuite.java b/community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexProviderCompatibilityTestSuite.java index 3360aaa6e4a0..8d515a49d511 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexProviderCompatibilityTestSuite.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexProviderCompatibilityTestSuite.java @@ -26,11 +26,9 @@ import java.io.File; -import org.neo4j.function.ThrowingConsumer; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.schema.index.IndexDescriptor; -import org.neo4j.test.rule.PageCacheAndDependenciesRule; +import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; import org.neo4j.test.runner.ParameterizedSuiteRunner; @@ -48,12 +46,14 @@ } ) public abstract class IndexProviderCompatibilityTestSuite { - protected abstract SchemaIndexProvider createIndexProvider( PageCache pageCache, FileSystemAbstraction fs, File graphDbDir ); + protected abstract SchemaIndexProvider createIndexProvider( FileSystemAbstraction fs, File graphDbDir ); public abstract static class Compatibility { @Rule - public PageCacheAndDependenciesRule pageCacheAndDependenciesRule = new PageCacheAndDependenciesRule( DefaultFileSystemRule::new ); + public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); + @Rule + public final TestDirectory testDir = TestDirectory.testDirectory( getClass() ); protected File graphDbDir; protected FileSystemAbstraction fs; @@ -64,10 +64,9 @@ public abstract static class Compatibility @Before public void setup() { - fs = pageCacheAndDependenciesRule.fileSystem(); - graphDbDir = pageCacheAndDependenciesRule.directory().graphDbDir(); - PageCache pageCache = pageCacheAndDependenciesRule.pageCache(); - indexProvider = testSuite.createIndexProvider( pageCache, fs, graphDbDir ); + fs = fileSystemRule.get(); + graphDbDir = testDir.graphDbDir(); + indexProvider = testSuite.createIndexProvider( fs, graphDbDir ); } public Compatibility( IndexProviderCompatibilityTestSuite testSuite, IndexDescriptor descriptor ) @@ -75,30 +74,5 @@ public Compatibility( IndexProviderCompatibilityTestSuite testSuite, IndexDescri this.testSuite = testSuite; this.descriptor = descriptor; } - - protected void withPopulator( IndexPopulator populator, ThrowingConsumer runWithPopulator ) throws Exception - { - try - { - runWithPopulator.accept( populator ); - } - finally - { - try - { - populator.close( true ); - } - catch ( Exception e ) - { // ignore - } - try - { - populator.close( false ); - } - catch ( Exception e ) - { // ignore - } - } - } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/index/SimpleIndexPopulatorCompatibility.java b/community/kernel/src/test/java/org/neo4j/kernel/api/index/SimpleIndexPopulatorCompatibility.java index 0756dab89c85..28a0475a5bd2 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/index/SimpleIndexPopulatorCompatibility.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/index/SimpleIndexPopulatorCompatibility.java @@ -38,9 +38,7 @@ import org.neo4j.values.storable.Values; import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.neo4j.helpers.collection.Iterators.asSet; import static org.neo4j.kernel.api.index.IndexEntryUpdate.add; @@ -64,18 +62,15 @@ public void shouldStorePopulationFailedForRetrievalFromProviderLater() throws Ex { // GIVEN IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - String failure = "The contrived failure"; - p.create(); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + String failure = "The contrived failure"; + populator.create(); - // WHEN - p.markAsFailed( failure ); - p.close( false ); + // WHEN + populator.markAsFailed( failure ); - // THEN - assertThat( indexProvider.getPopulationFailure( 17 ), containsString( failure ) ); - } ); + // THEN + assertEquals( failure, indexProvider.getPopulationFailure( 17 ) ); } @Test @@ -83,17 +78,15 @@ public void shouldReportInitialStateAsFailedIfPopulationFailed() throws Exceptio { // GIVEN IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - String failure = "The contrived failure"; - p.create(); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + String failure = "The contrived failure"; + populator.create(); - // WHEN - p.markAsFailed( failure ); + // WHEN + populator.markAsFailed( failure ); - // THEN - assertEquals( FAILED, indexProvider.getInitialState( 17, descriptor ) ); - } ); + // THEN + assertEquals( FAILED, indexProvider.getInitialState( 17, descriptor ) ); } @Test @@ -101,15 +94,13 @@ public void shouldBeAbleToDropAClosedIndexPopulator() throws Exception { // GIVEN IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - p.close( false ); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + populator.close( false ); - // WHEN - p.drop(); + // WHEN + populator.drop(); - // THEN - no exception should be thrown (it's been known to!) - } ); + // THEN - no exception should be thrown (it's been known to!) } @Test @@ -117,37 +108,33 @@ public void shouldApplyUpdatesIdempotently() throws Exception { // GIVEN IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + populator.create(); + populator.configureSampling( true ); + long nodeId = 1; final Value propertyValue = Values.of( "value1" ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> + PropertyAccessor propertyAccessor = + ( nodeId1, propertyKeyId ) -> propertyValue; + + // this update (using add())... + populator.add( singletonList( IndexEntryUpdate.add( nodeId, descriptor.schema(), propertyValue ) ) ); + // ...is the same as this update (using update()) + try ( IndexUpdater updater = populator.newPopulatingUpdater( propertyAccessor ) ) { - p.create(); - p.configureSampling( true ); - long nodeId = 1; - PropertyAccessor propertyAccessor = - ( nodeId1, propertyKeyId ) -> propertyValue; - - // this update (using add())... - p.add( singletonList( IndexEntryUpdate.add( nodeId, descriptor.schema(), propertyValue ) ) ); - // ...is the same as this update (using update()) - try ( IndexUpdater updater = p.newPopulatingUpdater( propertyAccessor ) ) - { - updater.process( add( nodeId, descriptor.schema(), propertyValue ) ); - } + updater.process( add( nodeId, descriptor.schema(), propertyValue ) ); + } - p.close( true ); - } ); + populator.close( true ); // then - try ( IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ) ) + IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ); + try ( IndexReader reader = accessor.newReader() ) { - try ( IndexReader reader = accessor.newReader() ) - { - int propertyKeyId = descriptor.schema().getPropertyId(); - PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( propertyKeyId, propertyValue ) ); - assertEquals( asSet( 1L ), PrimitiveLongCollections.toSet( nodes ) ); - } - accessor.close(); + int propertyKeyId = descriptor.schema().getPropertyId(); + PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( propertyKeyId, propertyValue ) ); + assertEquals( asSet( 1L ), PrimitiveLongCollections.toSet( nodes ) ); } + accessor.close(); } @Ignore( "Not a test. This is a compatibility suite" ) @@ -163,26 +150,22 @@ public void shouldProvidePopulatorThatAcceptsDuplicateEntries() throws Exception { // when IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + populator.create(); Value value = Values.of( "value1" ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> - { - p.create(); - p.add( Arrays.asList( - IndexEntryUpdate.add( 1, descriptor.schema(), value ), - IndexEntryUpdate.add( 2, descriptor.schema(), value ) ) ); - p.close( true ); - } ); + populator.add( Arrays.asList( + IndexEntryUpdate.add( 1, descriptor.schema(), value ), + IndexEntryUpdate.add( 2, descriptor.schema(), value ) ) ); + populator.close( true ); // then - try ( IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ) ) + IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, descriptor, indexSamplingConfig ); + try ( IndexReader reader = accessor.newReader() ) { - try ( IndexReader reader = accessor.newReader() ) - { - PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( 1, value ) ); - assertEquals( asSet( 1L, 2L ), PrimitiveLongCollections.toSet( nodes ) ); - } - accessor.close(); + PrimitiveLongIterator nodes = reader.query( IndexQuery.exact( 1, value ) ); + assertEquals( asSet( 1L, 2L ), PrimitiveLongCollections.toSet( nodes ) ); } + accessor.close(); } } @@ -206,29 +189,28 @@ public void shouldProvidePopulatorThatEnforcesUniqueConstraints() throws Excepti int nodeId2 = 2; IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.defaults() ); - withPopulator( indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ), p -> + IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexSamplingConfig ); + + populator.create(); + populator.add( Arrays.asList( + IndexEntryUpdate.add( nodeId1, descriptor.schema(), value ), + IndexEntryUpdate.add( nodeId2, descriptor.schema(), value ) ) ); + try { - p.create(); - p.add( Arrays.asList( - IndexEntryUpdate.add( nodeId1, descriptor.schema(), value ), - IndexEntryUpdate.add( nodeId2, descriptor.schema(), value ) ) ); - try - { - NodePropertyAccessor propertyAccessor = - new NodePropertyAccessor( nodeId1, descriptor.schema(), value ); - propertyAccessor.addNode( nodeId2, descriptor.schema(), value ); - p.verifyDeferredConstraints( propertyAccessor ); - - fail( "expected exception" ); - } - // then - catch ( IndexEntryConflictException conflict ) - { - assertEquals( nodeId1, conflict.getExistingNodeId() ); - assertEquals( ValueTuple.of( value ), conflict.getPropertyValues() ); - assertEquals( nodeId2, conflict.getAddedNodeId() ); - } - } ); + NodePropertyAccessor propertyAccessor = + new NodePropertyAccessor( nodeId1, descriptor.schema(), value ); + propertyAccessor.addNode( nodeId2, descriptor.schema(), value ); + populator.verifyDeferredConstraints( propertyAccessor ); + + fail( "expected exception" ); + } + // then + catch ( IndexEntryConflictException conflict ) + { + assertEquals( nodeId1, conflict.getExistingNodeId() ); + assertEquals( ValueTuple.of( value ), conflict.getPropertyValues() ); + assertEquals( nodeId2, conflict.getAddedNodeId() ); + } } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/index/UniqueConstraintCompatibility.java b/community/kernel/src/test/java/org/neo4j/kernel/api/index/UniqueConstraintCompatibility.java index 1a3159ad9e85..bf52cae74859 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/index/UniqueConstraintCompatibility.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/index/UniqueConstraintCompatibility.java @@ -59,9 +59,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertThat; - -import static java.util.Arrays.asList; - import static org.neo4j.kernel.impl.locking.LockService.LockType; @Ignore( "Not a test. This is a compatibility suite that provides test cases for verifying" + @@ -679,8 +676,8 @@ private Future applyChangesToPopulatingUpdater( private static final long COLLISION_Y = 4611686018427387907L; private static final ExecutorService executor = Executors.newCachedThreadPool(); - private final Label label = Label.label( "Cybermen" ); - private final String property = "name"; + private Label label = Label.label( "Cybermen" ); + private String property = "name"; private Node a; private Node b; private Node c; @@ -992,7 +989,7 @@ public void setUp() { File storeDir = testDirectory.graphDbDir(); TestGraphDatabaseFactory dbfactory = new TestGraphDatabaseFactory(); - dbfactory.setKernelExtensions( asList( new PredefinedSchemaIndexProviderFactory( indexProvider ) ) ); + dbfactory.addKernelExtension( new PredefinedSchemaIndexProviderFactory( indexProvider ) ); db = dbfactory.newImpermanentDatabase( storeDir ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java index 469afe44f56f..000e03a96f71 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java @@ -87,7 +87,7 @@ public void populateFromQueueDoesNothingIfThresholdNotReached() throws Exception setProperty( QUEUE_THRESHOLD_NAME, 5 ); BatchingMultipleIndexPopulator batchingPopulator = new BatchingMultipleIndexPopulator( - mock( IndexStoreView.class ), immediateExecutor(), NullLogProvider.getInstance() ); + mock( IndexStoreView.class ), mock( ExecutorService.class ), NullLogProvider.getInstance() ); IndexPopulator populator = addPopulator( batchingPopulator, index1 ); IndexUpdater updater = mock( IndexUpdater.class ); @@ -116,7 +116,7 @@ public void populateFromQueuePopulatesWhenThresholdReached() throws Exception NeoStoreIndexStoreView storeView = new NeoStoreIndexStoreView( LockService.NO_LOCK_SERVICE, neoStores ); BatchingMultipleIndexPopulator batchingPopulator = new BatchingMultipleIndexPopulator( - storeView, immediateExecutor(), NullLogProvider.getInstance() ); + storeView, mock( ExecutorService.class ), NullLogProvider.getInstance() ); IndexPopulator populator1 = addPopulator( batchingPopulator, index1 ); IndexUpdater updater1 = mock( IndexUpdater.class ); @@ -147,7 +147,7 @@ public void executorShutdownAfterStoreScanCompletes() throws Exception NodeUpdates update = nodeUpdates( 1, propertyId, "foo", labelId ); IndexStoreView storeView = newStoreView( update ); - ExecutorService executor = immediateExecutor(); + ExecutorService executor = mock( ExecutorService.class ); when( executor.awaitTermination( anyLong(), any() ) ).thenReturn( true ); BatchingMultipleIndexPopulator batchingPopulator = new BatchingMultipleIndexPopulator( storeView, @@ -171,7 +171,7 @@ public void executorForcefullyShutdownIfStoreScanFails() throws Exception doThrow( scanError ).when( failingStoreScan ).run(); when( storeView.visitNodes( any(), any(), any(), any(), anyBoolean() ) ).thenReturn( failingStoreScan ); - ExecutorService executor = immediateExecutor(); + ExecutorService executor = mock( ExecutorService.class ); when( executor.awaitTermination( anyLong(), any() ) ).thenReturn( true ); BatchingMultipleIndexPopulator batchingPopulator = new BatchingMultipleIndexPopulator( storeView, @@ -348,7 +348,7 @@ private static IndexStoreView newStoreView( NodeUpdates... updates ) private static ExecutorService sameThreadExecutor() throws InterruptedException { - ExecutorService executor = immediateExecutor(); + ExecutorService executor = mock( ExecutorService.class ); when( executor.awaitTermination( anyLong(), any() ) ).thenReturn( true ); doAnswer( invocation -> { @@ -368,17 +368,6 @@ private static void clearProperty( String name ) FeatureToggles.clear( BatchingMultipleIndexPopulator.class, name ); } - private static ExecutorService immediateExecutor() - { - ExecutorService result = mock( ExecutorService.class ); - doAnswer( invocation -> - { - invocation.getArgumentAt( 0, Runnable.class ).run(); - return null; - } ).when( result ).execute( any( Runnable.class ) ); - return result; - } - private static class IndexEntryUpdateScan implements StoreScan { final NodeUpdates[] updates; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexCRUDIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexCRUDIT.java index 7711836f6e8b..fec1eea986d1 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexCRUDIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexCRUDIT.java @@ -168,12 +168,11 @@ private Node createNode( Map properties, Label ... labels ) @Before public void before() throws Exception { - when( mockedIndexProvider.getProviderDescriptor() ).thenReturn( PROVIDER_DESCRIPTOR ); when( mockedIndexProvider.storeMigrationParticipant( any( FileSystemAbstraction.class ), any( PageCache.class ) ) ) .thenReturn( StoreMigrationParticipant.NOT_PARTICIPATING ); TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory(); factory.setFileSystem( fs.get() ); - factory.setKernelExtensions( + factory.addKernelExtensions( Collections.singletonList( mockedIndexProviderFactory ) ); db = (GraphDatabaseAPI) factory.newImpermanentDatabase(); ctxSupplier = db.getDependencyResolver().resolveDependency( ThreadToStatementContextBridge.class ); @@ -185,6 +184,7 @@ private GatheringIndexWriter newWriter() throws IOException when( mockedIndexProvider.getPopulator( anyLong(), any( IndexDescriptor.class ), any( IndexSamplingConfig.class ) ) ).thenReturn( writer ); + when( mockedIndexProvider.getProviderDescriptor() ).thenReturn( PROVIDER_DESCRIPTOR ); when( mockedIndexProvider.getOnlineAccessor( anyLong(), any( IndexDescriptor.class ), any( IndexSamplingConfig.class ) ) ).thenReturn( writer ); @@ -215,6 +215,13 @@ public void add( Collection> updates ) updatesCommitted.addAll( updates ); } + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + ReadOperations statement = ctxSupplier.get().readOperations(); + updatesCommitted.add( update ); + } + @Override public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) throws IndexEntryConflictException, IOException { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java index b2d67ccf5145..3783c759f818 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java @@ -162,7 +162,7 @@ public void shouldPopulateIndexWithOneNode() throws Exception verify( populator ).create(); verify( populator ).configureSampling( false ); verify( populator ).includeSample( update ); - verify( populator ).add( any( Collection.class) ); + verify( populator ).add( any( IndexEntryUpdate.class) ); verify( populator ).sampleResult(); verify( populator ).close( true ); @@ -204,14 +204,14 @@ public void shouldPopulateIndexWithASmallDataset() throws Exception job.run(); // THEN - IndexEntryUpdate update1 = add( node1, descriptor, Values.of( value ) ); + IndexEntryUpdate update1 = IndexEntryUpdate.add( node1, descriptor, Values.of( value ) ); IndexEntryUpdate update2 = add( node4, descriptor, Values.of( value ) ); verify( populator ).create(); verify( populator ).configureSampling( false ); verify( populator ).includeSample( update1 ); verify( populator ).includeSample( update2 ); - verify( populator ).add( Matchers.anyCollection() ); + verify( populator, times( 2 ) ).add( any( IndexEntryUpdate.class ) ); verify( populator ).sampleResult(); verify( populator ).close( true ); @@ -477,7 +477,8 @@ public void add( Collection> updates ) } } - void add( IndexEntryUpdate update ) + @Override + public void add( IndexEntryUpdate update ) { if ( update.getEntityId() == 2 ) { @@ -554,7 +555,8 @@ public void add( Collection> updates ) } } - void add( IndexEntryUpdate update ) + @Override + public void add( IndexEntryUpdate update ) { if ( update.getEntityId() == 2 ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRecoveryIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRecoveryIT.java index 4af3161d72f7..d6d84af447e3 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRecoveryIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRecoveryIT.java @@ -270,7 +270,7 @@ private void startDb() TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory(); factory.setFileSystem( fs.get() ); - factory.setKernelExtensions( Arrays.asList( mockedIndexProviderFactory ) ); + factory.addKernelExtensions( Arrays.asList( mockedIndexProviderFactory ) ); db = (GraphDatabaseAPI) factory.newImpermanentDatabase(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java index 4f652354a070..e12f9695195a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java @@ -67,7 +67,7 @@ public void before() throws Exception { factory = new TestGraphDatabaseFactory(); factory.setFileSystem( new UncloseableDelegatingFileSystemAbstraction( fs.get() ) ); - factory.setKernelExtensions( Collections.singletonList( + factory.addKernelExtensions( Collections.singletonList( singleInstanceSchemaIndexProviderFactory( "test", provider ) ) ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexSamplingCancellationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexSamplingCancellationTest.java new file mode 100644 index 000000000000..fe60dd6dcddf --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexSamplingCancellationTest.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api.index; + +import org.junit.Rule; +import org.junit.Test; + +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.schema.IndexDefinition; +import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException; +import org.neo4j.kernel.api.index.DelegatingIndexReader; +import org.neo4j.kernel.api.index.IndexAccessor; +import org.neo4j.kernel.api.schema.index.IndexDescriptor; +import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; +import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProviderFactory; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.storageengine.api.schema.IndexReader; +import org.neo4j.storageengine.api.schema.IndexSample; +import org.neo4j.storageengine.api.schema.IndexSampler; +import org.neo4j.test.Barrier; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.DatabaseRule; +import org.neo4j.test.rule.ImpermanentDatabaseRule; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.Label.label; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.index_background_sampling_enabled; +import static org.neo4j.kernel.configuration.Settings.FALSE; +import static org.neo4j.kernel.impl.api.index.sampling.IndexSamplingMode.TRIGGER_REBUILD_ALL; + +public class IndexSamplingCancellationTest +{ + private final Barrier.Control samplingStarted = new Barrier.Control(); + private final Barrier.Control samplingDone = new Barrier.Control(); + private volatile Throwable samplingException; + private final InMemoryIndexProvider index = new TestInMemoryIndexProvider(); + @Rule + public final DatabaseRule db = new ImpermanentDatabaseRule() + { + @Override + protected void configure( GraphDatabaseFactory factory ) + { + ((TestGraphDatabaseFactory) factory).addKernelExtension( new InMemoryIndexProviderFactory( index ) ); + } + + @Override + protected void configure( GraphDatabaseBuilder builder ) + { + builder.setConfig( index_background_sampling_enabled, FALSE ); + } + }; + + @Test + public void shouldStopSamplingWhenIndexIsDropped() throws Exception + { + // given + IndexDefinition index = awaitOnline( indexOn( label( "Foo" ), "bar" ) ); + + // when + db.resolveDependency( IndexingService.class ).triggerIndexSampling( TRIGGER_REBUILD_ALL ); + samplingStarted.await(); + drop( index ); + samplingDone.release(); + samplingStarted.release(); + samplingDone.await(); + + // then + Throwable exception = samplingException; + assertThat( exception, instanceOf( IndexNotFoundKernelException.class ) ); + assertEquals( "Index dropped while sampling.", exception.getMessage() ); + } + + private void drop( IndexDefinition index ) + { + try ( Transaction tx = db.beginTx() ) + { + index.drop(); + tx.success(); + } + } + + private IndexDefinition indexOn( Label label, String propertyKey ) + { + IndexDefinition index; + try ( Transaction tx = db.beginTx() ) + { + index = db.schema().indexFor( label ).on( propertyKey ).create(); + tx.success(); + } + return index; + } + + private IndexDefinition awaitOnline( IndexDefinition index ) + { + try ( Transaction tx = db.beginTx() ) + { + db.schema().awaitIndexOnline( index, 3, SECONDS ); + + tx.success(); + } + return index; + } + + private class TestInMemoryIndexProvider extends InMemoryIndexProvider + { + TestInMemoryIndexProvider() + { + super( 100 ); + } + + @Override + public IndexAccessor getOnlineAccessor( long indexId, IndexDescriptor descriptor, + IndexSamplingConfig samplingConfig ) + { + return new DelegatingIndexAccessor( super.getOnlineAccessor( indexId, descriptor, samplingConfig ) ); + } + } + + private class DelegatingIndexAccessor extends IndexAccessor.Delegator + { + DelegatingIndexAccessor( IndexAccessor delegate ) + { + super( delegate ); + } + + @Override + public IndexReader newReader() + { + return new TestIndexReader( super.newReader() ); + } + } + + private class TestIndexReader extends DelegatingIndexReader + { + TestIndexReader( IndexReader delegate ) + { + super( delegate ); + } + + @Override + public IndexSampler createSampler() + { + samplingStarted.reached(); + IndexSampler sampler = super.createSampler(); + return new DelegatingIndexSampler( sampler ); + } + } + + private class DelegatingIndexSampler implements IndexSampler + { + final IndexSampler delegate; + + DelegatingIndexSampler( IndexSampler delegate ) + { + this.delegate = delegate; + } + + @Override + public IndexSample sampleIndex() throws IndexNotFoundKernelException + { + try + { + return delegate.sampleIndex(); + } + catch ( Throwable e ) + { + samplingException = e; + throw e; + } + finally + { + samplingDone.reached(); + } + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsIT.java index a8f081d4e87a..beda7c31a3c1 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexStatisticsIT.java @@ -52,9 +52,6 @@ import org.neo4j.test.rule.fs.EphemeralFileSystemRule; import static org.junit.Assert.assertEquals; - -import static java.util.Arrays.asList; - import static org.neo4j.graphdb.Label.label; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.index_background_sampling_enabled; import static org.neo4j.logging.AssertableLogProvider.inLog; @@ -213,7 +210,7 @@ private void setupDb( EphemeralFileSystemAbstraction fs ) { db = new TestGraphDatabaseFactory().setInternalLogProvider( logProvider ) .setFileSystem( new UncloseableDelegatingFileSystemAbstraction( fs ) ) - .setKernelExtensions( asList( new InMemoryIndexProviderFactory( indexProvider ) ) ) + .addKernelExtension( new InMemoryIndexProviderFactory( indexProvider ) ) .newImpermanentDatabaseBuilder() .setConfig( index_background_sampling_enabled, "false" ) .newGraphDatabase(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java index 5fcc1d80f298..faec775f7781 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java @@ -231,7 +231,7 @@ public void shouldDeliverUpdatesThatOccurDuringPopulationToPopulator() throws Ex CountDownLatch latch = new CountDownLatch( 1 ); AwaitAnswer awaitAnswer = afterAwaiting( latch ); - doAnswer( awaitAnswer ).when( populator ).add( any( Collection.class ) ); + doAnswer( awaitAnswer ).when( populator ).add( any( IndexEntryUpdate.class ) ); IndexingService indexingService = newIndexingServiceWithMockedDependencies( populator, accessor, withData( addNodeUpdate( 1, "value1" ) ) ); @@ -259,7 +259,7 @@ public void shouldDeliverUpdatesThatOccurDuringPopulationToPopulator() throws Ex InOrder order = inOrder( populator, accessor, updater); order.verify( populator ).create(); order.verify( populator ).includeSample( add( 1, "value1" ) ); - order.verify( populator ).add( any( Collection.class ) ); + order.verify( populator ).add( any( IndexEntryUpdate.class ) ); // invoked from indexAllNodes(), empty because the id we added (2) is bigger than the one we indexed (1) // diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java index 935d5665c3c9..8465600f68b7 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java @@ -290,7 +290,7 @@ public PrimitiveLongIterator query( IndexQuery... predicates ) } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { return false; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndex.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndex.java index f4053ee6298a..9b705a929cc6 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndex.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndex.java @@ -137,6 +137,12 @@ public void add( Collection> updates ) } } + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + InMemoryIndex.this.add( update.getEntityId(), update.values(), false ); + } + @Override public void verifyDeferredConstraints( PropertyAccessor accessor ) throws IndexEntryConflictException, IOException { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndexProviderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndexProviderTest.java index 7d53c1a93a12..027a8c038767 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndexProviderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/InMemoryIndexProviderTest.java @@ -22,14 +22,13 @@ import java.io.File; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.index.IndexProviderCompatibilityTestSuite; import org.neo4j.kernel.api.index.SchemaIndexProvider; public class InMemoryIndexProviderTest extends IndexProviderCompatibilityTestSuite { @Override - protected SchemaIndexProvider createIndexProvider( PageCache pageCache, FileSystemAbstraction fs, File graphDbDir ) + protected SchemaIndexProvider createIndexProvider( FileSystemAbstraction fs, File graphDbDir ) { return new InMemoryIndexProvider(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/state/StateHandlingStatementOperationsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/state/StateHandlingStatementOperationsTest.java index 002b1ba4c261..3b2f21a7e1dc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/state/StateHandlingStatementOperationsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/state/StateHandlingStatementOperationsTest.java @@ -608,7 +608,7 @@ public void shouldNotDecorateNumberQuerResultsWIthLookupFilterIfIndexHasFullNumb } ); when( kernelStatement.getStoreStatement() ).thenReturn( storeStatement ); IndexReader indexReader = mock( IndexReader.class ); - when( indexReader.hasFullNumberPrecision( anyVararg() ) ).thenReturn( true ); + when( indexReader.hasFullNumberPrecision() ).thenReturn( true ); when( indexReader.query( anyVararg() ) ) .thenAnswer( invocation -> PrimitiveLongCollections.iterator( nodeId ) ); when( storeStatement.getFreshIndexReader( any() ) ).thenReturn( indexReader ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaNumberIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaNumberIndexPopulatorTest.java index 46441edbe526..264e4f030bee 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaNumberIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaNumberIndexPopulatorTest.java @@ -68,14 +68,13 @@ public void updaterShouldApplyDuplicateValues() throws Exception { // given populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); IndexEntryUpdate[] updates = layoutUtil.someUpdatesWithDuplicateValues(); - try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) ) + + // when + for ( IndexEntryUpdate update : updates ) { - // when - for ( IndexEntryUpdate update : updates ) - { - updater.process( update ); - } + updater.process( update ); } // then diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulatorTest.java index 6e28e51dba0c..b1b387200433 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaNumberIndexPopulatorTest.java @@ -26,9 +26,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -50,7 +48,6 @@ import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.values.storable.Values; -import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -177,14 +174,14 @@ public void updaterShouldApplyUpdates() throws Exception { // given populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); + @SuppressWarnings( "unchecked" ) IndexEntryUpdate[] updates = layoutUtil.someUpdates(); - try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) ) + + // when + for ( IndexEntryUpdate update : updates ) { - // when - for ( IndexEntryUpdate update : updates ) - { - updater.process( update ); - } + updater.process( update ); } // then @@ -220,11 +217,12 @@ public void shouldApplyInterleavedUpdatesFromAddAndUpdater() throws Exception { // given populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); @SuppressWarnings( "unchecked" ) IndexEntryUpdate[] updates = layoutUtil.someUpdates(); // when - applyInterleaved( updates, populator ); + applyInterleaved( updates, updater, populator ); // then populator.close( true ); @@ -504,7 +502,7 @@ private int interleaveLargeAmountOfUpdates( Random updaterRandom, } } } - populator.add( asList( updates.next() ) ); + populator.add( updates.next() ); count++; } return count; @@ -542,45 +540,20 @@ private String longString( int length ) return RandomStringUtils.random( length, true, true ); } - private void applyInterleaved( IndexEntryUpdate[] updates, NativeSchemaNumberIndexPopulator populator ) - throws IOException, IndexEntryConflictException + private void applyInterleaved( IndexEntryUpdate[] updates, IndexUpdater updater, + NativeSchemaNumberIndexPopulator populator ) throws IOException, IndexEntryConflictException { - boolean useUpdater = true; - Collection> populatorBatch = new ArrayList<>(); - IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); for ( IndexEntryUpdate update : updates ) { - if ( random.nextInt( 100 ) < 20 ) - { - if ( useUpdater ) - { - updater.close(); - populatorBatch = new ArrayList<>(); - } - else - { - populator.add( populatorBatch ); - updater = populator.newPopulatingUpdater( null_property_accessor ); - } - useUpdater = !useUpdater; - } - if ( useUpdater ) + if ( random.nextBoolean() ) { - updater.process( update ); + populator.add( update ); } else { - populatorBatch.add( update ); + updater.process( update ); } } - if ( useUpdater ) - { - updater.close(); - } - else - { - populator.add( populatorBatch ); - } } private void verifyUpdates( Iterator> indexEntryUpdateIterator, int count ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeUniqueSchemaNumberIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeUniqueSchemaNumberIndexPopulatorTest.java index ab61e261f751..8bd69a275988 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeUniqueSchemaNumberIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeUniqueSchemaNumberIndexPopulatorTest.java @@ -24,7 +24,6 @@ import java.io.File; import java.util.Arrays; -import org.neo4j.helpers.Exceptions; import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; @@ -34,9 +33,7 @@ import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.storageengine.api.schema.IndexSample; -import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class NativeUniqueSchemaNumberIndexPopulatorTest extends NativeSchemaNumberIndexPopulatorTest @@ -68,15 +65,12 @@ public void addShouldThrowOnDuplicateValues() throws Exception populator.add( Arrays.asList( updates ) ); fail( "Updates should have conflicted" ); } - catch ( Throwable e ) + catch ( IndexEntryConflictException e ) { - // then - assertTrue( Exceptions.contains( e, IndexEntryConflictException.class ) ); - } - finally - { - populator.close( true ); + // then good } + + populator.close( true ); } @Test @@ -85,27 +79,21 @@ public void updaterShouldThrowOnDuplicateValues() throws Exception // given populator.create(); IndexEntryUpdate[] updates = layoutUtil.someUpdatesWithDuplicateValues(); - IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); - - // when - for ( IndexEntryUpdate update : updates ) + try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) ) { - updater.process( update ); - } - try - { - updater.close(); + // when + for ( IndexEntryUpdate update : updates ) + { + updater.process( update ); + } fail( "Updates should have conflicted" ); } - catch ( Throwable e ) + catch ( IndexEntryConflictException e ) { - // then - assertTrue( e.getMessage(), Exceptions.contains( e, IndexEntryConflictException.class ) ); - } - finally - { - populator.close( true ); + // then good } + + populator.close( true ); } @Test @@ -117,9 +105,9 @@ public void shouldSampleUpdates() throws Exception IndexEntryUpdate[] updates = layoutUtil.someUpdates(); // WHEN - populator.add( asList( updates ) ); for ( IndexEntryUpdate update : updates ) { + populator.add( update ); populator.includeSample( update ); } IndexSample sample = populator.sampleResult(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java index 2c3c13fe5ec7..ee1a707de8a4 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.Collection; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.index.IndexEntryUpdate; @@ -45,8 +44,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static java.util.Arrays.asList; - import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexTestHelp.add; import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexTestHelp.verifyCallFail; @@ -181,7 +178,7 @@ public void addMustSelectCorrectPopulator() throws Exception private void verifyAddWithCorrectPopulator( IndexPopulator correctPopulator, IndexPopulator wrongPopulator, Value... numberValues ) throws IndexEntryConflictException, IOException { - Collection> update = asList( add( numberValues ) ); + IndexEntryUpdate update = add( numberValues ); fusionIndexPopulator.add( update ); verify( correctPopulator, times( 1 ) ).add( update ); verify( wrongPopulator, times( 0 ) ).add( update ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProviderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProviderTest.java index 717181db733d..0bd123e3194a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProviderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionSchemaIndexProviderTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.impl.index.schema.fusion; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -35,6 +34,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -42,31 +42,66 @@ import static org.mockito.Matchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.neo4j.helpers.ArrayUtil.array; public class FusionSchemaIndexProviderTest { - private static final SchemaIndexProvider.Descriptor DESCRIPTOR = new SchemaIndexProvider.Descriptor( "test-fusion", "1" ); - - private SchemaIndexProvider nativeProvider; - private SchemaIndexProvider luceneProvider; + @Rule + public RandomRule random = new RandomRule(); - @Before - public void setup() + @Test + public void mustThrowForMixedAndReportCorrectForMatchingInitialState() throws Exception { - nativeProvider = mock( SchemaIndexProvider.class ); - luceneProvider = mock( SchemaIndexProvider.class ); + // given + SchemaIndexProvider nativeProvider = mock( SchemaIndexProvider.class ); + SchemaIndexProvider luceneProvider = mock( SchemaIndexProvider.class ); when( nativeProvider.getProviderDescriptor() ).thenReturn( new SchemaIndexProvider.Descriptor( "native", "1" ) ); when( luceneProvider.getProviderDescriptor() ).thenReturn( new SchemaIndexProvider.Descriptor( "lucene", "1" ) ); + FusionSchemaIndexProvider fusionSchemaIndexProvider = + new FusionSchemaIndexProvider( nativeProvider, luceneProvider, new NativeSelector() ); + IndexDescriptor anyIndexDescriptor = IndexDescriptorFactory.forLabel( 0, 0 ); + + for ( InternalIndexState nativeState : InternalIndexState.values() ) + { + setInitialState( nativeProvider, nativeState ); + for ( InternalIndexState luceneState : InternalIndexState.values() ) + { + setInitialState( luceneProvider, luceneState ); + + // when + if ( nativeState != luceneState ) + { + // then + try + { + fusionSchemaIndexProvider.getInitialState( 0, anyIndexDescriptor ); + fail( "Should have failed" ); + } + catch ( IllegalStateException e ) + { + // good + } + } + else + { + // or then + assertSame( nativeState, fusionSchemaIndexProvider.getInitialState( 0, anyIndexDescriptor ) ); + } + } + } } - @Rule - public RandomRule random = new RandomRule(); + private void setInitialState( SchemaIndexProvider mockedProvider, InternalIndexState state ) + { + when( mockedProvider.getInitialState( anyLong(), any( IndexDescriptor.class ) ) ).thenReturn( state ); + } @Test public void mustSelectCorrectTargetForAllGivenValueCombinations() throws Exception { // given + SchemaIndexProvider nativeProvider = mock( SchemaIndexProvider.class ); + SchemaIndexProvider luceneProvider = mock( SchemaIndexProvider.class ); + Value[] numberValues = FusionIndexTestHelp.valuesSupportedByNative(); Value[] otherValues = FusionIndexTestHelp.valuesNotSupportedByNative(); Value[] allValues = FusionIndexTestHelp.allValues(); @@ -130,67 +165,41 @@ public void mustCombineSamples() throws Exception } @Test - public void getPopulationFailureMustThrowIfNoFailure() throws Exception + public void mustReportPopulationFailure() throws Exception { // given - FusionSchemaIndexProvider fusionSchemaIndexProvider = fusionProvider(); + SchemaIndexProvider nativeProvider = mock( SchemaIndexProvider.class ); + SchemaIndexProvider luceneProvider = mock( SchemaIndexProvider.class ); + when( nativeProvider.getProviderDescriptor() ).thenReturn( new SchemaIndexProvider.Descriptor( "native", "1" ) ); + when( luceneProvider.getProviderDescriptor() ).thenReturn( new SchemaIndexProvider.Descriptor( "lucene", "1" ) ); + FusionSchemaIndexProvider fusionSchemaIndexProvider = + new FusionSchemaIndexProvider( nativeProvider, luceneProvider, new NativeSelector() ); // when // ... no failure - IllegalStateException nativeThrow = new IllegalStateException( "no native failure" ); - IllegalStateException luceneThrow = new IllegalStateException( "no lucene failure" ); - when( nativeProvider.getPopulationFailure( anyLong() ) ).thenThrow( nativeThrow ); - when( luceneProvider.getPopulationFailure( anyLong() ) ).thenThrow( luceneThrow ); + when( nativeProvider.getPopulationFailure( anyLong() ) ).thenReturn( null ); + when( luceneProvider.getPopulationFailure( anyLong() ) ).thenReturn( null ); // then - try - { - fusionSchemaIndexProvider.getPopulationFailure( 0 ); - fail( "Should have failed" ); - } - catch ( IllegalStateException e ) - { // good - } - } - - @Test - public void getPopulationFailureMustReportFailureWhenNativeFailure() throws Exception - { - FusionSchemaIndexProvider fusionSchemaIndexProvider = fusionProvider(); + assertNull( fusionSchemaIndexProvider.getPopulationFailure( 0 ) ); // when // ... native failure String nativeFailure = "native failure"; - IllegalStateException luceneThrow = new IllegalStateException( "no lucene failure" ); when( nativeProvider.getPopulationFailure( anyLong() ) ).thenReturn( nativeFailure ); - when( luceneProvider.getPopulationFailure( anyLong() ) ).thenThrow( luceneThrow ); + when( luceneProvider.getPopulationFailure( anyLong() ) ).thenReturn( null ); // then - assertThat( fusionSchemaIndexProvider.getPopulationFailure( 0 ), containsString( nativeFailure ) ); - } - - @Test - public void getPopulationFailureMustReportFailureWhenLuceneFailure() throws Exception - { - FusionSchemaIndexProvider fusionSchemaIndexProvider = fusionProvider(); + assertEquals( nativeFailure, fusionSchemaIndexProvider.getPopulationFailure( 0 ) ); // when // ... lucene failure String luceneFailure = "lucene failure"; - IllegalStateException nativeThrow = new IllegalStateException( "no native failure" ); - when( nativeProvider.getPopulationFailure( anyLong() ) ).thenThrow( nativeThrow ); + when( nativeProvider.getPopulationFailure( anyLong() ) ).thenReturn( null ); when( luceneProvider.getPopulationFailure( anyLong() ) ).thenReturn( luceneFailure ); // then - assertThat( fusionSchemaIndexProvider.getPopulationFailure( 0 ), containsString( luceneFailure ) ); - } - - @Test - public void getPopulationFailureMustReportFailureWhenBothFailure() throws Exception - { - FusionSchemaIndexProvider fusionSchemaIndexProvider = fusionProvider(); + assertEquals( luceneFailure, fusionSchemaIndexProvider.getPopulationFailure( 0 ) ); // when // ... native and lucene failure - String luceneFailure = "lucene failure"; - String nativeFailure = "native failure"; when( nativeProvider.getPopulationFailure( anyLong() ) ).thenReturn( nativeFailure ); when( luceneProvider.getPopulationFailure( anyLong() ) ).thenReturn( luceneFailure ); // then @@ -198,62 +207,4 @@ public void getPopulationFailureMustReportFailureWhenBothFailure() throws Except assertThat( populationFailure, containsString( nativeFailure ) ); assertThat( populationFailure, containsString( luceneFailure ) ); } - - @Test - public void shouldReportFailedIfAnyIsFailed() throws Exception - { - // given - SchemaIndexProvider provider = fusionProvider(); - IndexDescriptor indexDescriptor = IndexDescriptorFactory.forLabel( 1, 1 ); - - for ( InternalIndexState state : InternalIndexState.values() ) - { - // when - setInitialState( nativeProvider, InternalIndexState.FAILED ); - setInitialState( luceneProvider, state ); - InternalIndexState failed1 = provider.getInitialState( 0, indexDescriptor ); - - setInitialState( nativeProvider, state ); - setInitialState( luceneProvider, InternalIndexState.FAILED ); - InternalIndexState failed2 = provider.getInitialState( 0, indexDescriptor ); - - // then - assertEquals( InternalIndexState.FAILED, failed1 ); - assertEquals( InternalIndexState.FAILED, failed2 ); - } - } - - @Test - public void shouldReportPopulatingIfAnyIsPopulating() throws Exception - { - // given - SchemaIndexProvider provider = fusionProvider(); - IndexDescriptor indexDescriptor = IndexDescriptorFactory.forLabel( 1, 1 ); - - for ( InternalIndexState state : array( InternalIndexState.ONLINE, InternalIndexState.POPULATING ) ) - { - // when - setInitialState( nativeProvider, InternalIndexState.POPULATING ); - setInitialState( luceneProvider, state ); - InternalIndexState failed1 = provider.getInitialState( 0, indexDescriptor ); - - setInitialState( nativeProvider, state ); - setInitialState( luceneProvider, InternalIndexState.POPULATING ); - InternalIndexState failed2 = provider.getInitialState( 0, indexDescriptor ); - - // then - assertEquals( InternalIndexState.POPULATING, failed1 ); - assertEquals( InternalIndexState.POPULATING, failed2 ); - } - } - - private FusionSchemaIndexProvider fusionProvider() - { - return new FusionSchemaIndexProvider( nativeProvider, luceneProvider, new NativeSelector(), DESCRIPTOR, 10 ); - } - - private void setInitialState( SchemaIndexProvider mockedProvider, InternalIndexState state ) - { - when( mockedProvider.getInitialState( anyLong(), any( IndexDescriptor.class ) ) ).thenReturn( state ); - } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/PartialTransactionFailureIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/PartialTransactionFailureIT.java index f49e8465f1bd..7a76fd3f97a0 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/PartialTransactionFailureIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/PartialTransactionFailureIT.java @@ -38,6 +38,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProviderFactory; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; @@ -50,6 +51,7 @@ import org.neo4j.kernel.internal.EmbeddedGraphDatabase; import org.neo4j.test.rule.TestDirectory; +import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @@ -209,6 +211,7 @@ private static class TestEmbeddedGraphDatabase extends EmbeddedGraphDatabase private static GraphDatabaseFacadeFactory.Dependencies dependencies() { GraphDatabaseFactoryState state = new GraphDatabaseFactoryState(); + state.addKernelExtensions( singletonList( new InMemoryIndexProviderFactory() ) ); return state.databaseDependencies(); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMapTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMapTest.java deleted file mode 100644 index 73b4730382e1..000000000000 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/DefaultSchemaIndexProviderMapTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.transaction.state; - -import org.junit.Test; - -import org.neo4j.kernel.api.index.SchemaIndexProvider; - -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import static java.util.Arrays.asList; - -public class DefaultSchemaIndexProviderMapTest -{ - @Test - public void shouldNotSupportMultipleProvidersWithSameDescriptor() throws Exception - { - // given - SchemaIndexProvider.Descriptor descriptor = new SchemaIndexProvider.Descriptor( "provider", "1.2" ); - SchemaIndexProvider provider1 = mock( SchemaIndexProvider.class ); - when( provider1.getProviderDescriptor() ).thenReturn( descriptor ); - SchemaIndexProvider provider2 = mock( SchemaIndexProvider.class ); - when( provider2.getProviderDescriptor() ).thenReturn( descriptor ); - - // when - try - { - new DefaultSchemaIndexProviderMap( provider1, asList( provider2 ) ); - fail( "Should have failed" ); - } - catch ( IllegalArgumentException e ) - { - // then good - } - } -} diff --git a/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/CollectionMatcher.java b/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/CollectionMatcher.java deleted file mode 100644 index 674499f49645..000000000000 --- a/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/CollectionMatcher.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.test.mockito.matcher; - -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; - -import java.util.Collection; -import static java.util.Arrays.asList; - -/** - * An org.hamcrest Matcher that matches {@link Collection collections}. - * @param The parameter of the Collection to match - */ -public class CollectionMatcher extends TypeSafeMatcher> -{ - private final Collection toMatch; - - private CollectionMatcher( Collection toMatch ) - { - this.toMatch = toMatch; - } - - @Override - protected boolean matchesSafely( Collection objects ) - { - return IterableMatcher.itemsMatches( toMatch, objects ); - } - - @Override - public void describeTo( Description description ) - { - description.appendValueList( "Collection [", ",", "]", toMatch ); - } - - public static CollectionMatcher matchesCollection( Collection toMatch ) - { - return new CollectionMatcher<>( toMatch ); - } - - @SafeVarargs - public static CollectionMatcher matchesCollection( T... toMatch ) - { - return new CollectionMatcher<>( asList( toMatch ) ); - } -} diff --git a/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/IterableMatcher.java b/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/IterableMatcher.java index 0ed2620a5e96..bf71f70f207c 100644 --- a/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/IterableMatcher.java +++ b/community/kernel/src/test/java/org/neo4j/test/mockito/matcher/IterableMatcher.java @@ -42,17 +42,12 @@ private IterableMatcher( Iterable toMatch ) @Override protected boolean matchesSafely( Iterable objects ) { - return itemsMatches( toMatch, objects ); - } - - static boolean itemsMatches( Iterable expected, Iterable actual ) - { - if ( Iterables.count( expected ) != Iterables.count( actual ) ) + if ( Iterables.count( toMatch ) != Iterables.count( objects ) ) { return false; } - Iterator original = expected.iterator(); - Iterator matched = actual.iterator(); + Iterator original = toMatch.iterator(); + Iterator matched = objects.iterator(); T fromOriginal; T fromToMatch; for ( ; original.hasNext() && matched.hasNext(); ) @@ -75,6 +70,6 @@ public void describeTo( Description description ) public static IterableMatcher matchesIterable( Iterable toMatch ) { - return new IterableMatcher<>( toMatch ); + return new IterableMatcher( toMatch ); } } diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/DatabaseRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/DatabaseRule.java index 4d4736b1ed25..e255fdaddc73 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/DatabaseRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/DatabaseRule.java @@ -74,7 +74,7 @@ public abstract class DatabaseRule extends ExternalResource implements GraphData /** * Means the database will be started on first {@link #getGraphDatabaseAPI()}} - * or {@link #ensureStarted(String...)} call. + * or {@link #ensureStarted()} call. */ public DatabaseRule startLazily() { diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java index 7f6c3c956998..266eff9fb497 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java @@ -50,7 +50,6 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdReuseEligibility; import org.neo4j.kernel.impl.store.id.configuration.CommunityIdTypeConfigurationProvider; -import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; import org.neo4j.kernel.impl.util.IdOrderingQueue; import org.neo4j.kernel.impl.util.Neo4jJobScheduler; import org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue; @@ -208,8 +207,7 @@ private class ExtendedRecordStorageEngine extends RecordStorageEngine { super( storeDir, config, pageCache, fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaState, constraintSemantics, scheduler, tokenNameLookup, - lockService, new DefaultSchemaIndexProviderMap( indexProvider ), - indexingServiceMonitor, databaseHealth, + lockService, indexProvider, indexingServiceMonitor, databaseHealth, legacyIndexProviderLookup, indexConfigStore, legacyIndexTransactionOrdering, idGeneratorFactory, idController, monitors, recoveryCleanupWorkCollector, operationalMode ); this.transactionApplierTransformer = transactionApplierTransformer; diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java index 70ca38787ec0..b14143599789 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/batchinsert/internal/BatchInsertTest.java @@ -61,6 +61,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexEntryUpdate; import org.neo4j.kernel.api.index.IndexPopulator; import org.neo4j.kernel.api.index.PropertyAccessor; import org.neo4j.kernel.api.index.SchemaIndexProvider; @@ -114,7 +115,6 @@ import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -128,11 +128,9 @@ import static org.neo4j.helpers.collection.Iterators.iterator; import static org.neo4j.helpers.collection.MapUtil.map; import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.api.index.IndexEntryUpdate.add; import static org.neo4j.kernel.impl.api.index.SchemaIndexTestHelper.singleInstanceSchemaIndexProviderFactory; import static org.neo4j.kernel.impl.store.RecordStore.getRecord; import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; -import static org.neo4j.test.mockito.matcher.CollectionMatcher.matchesCollection; import static org.neo4j.test.mockito.matcher.Neo4jMatchers.hasProperty; import static org.neo4j.test.mockito.matcher.Neo4jMatchers.inTx; @@ -930,7 +928,7 @@ public void shouldRunIndexPopulationJobAtShutdown() throws Throwable verify( provider ).start(); verify( provider ).getPopulator( anyLong(), any( IndexDescriptor.class ), any( IndexSamplingConfig.class ) ); verify( populator ).create(); - verify( populator ).add( argThat( matchesCollection( add( nodeId, internalIndex.schema(), Values.of( "Jakewins" ) ) ) ) ); + verify( populator ).add( IndexEntryUpdate.add( nodeId, internalIndex.schema(), Values.of( "Jakewins" ) ) ); verify( populator ).verifyDeferredConstraints( any( PropertyAccessor.class ) ); verify( populator ).close( true ); verify( provider ).stop(); @@ -964,7 +962,8 @@ public void shouldRunConstraintPopulationJobAtShutdown() throws Throwable verify( provider ).start(); verify( provider ).getPopulator( anyLong(), any( IndexDescriptor.class ), any( IndexSamplingConfig.class ) ); verify( populator ).create(); - verify( populator ).add( argThat( matchesCollection( add( nodeId, internalUniqueIndex.schema(), Values.of( "Jakewins" ) ) ) ) ); + verify( populator ).add( + IndexEntryUpdate.add( nodeId, internalUniqueIndex.schema(), Values.of( "Jakewins" ) ) ); verify( populator ).verifyDeferredConstraints( any( PropertyAccessor.class ) ); verify( populator ).close( true ); verify( provider ).stop(); @@ -998,9 +997,8 @@ public void shouldRepopulatePreexistingIndexed() throws Throwable verify( provider ).start(); verify( provider ).getPopulator( anyLong(), any( IndexDescriptor.class ), any( IndexSamplingConfig.class ) ); verify( populator ).create(); - verify( populator ).add( argThat( matchesCollection( - add( jakewins, internalIndex.schema(), Values.of( "Jakewins" ) ), - add( boggle, internalIndex.schema(), Values.of( "b0ggl3" ) ) ) ) ); + verify( populator ).add( IndexEntryUpdate.add( jakewins, internalIndex.schema(), Values.of( "Jakewins" ) ) ); + verify( populator ).add( IndexEntryUpdate.add( boggle, internalIndex.schema(), Values.of( "b0ggl3" ) ) ); verify( populator ).verifyDeferredConstraints( any( PropertyAccessor.class ) ); verify( populator ).close( true ); verify( provider ).stop(); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProvider.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProvider.java index c07bb60aa463..1484b3830415 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProvider.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProvider.java @@ -48,8 +48,6 @@ public class LuceneSchemaIndexProvider extends SchemaIndexProvider { - static final int PRIORITY = 1; - private final IndexStorageFactory indexStorageFactory; private final Log log; private Config config; @@ -60,7 +58,7 @@ public LuceneSchemaIndexProvider( FileSystemAbstraction fileSystem, DirectoryFac File storeDir, LogProvider logging, Config config, OperationalMode operationalMode ) { - super( LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR, PRIORITY ); + super( LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR, 1 ); File schemaIndexStoreFolder = getSchemaIndexStoreDirectory( storeDir ); this.indexStorageFactory = buildIndexStorageFactory( fileSystem, directoryFactory, schemaIndexStoreFolder ); this.fileSystem = fileSystem; diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderFactory.java index e6efe47f3ffb..2766e3a0f57b 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderFactory.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderFactory.java @@ -19,8 +19,6 @@ */ package org.neo4j.kernel.api.impl.schema; -import java.io.File; - import org.neo4j.helpers.Service; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; @@ -28,7 +26,6 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; -import org.neo4j.kernel.impl.factory.OperationalMode; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.logging.LogProvider; @@ -61,20 +58,14 @@ public LuceneSchemaIndexProviderFactory() @Override public LuceneSchemaIndexProvider newInstance( KernelContext context, Dependencies dependencies ) throws Throwable { - FileSystemAbstraction fileSystemAbstraction = dependencies.fileSystem(); - File storeDir = context.storeDir(); Config config = dependencies.getConfig(); - LogService logging = dependencies.getLogging(); - LogProvider internalLogProvider = logging.getInternalLogProvider(); - OperationalMode operationalMode = context.databaseInfo().operationalMode; - return create( fileSystemAbstraction, storeDir, internalLogProvider, config, operationalMode ); - } - - public static LuceneSchemaIndexProvider create( FileSystemAbstraction fileSystemAbstraction, File storeDir, - LogProvider logProvider, Config config, OperationalMode operationalMode ) - { + LogProvider logging = dependencies.getLogging().getInternalLogProvider(); boolean ephemeral = config.get( GraphDatabaseFacadeFactory.Configuration.ephemeral ); - DirectoryFactory directoryFactory = directoryFactory( ephemeral, fileSystemAbstraction ); - return new LuceneSchemaIndexProvider( fileSystemAbstraction, directoryFactory, storeDir, logProvider, config, operationalMode ); + + FileSystemAbstraction fileSystem = dependencies.fileSystem(); + DirectoryFactory directoryFactory = directoryFactory( ephemeral, fileSystem ); + + return new LuceneSchemaIndexProvider( fileSystem, directoryFactory, context.storeDir(), logging, config, + context.databaseInfo().operationalMode ); } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionSchemaIndexProviderFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionSchemaIndexProviderFactory.java deleted file mode 100644 index 229d568d72d3..000000000000 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionSchemaIndexProviderFactory.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.schema; - -import java.io.File; - -import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.Service; -import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.extension.KernelExtensionFactory; -import org.neo4j.kernel.impl.factory.OperationalMode; -import org.neo4j.kernel.impl.index.schema.NativeSchemaNumberIndexProvider; -import org.neo4j.kernel.impl.index.schema.NativeSelector; -import org.neo4j.kernel.impl.index.schema.fusion.FusionSchemaIndexProvider; -import org.neo4j.kernel.impl.spi.KernelContext; -import org.neo4j.logging.LogProvider; - -@Service.Implementation( KernelExtensionFactory.class ) -public class NativeLuceneFusionSchemaIndexProviderFactory - extends KernelExtensionFactory -{ - public static final String KEY = LuceneSchemaIndexProviderFactory.KEY + "+" + NativeSchemaNumberIndexProvider.KEY; - private static final int PRIORITY = LuceneSchemaIndexProvider.PRIORITY + 1; - - private static final SchemaIndexProvider.Descriptor DESCRIPTOR = new SchemaIndexProvider.Descriptor( KEY, "0.1" ); - - public interface Dependencies extends LuceneSchemaIndexProviderFactory.Dependencies - { - PageCache pageCache(); - - RecoveryCleanupWorkCollector recoveryCleanupWorkCollector(); - } - - public NativeLuceneFusionSchemaIndexProviderFactory() - { - super( KEY ); - } - - @Override - public FusionSchemaIndexProvider newInstance( KernelContext context, Dependencies dependencies ) throws Throwable - { - PageCache pageCache = dependencies.pageCache(); - File storeDir = context.storeDir(); - FileSystemAbstraction fs = dependencies.fileSystem(); - LogProvider logProvider = dependencies.getLogging().getInternalLogProvider(); - Config config = dependencies.getConfig(); - OperationalMode operationalMode = context.databaseInfo().operationalMode; - RecoveryCleanupWorkCollector recoveryCleanupWorkCollector = dependencies.recoveryCleanupWorkCollector(); - return newInstance( pageCache, storeDir, fs, logProvider, config, operationalMode, recoveryCleanupWorkCollector ); - } - - public static FusionSchemaIndexProvider newInstance( PageCache pageCache, File storeDir, FileSystemAbstraction fs, - LogProvider logProvider, Config config, OperationalMode operationalMode, - RecoveryCleanupWorkCollector recoveryCleanupWorkCollector ) - { - boolean readOnly = isReadOnly( config, operationalMode ); - NativeSchemaNumberIndexProvider nativeProvider = - new NativeSchemaNumberIndexProvider( pageCache, storeDir, logProvider, recoveryCleanupWorkCollector, readOnly ); - LuceneSchemaIndexProvider luceneProvider = LuceneSchemaIndexProviderFactory.create( fs, storeDir, logProvider, config, - operationalMode ); - boolean useNativeIndex = config.get( GraphDatabaseSettings.enable_native_schema_index ); - int priority = useNativeIndex ? PRIORITY : 0; - return new FusionSchemaIndexProvider( nativeProvider, luceneProvider, new NativeSelector(), DESCRIPTOR, priority ); - } - - private static boolean isReadOnly( Config config, OperationalMode operationalMode ) - { - return config.get( GraphDatabaseSettings.read_only ) && (OperationalMode.single == operationalMode); - } -} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java index 02717d602926..5bda6af3848d 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java @@ -70,6 +70,12 @@ public void add( Collection> updates ) throws Inde .iterator() ); } + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + writer.addDocument( updateAsDocument( update ) ); + } + @Override public void close( boolean populationCompletedSuccessfully ) throws IOException { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/PartitionedIndexReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/PartitionedIndexReader.java index f0ca18ab5c75..9fa19de1d6aa 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/PartitionedIndexReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/PartitionedIndexReader.java @@ -79,7 +79,7 @@ public PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotAp } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { return false; } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/SimpleIndexReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/SimpleIndexReader.java index 3d6e186a708e..80df7d8c573f 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/SimpleIndexReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/reader/SimpleIndexReader.java @@ -139,7 +139,7 @@ public PrimitiveLongIterator query( IndexQuery... predicates ) throws IndexNotAp } @Override - public boolean hasFullNumberPrecision( IndexQuery... predicates ) + public boolean hasFullNumberPrecision() { return false; } diff --git a/community/lucene-index/src/main/resources/META-INF/services/org.neo4j.kernel.extension.KernelExtensionFactory b/community/lucene-index/src/main/resources/META-INF/services/org.neo4j.kernel.extension.KernelExtensionFactory index ee53bd45c045..1ae2ce9d50b8 100644 --- a/community/lucene-index/src/main/resources/META-INF/services/org.neo4j.kernel.extension.KernelExtensionFactory +++ b/community/lucene-index/src/main/resources/META-INF/services/org.neo4j.kernel.extension.KernelExtensionFactory @@ -1,3 +1,3 @@ org.neo4j.index.lucene.LuceneKernelExtensionFactory + org.neo4j.kernel.api.impl.labelscan.LuceneLabelScanStoreExtension org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProviderFactory -org.neo4j.kernel.api.impl.schema.NativeLuceneFusionSchemaIndexProviderFactory diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/NonUniqueIndexTest.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/NonUniqueIndexTest.java index c9c540f1ee70..29a46f7f6728 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/NonUniqueIndexTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/NonUniqueIndexTest.java @@ -32,8 +32,8 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; -import org.neo4j.io.fs.DefaultFileSystemAbstraction; -import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProviderFactory; +import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; import org.neo4j.kernel.api.index.IndexAccessor; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.schema.IndexQuery; @@ -151,11 +151,9 @@ private static Runnable slowRunnable( final Runnable target ) private List nodeIdsInIndex( int indexId, String value ) throws Exception { Config config = Config.defaults(); - DefaultFileSystemAbstraction fs = fileSystemRule.get(); - File storeDir = directory.graphDbDir(); - NullLogProvider logProvider = NullLogProvider.getInstance(); - OperationalMode operationalMode = OperationalMode.single; - SchemaIndexProvider indexProvider = LuceneSchemaIndexProviderFactory.create( fs, storeDir, logProvider, config, operationalMode ); + SchemaIndexProvider indexProvider = new LuceneSchemaIndexProvider( fileSystemRule.get(), + DirectoryFactory.PERSISTENT, directory.graphDbDir(), NullLogProvider.getInstance(), + Config.defaults(), OperationalMode.single ); IndexSamplingConfig samplingConfig = new IndexSamplingConfig( config ); try ( IndexAccessor accessor = indexProvider.getOnlineAccessor( indexId, IndexDescriptorFactory.forLabel( 0, 0 ), samplingConfig ); diff --git a/community/lucene-index/src/test/java/org/neo4j/index/lucene/ConstraintIndexFailureIT.java b/community/lucene-index/src/test/java/org/neo4j/index/lucene/ConstraintIndexFailureIT.java index 329bd064683f..6807d0a944be 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/lucene/ConstraintIndexFailureIT.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/lucene/ConstraintIndexFailureIT.java @@ -35,17 +35,14 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.core.IsEqual.equalTo; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.neo4j.graphdb.Label.label; public class ConstraintIndexFailureIT { - private static final String INJECTED_FAILURE = "Injected failure"; - @Rule public final TestDirectory storeDir = TestDirectory.testDirectory(); @Rule @@ -56,7 +53,7 @@ public void shouldFailToValidateConstraintsIfUnderlyingIndexIsFailed() throws Ex { // given dbWithConstraint(); - storeIndexFailure( INJECTED_FAILURE ); + storeIndexFailure( "Injected failure" ); // when GraphDatabaseService db = startDatabase(); @@ -72,9 +69,7 @@ public void shouldFailToValidateConstraintsIfUnderlyingIndexIsFailed() throws Ex catch ( ConstraintViolationException e ) { assertThat( e.getCause(), instanceOf( UnableToValidateConstraintException.class ) ); - assertThat( e.getCause().getCause().getMessage(), allOf( - containsString( "The index is in a failed state:" ), - containsString( INJECTED_FAILURE ) ) ); + assertThat( e.getCause().getCause().getMessage(), equalTo( "The index is in a failed state: 'Injected failure'.") ); } } finally diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionIndexIT.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionIndexIT.java deleted file mode 100644 index a37e7aea62e1..000000000000 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionIndexIT.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.schema; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.util.concurrent.TimeUnit; - -import org.neo4j.graphdb.Label; -import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.schema.IndexDefinition; -import org.neo4j.helpers.collection.Iterators; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.impl.index.schema.NativeSchemaNumberIndexProvider; -import org.neo4j.kernel.internal.GraphDatabaseAPI; -import org.neo4j.test.rule.DatabaseRule; -import org.neo4j.test.rule.EmbeddedDatabaseRule; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class FusionIndexIT -{ - @Rule - public DatabaseRule db = new EmbeddedDatabaseRule(); - - private File storeDir; - private Label label = Label.label( "label" ); - private String propKey = "propKey"; - private FileSystemAbstraction fs; - - @Before - public void setup() - { - storeDir = db.getStoreDir(); - fs = db.getDependencyResolver().resolveDependency( FileSystemAbstraction.class ); - } - - @Test - public void mustRebuildFusionIndexIfNativePartIsMissing() throws Exception - { - // given - initializeIndexWithDataAndShutdown(); - - // when - SchemaIndexProvider.Descriptor descriptor = NativeSchemaNumberIndexProvider.NATIVE_PROVIDER_DESCRIPTOR; - deleteIndexFilesFor( descriptor ); - - // then - // ... should rebuild - verifyContent(); - } - - @Test - public void mustRebuildFusionIndexIfLucenePartIsMissing() throws Exception - { - // given - initializeIndexWithDataAndShutdown(); - - // when - SchemaIndexProvider.Descriptor descriptor = LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR; - deleteIndexFilesFor( descriptor ); - - // then - // ... should rebuild - verifyContent(); - } - - @Test - public void mustRebuildFusionIndexIfCompletelyMissing() throws Exception - { - // given - initializeIndexWithDataAndShutdown(); - - // when - SchemaIndexProvider.Descriptor luceneDescriptor = LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR; - SchemaIndexProvider.Descriptor nativeDescriptor = NativeSchemaNumberIndexProvider.NATIVE_PROVIDER_DESCRIPTOR; - deleteIndexFilesFor( luceneDescriptor ); - deleteIndexFilesFor( nativeDescriptor ); - - // then - // ... should rebuild - verifyContent(); - } - - private void verifyContent() - { - GraphDatabaseAPI newDb = db.getGraphDatabaseAPI(); - try ( Transaction tx = newDb.beginTx() ) - { - assertEquals( 1L, Iterators.stream( newDb.schema().getIndexes( label ).iterator() ).count() ); - assertNotNull( newDb.findNode( label, propKey, 1 ) ); - assertNotNull( newDb.findNode( label, propKey, "string" ) ); - tx.success(); - } - } - - private void deleteIndexFilesFor( SchemaIndexProvider.Descriptor descriptor ) - { - File nativeIndexDirectory = SchemaIndexProvider.getSchemaIndexStoreDirectory( storeDir, descriptor ); - File[] files = fs.listFiles( nativeIndexDirectory ); - for ( File indexFile : files ) - { - fs.deleteFile( indexFile ); - } - } - - private void initializeIndexWithDataAndShutdown() - { - createIndex(); - try ( Transaction tx = db.beginTx() ) - { - db.createNode( label ).setProperty( propKey, 1 ); - db.createNode( label ).setProperty( propKey, "string" ); - tx.success(); - } - db.shutdown(); - } - - private void createIndex() - { - try ( Transaction tx = db.beginTx() ) - { - IndexDefinition indexDefinition = db.schema().indexFor( label ).on( propKey ).create(); - tx.success(); - } - try ( Transaction tx = db.beginTx() ) - { - db.schema().awaitIndexesOnline( 10, TimeUnit.SECONDS ); - tx.success(); - } - } -} diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionSchemaIndexProviderCompatibilitySuiteTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionSchemaIndexProviderCompatibilitySuiteTest.java deleted file mode 100644 index 1864add08ee9..000000000000 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/FusionSchemaIndexProviderCompatibilitySuiteTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.schema; - -import java.io.File; - -import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; -import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.api.index.IndexProviderCompatibilityTestSuite; -import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.Settings; -import org.neo4j.kernel.impl.factory.OperationalMode; -import org.neo4j.logging.NullLogProvider; - -import static org.neo4j.helpers.collection.MapUtil.stringMap; - -public class FusionSchemaIndexProviderCompatibilitySuiteTest extends IndexProviderCompatibilityTestSuite -{ - @Override - protected SchemaIndexProvider createIndexProvider( PageCache pageCache, FileSystemAbstraction fs, File graphDbDir ) - { - NullLogProvider logProvider = NullLogProvider.getInstance(); - Config config = Config.defaults( stringMap( GraphDatabaseSettings.enable_native_schema_index.name(), Settings.TRUE ) ); - return NativeLuceneFusionSchemaIndexProviderFactory - .newInstance( pageCache, graphDbDir, fs, logProvider, config, OperationalMode.single, - RecoveryCleanupWorkCollector.IMMEDIATE ); - } -} diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneIndexRecoveryIT.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneIndexRecoveryIT.java index cbfd5d66bda5..ec0d81c5b24e 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneIndexRecoveryIT.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneIndexRecoveryIT.java @@ -219,7 +219,7 @@ private void startDb( KernelExtensionFactory indexProviderFactory ) TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory(); factory.setFileSystem( fs.get() ); - factory.setKernelExtensions( Arrays.asList( indexProviderFactory ) ); + factory.addKernelExtensions( Arrays.asList( indexProviderFactory ) ); db = (GraphDatabaseAPI) factory.newImpermanentDatabase(); } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderCompatibilitySuiteTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderCompatibilitySuiteTest.java index 81c134e35251..d23c6bfc127d 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderCompatibilitySuiteTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProviderCompatibilitySuiteTest.java @@ -21,27 +21,23 @@ import java.io.File; -import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; import org.neo4j.kernel.api.index.IndexProviderCompatibilityTestSuite; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.factory.OperationalMode; import org.neo4j.logging.NullLogProvider; -import static org.neo4j.helpers.collection.MapUtil.stringMap; - public class LuceneSchemaIndexProviderCompatibilitySuiteTest extends IndexProviderCompatibilityTestSuite { @Override - protected LuceneSchemaIndexProvider createIndexProvider( PageCache pageCache, FileSystemAbstraction fs, File graphDbDir ) + protected LuceneSchemaIndexProvider createIndexProvider( FileSystemAbstraction fs, File graphDbDir ) { DirectoryFactory.InMemoryDirectoryFactory directoryFactory = new DirectoryFactory.InMemoryDirectoryFactory(); NullLogProvider logging = NullLogProvider.getInstance(); - Config config = Config.defaults( stringMap( GraphDatabaseSettings.enable_native_schema_index.name(), Settings.FALSE ) ); + Config config = Config.defaults(); OperationalMode mode = OperationalMode.single; - return LuceneSchemaIndexProviderFactory.create( fs, graphDbDir, logging, config, mode ); + return new LuceneSchemaIndexProvider( fs, directoryFactory, graphDbDir, logging, config, mode ); + } } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueDatabaseIndexPopulatorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueDatabaseIndexPopulatorTest.java index 19faac2aa1d9..4baad31d4e11 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueDatabaseIndexPopulatorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueDatabaseIndexPopulatorTest.java @@ -556,8 +556,8 @@ private UniqueLuceneIndexPopulator newPopulator() throws IOException private static void addUpdate( UniqueLuceneIndexPopulator populator, long nodeId, Object value ) throws IOException, IndexEntryConflictException { - IndexEntryUpdate update = add( nodeId, descriptor.schema(), value ); - populator.add( asList( update ) ); + IndexEntryUpdate update = add( nodeId, descriptor.schema(), value ); + populator.add( update ); } private List getAllNodes( Directory directory, Object value ) throws IOException diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceIntegrationTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceIntegrationTest.java index 514f637f4bb3..f86231519cf4 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceIntegrationTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceIntegrationTest.java @@ -54,8 +54,6 @@ import static org.junit.Assert.assertEquals; -import static java.util.Arrays.asList; - public class IndexingServiceIntegrationTest { private static final String FOOD_LABEL = "food"; @@ -77,7 +75,7 @@ public void setUp() EphemeralFileSystemAbstraction fileSystem = fileSystemRule.get(); database = new TestGraphDatabaseFactory() .setFileSystem( fileSystem ) - .setKernelExtensions( asList( new LuceneSchemaIndexProviderFactory() ) ) + .addKernelExtension( new LuceneSchemaIndexProviderFactory() ) .newImpermanentDatabase(); createData( database, 100 ); } diff --git a/community/lucene-index/src/test/java/org/neo4j/unsafe/batchinsert/LuceneSchemaBatchInsertTest.java b/community/lucene-index/src/test/java/org/neo4j/unsafe/batchinsert/LuceneSchemaBatchInsertTest.java new file mode 100644 index 000000000000..8feb89022f4b --- /dev/null +++ b/community/lucene-index/src/test/java/org/neo4j/unsafe/batchinsert/LuceneSchemaBatchInsertTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.unsafe.batchinsert; + +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.schema.IndexDefinition; +import org.neo4j.graphdb.schema.Schema; +import org.neo4j.helpers.collection.Iterables; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; +import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.extension.dependency.HighestSelectionStrategy; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.neo4j.graphdb.Label.label; +import static org.neo4j.helpers.collection.MapUtil.map; + +public class LuceneSchemaBatchInsertTest +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + @Rule + public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); + + private static final Label LABEL = label( "Person" ); + + @Test + public void shouldLoadAndUseLuceneProvider() throws Exception + { + // GIVEN + File storeDir = testDirectory.graphDbDir(); + BatchInserter inserter = BatchInserters.inserter( storeDir, fileSystemRule.get() ); + inserter.createDeferredSchemaIndex( LABEL ).on( "name" ).create(); + + // WHEN + inserter.createNode( map( "name", "Mattias" ), LABEL ); + inserter.shutdown(); + + // THEN + GraphDatabaseFactory graphDatabaseFactory = new TestGraphDatabaseFactory(); + GraphDatabaseAPI db = (GraphDatabaseAPI) graphDatabaseFactory.newEmbeddedDatabase( storeDir ); + DependencyResolver dependencyResolver = db.getDependencyResolver(); + SchemaIndexProvider schemaIndexProvider = dependencyResolver.resolveDependency( + SchemaIndexProvider.class, HighestSelectionStrategy.getInstance() ); + + // assert the indexProvider is a Lucene one + try ( Transaction ignore = db.beginTx() ) + { + IndexDefinition indexDefinition = Iterables.single( db.schema().getIndexes( LABEL ) ); + assertThat( db.schema().getIndexState( indexDefinition ), is( Schema.IndexState.ONLINE ) ); + assertThat( schemaIndexProvider, instanceOf( LuceneSchemaIndexProvider.class ) ); + } + + // CLEANUP + db.shutdown(); + } +} diff --git a/community/neo4j/src/test/java/counts/RebuildCountsTest.java b/community/neo4j/src/test/java/counts/RebuildCountsTest.java index 88c78390b56d..8ff5091bd93d 100644 --- a/community/neo4j/src/test/java/counts/RebuildCountsTest.java +++ b/community/neo4j/src/test/java/counts/RebuildCountsTest.java @@ -50,9 +50,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; - -import static java.util.Arrays.asList; - import static org.neo4j.graphdb.Label.label; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.index_background_sampling_enabled; import static org.neo4j.logging.AssertableLogProvider.inLog; @@ -195,7 +192,7 @@ private void restart( FileSystemAbstraction fs ) throws IOException db = dbFactory.setUserLogProvider( userLogProvider ) .setInternalLogProvider( internalLogProvider ) .setFileSystem( new UncloseableDelegatingFileSystemAbstraction( fs ) ) - .setKernelExtensions( asList( new InMemoryIndexProviderFactory( indexProvider ) ) ) + .addKernelExtension( new InMemoryIndexProviderFactory( indexProvider ) ) .newImpermanentDatabaseBuilder( storeDir ) .setConfig( index_background_sampling_enabled, "false" ) .newGraphDatabase(); diff --git a/community/neo4j/src/test/java/recovery/CountsStoreRecoveryTest.java b/community/neo4j/src/test/java/recovery/CountsStoreRecoveryTest.java index b383ea5f47de..aac66bd1d390 100644 --- a/community/neo4j/src/test/java/recovery/CountsStoreRecoveryTest.java +++ b/community/neo4j/src/test/java/recovery/CountsStoreRecoveryTest.java @@ -44,9 +44,6 @@ import org.neo4j.test.rule.fs.EphemeralFileSystemRule; import static org.junit.Assert.assertEquals; - -import static java.util.Arrays.asList; - import static org.neo4j.graphdb.Label.label; public class CountsStoreRecoveryTest @@ -139,7 +136,7 @@ public void before() private TestGraphDatabaseFactory databaseFactory( FileSystemAbstraction fs, InMemoryIndexProvider indexProvider ) { return new TestGraphDatabaseFactory() - .setFileSystem( fs ).setKernelExtensions( asList( new InMemoryIndexProviderFactory( indexProvider ) ) ); + .setFileSystem( fs ).addKernelExtension( new InMemoryIndexProviderFactory( indexProvider ) ); } @After diff --git a/community/neo4j/src/test/java/recovery/TestRecoveryScenarios.java b/community/neo4j/src/test/java/recovery/TestRecoveryScenarios.java index adf21ba79049..8473fe5d8b33 100644 --- a/community/neo4j/src/test/java/recovery/TestRecoveryScenarios.java +++ b/community/neo4j/src/test/java/recovery/TestRecoveryScenarios.java @@ -52,7 +52,6 @@ import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; -import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -302,7 +301,7 @@ public void before() private TestGraphDatabaseFactory databaseFactory( FileSystemAbstraction fs, InMemoryIndexProvider indexProvider ) { return new TestGraphDatabaseFactory() - .setFileSystem( fs ).setKernelExtensions( asList( new InMemoryIndexProviderFactory( indexProvider ) ) ); + .setFileSystem( fs ).addKernelExtension( new InMemoryIndexProviderFactory( indexProvider ) ); } @After diff --git a/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java b/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java index fb3d53391948..67795ae48a11 100644 --- a/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java +++ b/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java @@ -46,7 +46,6 @@ import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException; import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException; -import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProviderFactory; import org.neo4j.kernel.api.index.IndexEntryUpdate; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; @@ -313,7 +312,7 @@ private IndexRule[] createIndexRules( Map labelNameIdMap, int pr { return labelNameIdMap.values().stream() .map( index -> IndexRule.indexRule( index, IndexDescriptorFactory.forLabel( index, propertyId ), - LuceneSchemaIndexProviderFactory.PROVIDER_DESCRIPTOR ) ) + new SchemaIndexProvider.Descriptor( "lucene", "version" ) ) ) .toArray( IndexRule[]::new ); } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java index c49056fe5713..91dac6e7d8b0 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java @@ -25,8 +25,8 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.NoSuchElementException; @@ -45,12 +45,11 @@ import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.graphdb.schema.Schema.IndexState; import org.neo4j.helpers.collection.Iterables; -import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; -import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; -import org.neo4j.kernel.api.impl.schema.NativeLuceneFusionSchemaIndexProviderFactory; +import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; import org.neo4j.kernel.api.index.IndexAccessor; import org.neo4j.kernel.api.index.IndexEntryUpdate; import org.neo4j.kernel.api.index.IndexPopulator; @@ -65,10 +64,8 @@ import org.neo4j.kernel.ha.UpdatePuller; import org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; -import org.neo4j.kernel.impl.factory.OperationalMode; import org.neo4j.kernel.impl.ha.ClusterManager; import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; -import org.neo4j.kernel.impl.index.schema.fusion.FusionSchemaIndexProvider; import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant; import org.neo4j.kernel.lifecycle.Lifecycle; @@ -445,6 +442,13 @@ public void add( Collection> updates ) latch.startAndWaitForAllToStartAndFinish(); } + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + delegate.add( update ); + latch.startAndWaitForAllToStartAndFinish(); + } + @Override public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) throws IndexEntryConflictException, IOException @@ -543,8 +547,6 @@ interface IndexProviderDependencies { GraphDatabaseService db(); Config config(); - PageCache pageCache(); - RecoveryCleanupWorkCollector recoveryCleanupWorkCollector(); } private static class ControllingIndexProviderFactory extends KernelExtensionFactory @@ -563,26 +565,20 @@ private static class ControllingIndexProviderFactory extends KernelExtensionFact @Override public Lifecycle newInstance( KernelContext context, SchemaIndexHaIT.IndexProviderDependencies deps ) throws Throwable { - PageCache pageCache = deps.pageCache(); - File storeDir = context.storeDir(); - DefaultFileSystemAbstraction fs = fileSystemRule.get(); - NullLogProvider logProvider = NullLogProvider.getInstance(); - Config config = deps.config(); - OperationalMode operationalMode = context.databaseInfo().operationalMode; - RecoveryCleanupWorkCollector recoveryCleanupWorkCollector = deps.recoveryCleanupWorkCollector(); - - FusionSchemaIndexProvider fusionSchemaIndexProvider = NativeLuceneFusionSchemaIndexProviderFactory - .newInstance( pageCache, storeDir, fs, logProvider, config, operationalMode, recoveryCleanupWorkCollector ); - if ( injectLatchPredicate.test( deps.db() ) ) { - ControlledSchemaIndexProvider provider = new ControlledSchemaIndexProvider( fusionSchemaIndexProvider ); + ControlledSchemaIndexProvider provider = new ControlledSchemaIndexProvider( + new LuceneSchemaIndexProvider( fileSystemRule.get(), + DirectoryFactory.PERSISTENT, context.storeDir(), NullLogProvider.getInstance(), + deps.config(), context.databaseInfo().operationalMode ) ); perDbIndexProvider.put( deps.db(), provider ); return provider; } else { - return fusionSchemaIndexProvider; + return new LuceneSchemaIndexProvider( fileSystemRule.get(), + DirectoryFactory.PERSISTENT, context.storeDir(), NullLogProvider.getInstance(), deps.config(), + context.databaseInfo().operationalMode ); } } } @@ -594,19 +590,18 @@ private static class ControlledGraphDatabaseFactory extends TestHighlyAvailableG ControlledGraphDatabaseFactory() { - this( Predicates.alwaysTrue() ); + factory = new ControllingIndexProviderFactory( perDbIndexProvider, Predicates.alwaysTrue() ); } private ControlledGraphDatabaseFactory( Predicate dbsToControlIndexingOn ) { factory = new ControllingIndexProviderFactory( perDbIndexProvider, dbsToControlIndexingOn ); - getCurrentState().removeKernelExtensions( kef -> kef.getClass().getSimpleName().contains( "IndexProvider" ) ); - getCurrentState().addKernelExtensions( Collections.singletonList( factory ) ); } @Override public GraphDatabaseBuilder newEmbeddedDatabaseBuilder( File file ) { + getCurrentState().addKernelExtensions( Arrays.asList( factory ) ); return super.newEmbeddedDatabaseBuilder( file ); } diff --git a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java index 3ae30aac85b2..99ec28ddffd4 100644 --- a/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java +++ b/tools/src/main/java/org/neo4j/tools/migration/StoreMigration.java @@ -35,8 +35,7 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.extension.KernelExtensions; -import org.neo4j.kernel.extension.dependency.AllByPrioritySelectionStrategy; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; +import org.neo4j.kernel.extension.dependency.HighestSelectionStrategy; import org.neo4j.kernel.impl.factory.DatabaseInfo; import org.neo4j.kernel.impl.logging.StoreLogService; import org.neo4j.kernel.impl.spi.KernelContext; @@ -46,7 +45,6 @@ import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; -import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.monitoring.Monitors; @@ -123,15 +121,11 @@ public void run( final FileSystemAbstraction fs, final File storeDirectory, Conf // Add the kernel store migrator life.start(); - AllByPrioritySelectionStrategy indexProviderSelection = new AllByPrioritySelectionStrategy<>(); - SchemaIndexProvider defaultIndexProvider = kernelExtensions.resolveDependency( SchemaIndexProvider.class, - indexProviderSelection ); - SchemaIndexProviderMap schemaIndexProviderMap = new DefaultSchemaIndexProviderMap( defaultIndexProvider, - indexProviderSelection.lowerPrioritizedCandidates() ); - + SchemaIndexProvider schemaIndexProvider = kernelExtensions.resolveDependency( SchemaIndexProvider.class, + HighestSelectionStrategy.getInstance() ); long startTime = System.currentTimeMillis(); DatabaseMigrator migrator = new DatabaseMigrator( progressMonitor, fs, config, logService, - schemaIndexProviderMap, legacyIndexProvider.getIndexProviders(), + schemaIndexProvider, legacyIndexProvider.getIndexProviders(), pageCache, RecordFormatSelector.selectForConfig( config, userLogProvider ) ); migrator.migrate( storeDirectory ); long duration = System.currentTimeMillis() - startTime; diff --git a/tools/src/main/java/org/neo4j/tools/rebuild/RebuildFromLogs.java b/tools/src/main/java/org/neo4j/tools/rebuild/RebuildFromLogs.java index e8bf2fe391be..e58b6f76c748 100644 --- a/tools/src/main/java/org/neo4j/tools/rebuild/RebuildFromLogs.java +++ b/tools/src/main/java/org/neo4j/tools/rebuild/RebuildFromLogs.java @@ -42,12 +42,12 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory; import org.neo4j.kernel.api.direct.DirectStoreAccess; +import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionQueue; import org.neo4j.kernel.impl.api.TransactionToApply; -import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.StoreAccess; @@ -278,14 +278,14 @@ private static class ConsistencyChecker implements AutoCloseable private final GraphDatabaseAPI graphdb; private final LabelScanStore labelScanStore; private final Config tuningConfiguration = Config.defaults(); - private final SchemaIndexProviderMap indexes; + private final SchemaIndexProvider indexes; ConsistencyChecker( File dbDirectory, PageCache pageCache ) { this.graphdb = startTemporaryDb( dbDirectory.getAbsoluteFile(), pageCache ); DependencyResolver resolver = graphdb.getDependencyResolver(); this.labelScanStore = resolver.resolveDependency( LabelScanStore.class ); - this.indexes = resolver.resolveDependency( SchemaIndexProviderMap.class ); + this.indexes = resolver.resolveDependency( SchemaIndexProvider.class ); } private void checkConsistency() throws ConsistencyCheckIncompleteException, InconsistentStoreException