Update store migration to remove schema and label indexes as part of …

…3.0 migration.

Lucene 5.x does not support existent 3.x indexes, so as part of migration we will remove all old indexes, so they will be rebuild.
Restore all ignored upgrage tests.
Current path are only valid for schema and label indexes, legacy indexes will be addressed in a separate commit.

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -47,6 +47,7 @@
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.extension.dependency.HighestSelectionStrategy;
 import org.neo4j.kernel.guard.Guard;
 import org.neo4j.kernel.impl.api.CommitProcessFactory;
 import org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations;
@@ -88,6 +89,8 @@
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.StoreId;
 import org.neo4j.kernel.impl.store.UnderlyingStorageException;
+import org.neo4j.kernel.impl.store.record.SchemaRule;
+import org.neo4j.kernel.impl.storemigration.StoreMigrator;
 import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
 import org.neo4j.kernel.impl.storemigration.StoreVersionCheck;
 import org.neo4j.kernel.impl.storemigration.UpgradableDatabase;
@@ -294,6 +297,7 @@ boolean applicable( DiagnosticsPhase phase )
     private final IndexingService.Monitor indexingServiceMonitor;
     private final FileSystemAbstraction fs;
     private final StoreUpgrader storeMigrationProcess;
+    private StoreMigrator storeMigrator;
     private final TransactionMonitor transactionMonitor;
     private final DatabaseHealth databaseHealth;
     private final PhysicalLogFile.Monitor physicalLogMonitor;
@@ -354,6 +358,7 @@ public NeoStoreDataSource(
             IndexingService.Monitor indexingServiceMonitor,
             FileSystemAbstraction fs,
             StoreUpgrader storeMigrationProcess,
+            StoreMigrator storeMigrator,
             TransactionMonitor transactionMonitor,
             DatabaseHealth databaseHealth,
             PhysicalLogFile.Monitor physicalLogMonitor,
@@ -382,6 +387,7 @@ public NeoStoreDataSource(
         this.indexingServiceMonitor = indexingServiceMonitor;
         this.fs = fs;
         this.storeMigrationProcess = storeMigrationProcess;
+        this.storeMigrator = storeMigrator;
         this.transactionMonitor = transactionMonitor;
         this.databaseHealth = databaseHealth;
         this.physicalLogMonitor = physicalLogMonitor;
@@ -439,7 +445,11 @@ public void start() throws IOException
         life = new LifeSupport();
 
         indexProvider = dependencyResolver.resolveDependency( SchemaIndexProvider.class,
-                SchemaIndexProvider.HIGHEST_PRIORITIZED_OR_NONE );
+                HighestSelectionStrategy.getInstance() );
+
+        LabelScanStoreProvider labelScanStoreProvider =
+                dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
+                        HighestSelectionStrategy.getInstance() );
 
         IndexConfigStore indexConfigStore = new IndexConfigStore( storeDir, fs );
         dependencies.satisfyDependency( lockService );
@@ -454,7 +464,7 @@ public void start() throws IOException
         life.add( new Lifecycle.Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
 
         // Upgrade the store before we begin
-        upgradeStore( storeDir, storeMigrationProcess, indexProvider );
+        upgradeStore( storeDir, storeMigrationProcess, indexProvider, labelScanStoreProvider );
 
         // Build all modules and their services
         StorageEngine storageEngine = null;
@@ -561,13 +571,14 @@ public void start() throws IOException
     // Startup sequence
     // By doing this sequence of method calls we can ensure that no dependency cycles exist, and get a clearer view
     // of the dependency tree, starting at the bottom
-    private void upgradeStore( File storeDir, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider )
+    private void upgradeStore( File storeDir, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider,
+            LabelScanStoreProvider labelScanStoreProvider )
     {
         UpgradableDatabase upgradableDatabase =
                 new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
-        storeMigrationProcess
-                .addParticipant( indexProvider.storeMigrationParticipant( fs, pageCache ) );
-        storeMigrationProcess.migrateIfNeeded( storeDir, upgradableDatabase, indexProvider );
+        storeMigrationProcess.addParticipant( indexProvider.storeMigrationParticipant( fs, pageCache ) );
+        storeMigrationProcess.addParticipant( storeMigrator );
+        storeMigrationProcess.migrateIfNeeded( storeDir, upgradableDatabase, indexProvider, labelScanStoreProvider );
     }
 
     private StorageEngine buildStorageEngine(
@@ -577,7 +588,7 @@ private StorageEngine buildStorageEngine(
             Runnable schemaStateChangeCallback )
     {
         LabelScanStoreProvider labelScanStore = dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
-                LabelScanStoreProvider.HIGHEST_PRIORITIZED );
+                HighestSelectionStrategy.getInstance());
         return life.add(
                 new RecordStorageEngine( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, propertyKeyTokenHolder,
                         labelTokens, relationshipTypeTokens, schemaStateChangeCallback, constraintSemantics, scheduler,

community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java

@@ -137,24 +137,6 @@ public String getPopulationFailure( long indexId ) throws IllegalStateException
                 }
             };
 
-    public static final SelectionStrategy HIGHEST_PRIORITIZED_OR_NONE =
-            new SelectionStrategy()
-    {
-        @Override
-        @SuppressWarnings("unchecked")
-        public <T> T select( Class<T> type, Iterable<T> candidates ) throws IllegalArgumentException
-        {
-            List<Comparable> all = (List<Comparable>) addToCollection( candidates, new ArrayList<T>() );
-            if ( all.isEmpty() )
-            {
-                throw new IllegalArgumentException( "No schema index provider " +
-                        SchemaIndexProvider.class.getName() + " found. " + servicesClassPathEntryInformation() );
-            }
-            Collections.sort( all );
-            return (T) all.get( all.size()-1 );
-        }
-    };
-
     protected final int priority;
     private final Descriptor providerDescriptor;
 
@@ -233,9 +215,9 @@ public int hashCode()
         return result;
     }
 
-    public static File getRootDirectory( File storeDir, String key )
+    public File getStoreDirectory( File storeDir )
     {
-        return new File( new File( new File( storeDir, "schema" ), "index" ), key );
+        return new File( new File( new File( storeDir, "schema" ), "index" ), getProviderDescriptor().getKey() );
     }
 
     public abstract StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache );

community/kernel/src/main/java/org/neo4j/kernel/extension/dependency/HighestSelectionStrategy.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2002-2015 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.extension.dependency;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.neo4j.graphdb.DependencyResolver;
+import org.neo4j.helpers.collection.Iterables;
+import org.neo4j.kernel.extension.KernelExtensions;
+
+import static org.neo4j.kernel.extension.KernelExtensionUtil.servicesClassPathEntryInformation;
+
+/**
+ * SelectionStrategy for {@link KernelExtensions kernel extensions loading} where the one with highest
+ * natural order will be selected. If there are no such stores then an {@link IllegalStateException} will be
+ * thrown.
+ *
+ * @see Comparable
+ */
+public class HighestSelectionStrategy implements DependencyResolver.SelectionStrategy
+{
+    private static final DependencyResolver.SelectionStrategy instance = new HighestSelectionStrategy();
+
+    public static DependencyResolver.SelectionStrategy getInstance()
+    {
+        return instance;
+    }
+
+    private HighestSelectionStrategy()
+    {
+    }
+
+    public <T> T select( Class<T> type, Iterable<T> candidates )
+            throws IllegalArgumentException
+    {
+        List<Comparable> all = (List<Comparable>) Iterables.toList( candidates );
+        if ( all.isEmpty() )
+        {
+            throw new IllegalArgumentException( "Could not resolve dependency of type: " +
+                                                type.getName() + ". " + servicesClassPathEntryInformation() );
+        }
+        Collections.sort( all );
+        return (T) all.get( all.size() - 1 );
+    }
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/api/scan/LabelScanStoreProvider.java

@@ -19,12 +19,9 @@
  */
 package org.neo4j.kernel.impl.api.scan;
 
-import java.util.ArrayList;
-import java.util.Collections;
+import java.io.File;
 import java.util.Iterator;
-import java.util.List;
 
-import org.neo4j.graphdb.DependencyResolver.SelectionStrategy;
 import org.neo4j.helpers.collection.PrefetchingIterator;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
@@ -39,8 +36,6 @@
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
 import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY;
-import static org.neo4j.helpers.collection.IteratorUtil.addToCollection;
-import static org.neo4j.kernel.extension.KernelExtensionUtil.servicesClassPathEntryInformation;
 
 /**
  * Used by a {@link KernelExtensions} to provide access a {@link LabelScanStore} and prioritize against other.
@@ -56,31 +51,9 @@
  */
 public class LabelScanStoreProvider extends LifecycleAdapter implements Comparable<LabelScanStoreProvider>
 {
-    /**
-     * SelectionStrategy for {@link KernelExtensions kernel extensions loading} where the one with highest
-     * {@link #priority} will be selected. If there are no such stores  then an {@link IllegalStateException} will be
-     * thrown.
-     */
-    public static SelectionStrategy HIGHEST_PRIORITIZED =
-            new SelectionStrategy()
-    {
-        @Override
-        public <T> T select( Class<T> type, Iterable<T> candidates )
-                throws IllegalArgumentException
-        {
-            List<Comparable> all = (List<Comparable>) addToCollection( candidates, new ArrayList<T>() );
-            if ( all.isEmpty() )
-            {
-                throw new IllegalArgumentException( "No label scan store provider " +
-                        LabelScanStoreProvider.class.getName() + " found. " + servicesClassPathEntryInformation() );
-            }
-            Collections.sort( all );
-            return (T) all.get( all.size()-1 );
-        }
-    };
+    private static final String KEY = "lucene";
 
     private final LabelScanStore labelScanStore;
-
     private final int priority;
 
     public LabelScanStoreProvider( LabelScanStore labelScanStore, int priority )
@@ -89,6 +62,11 @@ public LabelScanStoreProvider( LabelScanStore labelScanStore, int priority )
         this.priority = priority;
     }
 
+    public static File getStoreDirectory( File storeRootDir )
+    {
+        return new File( new File( new File( storeRootDir, "schema" ), "label" ), KEY );
+    }
+
     public LabelScanStore getLabelScanStore()
     {
         return labelScanStore;

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java

@@ -168,8 +168,8 @@ public GraphDatabaseService getGraphDatabaseService()
 
         VisibleMigrationProgressMonitor progressMonitor =
                 new VisibleMigrationProgressMonitor( logging.getInternalLog( StoreMigrator.class ) );
-        storeMigrationProcess.addParticipant(
-                new StoreMigrator( progressMonitor, fileSystem, pageCache, config, logging ) );
+
+        StoreMigrator storeMigrator = new StoreMigrator( progressMonitor, fileSystem, pageCache, config, logging );
 
         Guard guard = config.get( execution_guard_enabled ) ?
                       deps.satisfyDependency( new Guard( logging.getInternalLog( Guard.class ) ) ) :
@@ -190,7 +190,7 @@ public GraphDatabaseService getGraphDatabaseService()
                 deps, editionModule.propertyKeyTokenHolder, editionModule.labelTokenHolder, relationshipTypeTokenHolder,
                 editionModule.lockManager, schemaWriteGuard, transactionEventHandlers,
                 platformModule.monitors.newMonitor( IndexingService.Monitor.class ), fileSystem,
-                storeMigrationProcess, platformModule.transactionMonitor, databaseHealth,
+                storeMigrationProcess, storeMigrator, platformModule.transactionMonitor, databaseHealth,
                 platformModule.monitors.newMonitor( PhysicalLogFile.Monitor.class ),
                 editionModule.headerInformationFactory, startupStatistics, nodeManager, guard,
                 editionModule.commitProcessFactory, pageCache, editionModule.constraintSemantics,

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java

@@ -21,106 +21,53 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
 
 import org.neo4j.io.fs.FileSystemAbstraction;
-import org.neo4j.io.pagecache.PageCache;
-import org.neo4j.kernel.DefaultIdGeneratorFactory;
-import org.neo4j.kernel.api.index.IndexAccessor;
-import org.neo4j.kernel.api.index.IndexConfiguration;
-import org.neo4j.kernel.api.index.IndexReader;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
-import org.neo4j.kernel.configuration.Config;
-import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
-import org.neo4j.kernel.impl.store.NeoStores;
-import org.neo4j.kernel.impl.store.SchemaStore;
-import org.neo4j.kernel.impl.store.StoreFactory;
-import org.neo4j.kernel.impl.store.StoreType;
-import org.neo4j.kernel.impl.store.record.SchemaRule;
+import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
 import org.neo4j.kernel.impl.storemigration.legacystore.v19.Legacy19Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v20.Legacy20Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store;
 import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store;
-import org.neo4j.logging.NullLogProvider;
-
-import static org.neo4j.kernel.api.index.SchemaIndexProvider.getRootDirectory;
-import static org.neo4j.kernel.impl.store.record.SchemaRule.Kind.UNIQUENESS_CONSTRAINT;
 
 public class SchemaIndexMigrator implements StoreMigrationParticipant
 {
     private final FileSystemAbstraction fileSystem;
-    private final StoreFactory storeFactory;
 
-    public SchemaIndexMigrator( FileSystemAbstraction fileSystem, PageCache pageCache, StoreFactory storeFactory )
+    public SchemaIndexMigrator( FileSystemAbstraction fileSystem )
     {
         this.fileSystem = fileSystem;
-        this.storeFactory = storeFactory;
-
-        storeFactory.setConfig( new Config() );
-        storeFactory.setFileSystemAbstraction( fileSystem );
-        storeFactory.setIdGeneratorFactory( new DefaultIdGeneratorFactory( fileSystem ) );
-        storeFactory.setLogProvider( NullLogProvider.getInstance() );
-        storeFactory.setPageCache( pageCache );
     }
 
     @Override
     public void migrate( File storeDir, File migrationDir, SchemaIndexProvider schemaIndexProvider,
-            String versionToMigrateFrom ) throws IOException
+            LabelScanStoreProvider labelScanStoreProvider, String versionToMigrateFrom ) throws IOException
     {
         switch ( versionToMigrateFrom )
         {
         case Legacy19Store.LEGACY_VERSION:
-            break;
         case Legacy20Store.LEGACY_VERSION:
         case Legacy21Store.LEGACY_VERSION:
-            deleteIndexesContainingArrayValues( storeDir, schemaIndexProvider );
-            break;
         case Legacy22Store.LEGACY_VERSION:
         case Legacy23Store.LEGACY_VERSION:
+            deleteIndexes( storeDir, schemaIndexProvider, labelScanStoreProvider );
             break;
         default:
             throw new IllegalStateException( "Unknown version to upgrade from: " + versionToMigrateFrom );
         }
     }
 
-    private void deleteIndexesContainingArrayValues( File storeDir,
-                                                     SchemaIndexProvider schemaIndexProvider ) throws IOException
+    private void deleteIndexes( File storeDir, SchemaIndexProvider schemaIndexProvider,
+            LabelScanStoreProvider labelScanStoreProvider ) throws IOException
     {
-        File indexRoot = getRootDirectory( storeDir, schemaIndexProvider.getProviderDescriptor().getKey() );
-        IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
-        List<File> indexesToBeDeleted = new ArrayList<>();
-        storeFactory.setStoreDir( storeDir );
-        try ( NeoStores neoStores = storeFactory.openNeoStores( StoreType.SCHEMA ) )
-        {
-            SchemaStore schema = neoStores.getSchemaStore();
-            Iterator<SchemaRule> rules = schema.loadAllSchemaRules();
-            while ( rules.hasNext() )
-            {
-                SchemaRule rule = rules.next();
-                IndexConfiguration indexConfig = new IndexConfiguration( rule.getKind() == UNIQUENESS_CONSTRAINT );
-                try ( IndexAccessor accessor =
-                              schemaIndexProvider.getOnlineAccessor( rule.getId(), indexConfig, samplingConfig ) )
-                {
-                    try ( IndexReader reader = accessor.newReader() )
-                    {
-                        if ( reader.valueTypesInIndex().contains( Array.class ) )
-                        {
-                            indexesToBeDeleted.add( new File( indexRoot, "" + rule.getId() ) );
-                        }
-                    }
-                }
-            }
-        }
-
-        for ( File index : indexesToBeDeleted )
-        {
-            fileSystem.deleteRecursively( index );
-        }
+        deleteIndexes( schemaIndexProvider.getStoreDirectory( storeDir ) );
+        deleteIndexes( labelScanStoreProvider.getStoreDirectory( storeDir ) );
+    }
 
+    private void deleteIndexes( File indexRootDirectory ) throws IOException
+    {
+        fileSystem.deleteRecursively( indexRootDirectory );
     }
 
     @Override