Fixes a legacy index lifecycle issue around store copy

where previously a LuceneIndexImplementation/LuceneDataSource would not be
restarted after a store copy, which means that indexes might refer to
stale files or files that doesn't exist after the store copy.

Now the licecycle management is better, although not perfect since
IndexImplementation can expect to see multiple calls to init()/shutdown().
Other than that the lifecycle of legacy indexes now work in and around
store copy. Further changing the legacy index lifecycle structure to
avoid this anomaly would require a bigger change and can be done later.

community/kernel/src/main/java/org/neo4j/graphdb/index/IndexImplementation.java

@@ -25,14 +25,15 @@
 
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler;
+import org.neo4j.kernel.lifecycle.Lifecycle;
 
 /**
  * An index provider which can create and give access to index transaction state and means of applying
  * updates to indexes it provides.
  * An {@link IndexImplementation} is typically tied to one implementation, f.ex.
  * lucene, http://lucene.apache.org/java.
  */
-public interface IndexImplementation
+public interface IndexImplementation extends Lifecycle
 {
     /**
      * Returns a {@link LegacyIndexProviderTransaction} that keeps transaction state for all
@@ -71,4 +72,28 @@ public interface IndexImplementation
      * @throws IOException
      */
     ResourceIterator<File> listStoreFiles() throws IOException;
+
+    /**
+     * Makes available index resource for recovery.
+     */
+    @Override
+    void init() throws Throwable;
+
+    /**
+     * Makes available index resource for online transaction processing.
+     */
+    @Override
+    void start() throws Throwable;
+
+    /**
+     * Makes unavailable index resource from online transaction processing.
+     */
+    @Override
+    void stop() throws Throwable;
+
+    /**
+     * Makes unavailable the index resource as a whole.
+     */
+    @Override
+    void shutdown() throws Throwable;
 }

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -150,6 +150,7 @@
 import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
+import org.neo4j.kernel.lifecycle.Lifecycles;
 import org.neo4j.kernel.logging.Logging;
 import org.neo4j.kernel.monitoring.Monitors;
 import org.neo4j.kernel.monitoring.tracing.Tracers;
@@ -456,6 +457,8 @@ public void transactionRecovered( long txId )
             }
         } );
 
+        life.add( new Lifecycle.Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
+
         // Upgrade the store before we begin
         upgradeStore( store, storeMigrationProcess, indexProvider );
 

community/kernel/src/main/java/org/neo4j/kernel/lifecycle/Lifecycle.java

@@ -20,11 +20,11 @@
 package org.neo4j.kernel.lifecycle;
 
 /**
- * Lifecycle interface for kernel components. Init is called first, 
- * followed by start, 
+ * Lifecycle interface for kernel components. Init is called first,
+ * followed by start,
  * and then any number of stop-start sequences,
  * and finally stop and shutdown.
- * 
+ *
  * As a stop-start cycle could be due to change of configuration, please perform anything that depends on config
  * in start().
  *
@@ -38,13 +38,47 @@ public interface Lifecycle
 {
     void init()
         throws Throwable;
-    
+
     void start()
         throws Throwable;
-    
+
     void stop()
         throws Throwable;
-    
+
     void shutdown()
         throws Throwable;
+
+    class Delegate implements Lifecycle
+    {
+        private final Lifecycle delegate;
+
+        public Delegate( Lifecycle delegate )
+        {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void init() throws Throwable
+        {
+            delegate.init();
+        }
+
+        @Override
+        public void start() throws Throwable
+        {
+            delegate.start();
+        }
+
+        @Override
+        public void stop() throws Throwable
+        {
+            delegate.stop();
+        }
+
+        @Override
+        public void shutdown() throws Throwable
+        {
+            delegate.shutdown();
+        }
+    }
 }

community/kernel/src/main/java/org/neo4j/kernel/lifecycle/Lifecycles.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2002-2015 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.lifecycle;
+
+public class Lifecycles
+{
+    private Lifecycles()
+    {   // No instances allowed or even necessary
+    }
+
+    public static Lifecycle multiple( final Iterable<? extends Lifecycle> lifecycles )
+    {
+        return new Lifecycle()
+        {
+            @Override
+            public void init() throws Throwable
+            {
+                for ( Lifecycle lifecycle : lifecycles )
+                {
+                    lifecycle.init();
+                }
+            }
+
+            @Override
+            public void start() throws Throwable
+            {
+                for ( Lifecycle lifecycle : lifecycles )
+                {
+                    lifecycle.start();
+                }
+            }
+
+            @Override
+            public void stop() throws Throwable
+            {
+                for ( Lifecycle lifecycle : lifecycles )
+                {
+                    lifecycle.stop();
+                }
+            }
+
+            @Override
+            public void shutdown() throws Throwable
+            {
+                for ( Lifecycle lifecycle : lifecycles )
+                {
+                    lifecycle.shutdown();
+                }
+            }
+        };
+    }
+}

community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexExtensionFactory.java

@@ -67,16 +67,6 @@ public void init() throws Throwable
             indexProviders.registerIndexProvider( IDENTIFIER, new DummyIndexImplementation() );
         }
 
-        @Override
-        public void start() throws Throwable
-        {
-        }
-
-        @Override
-        public void stop() throws Throwable
-        {
-        }
-
         @Override
         public void shutdown() throws Throwable
         {

community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexImplementation.java

@@ -31,8 +31,9 @@
 import org.neo4j.kernel.api.LegacyIndex;
 import org.neo4j.kernel.api.LegacyIndexHits;
 import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler;
+import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
-public class DummyIndexImplementation implements IndexImplementation
+public class DummyIndexImplementation extends LifecycleAdapter implements IndexImplementation
 {
     @Override
     public Map<String, String> fillInDefaults( Map<String, String> config )

community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/LuceneDataSource.java

@@ -66,14 +66,14 @@
 import org.neo4j.kernel.impl.cache.LruCache;
 import org.neo4j.kernel.impl.index.IndexConfigStore;
 import org.neo4j.kernel.impl.index.IndexEntityType;
-import org.neo4j.kernel.lifecycle.Lifecycle;
+import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
 import static org.neo4j.index.impl.lucene.MultipleBackupDeletionPolicy.SNAPSHOT_ID;
 
 /**
  * An DataSource optimized for the {@link LuceneIndexImplementation}.
  */
-public class LuceneDataSource implements Lifecycle
+public class LuceneDataSource extends LifecycleAdapter
 {
     private final Config config;
     private final FileSystemAbstraction fileSystemAbstraction;
@@ -142,11 +142,6 @@ public LuceneDataSource( Config config, IndexConfigStore indexStore, FileSystemA
 
     @Override
     public void init()
-    {
-    }
-
-    @Override
-    public void start()
     {
         this.filesystemFacade = config.get( Configuration.ephemeral ) ? LuceneFilesystemFacade.MEMORY
                 : LuceneFilesystemFacade.FS;
@@ -171,7 +166,7 @@ IndexType getType( IndexIdentifier identifier, boolean recovery )
     }
 
     @Override
-    public void stop() throws IOException
+    public void shutdown() throws IOException
     {
         synchronized ( this )
         {
@@ -188,11 +183,6 @@ public void stop() throws IOException
         }
     }
 
-    @Override
-    public void shutdown()
-    {
-    }
-
     private synchronized IndexReference[] getAllIndexes()
     {
         Collection<IndexReference> indexReferences = indexSearchers.values();
@@ -209,7 +199,8 @@ void force()
             }
             catch ( IOException e )
             {
-                throw new RuntimeException( "unable to commit changes to " + index.getIdentifier(), e );
+                throw new RuntimeException( "Unable to commit changes to " + index.getIdentifier() + " in " +
+                        config.get( GraphDatabaseSettings.store_dir ).getAbsolutePath(), e );
             }
         }
     }
@@ -544,25 +535,30 @@ public ResourceIterator<File> listStoreFiles( boolean includeLogicalLogs ) throw
             SnapshotDeletionPolicy deletionPolicy = (SnapshotDeletionPolicy) writer.getWriter().getConfig()
                     .getIndexDeletionPolicy();
             File indexDirectory = getFileDirectory( baseStorePath, writer.getIdentifier() );
+            IndexCommit commit;
             try
             {
                 // Throws IllegalStateException if no commits yet
-                IndexCommit commit = deletionPolicy.snapshot( SNAPSHOT_ID );
-                for ( String fileName : commit.getFileNames() )
-                {
-                    files.add( new File( indexDirectory, fileName ) );
-                }
-                snapshots.add( deletionPolicy );
+                commit = deletionPolicy.snapshot( SNAPSHOT_ID );
             }
             catch ( IllegalStateException e )
             {
-                // TODO Review this
                 /*
                  * This is insane but happens if we try to snapshot an existing index
                  * that has no commits. This is a bad API design - it should return null
                  * or something. This is not exceptional.
+                 *
+                 * For the time being we just do a commit and try again.
                  */
+                writer.getWriter().commit();
+                commit = deletionPolicy.snapshot( SNAPSHOT_ID );
+            }
+
+            for ( String fileName : commit.getFileNames() )
+            {
+                files.add( new File( indexDirectory, fileName ) );
             }
+            snapshots.add( deletionPolicy );
         }
         return new PrefetchingResourceIterator<File>()
         {

community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/LuceneIndexImplementation.java

@@ -31,9 +31,13 @@
 import org.neo4j.graphdb.index.IndexManager;
 import org.neo4j.graphdb.index.LegacyIndexProviderTransaction;
 import org.neo4j.helpers.collection.MapUtil;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.impl.index.IndexConfigStore;
 import org.neo4j.kernel.impl.transaction.command.NeoCommandHandler;
+import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
-public class LuceneIndexImplementation implements IndexImplementation
+public class LuceneIndexImplementation extends LifecycleAdapter implements IndexImplementation
 {
     static final String KEY_TYPE = "type";
     static final String KEY_ANALYZER = "analyzer";
@@ -50,15 +54,43 @@ public class LuceneIndexImplementation implements IndexImplementation
                     IndexManager.PROVIDER, SERVICE_NAME, KEY_TYPE, "fulltext",
                     KEY_TO_LOWER_CASE, "true" ) );
 
-    public static final int DEFAULT_LAZY_THRESHOLD = 100;
+    private LuceneDataSource dataSource;
+    private final Config config;
+    private final IndexConfigStore indexStore;
+    private final FileSystemAbstraction fileSystemAbstraction;
 
-    private final LuceneDataSource dataSource;
-    final int lazynessThreshold;
+    public LuceneIndexImplementation( Config config, IndexConfigStore indexStore,
+            FileSystemAbstraction fileSystemAbstraction )
+    {
+        this.config = config;
+        this.indexStore = indexStore;
+        this.fileSystemAbstraction = fileSystemAbstraction;
+    }
+
+    @Override
+    public void init() throws Throwable
+    {
+        this.dataSource = new LuceneDataSource( config, indexStore, fileSystemAbstraction );
+        this.dataSource.init();
+    }
 
-    public LuceneIndexImplementation( LuceneDataSource dataSource )
+    @Override
+    public void start() throws Throwable
+    {
+        this.dataSource.start();
+    }
+
+    @Override
+    public void stop() throws Throwable
+    {
+        this.dataSource.stop();
+    }
+
+    @Override
+    public void shutdown() throws Throwable
     {
-        this.dataSource = dataSource;
-        this.lazynessThreshold = DEFAULT_LAZY_THRESHOLD;
+        this.dataSource.shutdown();
+        this.dataSource = null;
     }
 
     @Override