efactored the server startup healtchecks, combining it with the PreS…

…tartupStoreUpgrader code. Checks and tasks performed before proper server

start are now uniformly called PreFlightTasks. These can take arbitrary time to perform (eg. upgrades, recovery).

After the preflight checks, the main server startup uses a timer thread to keep track of startup time. If startup takes longer than the configured timeout,
the startup process is interrupted.

This is all done within the main server process, which means we can remove the code in startup scripts that start a bootstrap JVM separately, as well
as the code that keeps track of startup timeouts.

kernel/src/main/java/org/neo4j/kernel/impl/recovery/StoreRecoverer.java

@@ -0,0 +1,117 @@
+/**
+ * Copyright (c) 2002-2012 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.recovery;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.Map;
+
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.factory.GraphDatabaseFactory;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.kernel.DefaultFileSystemAbstraction;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.impl.nioneo.store.FileSystemAbstraction;
+import org.neo4j.kernel.impl.transaction.xaframework.XaLogicalLogFiles;
+import org.neo4j.kernel.impl.transaction.xaframework.XaLogicalLogRecoveryCheck;
+
+/**
+ * For now, an external tool that can determine if a given store
+ * will need recovery, and perform recovery on given stores.
+ */
+public class StoreRecoverer {
+
+	private FileSystemAbstraction fs;
+
+	public StoreRecoverer()
+	{
+		this(new DefaultFileSystemAbstraction());
+	}
+
+	public StoreRecoverer(FileSystemAbstraction fs)
+	{
+		this.fs = fs;
+	}
+
+	public boolean recoveryNeededAt(File dataDir, Map<String,String> params) throws IOException
+	{
+		// We need config to determine where the logical log files are
+		params.put(GraphDatabaseSettings.store_dir.name(), dataDir.getAbsolutePath());
+		Config config = new Config(params, GraphDatabaseSettings.class);
+
+		String baseLogPath = config.get(GraphDatabaseSettings.logical_log);
+		XaLogicalLogFiles logFiles = new XaLogicalLogFiles(baseLogPath, fs);
+
+		String log;
+		switch(logFiles.determineState()) 
+        {
+	    case CLEAN:
+	        return false;
+
+        case NO_ACTIVE_FILE:
+        case DUAL_LOGS_LOG_1_ACTIVE:
+        case DUAL_LOGS_LOG_2_ACTIVE:
+        	return true;
+
+        case LEGACY_WITHOUT_LOG_ROTATION:
+            log = baseLogPath;
+            break;
+
+        case LOG_1_ACTIVE:
+            log = logFiles.getLog1FileName();
+            break;
+
+        case LOG_2_ACTIVE:
+        	log = logFiles.getLog2FileName();
+            break;
+
+        default:
+            return true;
+        }
+
+		FileChannel logChannel = null;
+		try 
+		{
+			logChannel = fs.open(log, "r");
+			return new XaLogicalLogRecoveryCheck(logChannel).recoveryRequired();
+		} finally 
+		{
+			if(logChannel != null)
+			{
+				logChannel.close();
+			}
+		}
+	}
+
+	public void recover(File dataDir, Map<String,String> params) throws IOException
+	{
+		// For now, just launch a full embedded database on top of the directory.
+		// In a perfect world, to be expanded to only do recovery, and to be used
+		// as a component of the database, rather than something that is bolted
+		// on outside it like this.
+		GraphDatabaseService db = 
+				new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(dataDir.getCanonicalPath())
+				.setConfig(params).newGraphDatabase();
+
+		db.shutdown();
+	}
+
+}

kernel/src/main/java/org/neo4j/kernel/impl/transaction/xaframework/XaLogicalLog.java

@@ -133,7 +133,6 @@ public XaLogicalLog( String fileName, XaResourceManager xaRm, XaCommandFactory c
 
     synchronized void open() throws IOException
     {
-
         switch(logFiles.determineState()) 
         {
         case LEGACY_WITHOUT_LOG_ROTATION:
@@ -1019,7 +1018,8 @@ public synchronized Pair<Integer, Long> getMasterForCommittedTransaction( long t
      * @return The channel
      * @throws IOException If an IO error occurs when reading the log file
      */
-    public ReadableByteChannel getLogicalLogOrMyselfCommitted( long version, long position )
+    @Override
+	public ReadableByteChannel getLogicalLogOrMyselfCommitted( long version, long position )
             throws IOException
     {
         synchronized ( this )
@@ -1592,7 +1592,8 @@ public long getLogicalLogTargetSize()
         return this.rotateAtSize;
     }
 
-    public String getFileName( long version )
+    @Override
+	public String getFileName( long version )
     {
         return fileName + ".v" + version;
     }
@@ -1640,7 +1641,8 @@ public boolean wasNonClean()
         return nonCleanShutdown;
     }
 
-    public long getHighestLogVersion()
+    @Override
+	public long getHighestLogVersion()
     {
         return logVersion;
     }

kernel/src/test/java/org/neo4j/kernel/impl/recovery/TestStoreRecoverer.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) 2002-2012 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.recovery;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.Test;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.factory.GraphDatabaseFactory;
+import org.neo4j.kernel.impl.util.FileUtils;
+
+
+public class TestStoreRecoverer {
+
+	@Test
+	public void shouldNotWantToRecoverIntactStore() throws Exception
+	{
+		File store = null;
+		try
+		{
+			store = createIntactStore();
+
+			StoreRecoverer recoverer = new StoreRecoverer();
+
+			assertThat(recoverer.recoveryNeededAt(store, new HashMap<String,String>()), is(false));
+
+		} finally 
+		{
+			if(store != null)
+			{
+				FileUtils.deleteRecursively(store);
+			}
+		}
+
+	}
+
+	@Test
+	public void shouldWantToRecoverBrokenStore() throws Exception
+	{
+		File store = null;
+		try
+		{
+			store = createIntactStore();
+			new File(store,"nioneo_logical.log.active").delete();
+
+			StoreRecoverer recoverer = new StoreRecoverer();
+
+			assertThat(recoverer.recoveryNeededAt(store, new HashMap<String,String>()), is(true));
+
+		} finally 
+		{
+			if(store != null)
+			{
+				FileUtils.deleteRecursively(store);
+			}
+		}
+
+	}
+
+	@Test
+	public void shouldBeAbleToRecoverBrokenStore() throws Exception
+	{
+		File store = null;
+		try
+		{
+			store = createIntactStore();
+			new File(store,"nioneo_logical.log.active").delete();
+
+			StoreRecoverer recoverer = new StoreRecoverer();
+
+			assertThat(recoverer.recoveryNeededAt(store, new HashMap<String,String>()), is(true));
+
+			recoverer.recover(store, new HashMap<String,String>());
+
+			assertThat(recoverer.recoveryNeededAt(store, new HashMap<String,String>()), is(false));
+
+		} finally 
+		{
+			if(store != null)
+			{
+				FileUtils.deleteRecursively(store);
+			}
+		}
+
+	}
+
+	private File createIntactStore() throws IOException {
+		File tmpFile = File.createTempFile( "neo4j-test", "" );
+        tmpFile.delete();
+		GraphDatabaseService db = 
+				new GraphDatabaseFactory().newEmbeddedDatabase(tmpFile.getCanonicalPath());
+
+		db.shutdown();
+		return tmpFile;
+	}
+
+}

server/src/functionaltest/java/org/neo4j/server/StartupHealthcheckFunctionalTest.java → server/src/functionaltest/java/org/neo4j/server/PreflightTasksFunctionalTest.java

@@ -19,23 +19,21 @@
  */
 package org.neo4j.server;
 
-import java.io.IOException;
-
 import org.junit.Test;
 import org.neo4j.server.helpers.ServerBuilder;
-import org.neo4j.server.startup.healthcheck.StartupHealthCheckFailedException;
+import org.neo4j.server.preflight.PreflightFailedException;
 import org.neo4j.test.server.ExclusiveServerTestBase;
 
-public class StartupHealthcheckFunctionalTest extends ExclusiveServerTestBase
+public class PreflightTasksFunctionalTest extends ExclusiveServerTestBase
 {
 
     private NeoServer server;
 
-    @Test( expected = StartupHealthCheckFailedException.class )
-    public void shouldExitWhenFailedStartupHealthCheck() throws IOException
+    @Test( expected = PreflightFailedException.class )
+    public void shouldExitWhenFailedStartupHealthCheck() throws Throwable
     {
         server = ServerBuilder.server()
-                .withFailingStartupHealthcheck()
+                .withFailingPreflightTasks()
                 .build();
         server.start();
     }