Prevent database healing in case of critical errors

Do not allow to heal database health in case of critical exception was
observed.
So far only OutOfMemoryError considered as critical exception.

community/kernel/src/main/java/org/neo4j/kernel/internal/DatabaseHealth.java

@@ -20,6 +20,7 @@
 package org.neo4j.kernel.internal;
 
 import org.neo4j.graphdb.event.ErrorState;
+import org.neo4j.helpers.Exceptions;
 import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator;
 import org.neo4j.logging.Log;
 
@@ -29,9 +30,9 @@ public class DatabaseHealth
 {
     private static final String panicMessage = "Database has encountered some problem, "
             + "please perform necessary action (tx recovery/restart)";
+    private static final Class<?>[] CRITICAL_EXCEPTIONS = new Class[]{OutOfMemoryError.class};
 
-    // Keep that cozy name for legacy purposes
-    private volatile boolean tmOk = true; // TODO rather skip volatile if possible here.
+    private volatile boolean healthy = true;
     private final DatabasePanicEventGenerator dbpe;
     private final Log log;
     private Throwable causeOfPanic;
@@ -51,7 +52,7 @@ public DatabaseHealth( DatabasePanicEventGenerator dbpe, Log log )
      */
     public <EXCEPTION extends Throwable> void assertHealthy( Class<EXCEPTION> panicDisguise ) throws EXCEPTION
     {
-        if ( !tmOk )
+        if ( !healthy )
         {
             EXCEPTION exception;
             try
@@ -78,7 +79,7 @@ public <EXCEPTION extends Throwable> void assertHealthy( Class<EXCEPTION> panicD
 
     public void panic( Throwable cause )
     {
-        if ( !tmOk )
+        if ( !healthy )
         {
             return;
         }
@@ -88,21 +89,35 @@ public void panic( Throwable cause )
             throw new IllegalArgumentException( "Must provide a cause for the database panic" );
         }
         this.causeOfPanic = cause;
-        this.tmOk = false;
+        this.healthy = false;
         log.error( "Database panic: " + panicMessage, cause );
         dbpe.generateEvent( ErrorState.TX_MANAGER_NOT_OK, causeOfPanic );
     }
 
     public boolean isHealthy()
     {
-        return tmOk;
+        return healthy;
     }
 
-    public void healed()
+    public boolean healed()
     {
-        tmOk = true;
-        causeOfPanic = null;
-        log.info( "Database health set to OK" );
+        if ( hasCriticalFailure() )
+        {
+            log.error( "Database encountered a critical error and can't be healed. Restart required." );
+            return false;
+        }
+        else
+        {
+            healthy = true;
+            causeOfPanic = null;
+            log.info( "Database health set to OK" );
+            return true;
+        }
+    }
+
+    private boolean hasCriticalFailure()
+    {
+        return !isHealthy() && Exceptions.contains( causeOfPanic, CRITICAL_EXCEPTIONS );
     }
 
     public Throwable cause()

community/kernel/src/test/java/org/neo4j/kernel/internal/DatabaseHealthTest.java

@@ -21,12 +21,16 @@
 
 import org.junit.Test;
 
+import java.io.IOException;
+
 import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator;
 import org.neo4j.logging.AssertableLogProvider;
 import org.neo4j.logging.NullLogProvider;
 
 import static org.hamcrest.Matchers.sameInstance;
 import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -36,7 +40,7 @@
 public class DatabaseHealthTest
 {
     @Test
-    public void shouldGenerateKernelPanicEvents() throws Exception
+    public void shouldGenerateDatabasePanicEvents() throws Exception
     {
         // GIVEN
         DatabasePanicEventGenerator generator = mock( DatabasePanicEventGenerator.class );
@@ -53,7 +57,7 @@ public void shouldGenerateKernelPanicEvents() throws Exception
     }
 
     @Test
-    public void shouldLogKernelPanicEvent() throws Exception
+    public void shouldLogDatabasePanicEvent() throws Exception
     {
         // GIVEN
         AssertableLogProvider logProvider = new AssertableLogProvider();
@@ -74,4 +78,40 @@ public void shouldLogKernelPanicEvent() throws Exception
                 )
         );
     }
+
+    @Test
+    public void healDatabaseWithoutCriticalErrors()
+    {
+        AssertableLogProvider logProvider = new AssertableLogProvider();
+        DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ),
+                logProvider.getLog( DatabaseHealth.class ) );
+
+        assertTrue( databaseHealth.isHealthy() );
+
+        databaseHealth.panic( new IOException( "Space exception." ) );
+
+        assertFalse( databaseHealth.isHealthy() );
+        assertTrue( databaseHealth.healed() );
+        logProvider.assertContainsLogCallContaining( "Database health set to OK" );
+        logProvider.assertNoMessagesContaining( "Database encountered a critical error and can't be healed. Restart required." );
+    }
+
+    @Test
+    public void databaseWithCriticalErrorsCanNotBeHealed()
+    {
+        AssertableLogProvider logProvider = new AssertableLogProvider();
+        DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ),
+                logProvider.getLog( DatabaseHealth.class ) );
+
+        assertTrue( databaseHealth.isHealthy() );
+
+        IOException criticalException = new IOException( "Space exception.", new OutOfMemoryError( "Out of memory." ) );
+        databaseHealth.panic( criticalException );
+
+        assertFalse( databaseHealth.isHealthy() );
+        assertFalse( databaseHealth.healed() );
+        logProvider.assertNoMessagesContaining( "Database health set to OK" );
+        logProvider.assertContainsLogCallContaining(
+                "Database encountered a critical error and can't be healed. Restart required." );
+    }
 }