HA cluster members will now become read only on disconnect from cluster

As a means to ensure safety, instances that get detached from a majority
 of the cluster will revert to PENDING state. That implicitly raises the
 barrier disallowing transactions to start, exluding that way the
 possibility of doing read only operations, which are still possible
 in principle, albeit forfeiting up to date data.
This commit makes it so that cluster members which can no longer
 participate in quorum will revert to PENDING state, without
 losing the ability to perform reads. Writes are still of course
 disallowed.

As a side note, a test that tries to verify the behaviour of instances
 when connectivity to quorum is lost abruptly is included but @ignored.
 This is because while the behaviour tested is part of this patch, the
 fact that it fails is a bug in previous changes. That bug will be
 solved in a different commit and the test will be enabled.

enterprise/ha/src/main/java/org/neo4j/kernel/ha/DelegateInvocationHandler.java

@@ -138,8 +138,9 @@ public Object invoke( Object proxy, Method method, Object[] args ) throws Throwa
             if ( delegate == null )
             {
                 throw new TransientDatabaseFailureException(
-                        "Transaction state is not valid. Perhaps a state change of" +
-                        "the database has happened while this transaction was running?" );
+                        "Instance state is not valid. There is no master currently available. Possible causes " +
+                                "include unavailability of a majority of the cluster members or network failure " +
+                                "that caused this instance to be partitioned away from the cluster" );
             }
 
             return proxyInvoke( delegate, method, args );

enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberListener.java

@@ -20,7 +20,8 @@
 package org.neo4j.kernel.ha.cluster;
 
 /**
- * These callback methods correspond to the cluster
+ * These callback methods correspond to broadcasted HA events. The supplied event argument contains the
+ * result of the state change and required information, as interpreted by the HA state machine.
  */
 public interface HighAvailabilityMemberListener
 {
@@ -32,6 +33,13 @@ public interface HighAvailabilityMemberListener
 
     void instanceStops( HighAvailabilityMemberChangeEvent event );
 
+    /**
+     * This event is different than the rest, in the sense that it is not a response to a broadcasted message,
+     * rather than the interpretation of the loss of connectivity to other cluster members. This corresponds generally
+     * to a loss of quorum but a special case is the event of being partitioned away completely from the cluster.
+     */
+    void instanceDetached( HighAvailabilityMemberChangeEvent event );
+
     class Adapter implements HighAvailabilityMemberListener
     {
         @Override
@@ -53,5 +61,10 @@ public void slaveIsAvailable( HighAvailabilityMemberChangeEvent event )
         public void instanceStops( HighAvailabilityMemberChangeEvent event )
         {
         }
+
+        @Override
+        public void instanceDetached( HighAvailabilityMemberChangeEvent event )
+        {
+        }
     }
 }

enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachine.java

@@ -269,14 +269,14 @@ public void memberIsFailed( InstanceId instanceId )
             if ( !isQuorum( getAliveCount(), getTotalCount() ) )
             {
                 HighAvailabilityMemberState oldState = state;
-                changeStateToPending();
+                changeStateToDetached();
                 log.debug( "Got memberIsFailed(" + instanceId + ") and cluster lost quorum to continue, moved to "
                         + state + " from " + oldState );
             }
             else if ( instanceId.equals( context.getElectedMasterId() ) && state == HighAvailabilityMemberState.SLAVE )
             {
                 HighAvailabilityMemberState oldState = state;
-                changeStateToPending();
+                changeStateToDetached();
                 log.debug( "Got memberIsFailed(" + instanceId + ") which was the master and i am a slave, moved to "
                         + state + " from " + oldState );
             }
@@ -321,6 +321,21 @@ public void notify( HighAvailabilityMemberListener listener )
             context.setElectedMasterId( null );
         }
 
+        private void changeStateToDetached()
+        {
+            state = HighAvailabilityMemberState.PENDING;
+            final HighAvailabilityMemberChangeEvent event =
+                    new HighAvailabilityMemberChangeEvent( state, HighAvailabilityMemberState.PENDING, null, null );
+            Listeners.notifyListeners( memberListeners, new Listeners.Notification<HighAvailabilityMemberListener>()
+            {
+                @Override
+                public void notify( HighAvailabilityMemberListener listener )
+                {
+                    listener.instanceDetached( event );
+                }
+            } );
+        }
+
         private long getAliveCount()
         {
             return Iterables.count( members.getAliveMembers() );

enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcher.java

@@ -201,6 +201,12 @@ public void instanceStops( HighAvailabilityMemberChangeEvent event )
         stateChanged( event );
     }
 
+    @Override
+    public void instanceDetached( HighAvailabilityMemberChangeEvent event )
+    {
+        switchToDetached();
+    }
+
     @Override
     public void addModeSwitcher( ModeSwitcher modeSwitcher )
     {
@@ -484,6 +490,51 @@ public void notify( ModeSwitcher listener )
         }
     }
 
+    private void switchToDetached()
+    {
+        msgLog.info( "I am %s, moving to detached", instanceId );
+
+        startModeSwitching( new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                if ( cancellationHandle.cancellationRequested() )
+                {
+                    msgLog.info( "Switch to pending cancelled on start." );
+                    return;
+                }
+
+                Listeners.notifyListeners( modeSwitchListeners, new Listeners.Notification<ModeSwitcher>()
+                {
+                    @Override
+                    public void notify( ModeSwitcher listener )
+                    {
+                        listener.switchToSlave();
+                    }
+                } );
+                neoStoreDataSourceSupplier.getDataSource().beforeModeSwitch();
+
+                if ( cancellationHandle.cancellationRequested() )
+                {
+                    msgLog.info( "Switch to pending cancelled before ha communication shutdown." );
+                    return;
+                }
+
+                haCommunicationLife.shutdown();
+                haCommunicationLife = new LifeSupport();
+            }
+        }, new CancellationHandle() );
+
+        try
+        {
+            modeSwitcherFuture.get( 10, TimeUnit.SECONDS );
+        }
+        catch ( Exception ignored )
+        {
+        }
+    }
+
     private synchronized void startModeSwitching( Runnable switcher, CancellationHandle cancellationHandle )
     {
         if ( modeSwitcherFuture != null )

enterprise/ha/src/main/java/org/neo4j/kernel/ha/factory/HighlyAvailableEditionModule.java

@@ -750,13 +750,8 @@ protected KernelData createKernelData( Config config, GraphDatabaseAPI graphDb,
     protected void registerRecovery( final String editionName, final DependencyResolver dependencyResolver,
                                      final LogService logging )
     {
-        memberStateMachine.addHighAvailabilityMemberListener( new HighAvailabilityMemberListener()
+        memberStateMachine.addHighAvailabilityMemberListener( new HighAvailabilityMemberListener.Adapter()
         {
-            @Override
-            public void masterIsElected( HighAvailabilityMemberChangeEvent event )
-            {
-            }
-
             @Override
             public void masterIsAvailable( HighAvailabilityMemberChangeEvent event )
             {
@@ -777,11 +772,6 @@ public void slaveIsAvailable( HighAvailabilityMemberChangeEvent event )
                 }
             }
 
-            @Override
-            public void instanceStops( HighAvailabilityMemberChangeEvent event )
-            {
-            }
-
             private void doAfterRecoveryAndStartup( boolean isMaster )
             {
                 try