introduce robust hazelcast wrapper

enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/discovery/HazelcastInstanceNotActiveException.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering.discovery;
+
+/**
+ * A checked exception to make exceptions visible.
+ */
+class HazelcastInstanceNotActiveException extends Exception
+{
+    HazelcastInstanceNotActiveException( String message )
+    {
+        super( message );
+    }
+
+    HazelcastInstanceNotActiveException( Throwable cause )
+    {
+        super( cause );
+    }
+}

enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/discovery/RobustHazelcastWrapper.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering.discovery;
+
+import com.hazelcast.core.HazelcastInstance;
+
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+/**
+ * A class which attempts to capture behaviours necessary to make interacting
+ * with hazelcast robust, e.g. reconnect on failures. This class is not aimed
+ * at high-performance and thus uses synchronization heavily.
+ */
+class RobustHazelcastWrapper
+{
+    private final HazelcastConnector connector;
+    private HazelcastInstance hzInstance;
+    private boolean shutdown;
+
+    RobustHazelcastWrapper( HazelcastConnector connector )
+    {
+        this.connector = connector;
+    }
+
+    synchronized void shutdown()
+    {
+        if ( hzInstance != null )
+        {
+            hzInstance.shutdown();
+            hzInstance = null;
+            shutdown = true;
+        }
+    }
+
+    private synchronized HazelcastInstance tryEnsureConnection() throws HazelcastInstanceNotActiveException
+    {
+        if ( shutdown )
+        {
+            throw new HazelcastInstanceNotActiveException( "Shutdown" );
+        }
+
+        if ( hzInstance == null )
+        {
+            hzInstance = connector.connectToHazelcast();
+        }
+        return hzInstance;
+    }
+
+    private synchronized void invalidateConnection()
+    {
+        hzInstance = null;
+    }
+
+    synchronized <T> T apply( Function<HazelcastInstance,T> function ) throws HazelcastInstanceNotActiveException
+    {
+        HazelcastInstance hzInstance = tryEnsureConnection();
+
+        try
+        {
+            return function.apply( hzInstance );
+        }
+        catch ( com.hazelcast.core.HazelcastInstanceNotActiveException e )
+        {
+            invalidateConnection();
+            throw new HazelcastInstanceNotActiveException( e );
+        }
+    }
+
+    synchronized void perform( Consumer<HazelcastInstance> operation ) throws HazelcastInstanceNotActiveException
+    {
+        HazelcastInstance hzInstance = tryEnsureConnection();
+
+        try
+        {
+            operation.accept( hzInstance );
+        }
+        catch ( com.hazelcast.core.HazelcastInstanceNotActiveException e )
+        {
+            invalidateConnection();
+            throw new HazelcastInstanceNotActiveException( e );
+        }
+    }
+}

enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/discovery/RobustHazelcastWrapperTest.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering.discovery;
+
+import com.hazelcast.core.HazelcastInstance;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RobustHazelcastWrapperTest
+{
+    @Test
+    public void shouldReconnectIfHazelcastConnectionInvalidated() throws Exception
+    {
+        // given
+        HazelcastConnector connector = mock( HazelcastConnector.class );
+        HazelcastInstance hzInstance = mock( HazelcastInstance.class );
+
+        when( connector.connectToHazelcast() ).thenReturn( hzInstance );
+
+        RobustHazelcastWrapper hzWrapper = new RobustHazelcastWrapper( connector );
+
+        // when
+        hzWrapper.perform( ( hz ) ->
+        { /* nothing*/ } );
+
+        // then
+        verify( connector, times( 1 ) ).connectToHazelcast();
+
+        // then
+        try
+        {
+            hzWrapper.perform( ( hz ) ->
+            {
+                throw new com.hazelcast.core.HazelcastInstanceNotActiveException();
+            } );
+            fail();
+        }
+        catch ( HazelcastInstanceNotActiveException e )
+        {
+            // expected
+        }
+
+        // when
+        hzWrapper.perform( ( hz ) ->
+        { /* nothing*/ } );
+        hzWrapper.perform( ( hz ) ->
+        { /* nothing*/ } );
+
+        // then
+        verify( connector, times( 2 ) ).connectToHazelcast();
+    }
+}