Introduce InvocationTargetLeaveAware for raft operations

InvocationTargetLeaveAware is used for determining if a raft operation
is safe to retry on leader leave and it is ok to commit the operation
multiple times. If the operation is safe, it is retried automatically if
the leader returns commit response of the invoked operation. If the
operation is not known to be safe, i.e. it either does not implement
InvocationTargetLeaveAware or returns false from
InvocationTargetLeaveAware#isSafeToRetryOnTargetLeave(), then
retry is decided based on the
RaftAlgorithmConfig#failOnIndeterminateOperationState config parameter.

Safe to retry raft operations are expected to be idempotent.
For instance, lock, session-aware semaphore, and count down latch
operations are idempotent. It does not make a difference to commit an
idempotent operation once or multiple times. However, although some
raft operations are idempotent in terms of state changes they cause,
they may not return the same response on duplicate commits. They can be
still considered to be safe to retry on leader node leave, depending on
their context and how they are used.

Author: metanet

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/atomiclong/operation/AddAndGetOp.java

@@ -19,6 +19,7 @@
 import com.hazelcast.nio.ObjectDataInput;
 import com.hazelcast.nio.ObjectDataOutput;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.atomiclong.RaftAtomicLongDataSerializerHook;
 import com.hazelcast.raft.service.atomiclong.RaftAtomicLong;
 
@@ -27,7 +28,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class AddAndGetOp extends AbstractAtomicLongOp {
+public class AddAndGetOp extends AbstractAtomicLongOp implements InvocationTargetLeaveAware {
 
     private long delta;
 
@@ -45,6 +46,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return atomic.addAndGet(delta);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return delta == 0;
+    }
+
     @Override
     public void writeData(ObjectDataOutput out) throws IOException {
         super.writeData(out);

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/atomiclong/operation/GetAndAddOp.java

@@ -19,6 +19,7 @@
 import com.hazelcast.nio.ObjectDataInput;
 import com.hazelcast.nio.ObjectDataOutput;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.atomiclong.RaftAtomicLongDataSerializerHook;
 import com.hazelcast.raft.service.atomiclong.RaftAtomicLong;
 
@@ -27,7 +28,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class GetAndAddOp extends AbstractAtomicLongOp {
+public class GetAndAddOp extends AbstractAtomicLongOp implements InvocationTargetLeaveAware {
 
     private long delta;
 
@@ -45,6 +46,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return atomic.getAndAdd(delta);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return delta == 0;
+    }
+
     @Override
     public void writeData(ObjectDataOutput out) throws IOException {
         super.writeData(out);

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/atomicref/operation/ContainsOp.java

@@ -21,6 +21,7 @@
 import com.hazelcast.nio.serialization.Data;
 import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.atomicref.RaftAtomicReferenceDataSerializerHook;
 import com.hazelcast.raft.service.atomicref.RaftAtomicRef;
 
@@ -29,7 +30,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class ContainsOp extends AbstractAtomicRefOp implements IdentifiedDataSerializable {
+public class ContainsOp extends AbstractAtomicRefOp implements InvocationTargetLeaveAware, IdentifiedDataSerializable {
 
     private Data value;
 
@@ -47,6 +48,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return ref.contains(value);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftAtomicReferenceDataSerializerHook.CONTAINS_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/atomicref/operation/GetOp.java

@@ -18,12 +18,13 @@
 
 import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.atomicref.RaftAtomicReferenceDataSerializerHook;
 
 /**
  * TODO: Javadoc Pending...
  */
-public class GetOp extends AbstractAtomicRefOp implements IdentifiedDataSerializable {
+public class GetOp extends AbstractAtomicRefOp implements InvocationTargetLeaveAware, IdentifiedDataSerializable {
 
     public GetOp() {
     }
@@ -37,6 +38,10 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return getAtomicRef(groupId).get();
     }
 
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftAtomicReferenceDataSerializerHook.GET_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/countdownlatch/operation/AwaitOp.java

@@ -19,6 +19,7 @@
 import com.hazelcast.nio.ObjectDataInput;
 import com.hazelcast.nio.ObjectDataOutput;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.impl.util.PostponedResponse;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchDataSerializerHook;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchService;
@@ -28,7 +29,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class AwaitOp extends AbstractCountDownLatchOp {
+public class AwaitOp extends AbstractCountDownLatchOp implements InvocationTargetLeaveAware {
 
     private long timeoutMillis;
 
@@ -50,6 +51,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return timeoutMillis > 0 ? PostponedResponse.INSTANCE : false;
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftCountDownLatchDataSerializerHook.AWAIT_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/countdownlatch/operation/CountDownOp.java

@@ -19,6 +19,7 @@
 import com.hazelcast.nio.ObjectDataInput;
 import com.hazelcast.nio.ObjectDataOutput;
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchDataSerializerHook;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchService;
 
@@ -28,7 +29,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class CountDownOp extends AbstractCountDownLatchOp {
+public class CountDownOp extends AbstractCountDownLatchOp implements InvocationTargetLeaveAware {
 
     private int expectedRound;
     private UUID invocationUid;
@@ -48,6 +49,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return service.countDown(groupId, name, expectedRound, invocationUid);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftCountDownLatchDataSerializerHook.COUNT_DOWN_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/countdownlatch/operation/GetRemainingCountOp.java

@@ -17,13 +17,14 @@
 package com.hazelcast.raft.service.countdownlatch.operation;
 
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchDataSerializerHook;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchService;
 
 /**
  * TODO: Javadoc Pending...
  */
-public class GetRemainingCountOp extends AbstractCountDownLatchOp {
+public class GetRemainingCountOp extends AbstractCountDownLatchOp implements InvocationTargetLeaveAware {
 
     public GetRemainingCountOp() {
     }
@@ -38,6 +39,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return service.getRemainingCount(groupId, name);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftCountDownLatchDataSerializerHook.GET_REMAINING_COUNT_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/countdownlatch/operation/GetRoundOp.java

@@ -17,13 +17,14 @@
 package com.hazelcast.raft.service.countdownlatch.operation;
 
 import com.hazelcast.raft.RaftGroupId;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchDataSerializerHook;
 import com.hazelcast.raft.service.countdownlatch.RaftCountDownLatchService;
 
 /**
  * TODO: Javadoc Pending...
  */
-public class GetRoundOp extends AbstractCountDownLatchOp {
+public class GetRoundOp extends AbstractCountDownLatchOp implements InvocationTargetLeaveAware {
 
     public GetRoundOp() {
     }
@@ -38,6 +39,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return service.getRound(groupId, name);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public int getId() {
         return RaftCountDownLatchDataSerializerHook.GET_ROUND_OP;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/lock/operation/ForceUnlockOp.java

@@ -21,6 +21,7 @@
 import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
 import com.hazelcast.raft.RaftGroupId;
 import com.hazelcast.raft.impl.RaftOp;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.lock.RaftLockDataSerializerHook;
 import com.hazelcast.raft.service.lock.RaftLockService;
 
@@ -30,7 +31,7 @@
 /**
  * TODO: Javadoc Pending...
  */
-public class ForceUnlockOp extends RaftOp implements IdentifiedDataSerializable {
+public class ForceUnlockOp extends RaftOp implements InvocationTargetLeaveAware, IdentifiedDataSerializable {
 
     private String name;
     private long expectedFence;
@@ -52,6 +53,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return true;
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public final String getServiceName() {
         return RaftLockService.SERVICE_NAME;

hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/lock/operation/GetLockOwnershipStateOp.java

@@ -1,16 +1,36 @@
+/*
+ * Copyright (c) 2008-2018, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.hazelcast.raft.service.lock.operation;
 
 import com.hazelcast.nio.ObjectDataInput;
 import com.hazelcast.nio.ObjectDataOutput;
 import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
 import com.hazelcast.raft.RaftGroupId;
 import com.hazelcast.raft.impl.RaftOp;
+import com.hazelcast.raft.impl.service.proxy.InvocationTargetLeaveAware;
 import com.hazelcast.raft.service.lock.RaftLockDataSerializerHook;
 import com.hazelcast.raft.service.lock.RaftLockService;
 
 import java.io.IOException;
 
-public class GetLockOwnershipStateOp extends RaftOp implements IdentifiedDataSerializable {
+/**
+ * TODO: Javadoc Pending...
+ */
+public class GetLockOwnershipStateOp extends RaftOp implements InvocationTargetLeaveAware, IdentifiedDataSerializable {
 
     private String name;
 
@@ -27,6 +47,11 @@ public Object run(RaftGroupId groupId, long commitIndex) {
         return service.getLockOwnershipState(groupId, name);
     }
 
+    @Override
+    public boolean isSafeToRetryOnTargetLeave() {
+        return true;
+    }
+
     @Override
     public final String getServiceName() {
         return RaftLockService.SERVICE_NAME;