Skip to content

Commit 135162e

Browse files
metanetmdogan
metanet
authored and
mdogan
committed
Make increasePermits() and decreasePermits() idempontent for session aware semaphore
1 parent ba68a51 commit 135162e

File tree

13 files changed

+241
-68
lines changed

13 files changed

+241
-68
lines changed

‎hazelcast-raft-client/src/main/java/com/hazelcast/raft/service/semaphore/client/RaftSessionAwareSemaphoreProxy.java‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,13 +268,31 @@ public void reducePermits(int reduction) {
268268
return;
269269
}
270270

271-
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES
271+
long sessionId = acquireSession();
272+
if (sessionId == NO_SESSION_ID) {
273+
throw new IllegalStateException("No valid session!");
274+
}
275+
276+
long threadId = getThreadId();
277+
UUID invocationUid = newUnsecureUUID();
278+
279+
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES * 4
272280
+ Bits.INT_SIZE_IN_BYTES;
273-
ClientMessage msg = prepareClientMessage(groupId, name, NO_SESSION_ID, dataSize, CHANGE_PERMITS_TYPE);
281+
ClientMessage msg = prepareClientMessage(groupId, name, sessionId, dataSize, CHANGE_PERMITS_TYPE);
282+
msg.set(threadId);
283+
msg.set(invocationUid.getLeastSignificantBits());
284+
msg.set(invocationUid.getMostSignificantBits());
274285
msg.set(-reduction);
275286
msg.updateFrameLength();
276287

277-
invoke(msg, BOOLEAN_RESPONSE_DECODER).join();
288+
try {
289+
invoke(msg, BOOLEAN_RESPONSE_DECODER).join();
290+
} catch (SessionExpiredException e) {
291+
invalidateSession(sessionId);
292+
throw e;
293+
} finally {
294+
releaseSession(sessionId);
295+
}
278296
}
279297

280298
@Override
@@ -284,13 +302,31 @@ public void increasePermits(int increase) {
284302
return;
285303
}
286304

287-
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES
305+
long sessionId = acquireSession();
306+
if (sessionId == NO_SESSION_ID) {
307+
throw new IllegalStateException("No valid session!");
308+
}
309+
310+
long threadId = getThreadId();
311+
UUID invocationUid = newUnsecureUUID();
312+
313+
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES * 4
288314
+ Bits.INT_SIZE_IN_BYTES;
289-
ClientMessage msg = prepareClientMessage(groupId, name, NO_SESSION_ID, dataSize, CHANGE_PERMITS_TYPE);
315+
ClientMessage msg = prepareClientMessage(groupId, name, sessionId, dataSize, CHANGE_PERMITS_TYPE);
316+
msg.set(threadId);
317+
msg.set(invocationUid.getLeastSignificantBits());
318+
msg.set(invocationUid.getMostSignificantBits());
290319
msg.set(increase);
291320
msg.updateFrameLength();
292321

293-
invoke(msg, BOOLEAN_RESPONSE_DECODER).join();
322+
try {
323+
invoke(msg, BOOLEAN_RESPONSE_DECODER).join();
324+
} catch (SessionExpiredException e) {
325+
invalidateSession(sessionId);
326+
throw e;
327+
} finally {
328+
releaseSession(sessionId);
329+
}
294330
}
295331

296332
@Override

‎hazelcast-raft-client/src/main/java/com/hazelcast/raft/service/semaphore/client/RaftSessionlessSemaphoreProxy.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,9 +246,15 @@ public void reducePermits(int reduction) {
246246
return;
247247
}
248248

249-
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES
249+
long globalThreadId = getGlobalThreadId(groupId, globallyUniqueThreadIdCtor);
250+
UUID invocationUid = newUnsecureUUID();
251+
252+
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES * 4
250253
+ Bits.INT_SIZE_IN_BYTES;
251254
ClientMessage msg = prepareClientMessage(groupId, name, dataSize, CHANGE_PERMITS_TYPE);
255+
msg.set(globalThreadId);
256+
msg.set(invocationUid.getLeastSignificantBits());
257+
msg.set(invocationUid.getMostSignificantBits());
252258
msg.set(-reduction);
253259
msg.updateFrameLength();
254260

@@ -262,9 +268,15 @@ public void increasePermits(int increase) {
262268
return;
263269
}
264270

265-
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES
271+
long globalThreadId = getGlobalThreadId(groupId, globallyUniqueThreadIdCtor);
272+
UUID invocationUid = newUnsecureUUID();
273+
274+
int dataSize = ClientMessage.HEADER_SIZE + dataSize(groupId) + calculateDataSize(name) + Bits.LONG_SIZE_IN_BYTES * 4
266275
+ Bits.INT_SIZE_IN_BYTES;
267276
ClientMessage msg = prepareClientMessage(groupId, name, dataSize, CHANGE_PERMITS_TYPE);
277+
msg.set(globalThreadId);
278+
msg.set(invocationUid.getLeastSignificantBits());
279+
msg.set(invocationUid.getMostSignificantBits());
268280
msg.set(increase);
269281
msg.updateFrameLength();
270282

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/blocking/AbstractBlockingService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import com.hazelcast.raft.impl.session.SessionExpiredException;
2929
import com.hazelcast.raft.impl.util.Tuple2;
3030
import com.hazelcast.raft.service.blocking.operation.InvalidateWaitKeysOp;
31-
import com.hazelcast.raft.service.session.AbstractSessionManager;
3231
import com.hazelcast.raft.service.spi.RaftRemoteService;
3332
import com.hazelcast.spi.ExecutionService;
3433
import com.hazelcast.spi.ManagedService;
@@ -49,6 +48,7 @@
4948
import java.util.concurrent.ConcurrentMap;
5049
import java.util.concurrent.Future;
5150

51+
import static com.hazelcast.raft.service.session.AbstractSessionManager.NO_SESSION_ID;
5252
import static com.hazelcast.util.Preconditions.checkNotNull;
5353
import static java.util.concurrent.TimeUnit.MILLISECONDS;
5454

@@ -207,7 +207,7 @@ protected final void scheduleTimeout(RaftGroupId groupId, W waitKey, long timeou
207207
}
208208

209209
protected final void heartbeatSession(RaftGroupId groupId, long sessionId) {
210-
if (sessionId == AbstractSessionManager.NO_SESSION_ID) {
210+
if (sessionId == NO_SESSION_ID) {
211211
return;
212212
}
213213

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/semaphore/RaftSemaphore.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,19 +199,35 @@ AcquireResult drain(long sessionId, long threadId, UUID invocationUid) {
199199
return new AcquireResult(drained, cancelled);
200200
}
201201

202-
Tuple2<Boolean, Collection<SemaphoreInvocationKey>> change(int permits) {
202+
ReleaseResult change(long sessionId, long threadId, UUID invocationUid, int permits) {
203203
if (permits == 0) {
204-
Collection<SemaphoreInvocationKey> c = Collections.emptyList();
205-
return Tuple2.of(false, c);
204+
return ReleaseResult.failed(Collections.<SemaphoreInvocationKey>emptyList());
205+
}
206+
207+
Collection<SemaphoreInvocationKey> cancelled = cancelWaitKeys(sessionId, threadId);
208+
209+
if (sessionId != NO_SESSION_ID) {
210+
SessionState state = sessionStates.get(sessionId);
211+
if (state == null) {
212+
state = new SessionState();
213+
sessionStates.put(sessionId, state);
214+
}
215+
216+
if (state.invocationRefUids.containsKey(Tuple2.of(threadId, invocationUid))) {
217+
Collection<SemaphoreInvocationKey> c = Collections.emptyList();
218+
return ReleaseResult.successful(c, c);
219+
}
220+
221+
state.invocationRefUids.put(Tuple2.of(threadId, invocationUid), permits);
206222
}
207223

208224
available += permits;
209225
initialized = true;
210226

211-
Collection<SemaphoreInvocationKey> keys =
227+
Collection<SemaphoreInvocationKey> acquired =
212228
permits > 0 ? assignPermitsToWaitKeys() : Collections.<SemaphoreInvocationKey>emptyList();
213229

214-
return Tuple2.of(true, keys);
230+
return ReleaseResult.successful(acquired, cancelled);
215231
}
216232

217233
@Override

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/semaphore/RaftSemaphoreService.java‎

Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.hazelcast.core.ISemaphore;
2323
import com.hazelcast.raft.RaftGroupId;
2424
import com.hazelcast.raft.impl.service.RaftInvocationManager;
25-
import com.hazelcast.raft.impl.util.Tuple2;
2625
import com.hazelcast.raft.service.blocking.AbstractBlockingService;
2726
import com.hazelcast.raft.service.exception.WaitKeyCancelledException;
2827
import com.hazelcast.raft.service.semaphore.RaftSemaphore.AcquireResult;
@@ -81,11 +80,6 @@ public boolean initSemaphore(RaftGroupId groupId, String name, int permits) {
8180
Collection<SemaphoreInvocationKey> acquired = getOrInitRegistry(groupId).init(name, permits);
8281
notifyWaitKeys(groupId, acquired, true);
8382

84-
if (acquired.size() > 0 && logger.isFineEnabled()) {
85-
logger.fine("Semaphore[" + name + "] in " + groupId + " acquired permits: " + acquired
86-
+ " after initialized with " + permits + " permits");
87-
}
88-
8983
return true;
9084
} catch (IllegalStateException ignored) {
9185
return false;
@@ -103,15 +97,10 @@ public boolean acquirePermits(RaftGroupId groupId, long commitIndex, String name
10397
SemaphoreInvocationKey key = new SemaphoreInvocationKey(name, commitIndex, sessionId, threadId, invocationUid, permits);
10498
AcquireResult result = getOrInitRegistry(groupId).acquire(name, key, timeoutMs);
10599

106-
// if (logger.isFineEnabled()) {
107-
if (result.acquired > 0) {
108-
logger.warning("Semaphore[" + name + "] in " + groupId + " acquired permits: " + result.acquired
100+
if (logger.isFineEnabled()) {
101+
logger.fine("Semaphore[" + name + "] in " + groupId + " acquired permits: " + result.acquired
109102
+ " by <" + sessionId + ", " + threadId + ", " + invocationUid + ">");
110-
} else {
111-
logger.warning("Semaphore[" + name + "] in " + groupId + " NOT acquired permits by <" + sessionId + ", " + threadId
112-
+ ", " + invocationUid + ">");
113103
}
114-
// }
115104

116105
notifyCancelledWaitKeys(groupId, name, result.cancelled);
117106

@@ -129,28 +118,26 @@ public void releasePermits(RaftGroupId groupId, String name, long sessionId, lon
129118
notifyWaitKeys(groupId, result.acquired, true);
130119

131120
if (!result.success) {
132-
logger.warning("Semaphore[" + name + "] in " + groupId + " NOT released permits: " + permits
133-
+ " by <" + sessionId + ", " + threadId + ", " + invocationUid + ">");
134121
throw new IllegalArgumentException();
135-
} else {
136-
logger.warning("Semaphore[" + name + "] in " + groupId + " released permits: " + permits
137-
+ " by <" + sessionId + ", " + threadId + ", " + invocationUid + ">");
138-
logger.warning("Semaphore[" + name + "] in " + groupId + " acquired permits: " + result.acquired);
139122
}
140123
}
141124

142125
public int drainPermits(RaftGroupId groupId, String name, long sessionId, long threadId, UUID invocationUid) {
126+
heartbeatSession(groupId, sessionId);
143127
AcquireResult result = getOrInitRegistry(groupId).drainPermits(name, sessionId, threadId, invocationUid);
144128
notifyCancelledWaitKeys(groupId, name, result.cancelled);
145129

146130
return result.acquired;
147131
}
148132

149-
public boolean changePermits(RaftGroupId groupId, String name, int permits) {
150-
Tuple2<Boolean, Collection<SemaphoreInvocationKey>> t = getOrInitRegistry(groupId).changePermits(name, permits);
151-
notifyWaitKeys(groupId, t.element2, true);
133+
public boolean changePermits(RaftGroupId groupId, String name, long sessionId, long threadId, UUID invocationUid,
134+
int permits) {
135+
heartbeatSession(groupId, sessionId);
136+
ReleaseResult result = getOrInitRegistry(groupId).changePermits(name, sessionId, threadId, invocationUid, permits);
137+
notifyCancelledWaitKeys(groupId, name, result.cancelled);
138+
notifyWaitKeys(groupId, result.acquired, true);
152139

153-
return t.element1;
140+
return result.success;
154141
}
155142

156143
private void notifyCancelledWaitKeys(RaftGroupId groupId, String name, Collection<SemaphoreInvocationKey> waitKeys) {

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/semaphore/SemaphoreRegistry.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
2020
import com.hazelcast.raft.RaftGroupId;
21-
import com.hazelcast.raft.impl.util.Tuple2;
2221
import com.hazelcast.raft.service.blocking.ResourceRegistry;
2322
import com.hazelcast.raft.service.semaphore.RaftSemaphore.AcquireResult;
2423
import com.hazelcast.raft.service.semaphore.RaftSemaphore.ReleaseResult;
@@ -95,13 +94,17 @@ AcquireResult drainPermits(String name, long sessionId, long threadId, UUID invo
9594
return result;
9695
}
9796

98-
Tuple2<Boolean, Collection<SemaphoreInvocationKey>> changePermits(String name, int permits) {
99-
Tuple2<Boolean, Collection<SemaphoreInvocationKey>> t = getOrInitResource(name).change(permits);
100-
for (SemaphoreInvocationKey key : t.element2) {
97+
ReleaseResult changePermits(String name, long sessionId, long threadId, UUID invocationUid, int permits) {
98+
ReleaseResult result = getOrInitResource(name).change(sessionId, threadId, invocationUid, permits);
99+
for (SemaphoreInvocationKey key : result.acquired) {
100+
removeWaitKey(key);
101+
}
102+
103+
for (SemaphoreInvocationKey key : result.cancelled) {
101104
removeWaitKey(key);
102105
}
103106

104-
return t;
107+
return result;
105108
}
106109

107110
@Override

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/semaphore/client/ChangePermitsMessageTask.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,19 @@
1919
import com.hazelcast.client.impl.protocol.ClientMessage;
2020
import com.hazelcast.instance.Node;
2121
import com.hazelcast.nio.Connection;
22+
import com.hazelcast.raft.impl.RaftOp;
2223
import com.hazelcast.raft.impl.service.RaftInvocationManager;
2324
import com.hazelcast.raft.service.semaphore.operation.ChangePermitsOp;
2425

26+
import java.util.UUID;
27+
2528
/**
2629
* TODO: Javadoc Pending...
2730
*/
2831
public class ChangePermitsMessageTask extends AbstractSemaphoreMessageTask {
2932

33+
private long threadId;
34+
private UUID invocationUid;
3035
private int permits;
3136

3237
ChangePermitsMessageTask(ClientMessage clientMessage, Node node, Connection connection) {
@@ -36,12 +41,17 @@ public class ChangePermitsMessageTask extends AbstractSemaphoreMessageTask {
3641
@Override
3742
protected void processMessage() {
3843
RaftInvocationManager invocationManager = getRaftInvocationManager();
39-
invocationManager.invoke(groupId, new ChangePermitsOp(name, permits)).andThen(this);
44+
RaftOp op = new ChangePermitsOp(name, sessionId, threadId, invocationUid, permits);
45+
invocationManager.invoke(groupId, op).andThen(this);
4046
}
4147

4248
@Override
4349
protected Object decodeClientMessage(ClientMessage clientMessage) {
4450
super.decodeClientMessage(clientMessage);
51+
threadId = clientMessage.getLong();
52+
long least = clientMessage.getLong();
53+
long most = clientMessage.getLong();
54+
invocationUid = new UUID(most, least);
4555
permits = clientMessage.getInt();
4656
return null;
4757
}

‎hazelcast-raft-dataservices/src/main/java/com/hazelcast/raft/service/semaphore/operation/ChangePermitsOp.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,32 +20,31 @@
2020
import com.hazelcast.nio.ObjectDataOutput;
2121
import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
2222
import com.hazelcast.raft.RaftGroupId;
23-
import com.hazelcast.raft.impl.RaftOp;
2423
import com.hazelcast.raft.service.semaphore.RaftSemaphoreDataSerializerHook;
2524
import com.hazelcast.raft.service.semaphore.RaftSemaphoreService;
2625

2726
import java.io.IOException;
27+
import java.util.UUID;
2828

2929
/**
3030
* TODO: Javadoc Pending...
3131
*/
32-
public class ChangePermitsOp extends RaftOp implements IdentifiedDataSerializable {
32+
public class ChangePermitsOp extends AbstractSemaphoreOp implements IdentifiedDataSerializable {
3333

34-
private String name;
3534
private int permits;
3635

3736
public ChangePermitsOp() {
3837
}
3938

40-
public ChangePermitsOp(String name, int permits) {
41-
this.name = name;
39+
public ChangePermitsOp(String name, long sessionId, long threadId, UUID invocationUid, int permits) {
40+
super(name, sessionId, threadId, invocationUid);
4241
this.permits = permits;
4342
}
4443

4544
@Override
4645
public Object run(RaftGroupId groupId, long commitIndex) {
4746
RaftSemaphoreService service = getService();
48-
return service.changePermits(groupId, name, permits);
47+
return service.changePermits(groupId, name, sessionId, threadId, invocationUid, permits);
4948
}
5049

5150
@Override
@@ -65,19 +64,19 @@ public int getId() {
6564

6665
@Override
6766
public void writeData(ObjectDataOutput out) throws IOException {
68-
out.writeUTF(name);
67+
super.writeData(out);
6968
out.writeInt(permits);
7069
}
7170

7271
@Override
7372
public void readData(ObjectDataInput in) throws IOException {
74-
name = in.readUTF();
73+
super.readData(in);
7574
permits = in.readInt();
7675
}
7776

7877
@Override
7978
protected void toString(StringBuilder sb) {
80-
sb.append(", name=").append(name)
81-
.append(", permits=").append(permits);
79+
super.toString(sb);
80+
sb.append(", permits=").append(permits);
8281
}
8382
}

0 commit comments

Comments
 (0)