Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #16 from dgomezferro/replication

Reduce coupling around Shared Message Buffer
  • Loading branch information...
commit f6f7b87fd389603a34bce90d8d1cb5ab6a45b153 2 parents dfcc588 + 12d2337
@dgomezferro authored
Showing with 1,202 additions and 1,046 deletions.
  1. +1 −9 pom.xml
  2. +10 −9 src/main/java/com/yahoo/omid/client/TSOClient.java
  3. +25 −24 src/main/java/com/yahoo/omid/{tso/TSOBuffer.java → replication/ReadersAwareBuffer.java}
  4. +188 −0 src/main/java/com/yahoo/omid/replication/SharedMessageBuffer.java
  5. +273 −0 src/main/java/com/yahoo/omid/replication/Zipper.java
  6. +86 −0 src/main/java/com/yahoo/omid/replication/ZipperState.java
  7. +9 −9 src/main/java/com/yahoo/omid/tso/BufferPool.java
  8. +1 −2  src/main/java/com/yahoo/omid/tso/RowKey.java
  9. +22 −17 src/main/java/com/yahoo/omid/tso/TSOHandler.java
  10. +8 −7 src/main/java/com/yahoo/omid/tso/TSOMessage.java
  11. +2 −2 src/main/java/com/yahoo/omid/tso/TSOMessageBuffer.java
  12. +1 −1  src/main/java/com/yahoo/omid/tso/TSOPipelineFactory.java
  13. +0 −337 src/main/java/com/yahoo/omid/tso/TSOSharedMessageBuffer.java
  14. +2 −5 src/main/java/com/yahoo/omid/tso/TSOState.java
  15. +7 −150 src/main/java/com/yahoo/omid/tso/ThroughputMonitor.java
  16. +1 −1  src/main/java/com/yahoo/omid/tso/messages/AbortRequest.java
  17. +53 −35 src/main/java/com/yahoo/omid/tso/messages/AbortedTransactionReport.java
  18. +86 −0 src/main/java/com/yahoo/omid/tso/messages/CleanedTransactionReport.java
  19. +1 −7 src/main/java/com/yahoo/omid/tso/messages/CommitQueryRequest.java
  20. +1 −2  src/main/java/com/yahoo/omid/tso/messages/CommitQueryResponse.java
  21. +1 −16 src/main/java/com/yahoo/omid/tso/messages/CommitRequest.java
  22. +1 −2  src/main/java/com/yahoo/omid/tso/messages/CommitResponse.java
  23. +54 −104 src/main/java/com/yahoo/omid/tso/messages/CommittedTransactionReport.java
  24. +16 −20 src/main/java/com/yahoo/omid/tso/messages/{FullAbortReport.java → FullAbortRequest.java}
  25. +54 −37 src/main/java/com/yahoo/omid/tso/messages/LargestDeletedTimestampReport.java
  26. +11 −15 src/main/java/com/yahoo/omid/tso/messages/TimestampRequest.java
  27. +2 −7 src/main/java/com/yahoo/omid/tso/messages/TimestampResponse.java
  28. +54 −189 src/main/java/com/yahoo/omid/tso/serialization/TSODecoder.java
  29. +5 −2 src/main/java/com/yahoo/omid/tso/serialization/TSOEncoder.java
  30. +3 −8 src/test/java/com/yahoo/omid/OmidTestBase.java
  31. +136 −0 src/test/java/com/yahoo/omid/replication/TestSharedMessageBuffer.java
  32. +56 −0 src/test/java/com/yahoo/omid/replication/TestZipper.java
  33. +4 −3 src/test/java/com/yahoo/omid/tso/TestBasicTransaction.java
  34. +17 −16 src/test/java/com/yahoo/omid/tso/TestClientHandler.java
  35. +9 −8 src/test/java/com/yahoo/omid/tso/TestCommitAbortedReport.java
  36. +2 −2 src/test/java/com/yahoo/omid/tso/TestCommitQuery.java
View
10 pom.xml
@@ -34,6 +34,7 @@
<configuration>
<argLine>-Xmx1G</argLine>
<forkMode>pertest</forkMode>
+ <argLine>-Djava.library.path=${basedir}/src/main/native</argLine>
</configuration>
</plugin>
<plugin>
@@ -77,15 +78,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.7.1</version>
- <configuration>
- <!-- forkMode>always</forkMode -->
- <argLine>-Djava.library.path=${basedir}/src/main/native</argLine>
- </configuration>
- </plugin>
</plugins>
<pluginManagement>
<plugins>
View
19 src/main/java/com/yahoo/omid/client/TSOClient.java
@@ -49,17 +49,20 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.yahoo.omid.replication.Zipper;
+import com.yahoo.omid.replication.ZipperState;
import com.yahoo.omid.tso.Committed;
import com.yahoo.omid.tso.RowKey;
import com.yahoo.omid.tso.TSOMessage;
import com.yahoo.omid.tso.messages.AbortRequest;
import com.yahoo.omid.tso.messages.AbortedTransactionReport;
+import com.yahoo.omid.tso.messages.CleanedTransactionReport;
import com.yahoo.omid.tso.messages.CommitQueryRequest;
import com.yahoo.omid.tso.messages.CommitQueryResponse;
import com.yahoo.omid.tso.messages.CommitRequest;
import com.yahoo.omid.tso.messages.CommitResponse;
import com.yahoo.omid.tso.messages.CommittedTransactionReport;
-import com.yahoo.omid.tso.messages.FullAbortReport;
+import com.yahoo.omid.tso.messages.FullAbortRequest;
import com.yahoo.omid.tso.messages.LargestDeletedTimestampReport;
import com.yahoo.omid.tso.messages.TimestampRequest;
import com.yahoo.omid.tso.messages.TimestampResponse;
@@ -275,7 +278,7 @@ public void error(Exception e) {
public void execute(Channel channel) {
try {
- FullAbortReport far = new FullAbortReport();
+ FullAbortRequest far = new FullAbortRequest();
far.startTimestamp = transactionId;
ChannelFuture f = channel.write(far);
@@ -410,7 +413,7 @@ public void completeAbort(long transactionId, AbortCompleteCallback cb) throws I
@Override
synchronized
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) {
- e.getChannel().getPipeline().addFirst("decoder", new TSODecoder());
+ e.getChannel().getPipeline().addFirst("decoder", new TSODecoder(new Zipper()));
e.getChannel().getPipeline().addAfter("decoder", "encoder",
new TSOEncoder());
}
@@ -481,8 +484,6 @@ public boolean validRead(long transaction, long startTimestamp) throws IOExcepti
return transaction <= largestDeletedTimestamp;
if (transaction <= largestDeletedTimestamp)
return true;
-// System.out.format("Asking TSO... hasConnectionTimestamp: %s connectionTimestamp: %d transaction: %d startTimestamp: %d\n",
-// Boolean.valueOf(hasConnectionTimestamp).toString(), connectionTimestamp, transaction, startTimestamp);
askedTSO++;
SyncCommitQueryCallback cb = new SyncCommitQueryCallback();
isCommitted(startTimestamp, transaction, cb);
@@ -547,8 +548,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
} else if (msg instanceof CommittedTransactionReport) {
CommittedTransactionReport ctr = (CommittedTransactionReport) msg;
committed.commit(ctr.startTimestamp, ctr.commitTimestamp);
- } else if (msg instanceof FullAbortReport) {
- FullAbortReport r = (FullAbortReport) msg;
+ } else if (msg instanceof CleanedTransactionReport) {
+ CleanedTransactionReport r = (CleanedTransactionReport) msg;
aborted.remove(r.startTimestamp);
} else if (msg instanceof AbortedTransactionReport) {
AbortedTransactionReport r = (AbortedTransactionReport) msg;
@@ -557,6 +558,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
LargestDeletedTimestampReport r = (LargestDeletedTimestampReport) msg;
largestDeletedTimestamp = r.largestDeletedTimestamp;
committed.raiseLargestDeletedTransaction(r.largestDeletedTimestamp);
+ } else if (msg instanceof ZipperState) {
+ // ignore
} else {
LOG.error("Unknown message received " + msg);
}
@@ -634,8 +637,6 @@ public void bailout(Exception cause) {
}
protected void processMessage(TSOMessage msg) {
- // TODO Auto-generated method stub
-
}
}
View
49 src/main/java/com/yahoo/omid/tso/TSOBuffer.java → ...om/yahoo/omid/replication/ReadersAwareBuffer.java
@@ -14,48 +14,49 @@
* limitations under the License. See accompanying LICENSE file.
*/
-package com.yahoo.omid.tso;
-
-import java.util.TreeSet;
+package com.yahoo.omid.replication;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
-public class TSOBuffer {
- private static final Log LOG = LogFactory.getLog(TSOBuffer.class);
+
+public class ReadersAwareBuffer {
+ private static final Log LOG = LogFactory.getLog(ReadersAwareBuffer.class);
private static final int CAPACITY = 1024*1024;
public ChannelBuffer buffer;
- public TreeSet<TSOSharedMessageBuffer.ReadingBuffer> readingBuffers = new TreeSet<TSOSharedMessageBuffer.ReadingBuffer>();
- private int pendingWrites = 0;
+ private int pendingReaders = 0;
+ private boolean scheduledForPool;
public static long nBuffers;
-
-
- public TSOBuffer() {
- this(true);
- }
-
- public TSOBuffer(boolean allocateBuffer) {
+
+ public ReadersAwareBuffer() {
nBuffers++;
LOG.warn("Allocated buffer");
- if (allocateBuffer)
- buffer = ChannelBuffers.directBuffer(CAPACITY);
+ buffer = ChannelBuffers.directBuffer(CAPACITY);
}
- public TSOBuffer reading(TSOSharedMessageBuffer.ReadingBuffer buf) {
- readingBuffers.add(buf);
- return this;
+ public synchronized void increaseReaders() {
+ pendingReaders++;
}
- public synchronized void incrementPending() {
- ++pendingWrites;
+ public synchronized void decreaseReaders() {
+ pendingReaders--;
}
-
- public synchronized boolean decrementPending() {
- return --pendingWrites == 0;
+
+ public synchronized boolean isReadyForPool() {
+ return scheduledForPool && pendingReaders == 0;
+ }
+
+ public synchronized void scheduleForPool() {
+ scheduledForPool = true;
+ }
+
+ public synchronized void reset() {
+ pendingReaders = 0;
+ scheduledForPool = false;
}
}
View
188 src/main/java/com/yahoo/omid/replication/SharedMessageBuffer.java
@@ -0,0 +1,188 @@
+/**
+ * Copyright (c) 2011 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package com.yahoo.omid.replication;
+
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.Channels;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SharedMessageBuffer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SharedMessageBuffer.class);
+
+ private static final int MAX_MESSAGE_SIZE = 30;
+
+ ReadersAwareBuffer pastBuffer = new ReadersAwareBuffer();
+ ReadersAwareBuffer currentBuffer = new ReadersAwareBuffer();
+ ChannelBuffer writeBuffer = currentBuffer.buffer;
+ BlockingQueue<ReadersAwareBuffer> bufferPool = new LinkedBlockingQueue<ReadersAwareBuffer>();
+ Zipper zipper = new Zipper();
+ Set<ReadingBuffer> readingBuffers = new TreeSet<SharedMessageBuffer.ReadingBuffer>();
+
+ public class ReadingBuffer implements Comparable<ReadingBuffer> {
+ private ChannelBuffer readBuffer;
+ private int readerIndex = 0;
+ private ReadersAwareBuffer readingBuffer;
+ private ChannelHandlerContext ctx;
+ private Channel channel;
+
+ private ReadingBuffer(ChannelHandlerContext ctx) {
+ this.channel = ctx.getChannel();
+ this.ctx = ctx;
+ }
+
+ public void initializeIndexes() {
+ this.readingBuffer = currentBuffer;
+ this.readBuffer = readingBuffer.buffer;
+ this.readerIndex = readBuffer.writerIndex();
+ }
+
+ /**
+ * Computes and returns the deltaSO for the associated client.
+ *
+ * This function registers some callbacks in the future passed for cleanup purposes, so the ChannelFuture object
+ * must be notified after the communication is finished.
+ *
+ * @param future
+ * It registers some callbacks on it
+ * @return the deltaSO for the associated client
+ */
+ public ChannelBuffer flush(ChannelFuture future) {
+ int readable = readBuffer.readableBytes() - readerIndex;
+
+ if (readable == 0 && readingBuffer != pastBuffer) {
+ return ChannelBuffers.EMPTY_BUFFER;
+ }
+
+ ChannelBuffer deltaSO = readBuffer.slice(readerIndex, readable);
+ addFinishedWriteListener(future, readingBuffer);
+ readingBuffer.increaseReaders();
+ readerIndex += readable;
+ if (readingBuffer == pastBuffer) {
+ readingBuffer = currentBuffer;
+ readBuffer = readingBuffer.buffer;
+ readerIndex = 0;
+ readable = readBuffer.readableBytes();
+ deltaSO = ChannelBuffers.wrappedBuffer(deltaSO, readBuffer.slice(readerIndex, readable));
+ addFinishedWriteListener(future, readingBuffer);
+ readingBuffer.increaseReaders();
+ readerIndex += readable;
+ }
+
+ return deltaSO;
+ }
+
+ private void addFinishedWriteListener(ChannelFuture future, final ReadersAwareBuffer buffer) {
+ future.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ buffer.decreaseReaders();
+ if (buffer.isReadyForPool()) {
+ bufferPool.add(buffer);
+ }
+ }
+ });
+ }
+
+ public ZipperState getZipperState() {
+ return zipper.getZipperState();
+ }
+
+ @Override
+ public int compareTo(ReadingBuffer o) {
+ return this.channel.compareTo(o.channel);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ReadingBuffer))
+ return false;
+ ReadingBuffer buf = (ReadingBuffer) obj;
+ return this.channel.equals(buf.channel);
+ }
+
+ }
+
+ public ReadingBuffer getReadingBuffer(ChannelHandlerContext ctx) {
+ ReadingBuffer rb = new ReadingBuffer(ctx);
+ readingBuffers.add(rb);
+ return rb;
+ }
+
+ public void removeReadingBuffer(ChannelHandlerContext ctx) {
+ readingBuffers.remove(new ReadingBuffer(ctx));
+ }
+
+ public void writeCommit(long startTimestamp, long commitTimestamp) {
+ checkBufferSpace();
+ zipper.encodeCommit(writeBuffer, startTimestamp, commitTimestamp);
+ }
+
+ public void writeHalfAbort(long startTimestamp) {
+ checkBufferSpace();
+ zipper.encodeHalfAbort(writeBuffer, startTimestamp);
+ }
+
+ public void writeFullAbort(long startTimestamp) {
+ checkBufferSpace();
+ zipper.encodeFullAbort(writeBuffer, startTimestamp);
+ }
+
+ public void writeLargestIncrease(long largestTimestamp) {
+ checkBufferSpace();
+ zipper.encodeLargestIncrease(writeBuffer, largestTimestamp);
+ }
+
+ private void checkBufferSpace() {
+ if (writeBuffer.writableBytes() < MAX_MESSAGE_SIZE) {
+ nextBuffer();
+ }
+ }
+
+ private void nextBuffer() {
+ LOG.debug("Switching buffers");
+
+ // mark past buffer as scheduled for pool when all pending operations finish
+ pastBuffer.scheduleForPool();
+
+ for (final ReadingBuffer rb : readingBuffers) {
+ if (rb.readingBuffer == pastBuffer) {
+ ChannelFuture future = Channels.future(rb.channel);
+ ChannelBuffer cb = rb.flush(future);
+ Channels.write(rb.ctx, future, cb);
+ }
+ }
+
+ pastBuffer = currentBuffer;
+ currentBuffer = bufferPool.poll();
+ if (currentBuffer == null) {
+ currentBuffer = new ReadersAwareBuffer();
+ }
+ writeBuffer = currentBuffer.buffer;
+ }
+}
View
273 src/main/java/com/yahoo/omid/replication/Zipper.java
@@ -0,0 +1,273 @@
+package com.yahoo.omid.replication;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import com.yahoo.omid.tso.TSOMessage;
+import com.yahoo.omid.tso.messages.AbortedTransactionReport;
+import com.yahoo.omid.tso.messages.CleanedTransactionReport;
+import com.yahoo.omid.tso.messages.CommittedTransactionReport;
+
+/**
+ * Encodes replication messages efficiently.
+ *
+ * 1 byte msgs
+ *
+ * 00xx xxxx => commit report (commit diff = 1; startTS diff = xx xxxx)
+ * 010x xxxx => half abort report (startTS diff = x xxxx)
+ * 011x xxxx => full abort report (startTS diff = x xxxx)
+ *
+ * 2 byte msgs
+ *
+ * 10xx xxxx xxxx xxxx => commit report (ct diff = 1; st diff = xx...)
+ *
+ * Remaining msgs
+ *
+ * 11xx xxxx yyyy yyyy [yyyy yyyy ...] => header = 11xx xxxx; data = yyyy...
+ *
+ */
+
+public class Zipper {
+ long lastStartTimestamp = 0;
+ long lastCommitTimestamp = 0;
+
+ long lastHalfAbortedTimestamp = 0;
+ long lastFullAbortedTimestamp = 0;
+
+ public ZipperState getZipperState() {
+ return new ZipperState(lastStartTimestamp, lastCommitTimestamp, lastHalfAbortedTimestamp, lastFullAbortedTimestamp);
+ }
+
+ public void encodeCommit(ChannelBuffer buffer, long startTimestamp, long commitTimestamp) {
+ long startDiff = startTimestamp - lastStartTimestamp;
+ long commitDiff = commitTimestamp - lastCommitTimestamp;
+ if (commitDiff == 1 && startDiff >= -32 && startDiff <= 31) {
+ startDiff &= 0x3f;
+ buffer.writeByte((byte) startDiff);
+ } else if (commitDiff == 1 && startDiff >= -8192 && startDiff <= 8191) {
+ byte high = (byte) 0x80;
+ high |= (startDiff >> 8) & (byte) 0x3f;
+ byte low = (byte) (startDiff & (byte) 0xff);
+ buffer.writeByte(high);
+ buffer.writeByte(low);
+ } else if (commitDiff >= Byte.MIN_VALUE && commitDiff <= Byte.MAX_VALUE) {
+ if (startDiff >= Byte.MIN_VALUE && startDiff <= Byte.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportByteByte);
+ buffer.writeByte((byte) startDiff);
+ } else if (startDiff >= Short.MIN_VALUE && startDiff <= Short.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportShortByte);
+ buffer.writeShort((short) startDiff);
+ } else if (startDiff >= Integer.MIN_VALUE && startDiff <= Integer.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportIntegerByte);
+ buffer.writeInt((int) startDiff);
+ } else {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportLongByte);
+ buffer.writeLong((byte) startDiff);
+ }
+ buffer.writeByte((byte) commitDiff);
+ } else if (commitDiff >= Short.MIN_VALUE && commitDiff <= Short.MAX_VALUE) {
+ if (startDiff >= Byte.MIN_VALUE && startDiff <= Byte.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportByteShort);
+ buffer.writeByte((byte) startDiff);
+ } else if (startDiff >= Short.MIN_VALUE && startDiff <= Short.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportShortShort);
+ buffer.writeShort((short) startDiff);
+ } else if (startDiff >= Integer.MIN_VALUE && startDiff <= Integer.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportIntegerShort);
+ buffer.writeInt((int) startDiff);
+ } else {
+ buffer.writeByte(TSOMessage.CommittedTransactionReportLongShort);
+ buffer.writeLong((byte) startDiff);
+ }
+ buffer.writeShort((short) commitDiff);
+ } else {
+ buffer.writeByte(TSOMessage.CommittedTransactionReport);
+ buffer.writeLong(startTimestamp);
+ buffer.writeLong(commitTimestamp);
+ }
+ lastStartTimestamp = startTimestamp;
+ lastCommitTimestamp = commitTimestamp;
+ }
+
+ public void encodeHalfAbort(ChannelBuffer buffer, long startTimestamp) {
+ long diff = startTimestamp - lastHalfAbortedTimestamp;
+ if (diff >= -16 && diff <= 15) {
+ buffer.writeByte((byte) ((diff & 0x1f) | (0x40)));
+ } else if (diff >= Byte.MIN_VALUE && diff <= Byte.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.AbortedTransactionReportByte);
+ buffer.writeByte((byte) diff);
+ } else {
+ buffer.writeByte(TSOMessage.AbortedTransactionReport);
+ buffer.writeLong(startTimestamp);
+ }
+
+ lastHalfAbortedTimestamp = startTimestamp;
+ }
+
+ public void encodeFullAbort(ChannelBuffer buffer, long startTimestamp) {
+ long diff = startTimestamp - lastFullAbortedTimestamp;
+ if (diff >= -16 && diff <= 15) {
+ buffer.writeByte((byte) ((diff & 0x1f) | (0x60)));
+ } else if (diff >= Byte.MIN_VALUE && diff <= Byte.MAX_VALUE) {
+ buffer.writeByte(TSOMessage.CleanedTransactionReportByte);
+ buffer.writeByte((byte) diff);
+ } else {
+ buffer.writeByte(TSOMessage.CleanedTransactionReport);
+ buffer.writeLong(startTimestamp);
+ }
+
+ lastFullAbortedTimestamp = startTimestamp;
+ }
+
+ public void encodeLargestIncrease(ChannelBuffer buffer, long largestTimestamp) {
+ buffer.writeByte(TSOMessage.LargestDeletedTimestampReport);
+ buffer.writeLong(largestTimestamp);
+ }
+
+ public TSOMessage decodeMessage(ChannelBuffer buffer) {
+ byte type = buffer.readByte();
+ if ((type & 0xC0) == 0x00) { // 00xx xxxx
+ return decodeCommittedTransactionReport(type, buffer);
+ } else if ((type & 0xE0) == 0x40) { // 010x xxxx
+ return decodeHalfAbort(type);
+ } else if ((type & 0xE0) == 0x60) { // 011x xxxx
+ return decodeFullAbort(type);
+ } else if ((type & 0x40) == 0) { // 10xx xxxx [...]
+ return decodeCommittedTransactionReport(type, buffer);
+ } else if (type >= TSOMessage.CommittedTransactionReport) {
+ return decodeCommittedTransactionReport(type, buffer);
+ } else {
+ switch (type) {
+ case TSOMessage.CleanedTransactionReport:
+ case TSOMessage.CleanedTransactionReportByte:
+ return decodeFullAbort(type, buffer);
+ case TSOMessage.AbortedTransactionReport:
+ case TSOMessage.AbortedTransactionReportByte:
+ return decodeHalfAbort(type, buffer);
+ case TSOMessage.ZipperState:
+ return decodeZipperState(buffer);
+ default:
+ return null;
+ }
+ }
+ }
+
+ private TSOMessage decodeZipperState(ChannelBuffer buffer) {
+ ZipperState state = new ZipperState();
+ state.readObject(buffer);
+ this.lastCommitTimestamp = state.getLastCommitTimestamp();
+ this.lastStartTimestamp = state.getLastStartTimestamp();
+ this.lastFullAbortedTimestamp = state.getLastFullAbortedTimestamp();
+ this.lastHalfAbortedTimestamp = state.getLastHalfAbortedTimestamp();
+ return state;
+ }
+
+ private TSOMessage decodeHalfAbort(byte diff) {
+ // Half abort
+ lastHalfAbortedTimestamp += extractAbortedDifference(diff);
+ return new AbortedTransactionReport(lastHalfAbortedTimestamp);
+ }
+
+ private TSOMessage decodeFullAbort(byte diff) {
+ // Full abort
+ lastFullAbortedTimestamp += extractAbortedDifference(diff);
+ return new CleanedTransactionReport(lastFullAbortedTimestamp);
+ }
+
+ private int extractAbortedDifference(byte diff) {
+ // extract the difference
+ int extracted = diff & 0x1f;
+ // extend the sign
+ return (extracted << 27) >> 27;
+ }
+
+ private AbortedTransactionReport decodeHalfAbort(byte type, ChannelBuffer buffer) {
+ AbortedTransactionReport msg;
+ if (type == TSOMessage.AbortedTransactionReport) {
+ msg = new AbortedTransactionReport();
+ msg.readObject(buffer);
+ } else {
+ msg = new AbortedTransactionReport();
+ int diff = buffer.readByte();
+ msg.startTimestamp = lastHalfAbortedTimestamp + diff;
+ }
+ lastHalfAbortedTimestamp = msg.startTimestamp;
+
+ return msg;
+ }
+
+ private CleanedTransactionReport decodeFullAbort(byte type, ChannelBuffer buffer) {
+ CleanedTransactionReport msg;
+ if (type == TSOMessage.CleanedTransactionReport) {
+ msg = new CleanedTransactionReport();
+ msg.readObject(buffer);
+ } else {
+ msg = new CleanedTransactionReport();
+ int diff = buffer.readByte();
+ msg.startTimestamp = lastFullAbortedTimestamp + diff;
+ }
+ lastFullAbortedTimestamp = msg.startTimestamp;
+
+ return msg;
+ }
+
+ private CommittedTransactionReport decodeCommittedTransactionReport(byte high, ChannelBuffer aInputStream) {
+ long startTimestamp = 0;
+ long commitTimestamp = 0;
+ if (high >= 0) {
+ high = (byte) ((high << 26) >> 26);
+ startTimestamp = lastStartTimestamp + high;
+ commitTimestamp = lastCommitTimestamp + 1;
+ } else if ((high & 0x40) == 0) {
+ byte low = aInputStream.readByte();
+ long startDiff = low & 0xff;
+ startDiff |= ((high & 0x3f) << 26) >> 18;
+ startTimestamp = lastStartTimestamp + startDiff;
+ commitTimestamp = lastCommitTimestamp + 1;
+ } else {
+ switch (high) {
+ case TSOMessage.CommittedTransactionReportByteByte:
+ startTimestamp = lastStartTimestamp + aInputStream.readByte();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
+ break;
+ case TSOMessage.CommittedTransactionReportShortByte:
+ startTimestamp = lastStartTimestamp + aInputStream.readShort();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
+ break;
+ case TSOMessage.CommittedTransactionReportIntegerByte:
+ startTimestamp = lastStartTimestamp + aInputStream.readInt();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
+ break;
+ case TSOMessage.CommittedTransactionReportLongByte:
+ startTimestamp = lastStartTimestamp + aInputStream.readLong();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
+ break;
+
+ case TSOMessage.CommittedTransactionReportByteShort:
+ startTimestamp = lastStartTimestamp + aInputStream.readByte();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
+ break;
+ case TSOMessage.CommittedTransactionReportShortShort:
+ startTimestamp = lastStartTimestamp + aInputStream.readShort();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
+ break;
+ case TSOMessage.CommittedTransactionReportIntegerShort:
+ startTimestamp = lastStartTimestamp + aInputStream.readInt();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
+ break;
+ case TSOMessage.CommittedTransactionReportLongShort:
+ startTimestamp = lastStartTimestamp + aInputStream.readLong();
+ commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
+ break;
+ case TSOMessage.CommittedTransactionReport:
+ startTimestamp = aInputStream.readLong();
+ commitTimestamp = aInputStream.readLong();
+ }
+ }
+
+ lastStartTimestamp = startTimestamp;
+ lastCommitTimestamp = commitTimestamp;
+
+ return new CommittedTransactionReport(startTimestamp, commitTimestamp);
+ }
+
+}
View
86 src/main/java/com/yahoo/omid/replication/ZipperState.java
@@ -0,0 +1,86 @@
+package com.yahoo.omid.replication;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import com.yahoo.omid.tso.TSOMessage;
+
+public class ZipperState implements TSOMessage {
+
+ private long lastStartTimestamp;
+ private long lastCommitTimestamp;
+
+ private long lastHalfAbortedTimestamp;
+ private long lastFullAbortedTimestamp;
+
+ public ZipperState() {
+ }
+
+ public ZipperState(long lastStartTimestamp, long lastCommitTimestamp, long lastHalfAbortedTimestamp,
+ long lastFullAbortedTimestamp) {
+ this.lastStartTimestamp = lastStartTimestamp;
+ this.lastCommitTimestamp = lastCommitTimestamp;
+ this.lastHalfAbortedTimestamp = lastHalfAbortedTimestamp;
+ this.lastFullAbortedTimestamp = lastFullAbortedTimestamp;
+ }
+
+ @Override
+ public void readObject(ChannelBuffer buffer) {
+ lastStartTimestamp = buffer.readLong();
+ lastCommitTimestamp = buffer.readLong();
+
+ lastHalfAbortedTimestamp = buffer.readLong();
+ lastFullAbortedTimestamp = buffer.readLong();
+ }
+
+ @Override
+ public void writeObject(ChannelBuffer buffer) {
+ buffer.writeLong(lastStartTimestamp);
+ buffer.writeLong(lastCommitTimestamp);
+ buffer.writeLong(lastHalfAbortedTimestamp);
+ buffer.writeLong(lastFullAbortedTimestamp);
+ }
+
+ @Override
+ public void writeObject(DataOutputStream buffer) throws IOException {
+ buffer.writeLong(lastStartTimestamp);
+ buffer.writeLong(lastCommitTimestamp);
+ buffer.writeLong(lastHalfAbortedTimestamp);
+ buffer.writeLong(lastFullAbortedTimestamp);
+ }
+
+ public long getLastStartTimestamp() {
+ return lastStartTimestamp;
+ }
+
+ public void setLastStartTimestamp(long lastStartTimestamp) {
+ this.lastStartTimestamp = lastStartTimestamp;
+ }
+
+ public long getLastCommitTimestamp() {
+ return lastCommitTimestamp;
+ }
+
+ public void setLastCommitTimestamp(long lastCommitTimestamp) {
+ this.lastCommitTimestamp = lastCommitTimestamp;
+ }
+
+ public long getLastHalfAbortedTimestamp() {
+ return lastHalfAbortedTimestamp;
+ }
+
+ public void setLastHalfAbortedTimestamp(long lastHalfAbortedTimestamp) {
+ this.lastHalfAbortedTimestamp = lastHalfAbortedTimestamp;
+ }
+
+ public long getLastFullAbortedTimestamp() {
+ return lastFullAbortedTimestamp;
+ }
+
+ public void setLastFullAbortedTimestamp(long lastFullAbortedTimestamp) {
+ this.lastFullAbortedTimestamp = lastFullAbortedTimestamp;
+ }
+
+}
View
18 src/main/java/com/yahoo/omid/tso/BufferPool.java
@@ -17,22 +17,22 @@
package com.yahoo.omid.tso;
import java.io.ByteArrayOutputStream;
-import java.util.ArrayDeque;
-import java.util.Deque;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
public class BufferPool {
- private static Deque<ByteArrayOutputStream> pool = new ArrayDeque<ByteArrayOutputStream>();
+ private static BlockingQueue<ByteArrayOutputStream> pool = new LinkedBlockingQueue<ByteArrayOutputStream>();
- public static synchronized ByteArrayOutputStream getBuffer() {
- if (pool.isEmpty()) {
- return new ByteArrayOutputStream(1500);
- }
- return pool.pollLast();
+ public static ByteArrayOutputStream getBuffer() {
+ ByteArrayOutputStream baos = pool.poll();
+ if (baos != null)
+ return baos;
+ return new ByteArrayOutputStream(1500);
}
- public static synchronized void pushBuffer(ByteArrayOutputStream buffer) {
+ public static void pushBuffer(ByteArrayOutputStream buffer) {
pool.add(buffer);
}
}
View
3  src/main/java/com/yahoo/omid/tso/RowKey.java
@@ -50,8 +50,7 @@ public String toString() {
return new String(tableId) + ":" + new String(rowId);
}
- public static RowKey readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public static RowKey readObject(ChannelBuffer aInputStream) {
int hash = aInputStream.readInt();
short len = aInputStream.readByte();
// byte[] rowId = RowKeyBuffer.nextRowKey(len);
View
39 src/main/java/com/yahoo/omid/tso/TSOHandler.java
@@ -36,6 +36,7 @@
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.Channels;
@@ -44,17 +45,16 @@
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.channel.group.ChannelGroup;
-import com.yahoo.omid.tso.TSOSharedMessageBuffer.ReadingBuffer;
+import com.yahoo.omid.replication.SharedMessageBuffer.ReadingBuffer;
import com.yahoo.omid.tso.messages.AbortRequest;
import com.yahoo.omid.tso.messages.AbortedTransactionReport;
import com.yahoo.omid.tso.messages.CommitQueryRequest;
import com.yahoo.omid.tso.messages.CommitQueryResponse;
import com.yahoo.omid.tso.messages.CommitRequest;
import com.yahoo.omid.tso.messages.CommitResponse;
-import com.yahoo.omid.tso.messages.CommittedTransactionReport;
-import com.yahoo.omid.tso.messages.FullAbortReport;
-import com.yahoo.omid.tso.messages.LargestDeletedTimestampReport;
+import com.yahoo.omid.tso.messages.FullAbortRequest;
import com.yahoo.omid.tso.messages.TimestampRequest;
+import com.yahoo.omid.tso.messages.TimestampResponse;
import com.yahoo.omid.tso.persistence.LoggerAsyncCallback.AddRecordCallback;
import com.yahoo.omid.tso.persistence.LoggerException;
import com.yahoo.omid.tso.persistence.LoggerException.Code;
@@ -146,6 +146,13 @@ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) thr
channelGroup.add(ctx.getChannel());
}
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ synchronized (sharedMsgBufLock) {
+ sharedState.sharedMessageBuffer.removeReadingBuffer(ctx);
+ }
+ }
+
/**
* Handle receieved messages
*/
@@ -158,8 +165,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
} else if (msg instanceof CommitRequest) {
handle((CommitRequest) msg, ctx);
return;
- } else if (msg instanceof FullAbortReport) {
- handle((FullAbortReport) msg, ctx);
+ } else if (msg instanceof FullAbortRequest) {
+ handle((FullAbortRequest) msg, ctx);
return;
} else if (msg instanceof CommitQueryRequest) {
handle((CommitQueryRequest) msg, ctx);
@@ -199,15 +206,14 @@ public void handle(TimestampRequest msg, ChannelHandlerContext ctx) {
}
ReadingBuffer buffer;
- Channel channel = null;
+ Channel channel = ctx.getChannel();
boolean bootstrap = false;
synchronized (messageBuffersMap) {
buffer = messageBuffersMap.get(ctx.getChannel());
if (buffer == null) {
synchronized (sharedMsgBufLock) {
bootstrap = true;
- channel = ctx.getChannel();
- buffer = sharedState.sharedMessageBuffer.new ReadingBuffer(channel);
+ buffer = sharedState.sharedMessageBuffer.getReadingBuffer(ctx);
messageBuffersMap.put(channel, buffer);
channelGroup.add(channel);
LOG.warn("Channel connected: " + messageBuffersMap.size());
@@ -217,10 +223,7 @@ public void handle(TimestampRequest msg, ChannelHandlerContext ctx) {
if (bootstrap) {
synchronized (sharedState) {
synchronized (sharedMsgBufLock) {
- channel.write(new CommittedTransactionReport(sharedState.latestStartTimestamp, sharedState.latestCommitTimestamp));
- channel.write(new AbortedTransactionReport(sharedState.latestHalfAbortTimestamp));
- channel.write(new FullAbortReport(sharedState.latestFullAbortTimestamp));
- channel.write(new LargestDeletedTimestampReport(sharedState.largestDeletedTimestamp));
+ channel.write(buffer.getZipperState());
buffer.initializeIndexes();
}
}
@@ -228,11 +231,13 @@ public void handle(TimestampRequest msg, ChannelHandlerContext ctx) {
channel.write(new AbortedTransactionReport(halfAborted.getStartTimestamp()));
}
}
+ ChannelBuffer cb;
+ ChannelFuture future = Channels.future(channel);
synchronized (sharedMsgBufLock) {
- sharedState.sharedMessageBuffer.writeTimestamp(timestamp);
- buffer.flush();
- sharedState.sharedMessageBuffer.rollBackTimestamp();
+ cb = buffer.flush(future);
}
+ Channels.write(ctx, future, cb);
+ Channels.write(channel, new TimestampResponse(timestamp));
}
ChannelBuffer cb = ChannelBuffers.buffer(10);
@@ -523,7 +528,7 @@ private void queueLargestIncrease(long largestTimestamp) {
/**
* Handle the FullAbortReport message
*/
- public void handle(FullAbortReport msg, ChannelHandlerContext ctx) {
+ public void handle(FullAbortRequest msg, ChannelHandlerContext ctx) {
synchronized (sharedState) {
DataOutputStream toWAL = sharedState.toWAL;
try {
View
15 src/main/java/com/yahoo/omid/tso/TSOMessage.java
@@ -48,17 +48,18 @@
final public byte CommitQueryRequest = (byte) 0xc5;
final public byte CommitQueryResponse = (byte) 0xc6;
final public byte AbortedTransactionReport = (byte) 0xc7;
- final public byte LargestDeletedTimestampReport = (byte) 0xc8;
- final public byte FullAbortReportByte = (byte) 0xc9;
- final public byte AbortedTransactionReportByte = (byte) 0xca;
- final public byte AbortRequest = (byte) 0xcb;
+ final public byte CleanedTransactionReport = (byte) 0xc8;
+ final public byte LargestDeletedTimestampReport = (byte) 0xc9;
+ final public byte CleanedTransactionReportByte = (byte) 0xca;
+ final public byte AbortedTransactionReportByte = (byte) 0xcb;
+ final public byte AbortRequest = (byte) 0xcc;
+ final public byte ZipperState = (byte) 0xcd;
/*
* Deserialize function
- * I use ChannelBuffer instead of DataInputStream because the performance was better
+ * We use ChannelBuffer instead of DataInputStream because the performance is better
*/
- public void readObject(ChannelBuffer aInputStream)
- throws IOException;
+ public void readObject(ChannelBuffer aInputStream);
/*
* Serialize function
View
4 src/main/java/com/yahoo/omid/tso/TSOMessageBuffer.java
@@ -30,7 +30,7 @@
import com.yahoo.omid.tso.messages.CommitRequest;
import com.yahoo.omid.tso.messages.CommitResponse;
import com.yahoo.omid.tso.messages.CommittedTransactionReport;
-import com.yahoo.omid.tso.messages.FullAbortReport;
+import com.yahoo.omid.tso.messages.FullAbortRequest;
import com.yahoo.omid.tso.messages.LargestDeletedTimestampReport;
import com.yahoo.omid.tso.messages.TimestampRequest;
import com.yahoo.omid.tso.messages.TimestampResponse;
@@ -134,7 +134,7 @@ public static void encode(Object msg, ChannelBuffer buffer) {
buffer.writeByte(TSOMessage.CommitRequest);
} else if (msg instanceof CommitResponse) {
buffer.writeByte(TSOMessage.CommitResponse);
- } else if (msg instanceof FullAbortReport) {
+ } else if (msg instanceof FullAbortRequest) {
buffer.writeByte(TSOMessage.FullAbortReport);
} else if (msg instanceof CommitQueryRequest) {
buffer.writeByte(TSOMessage.CommitQueryRequest);
View
2  src/main/java/com/yahoo/omid/tso/TSOPipelineFactory.java
@@ -65,7 +65,7 @@ public TSOPipelineFactory(Executor pipelineExecutor, ChannelHandler handler) {
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = Channels.pipeline();
- pipeline.addLast("decoder", new TSODecoder());
+ pipeline.addLast("decoder", new TSODecoder(null));
pipeline.addLast("encoder", new TSOEncoder());
synchronized (this) {
if (x == null)
View
337 src/main/java/com/yahoo/omid/tso/TSOSharedMessageBuffer.java
@@ -1,337 +0,0 @@
-/**
- * Copyright (c) 2011 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-
-package com.yahoo.omid.tso;
-
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.Iterator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-import org.jboss.netty.channel.Channels;
-
-public class TSOSharedMessageBuffer {
-
- private static final Log LOG = LogFactory.getLog(TSOSharedMessageBuffer.class);
-
- private TSOState state;
-
- TSOBuffer pastBuffer = new TSOBuffer();
- TSOBuffer currentBuffer = new TSOBuffer();
- ChannelBuffer writeBuffer = currentBuffer.buffer;
- Deque<TSOBuffer> futureBuffer = new ArrayDeque<TSOBuffer>();
-
- static long _1B = 0;
- static long _2B = 0;
- static long _AB = 0;
- static long _AS = 0;
- static long _LL = 0;
- static long _Coms = 0;
- static long _Writes = 0;
- static double _Avg = 0;
- static double _Avg2 = 0;
-
- static long _ha = 0;
- static long _fa = 0;
- static long _li = 0;
-
- static long _overflows = 0;
- static long _emptyFlushes = 0;
-
- class ReadingBuffer implements Comparable<ReadingBuffer> {
- private ChannelBuffer readBuffer;
- private int readerIndex = 0;
- private TSOBuffer readingBuffer;
- private Channel channel;
-
- public ReadingBuffer(Channel channel) {
- this.channel = channel;
- }
-
- public void initializeIndexes() {
- this.readingBuffer = currentBuffer.reading(this);
- this.readBuffer = readingBuffer.buffer;
- this.readerIndex = readBuffer.writerIndex();
- }
-
- public void flush() {
- flush(true, false);
- }
-
- private void flush(boolean deleteRef, final boolean clearPast) {
- int readable = readBuffer.readableBytes() - readerIndex;
-
- ++_flushes;
- _flSize += readable;
- if (readable == 0 && readingBuffer != pastBuffer) {
- _emptyFlushes++;
- if (wrap) {
- Channels.write(channel, tBuffer);
- }
- return;
- }
-
- ChannelBuffer temp;
- if (wrap && readingBuffer != pastBuffer) {
- temp = ChannelBuffers.wrappedBuffer(readBuffer.slice(readerIndex, readable), tBuffer);
- } else {
- temp = readBuffer.slice(readerIndex, readable);
- }
- ChannelFuture future = Channels.write(channel, temp);
- readerIndex += readable;
- if (readingBuffer == pastBuffer) {
- readingBuffer = currentBuffer.reading(this);
- readBuffer = readingBuffer.buffer;
- readerIndex = 0;
- readable = readBuffer.readableBytes();
- if (wrap) {
- temp = ChannelBuffers.wrappedBuffer(readBuffer.slice(readerIndex, readable), tBuffer);
- } else {
- temp = readBuffer.slice(readerIndex, readable);
- }
- Channels.write(channel, temp);
- readerIndex += readable;
- _flSize += readable;
- if (deleteRef) {
- pastBuffer.readingBuffers.remove(this);
- }
- pastBuffer.incrementPending();
- final TSOBuffer pendingBuffer = pastBuffer;
- future.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- if (clearPast) {
- pendingBuffer.readingBuffers.clear();
- }
- if (pendingBuffer.decrementPending() && pendingBuffer.readingBuffers.size() == 0) {
- pendingBuffer.buffer.clear();
- synchronized (futureBuffer) {
- futureBuffer.add(pendingBuffer);
- }
- }
- }
- });
- }
-
- }
-
- @Override
- public int compareTo(ReadingBuffer o) {
- return this.channel.compareTo(o.channel);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof ReadingBuffer))
- return false;
- ReadingBuffer buf = (ReadingBuffer) obj;
- return this.channel.equals(buf.channel);
- }
-
- }
-
- public TSOSharedMessageBuffer(TSOState state) {
- this.state = state;
- }
-
- static private final byte m0x80 = (byte) 0x80;
- static private final byte m0x3f = (byte) 0x3f;
- static private final byte m0xff = (byte) 0xff;
-
- private ChannelBuffer tBuffer;
- private boolean wrap = false;
-
- public void writeTimestamp(long timestamp) {
- wrap = true;
- tBuffer = ChannelBuffers.buffer(9);
- tBuffer.writeByte(TSOMessage.TimestampResponse);
- tBuffer.writeLong(timestamp);
- }
-
- public void rollBackTimestamp() {
- wrap = false;
- }
-
- public void writeCommit(long startTimestamp, long commitTimestamp) {
- if (writeBuffer.writableBytes() < 30) {
- nextBuffer();
- }
- ++_Coms;
- ++_Writes;
- int readBefore = writeBuffer.readableBytes();
- long startDiff = startTimestamp - state.latestStartTimestamp;
- long commitDiff = commitTimestamp - state.latestCommitTimestamp;
- if (commitDiff == 1 && startDiff >= -32 && startDiff <= 31) {
- ++_1B;
- startDiff &= 0x3f;
- writeBuffer.writeByte((byte) startDiff);
- } else if (commitDiff == 1 && startDiff >= -8192 && startDiff <= 8191) {
- ++_2B;
- byte high = m0x80;
- high |= (startDiff >> 8) & m0x3f;
- byte low = (byte) (startDiff & m0xff);
- writeBuffer.writeByte(high);
- writeBuffer.writeByte(low);
- } else if (commitDiff >= Byte.MIN_VALUE && commitDiff <= Byte.MAX_VALUE) {
- if (startDiff >= Byte.MIN_VALUE && startDiff <= Byte.MAX_VALUE) {
- ++_AB;
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportByteByte);
- writeBuffer.writeByte((byte) startDiff);
- } else if (startDiff >= Short.MIN_VALUE && startDiff <= Short.MAX_VALUE) {
- ++_AS;
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportShortByte);
- writeBuffer.writeShort((short) startDiff);
- } else if (startDiff >= Integer.MIN_VALUE && startDiff <= Integer.MAX_VALUE) {
- ++_LL;
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportIntegerByte);
- writeBuffer.writeInt((int) startDiff);
- } else {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportLongByte);
- writeBuffer.writeLong((byte) startDiff);
- }
- writeBuffer.writeByte((byte) commitDiff);
- } else if (commitDiff >= Short.MIN_VALUE && commitDiff <= Short.MAX_VALUE) {
- if (startDiff >= Byte.MIN_VALUE && startDiff <= Byte.MAX_VALUE) {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportByteShort);
- writeBuffer.writeByte((byte) startDiff);
- } else if (startDiff >= Short.MIN_VALUE && startDiff <= Short.MAX_VALUE) {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportShortShort);
- writeBuffer.writeShort((short) startDiff);
- } else if (startDiff >= Integer.MIN_VALUE && startDiff <= Integer.MAX_VALUE) {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportIntegerShort);
- writeBuffer.writeInt((int) startDiff);
- } else {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReportLongShort);
- writeBuffer.writeLong((byte) startDiff);
- }
- writeBuffer.writeShort((short) commitDiff);
- } else {
- writeBuffer.writeByte(TSOMessage.CommittedTransactionReport);
- writeBuffer.writeLong(startTimestamp);
- writeBuffer.writeLong(commitTimestamp);
- }
- int written = writeBuffer.readableBytes() - readBefore;
-
- _Avg2 += (written - _Avg2) / _Writes;
- _Avg += (written - _Avg) / _Coms;
- state.latestStartTimestamp = startTimestamp;
- state.latestCommitTimestamp = commitTimestamp;
- }
-
- public void writeHalfAbort(long startTimestamp) {
- if (writeBuffer.writableBytes() < 30) {
- nextBuffer();
- }
- ++_Writes;
- int readBefore = writeBuffer.readableBytes();
- long diff = startTimestamp - state.latestHalfAbortTimestamp;
- if (diff >= -16 && diff <= 15) {
- writeBuffer.writeByte((byte)((diff & 0x1f) | (0x40)));
- } else if (diff >= Byte.MIN_VALUE && diff <= Byte.MAX_VALUE) {
- writeBuffer.writeByte(TSOMessage.AbortedTransactionReportByte);
- writeBuffer.writeByte((byte)diff);
- } else {
- writeBuffer.writeByte(TSOMessage.AbortedTransactionReport);
- writeBuffer.writeLong(startTimestamp);
- }
- ++_ha;
-
- state.latestHalfAbortTimestamp = startTimestamp;
- int written = writeBuffer.readableBytes() - readBefore;
- _Avg2 += (written - _Avg2) / _Writes;
- }
-
- public void writeFullAbort(long startTimestamp) {
- if (writeBuffer.writableBytes() < 30) {
- nextBuffer();
- }
- ++_Writes;
- int readBefore = writeBuffer.readableBytes();
- long diff = startTimestamp - state.latestFullAbortTimestamp;
- if (diff >= -16 && diff <= 15) {
- writeBuffer.writeByte((byte)((diff & 0x1f) | (0x60)));
- } else if (diff >= Byte.MIN_VALUE && diff <= Byte.MAX_VALUE) {
- writeBuffer.writeByte(TSOMessage.FullAbortReportByte);
- writeBuffer.writeByte((byte)diff);
- } else {
- writeBuffer.writeByte(TSOMessage.FullAbortReport);
- writeBuffer.writeLong(startTimestamp);
- }
- ++_fa;
-
- state.latestFullAbortTimestamp = startTimestamp;
- int written = writeBuffer.readableBytes() - readBefore;
- _Avg2 += (written - _Avg2) / _Writes;
- }
-
- public void writeLargestIncrease(long largestTimestamp) {
- if (writeBuffer.writableBytes() < 30) {
- nextBuffer();
- }
- ++_Writes;
- ++_li;
- int readBefore = writeBuffer.readableBytes();
- writeBuffer.writeByte(TSOMessage.LargestDeletedTimestampReport);
- writeBuffer.writeLong(largestTimestamp);
- int written = writeBuffer.readableBytes() - readBefore;
- _Avg2 += (written - _Avg2) / _Writes;
- }
-
- private void nextBuffer() {
- _overflows++;
- LOG.debug("Switching buffers");
- Iterator<ReadingBuffer> it = pastBuffer.readingBuffers.iterator();
- boolean moreBuffers = it.hasNext();
- while(moreBuffers) {
- ReadingBuffer buf = it.next();
- moreBuffers = it.hasNext();
- buf.flush(false, !moreBuffers);
- }
-
- pastBuffer = currentBuffer;
- currentBuffer = null;
- synchronized (futureBuffer) {
- if (!futureBuffer.isEmpty()) {
- currentBuffer = futureBuffer.removeLast();
- }
- }
- if (currentBuffer == null) {
- currentBuffer = new TSOBuffer();
- }
- writeBuffer = currentBuffer.buffer;
- }
-
- static long _flushes = 0;
- static long _flSize = 0;
-
- public void reset() {
- if (pastBuffer != null) {
- pastBuffer.readingBuffers.clear();
- pastBuffer.buffer.clear();
- }
- if (currentBuffer != null) {
- currentBuffer.readingBuffers.clear();
- currentBuffer.buffer.clear();
- }
- }
-}
-
View
7 src/main/java/com/yahoo/omid/tso/TSOState.java
@@ -25,6 +25,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.yahoo.omid.replication.SharedMessageBuffer;
import com.yahoo.omid.tso.persistence.LoggerAsyncCallback.AddRecordCallback;
import com.yahoo.omid.tso.persistence.LoggerException.Code;
import com.yahoo.omid.tso.persistence.StateLogger;
@@ -105,12 +106,8 @@ protected TimestampOracle getSO(){
*/
public long largestDeletedTimestamp = 0;
public long previousLargestDeletedTimestamp = 0;
- public long latestCommitTimestamp = 0;
- public long latestStartTimestamp = 0;
- public long latestHalfAbortTimestamp = 0;
- public long latestFullAbortTimestamp = 0;
- public TSOSharedMessageBuffer sharedMessageBuffer = new TSOSharedMessageBuffer(this);
+ public SharedMessageBuffer sharedMessageBuffer = new SharedMessageBuffer();
/**
* The hash map to to keep track of recently committed rows
View
157 src/main/java/com/yahoo/omid/tso/ThroughputMonitor.java
@@ -16,17 +16,15 @@
package com.yahoo.omid.tso;
-import java.util.Arrays;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.yahoo.omid.client.TSOClient;
import com.yahoo.omid.client.TransactionalTable;
+import com.yahoo.omid.replication.ReadersAwareBuffer;
/**
* Class for Throughput Monitoring
- * @author fbregier
*
*/
public class ThroughputMonitor extends Thread {
@@ -49,181 +47,40 @@ public void run() {
try {
long oldCounter = TSOHandler.getTransferredBytes();
long oldAbortCount = TSOHandler.abortCount;
- long oldHitCount = TSOHandler.hitCount;
long startTime = System.currentTimeMillis();
- // long oldWaitTime = TSOHandler.waitTime;
- long oldtotalput = CommitHashMap.gettotalput();
- long oldtotalget = CommitHashMap.gettotalget();
- long oldtotalwalkforget = CommitHashMap.gettotalwalkforget();
- long oldtotalwalkforput = CommitHashMap.gettotalwalkforput();
- long oldfull = TSOMessageBuffer.itWasFull;
- long oldflushes = TSOSharedMessageBuffer._flushes;
- long oldflusheSize = TSOSharedMessageBuffer._flSize;
- long oldwaited = TSOMessageBuffer.waited;
- long old1B = TSOSharedMessageBuffer._1B;
- long old2B = TSOSharedMessageBuffer._2B;
- long oldAB = TSOSharedMessageBuffer._AB;
- long oldAS = TSOSharedMessageBuffer._AS;
- long oldLL = TSOSharedMessageBuffer._LL;
- long oldComs = TSOSharedMessageBuffer._Coms;
- long oldHa = TSOSharedMessageBuffer._ha;
- long oldFa = TSOSharedMessageBuffer._fa;
- long oldLi = TSOSharedMessageBuffer._li;
- long oldWrites = TSOSharedMessageBuffer._Writes;
- long oldEmptyFlushes = TSOSharedMessageBuffer._emptyFlushes;
-
- long oldAskedTSO = TSOClient.askedTSO;
+
long oldQueries = TSOHandler.queries;
- long oldElementsRead = TransactionalTable.elementsRead;
- long oldExtraGetsPerformed = TransactionalTable.extraGetsPerformed;
-
- long oldOverflow = TSOSharedMessageBuffer._overflows;
for (;;) {
Thread.sleep(1000);
long endTime = System.currentTimeMillis();
long newCounter = TSOHandler.getTransferredBytes();
long newAbortCount = TSOHandler.abortCount;
- long newHitCount = TSOHandler.hitCount;
- // long newWaitTime = TSOHandler.waitTime;
- long newtotalput = CommitHashMap.gettotalput();
- long newtotalget = CommitHashMap.gettotalget();
- long newtotalwalkforget = CommitHashMap.gettotalwalkforget();
- long newtotalwalkforput = CommitHashMap.gettotalwalkforput();
-
- long newfull = TSOMessageBuffer.itWasFull;
- long newflushes = TSOSharedMessageBuffer._flushes;
- long newflusheSize = TSOSharedMessageBuffer._flSize;
- long newwaited = TSOMessageBuffer.waited;
-
- long new1B = TSOSharedMessageBuffer._1B;
- long new2B = TSOSharedMessageBuffer._2B;
- long newAB = TSOSharedMessageBuffer._AB;
- long newAS = TSOSharedMessageBuffer._AS;
- long newLL = TSOSharedMessageBuffer._LL;
- long newComs = TSOSharedMessageBuffer._Coms;
- long newHa = TSOSharedMessageBuffer._ha;
- long newFa = TSOSharedMessageBuffer._fa;
- long newLi = TSOSharedMessageBuffer._li;
- long newWrites = TSOSharedMessageBuffer._Writes;
- double avg = TSOSharedMessageBuffer._Avg;
- double avg2 = TSOSharedMessageBuffer._Avg2;
-
- long newOverflow = TSOSharedMessageBuffer._overflows;
- long newEmptyFlushes = TSOSharedMessageBuffer._emptyFlushes;
-
long newQueries = TSOHandler.queries;
- long newElementsRead = TransactionalTable.elementsRead;
- long newExtraGetsPerformed = TransactionalTable.extraGetsPerformed;
- long newAskedTSO = TSOClient.askedTSO;
-
- //System.err.format("%4.3f MiB/s%n", (newCounter - oldCounter) *
- //1000 / (endTime - startTime) / 1048576.0);
-// System.err.println( (newCounter - oldCounter) / (float)(endTime - startTime) );
-// LOG.trace(String.format("SERVER: %4.3f TPS, %4.6f Abort/s, %4.6f Hit/s "
-// + "walk/tnx: %2.4f walk/put: %2.4f walk/get: %2.4f puts %d gets %d txns %d ",
-// (newCounter - oldCounter) / (float)(endTime - startTime) * 1000,
-// (newAbortCount - oldAbortCount) / (float)(endTime - startTime) * 1000,
-// (newHitCount - oldHitCount) / (float)(endTime - startTime) * 1000,
-// (newtotalwalkforput + newtotalwalkforget - oldtotalwalkforput - oldtotalwalkforget)/(float)(newCounter - oldCounter),
-// (newtotalwalkforput - oldtotalwalkforput)/(float)(newtotalput - oldtotalput),
-// (newtotalwalkforget - oldtotalwalkforget)/(float)(newtotalget - oldtotalget),
-// (newtotalput - oldtotalput),
-// (newtotalget - oldtotalget),
-// (newCounter - oldCounter))
-// );
if (TSOPipelineFactory.bwhandler != null) {
TSOPipelineFactory.bwhandler.measure();
}
LOG.trace(String.format("SERVER: %4.3f TPS, %4.6f Abort/s "
- + "1B: %2.2f 2B: %2.2f AB: %2.2f AS: %2.2f LL: %2.2f Avg commit: %2.4f Avg flush: %5.2f "
- + "Avg write: %5.2f Tot writes: %d Avg diff flu: %5.2f Rec Bytes/s: %5.2fMBs Sent Bytes/s: %5.2fMBs %d ",
-// + "Abort Freqs: %s",
+ + " Avg diff flu: %5.2f Rec Bytes/s: %5.2fMBs Sent Bytes/s: %5.2fMBs %d "
+ + "Queries: %d CurrentBuffers: %d",
(newCounter - oldCounter) / (float)(endTime - startTime) * 1000,
(newAbortCount - oldAbortCount) / (float)(endTime - startTime) * 1000,
- (new1B - old1B) / (float)(newComs - oldComs) * 100,
- (new2B - old2B) / (float)(newComs - oldComs) * 100,
- (newAB - oldAB) / (float)(newComs - oldComs) * 100,
- (newAS - oldAS) / (float)(newComs - oldComs) * 100,
- (newLL - oldLL) / (float)(newComs - oldComs) * 100,
- avg,
- (newflusheSize - oldflusheSize) / (float)(newflushes - oldflushes),
- avg2,
- (newWrites - oldWrites),
0.0,
-// TSOSharedMessageBuffer._avgLT,
TSOPipelineFactory.bwhandler != null ? TSOPipelineFactory.bwhandler.getBytesReceivedPerSecond() / (double) (1024 * 1024) : 0,
TSOPipelineFactory.bwhandler != null ? TSOPipelineFactory.bwhandler.getBytesSentPerSecond() / (double) (1024 * 1024) : 0,
- state.largestDeletedTimestamp
-// Arrays.toString(TSOSharedMessageBuffer.freq)
- )
- );
- LOG.trace(String.format(" `--1 "
- + "Co: %2.2f Ha: %2.2f Fa: %2.2f Li: %2.2f Avg commit: %2.4f Avg flush: %5.2f "
- + "Avg write: %5.2f Tot overflows: %d Tot flushes: %d Tot empty flu: %d "
- + "Queries: %d CurrentBuffers: %d ExtraGets: %d AskedTSO: %d",
- (newComs - oldComs) / (float)(newWrites - oldWrites) * 100,
- (newHa - oldHa) / (float)(newWrites - oldWrites) * 100,
- (newFa - oldFa) / (float)(newWrites - oldWrites) * 100,
- (newLi - oldLi) / (float)(newWrites - oldWrites) * 100,
- avg,
- (newflusheSize - oldflusheSize) / (float)(newflushes - oldflushes),
- avg2,
- newOverflow - oldOverflow,
- (newflushes - oldflushes),
- newEmptyFlushes - oldEmptyFlushes,
+ state.largestDeletedTimestamp,
newQueries - oldQueries,
- TSOBuffer.nBuffers,
- newExtraGetsPerformed - oldExtraGetsPerformed,
- newAskedTSO - oldAskedTSO)
+ ReadersAwareBuffer.nBuffers
+ )
);
-// if (TSOPipelineFactory.bwhandler != null) {
-// TSOPipelineFactory.bwhandler.reset();
-// }
-// LOG.trace(String.format("SERVER: %4.3f TPS, %4.6f Abort/s, Flushes: %5d "
-// + "FullFlush/Flushes: %2.2f%c Avg flush size: %5.2f Waited: %5d",
-// (newCounter - oldCounter) / (float)(endTime - startTime) * 1000,
-// (newAbortCount - oldAbortCount) / (float)(endTime - startTime) * 1000,
-// (newflushes - oldflushes),
-// ((newfull - oldfull) / (float) (newflushes - oldflushes) * 100), '%',
-// ((newflushed - oldflushed) / (float) (newflushes - oldflushes)),
-// (newwaited - oldwaited)));
oldCounter = newCounter;
oldAbortCount = newAbortCount;
- oldHitCount = newHitCount;
startTime = endTime;
- // oldWaitTime = newWaitTime;
- oldtotalget = newtotalget;
- oldtotalput = newtotalput;
- oldtotalwalkforget = newtotalwalkforget;
- oldtotalwalkforput = newtotalwalkforput;
- oldfull = newfull;
- oldflushes = newflushes;
- oldflusheSize = newflusheSize;
- oldwaited = newwaited;
- oldOverflow = newOverflow;
-
-
- old1B = new1B;
- old2B = new2B;
- oldAB = newAB;
- oldAS = newAS;
- oldLL = newLL;
- oldComs = newComs;
- oldHa = newHa;
- oldFa = newFa;
- oldLi = newLi;
- oldWrites = newWrites;
- oldEmptyFlushes = newEmptyFlushes;
-
- oldAskedTSO = newAskedTSO;
oldQueries = newQueries;
- oldElementsRead = newElementsRead;
- oldExtraGetsPerformed = newExtraGetsPerformed;
}
} catch (InterruptedException e) {
// Stop monitoring asked
View
2  src/main/java/com/yahoo/omid/tso/messages/AbortRequest.java
@@ -26,7 +26,7 @@ public String toString() {
}
@Override
- public void readObject(ChannelBuffer aInputStream) throws IOException {
+ public void readObject(ChannelBuffer aInputStream) {
startTimestamp = aInputStream.readLong();
}
View
88 src/main/java/com/yahoo/omid/tso/messages/AbortedTransactionReport.java
@@ -24,45 +24,63 @@
import com.yahoo.omid.tso.TSOMessage;
/**
- * The message object for sending a commit request to TSO
- * @author maysam
- *
+ * The message object that notifies clients of an aborted transaction
+ *
*/
-public class AbortedTransactionReport implements TSOMessage {
- /**
- * Starting timestamp
- */
- public long startTimestamp;
+public class AbortedTransactionReport implements TSOMessage {
+ /**
+ * Starting timestamp
+ */
+ public long startTimestamp;
- public AbortedTransactionReport() {
- }
-
- public AbortedTransactionReport(long startTimestamp) {
- this.startTimestamp = startTimestamp;
- }
+ public AbortedTransactionReport() {
+ }
- @Override
- public String toString() {
- return "Aborted Transaction Report: T_s:" + startTimestamp;
- }
+ public AbortedTransactionReport(long startTimestamp) {
+ this.startTimestamp = startTimestamp;
+ }
- @Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
-
- startTimestamp = aInputStream.readLong();
- }
+ @Override
+ public String toString() {
+ return "Aborted Transaction Report: T_s:" + startTimestamp;
+ }
- @Override
- public void writeObject(DataOutputStream aOutputStream)
- throws IOException {
- aOutputStream.writeLong(startTimestamp);
- }
+ @Override
+ public void readObject(ChannelBuffer aInputStream) {
- @Override
- public void writeObject(ChannelBuffer buffer)
- {
- buffer.writeLong(startTimestamp);
- }
-}
+ startTimestamp = aInputStream.readLong();
+ }
+
+ @Override
+ public void writeObject(DataOutputStream aOutputStream) throws IOException {
+ aOutputStream.writeLong(startTimestamp);
+ }
+
+ @Override
+ public void writeObject(ChannelBuffer buffer) {
+ buffer.writeLong(startTimestamp);
+ }
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (startTimestamp ^ (startTimestamp >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ AbortedTransactionReport other = (AbortedTransactionReport) obj;
+ if (startTimestamp != other.startTimestamp)
+ return false;
+ return true;
+ }
+
+}
View
86 src/main/java/com/yahoo/omid/tso/messages/CleanedTransactionReport.java
@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) 2011 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package com.yahoo.omid.tso.messages;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import com.yahoo.omid.tso.TSOMessage;
+
+/**
+ * The message object that notifies clients of a cleaned up aborted transaction
+ *
+ */
+public class CleanedTransactionReport implements TSOMessage {
+ /**
+ * Starting timestamp
+ */
+ public long startTimestamp;
+
+ public CleanedTransactionReport() {
+ }
+
+ public CleanedTransactionReport(long startTimestamp) {
+ this.startTimestamp = startTimestamp;
+ }
+
+ @Override
+ public String toString() {
+ return "Cleaned up Transaction Report: T_s:" + startTimestamp;
+ }
+
+ @Override
+ public void readObject(ChannelBuffer aInputStream) {
+
+ startTimestamp = aInputStream.readLong();
+ }
+
+ @Override
+ public void writeObject(DataOutputStream aOutputStream) throws IOException {
+ aOutputStream.writeLong(startTimestamp);
+ }
+
+ @Override
+ public void writeObject(ChannelBuffer buffer) {
+ buffer.writeLong(startTimestamp);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (startTimestamp ^ (startTimestamp >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CleanedTransactionReport other = (CleanedTransactionReport) obj;
+ if (startTimestamp != other.startTimestamp)
+ return false;
+ return true;
+ }
+
+}
View
8 src/main/java/com/yahoo/omid/tso/messages/CommitQueryRequest.java
@@ -27,24 +27,19 @@
public class CommitQueryRequest implements TSOMessage {
public long startTimestamp;
public long queryTimestamp;
-// public RowKey rowkey;
public CommitQueryRequest() {
-// rowkey = new RowKey();;
}
public CommitQueryRequest(long startTimestamp, long queryTimestamp) {
this.startTimestamp = startTimestamp;
this.queryTimestamp = queryTimestamp;
-// this.rowkey = rowkey;
}
@Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public void readObject(ChannelBuffer aInputStream) {
startTimestamp = aInputStream.readLong();
queryTimestamp = aInputStream.readLong();
-// rowkey = RowKey.readObject(aInputStream);
}
@Override
@@ -52,7 +47,6 @@ public void writeObject(DataOutputStream aOutputStream)
throws IOException {
aOutputStream.writeLong(startTimestamp);
aOutputStream.writeLong(queryTimestamp);
-// rowkey.writeObject(aOutputStream);
}
@Override
View
3  src/main/java/com/yahoo/omid/tso/messages/CommitQueryResponse.java
@@ -50,8 +50,7 @@ public CommitQueryResponse(long startTimestamp, long queryTimestamp, boolean com
}
@Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public void readObject(ChannelBuffer aInputStream) {
startTimestamp = aInputStream.readLong();
queryTimestamp = aInputStream.readLong();
commitTimestamp = aInputStream.readLong();
View
17 src/main/java/com/yahoo/omid/tso/messages/CommitRequest.java
@@ -60,19 +60,11 @@ public String toString() {
}
- static private IndexOutOfBoundsException ex = new IndexOutOfBoundsException();
@Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
-// int totalSize = aInputStream.readInt();
-// if (totalSize < aInputStream.readableBytes()) {
-// throw ex;
-// }
+ public void readObject(ChannelBuffer aInputStream) {
long l = aInputStream.readLong();
startTimestamp = l;
- //LOG.error("tid: " + startTimestamp + " capacity: " + aInputStream.capacity());
int size = aInputStream.readInt();
- // LOG.error("size: " + size);
rows = new RowKey[size];
for (int i = 0; i < size; i++) {
rows[i] = RowKey.readObject(aInputStream);
@@ -86,13 +78,6 @@ public void writeObject(ChannelBuffer buffer) {
@Override
public void writeObject(DataOutputStream aOutputStream)
throws IOException {
-// int size = 12;
-// for (RowKey r: rows) {
-// size += 2;
-// size += r.getRow().length;
-// size += r.getTable().length;
-// }
-// aOutputStream.writeInt(size);
aOutputStream.writeLong(startTimestamp);
aOutputStream.writeInt(rows.length);
for (RowKey r: rows) {
View
3  src/main/java/com/yahoo/omid/tso/messages/CommitResponse.java
@@ -72,8 +72,7 @@ public String toString() {
}
@Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public void readObject(ChannelBuffer aInputStream) {
long l = aInputStream.readLong();
startTimestamp = l;
committed = aInputStream.readByte() == 1 ? true : false;
View
158 src/main/java/com/yahoo/omid/tso/messages/CommittedTransactionReport.java
@@ -18,121 +18,71 @@
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.jboss.netty.buffer.ChannelBuffer;
import com.yahoo.omid.tso.TSOMessage;
/**
- * The message object for sending a commit request to TSO
- * @author maysam
- *
+ * The message object that notifies clients of a committed transaction
+ *
*/
-public class CommittedTransactionReport implements TSOMessage {
- /**
- * Starting timestamp
- */
- public long startTimestamp;
- public long commitTimestamp;
- private byte high;
-
- long lastStartTimestamp = 0;
- long lastCommitTimestamp = 0;
-
- public CommittedTransactionReport() {
- }
-
- public CommittedTransactionReport(byte high) {
- this.high = high;
- }
-
- public CommittedTransactionReport(long startTimestamp, long commitTimestamp) {
- this.startTimestamp = startTimestamp;
- this.commitTimestamp = commitTimestamp;
- }
+public class CommittedTransactionReport implements TSOMessage {
+ /**
+ * Starting timestamp
+ */
+ public long startTimestamp;
+ public long commitTimestamp;
- @Override
- public String toString() {
- return "Committed Transaction Report: T_s:" + startTimestamp + " T_c:" + commitTimestamp;
- }
+ public CommittedTransactionReport() {
+ }
- @Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public CommittedTransactionReport(long startTimestamp, long commitTimestamp) {
+ this.startTimestamp = startTimestamp;
+ this.commitTimestamp = commitTimestamp;
+ }
- if (high >= 0) {
-// System.out.println("1 byte " + high);
- startTimestamp = lastStartTimestamp + high;
- commitTimestamp = lastCommitTimestamp + 1;
- } else if ((high & 0x40) == 0) {
- byte low = aInputStream.readByte();
-// System.out.println("2 bytes " + high + " " + low);
- long startDiff = low;
- startDiff |= (high << 4) & 0x3f0;
-// long commitDiff = (low & 0x0f);
+ @Override
+ public String toString() {
+ return "Committed Transaction Report: T_s:" + startTimestamp + " T_c:" + commitTimestamp;
+ }
- startTimestamp = lastStartTimestamp + startDiff;
-// commitTimestamp = lastCommitTimestamp + commitDiff;
- commitTimestamp = lastCommitTimestamp + 1;
- } else {
-// System.out.println("Else " + high);
- switch (high) {
- case TSOMessage.CommittedTransactionReportByteByte:
- startTimestamp = lastStartTimestamp + aInputStream.readByte();
- commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
- break;
- case TSOMessage.CommittedTransactionReportShortByte:
- startTimestamp = lastStartTimestamp + aInputStream.readShort();
- commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
- break;
- case TSOMessage.CommittedTransactionReportIntegerByte:
- startTimestamp = lastStartTimestamp + aInputStream.readInt();
- commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
- break;
- case TSOMessage.CommittedTransactionReportLongByte:
- startTimestamp = lastStartTimestamp + aInputStream.readLong();
- commitTimestamp = lastCommitTimestamp + aInputStream.readByte();
- break;
+ // (De)serialization handled on Zipper
+ @Override
+ public void readObject(ChannelBuffer aInputStream) {
+ }
- case TSOMessage.CommittedTransactionReportByteShort:
- startTimestamp = lastStartTimestamp + aInputStream.readByte();
- commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
- break;
- case TSOMessage.CommittedTransactionReportShortShort:
- startTimestamp = lastStartTimestamp + aInputStream.readShort();
- commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
- break;
- case TSOMessage.CommittedTransactionReportIntegerShort:
- startTimestamp = lastStartTimestamp + aInputStream.readInt();
- commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
- break;
- case TSOMessage.CommittedTransactionReportLongShort:
- startTimestamp = lastStartTimestamp + aInputStream.readLong();
- commitTimestamp = lastCommitTimestamp + aInputStream.readShort();
- break;
- case TSOMessage.CommittedTransactionReport:
- startTimestamp = aInputStream.readLong();
- commitTimestamp = aInputStream.readLong();
- }
- }
-
- lastStartTimestamp = startTimestamp;
- lastCommitTimestamp = commitTimestamp;
- }
+ @Override
+ public void writeObject(DataOutputStream aOutputStream) throws IOException {
+ }
- @Override
- public void writeObject(DataOutputStream aOutputStream)
- throws IOException {
- aOutputStream.writeLong(startTimestamp);
- aOutputStream.writeLong(commitTimestamp);
- }
-
- @Override
- public void writeObject(ChannelBuffer buffer)
- {
- buffer.writeLong(startTimestamp);
- buffer.writeLong(commitTimestamp);
- }
-}
+ @Override
+ public void writeObject(ChannelBuffer buffer) {
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (commitTimestamp ^ (commitTimestamp >>> 32));
+ result = prime * result + (int) (startTimestamp ^ (startTimestamp >>> 32));
+ return result;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CommittedTransactionReport other = (CommittedTransactionReport) obj;
+ if (commitTimestamp != other.commitTimestamp)
+ return false;
+ if (startTimestamp != other.startTimestamp)
+ return false;
+ return true;
+ }
+
+}
View
36 .../com/yahoo/omid/tso/messages/FullAbortReport.java → ...com/yahoo/omid/tso/messages/FullAbortRequest.java
@@ -16,8 +16,8 @@
package com.yahoo.omid.tso.messages;
-import java.io.*;
-import java.nio.ByteBuffer;
+import java.io.DataOutputStream;
+import java.io.IOException;
import org.jboss.netty.buffer.ChannelBuffer;
@@ -25,10 +25,11 @@
/**
* The message object for reporting the finish of cleanup after an abort
+ *
* @author maysam
- *
+ *
*/
-public class FullAbortReport implements TSOMessage {
+public class FullAbortRequest implements TSOMessage {
/**
* the start timestamp of the fully aborted transaction
@@ -38,40 +39,35 @@
/**
* Constructor from timestamp
*/
- public FullAbortReport() {
+ public FullAbortRequest() {
}
/**
* Constructor from timestamp
+ *
* @param t
*/
- public FullAbortReport(long t) {
+ public FullAbortRequest(long t) {
startTimestamp = t;
}
@Override
- public String toString() {
- return "FullAbortReport: T_s:" + startTimestamp;
- }
+ public String toString() {
+ return "FullAbortRequest: T_s:" + startTimestamp;
+ }
@Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
+ public void readObject(ChannelBuffer aInputStream) {
startTimestamp = aInputStream.readLong();
- }
+ }
@Override
- public void writeObject(DataOutputStream aOutputStream)
- throws IOException {
+ public void writeObject(DataOutputStream aOutputStream) throws IOException {
aOutputStream.writeLong(startTimestamp);
- }
+ }
-
@Override
- public void writeObject(ChannelBuffer buffer)
- {
+ public void writeObject(ChannelBuffer buffer) {
buffer.writeLong(startTimestamp);
}
}
-
-
View
91 src/main/java/com/yahoo/omid/tso/messages/LargestDeletedTimestampReport.java
@@ -18,52 +18,69 @@
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.jboss.netty.buffer.ChannelBuffer;
import com.yahoo.omid.tso.TSOMessage;
/**
- * The message object for sending a commit request to TSO
- * @author maysam
- *
+ * The message object that notifies clients about and increase of the largest deleted timestamp
+ *
*/
-public class LargestDeletedTimestampReport implements TSOMessage {
- /**
- * Starting timestamp
- */
- public long largestDeletedTimestamp;
-
- public LargestDeletedTimestampReport() {
- }
-
- public LargestDeletedTimestampReport(long largestDeletedTimestamp) {
- this.largestDeletedTimestamp = largestDeletedTimestamp;
- }
+public class LargestDeletedTimestampReport implements TSOMessage {
+ /**
+ * Starting timestamp
+ */
+ public long largestDeletedTimestamp;
- @Override
- public String toString() {
- return "Largest Deleted Timestamp Report: T_s:" + largestDeletedTimestamp;
- }
+ public LargestDeletedTimestampReport() {
+ }
- @Override
- public void readObject(ChannelBuffer aInputStream)
- throws IOException {
-
- largestDeletedTimestamp = aInputStream.readLong();
- }
+ public LargestDeletedTimestampReport(long largestDeletedTimestamp) {
+ this.largestDeletedTimestamp = largestDeletedTimestamp;
+ }
- @Override
- public void writeObject(DataOutputStream aOutputStream)
- throws IOException {
- aOutputStream.writeLong(largestDeletedTimestamp);
- }
+ @Override
+ public String toString() {
+ return "Largest Deleted Timestamp Report: T_s:" + largestDeletedTimestamp;
+ }
- @Override
- public void writeObject(ChannelBuffer buffer)
- {
- buffer.writeLong(largestDeletedTimestamp);
- }
-}
+ @Override
+ public void readObject(ChannelBuffer aInputStream) {
+
+ largestDeletedTimestamp = aInputStream.readLong();
+ }
+
+ @Override
+ public void writeObject(DataOutputStream aOutputStream) throws IOException {
+ aOutputStream.writeLong(largestDeletedTimestamp);
+ }
+ @Override
+ public void writeObject(ChannelBuffer buffer) {
+ buffer.writeLong(largestDeletedTimestamp);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;