Permalink
Browse files

- Added ConcurrentLinkedBlockingQueue2

- Added TP.SenderSendsBundler
- Added dont_bundle option

- Added UnicastTestTcp (TCP based equivalent of UnicastTest)

- Got rid of inefficient Data classes i UnicastTest*

- Added UUID.getByName()

- Set TCP.tcp_nodelay=false by default

- Added PERF protocol
- Changed Average to do round-robin storing of samples rather than random

- Added UnicastTestSharedLoopback to test unicast msg perf between 2 members within the same process (no network comm)

- Added Pool and PoolTest

- Creating an output stream per bundler

- Setting use_send_queue to false by default

- Added ByteBuffer{Input,Output}Stream and test

- Added TCPConnectionMap.flush(dest)

- Reverted used_flush_queues to true (default)
  • Loading branch information...
Bela Ban
Bela Ban committed Dec 16, 2013
1 parent b58fd4b commit 7747153d3f55f3f2722a186fdb08ffd92936f57c
View
@@ -68,4 +68,5 @@
<class id="106" name="org.jgroups.auth.X509Token"/>
<class id="107" name="org.jgroups.protocols.UNICAST3$Header"/>
<class id="108" name="org.jgroups.protocols.FORK$ForkHeader"/>
+ <class id="109" name="org.jgroups.protocols.PERF$PerfHeader"/>
</magic-number-class-mapping>
@@ -27,7 +27,7 @@ ParseFailure = JGRP000024: warning during parse
SyspropFailure = JGRP000025: failed getting system property for %s: %s
OnlyLoopbackFound = JGRP000026: unable to find an address other than loopback for IP version %s
PassUpFailure = JGRP000027: failed passing message up
-UnknownBundler = JGRP000028: bundler_type %s not known; using default bundler (new3)
+UnknownBundler = JGRP000028: bundler_type %s not known; using default bundler
SendFailure = JGRP000029: %s: failed sending message to %s (%d bytes): %s, headers: %s
IncomingMsgFailure = JGRP000030: %s: failed handling incoming message: %s
IncorrectDest = JGRP000031: %s: dropping unicast %s from %s to wrong destination %s, headers are: %s
View
@@ -59,6 +59,7 @@
<class id="63" name="org.jgroups.protocols.rules.SUPERVISOR"/>
<class id="64" name="org.jgroups.protocols.UNICAST3"/>
<class id="65" name="org.jgroups.protocols.FORK"/>
+ <class id="66" name="org.jgroups.protocols.PERF"/>
<!-- IDs reserved for building blocks -->
<class id="200" name="org.jgroups.blocks.RequestCorrelator"/> <!-- ID should be the same as Global.BLOCKS_START_ID -->
View
@@ -27,8 +27,8 @@
thread_pool.min_threads="1"
thread_pool.max_threads="10"
thread_pool.keep_alive_time="5000"
- thread_pool.queue_enabled="false"
- thread_pool.queue_max_size="100"
+ thread_pool.queue_enabled="true"
+ thread_pool.queue_max_size="10000"
thread_pool.rejection_policy="discard"
oob_thread_pool.enabled="true"
@@ -18,7 +18,6 @@
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
@@ -39,14 +38,14 @@
protected Log log=LogFactory.getLog(getClass());
protected int recv_buf_size=120000;
protected int send_buf_size=60000;
- protected int send_queue_size=0;
+ protected int send_queue_size=2000;
protected int sock_conn_timeout=1000; // max time in millis to wait for Socket.connect() to return
protected int peer_addr_read_timeout=2000; // max time in milliseconds to block on reading peer address
protected boolean tcp_nodelay=false;
protected int linger=-1;
protected final Thread acceptor;
protected final AtomicBoolean running=new AtomicBoolean(false);
- protected volatile boolean use_send_queues=false;
+ protected volatile boolean use_send_queues=true;
protected SocketFactory socket_factory=new DefaultSocketFactory();
@@ -110,7 +109,7 @@ else if(bind_addr != null)
acceptor=f.newThread(new Acceptor(),"ConnectionMap.Acceptor [" + local_addr + "]");
}
-
+
public Address getLocalAddress() {return local_addr;}
public Receiver getReceiver() {return recvr;}
public void setReceiver(Receiver receiver) {this.recvr=receiver;}
@@ -208,6 +207,13 @@ public void send(Address dest, byte[] data, int offset, int length) throws Excep
}
}
+ /** Flushes the TCPConnection associated with destination */
+ public void flush(Address destination) throws Exception {
+ TCPConnection conn=mapper.getConnection(destination);
+ if(conn != null)
+ conn.flush();
+ }
+
public void start() throws Exception {
if(running.compareAndSet(false, true)) {
acceptor.start();
@@ -349,7 +355,7 @@ protected static String explanation(boolean connection_existed, boolean replace)
public class TCPConnection implements Connection {
protected final Socket sock; // socket to/from peer (result of srv_sock.accept() or new Socket())
- protected final Lock send_lock=new ReentrantLock(); // serialize send()
+ protected final ReentrantLock send_lock=new ReentrantLock(); // serialize send()
protected final byte[] cookie= { 'b', 'e', 'l', 'a' };
protected DataOutputStream out;
protected DataInputStream in;
@@ -373,10 +379,10 @@ public TCPConnection(Socket s) throws Exception {
setSocketParameters(s);
this.out=new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));
this.in=new DataInputStream(new BufferedInputStream(s.getInputStream()));
- this.peer_addr=readPeerAddress(s);
+ this.peer_addr=readPeerAddress(s);
this.sock=s;
- }
-
+ }
+
protected Address getPeerAddress() {
return peer_addr;
}
@@ -446,7 +452,7 @@ protected void send(byte[] data, int offset, int length) throws Exception {
sender.addToQueue(tmp);
}
else
- _send(data, offset, length, true);
+ _send(data, offset, length, true, true);
}
/**
@@ -458,11 +464,11 @@ protected void send(byte[] data, int offset, int length) throws Exception {
* @param acquire_lock
* @throws Exception
*/
- protected void _send(byte[] data, int offset, int length, boolean acquire_lock) throws Exception {
+ protected void _send(byte[] data, int offset, int length, boolean acquire_lock, boolean flush) throws Exception {
if(acquire_lock)
send_lock.lock();
try {
- doSend(data, offset, length);
+ doSend(data, offset, length, acquire_lock, flush);
updateLastAccessed();
}
catch(InterruptedException iex) {
@@ -474,10 +480,17 @@ protected void _send(byte[] data, int offset, int length, boolean acquire_lock)
}
}
- protected void doSend(byte[] data, int offset, int length) throws Exception {
+ protected void doSend(byte[] data, int offset, int length, boolean acquire_lock, boolean flush) throws Exception {
out.writeInt(length); // write the length of the data buffer first
out.write(data,offset,length);
- out.flush(); // may not be very efficient (but safe)
+ if(!flush || (acquire_lock && send_lock.hasQueuedThreads()))
+ return; // don't flush as some of the waiting threads will do the flush, or flush is false
+ out.flush(); // may not be very efficient (but safe)
+ }
+
+ protected void flush() throws Exception {
+ if(out != null)
+ out.flush();
}
/**
@@ -643,7 +656,7 @@ public void run() {
if(data != null) {
try {
- _send(data, 0, data.length, false);
+ _send(data, 0, data.length, false, send_queue.isEmpty());
}
catch(Throwable ignored) {
}
@@ -31,7 +31,7 @@
protected boolean use_send_queues=true;
@Property(description="Max number of messages in a send queue")
- protected int send_queue_size=10000;
+ protected int send_queue_size=2000;
@Property(description="Receiver buffer size in bytes")
protected int recv_buf_size=150000;
@@ -46,7 +46,7 @@
protected int peer_addr_read_timeout=1000; // max time to block on reading of peer address
@Property(description="Should TCP no delay flag be turned on")
- protected boolean tcp_nodelay=true;
+ protected boolean tcp_nodelay=false;
@Property(description="SO_LINGER in msec. Default of -1 disables it")
protected int linger=-1; // SO_LINGER (number of ms, -1 disables it)
@@ -0,0 +1,117 @@
+package org.jgroups.protocols;
+
+import org.jgroups.*;
+import org.jgroups.annotations.MBean;
+import org.jgroups.annotations.ManagedAttribute;
+import org.jgroups.annotations.Property;
+import org.jgroups.stack.Protocol;
+import org.jgroups.util.Average;
+import org.jgroups.util.MessageBatch;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+
+/**
+ * Protocol measuring latency between stacks. On {@link Protocol#down(org.jgroups.Event)}, a header is added to the
+ * message with the ID of the PERF protocol and the start time is set in the header.
+ * On {@link Protocol#up(org.jgroups.Event)}, the time different is computed and a rolling average is updated in PERF.<p/>
+ * Note that we can have several measurements by inserting PERF protocols with different IDs (Protocol.id) into the stack.</p>
+ * If PERF is used to measure latency between nodes running on different physical boxes, it is important that the clocks
+ * are synchronized, or else latency cannot be computed correctly (may even be negative).
+ * @author Bela Ban
+ * @since 3.5
+ */
+@MBean(description="Measures latency between PERF instances")
+public class PERF extends Protocol {
+ protected Average avg;
+ protected Address local_addr;
+
+ @Property(description="Number of samples to maintain for rolling average")
+ protected int avg_size=20;
+
+ @ManagedAttribute(description="Average latency in ns")
+ public double latencyInNs() {return avg.getAverage();}
+
+ @ManagedAttribute(description="Average latency in ms")
+ public double latencyInMs() {return avg.getAverage() / 1000000.0;}
+
+ public void init() throws Exception {
+ super.init();
+ avg=new Average(avg_size);
+ }
+
+ public void resetStats() {
+ super.resetStats();
+ avg.clear();
+ }
+
+ public Object down(Event evt) {
+ switch(evt.getType()) {
+ case Event.MSG:
+ Message msg=(Message)evt.getArg();
+ msg.putHeader(id, new PerfHeader(System.nanoTime()));
+ break;
+ case Event.SET_LOCAL_ADDRESS:
+ local_addr=(Address)evt.getArg();
+ break;
+ }
+ return down_prot.down(evt);
+ }
+
+ public Object up(Event evt) {
+ if(evt.getType() == Event.MSG) {
+ Message msg=(Message)evt.getArg();
+ PerfHeader hdr=(PerfHeader)msg.getHeader(id);
+ if(hdr == null)
+ log.error("%s: no perf header found", local_addr);
+ else {
+ long time=System.nanoTime() - hdr.start_time;
+ if(time <= 0)
+ log.error("%s: time is <= 0");
+ else
+ avg.add(time);
+ }
+ }
+ return up_prot.up(evt);
+ }
+
+ public void up(MessageBatch batch) {
+ for(Message msg: batch) {
+ PerfHeader hdr=(PerfHeader)msg.getHeader(id);
+ if(hdr == null)
+ log.error("%s: no perf header found", local_addr);
+ else {
+ long time=System.nanoTime() - hdr.start_time;
+ if(time <= 0)
+ log.error("%s: time is <= 0");
+ else
+ avg.add(time);
+ }
+ }
+
+ super.up(batch);
+ }
+
+ protected static class PerfHeader extends Header {
+ protected long start_time; // in ns
+
+ public PerfHeader() {
+ }
+
+ public PerfHeader(long start_time) {
+ this.start_time=start_time;
+ }
+
+ public int size() {
+ return Global.LONG_SIZE;
+ }
+
+ public void writeTo(DataOutput out) throws Exception {
+ out.writeLong(start_time);
+ }
+
+ public void readFrom(DataInput in) throws Exception {
+ start_time=in.readLong();
+ }
+ }
+}
Oops, something went wrong.

0 comments on commit 7747153

Please sign in to comment.