Reduce write system calls, thus fewer TCP packets

Do a bit more copying around in user space before handing off
the data to the kernel, so that

Before this change, one connect, publish and disconnect
was 19 packets. After this change, it is 14 packets.
Ethernet/IP/TCP overhead is about 66 bytes. So with this
change, we use 73% of the bandwidth as before
(14*66/19*66 = 0.73).

Author: jeffall

messages.go

@@ -18,22 +18,30 @@ type Header struct {
 }
 
 func (hdr *Header) Encode(w io.Writer, msgType MessageType, remainingLength int32) error {
+	buf := new(bytes.Buffer)
+	err := hdr.encodeInto(buf, msgType, remainingLength)
+	if err != nil {
+					return err
+	}
+	_, err = w.Write(buf.Bytes())
+	return err
+}
+
+func (hdr *Header) encodeInto(buf *bytes.Buffer, msgType MessageType, remainingLength int32) error {
 	if !hdr.QosLevel.IsValid() {
 		return badQosError
 	}
 	if !msgType.IsValid() {
 		return badMsgTypeError
 	}
 
-	buf := new(bytes.Buffer)
 	val := byte(msgType) << 4
 	val |= (boolToByte(hdr.DupFlag) << 3)
 	val |= byte(hdr.QosLevel) << 1
 	val |= boolToByte(hdr.Retain)
 	buf.WriteByte(val)
 	encodeLength(remainingLength, buf)
-	_, err := w.Write(buf.Bytes())
-	return err
+	return nil
 }
 
 func (hdr *Header) Decode(r io.Reader) (msgType MessageType, remainingLength int32, err error) {
@@ -105,12 +113,14 @@ func writeMessage(w io.Writer, msgType MessageType, hdr *Header, payloadBuf *byt
 		return msgTooLongError
 	}
 
-	err := hdr.Encode(w, msgType, int32(totalPayloadLength))
+	buf := new(bytes.Buffer)
+	err := hdr.encodeInto(buf, msgType, int32(totalPayloadLength))
 	if err != nil {
 		return err
 	}
 
-	_, err = w.Write(payloadBuf.Bytes())
+	buf.Write(payloadBuf.Bytes())
+	_, err = w.Write(buf.Bytes())
 
 	return err
 }