Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

propagate trace transcripts, test transcripts, etc.

  • Loading branch information...
commit 97c204ec43ae634818cc4d7ee505d522faadd779 1 parent d095dec
@mariusae mariusae authored
View
5 finagle-core/src/main/scala/com/twitter/finagle/tracing/Host.scala
@@ -13,9 +13,8 @@ object Host {
dis.readInt
} catch {
case e =>
- log.warning(
- "Failed to retrieve local host address: %s".format(e))
- 0
+ log.warning("Failed to retrieve local host address: %s".format(e))
+ 0
}
def apply(): Int = localHost
View
39 finagle-core/src/main/scala/com/twitter/finagle/tracing/TraceContext.scala
@@ -8,18 +8,32 @@ package com.twitter.finagle.tracing
import scala.util.Random
-import com.twitter.util.Local
+import com.twitter.util.{Local, RichU64Long}
+
+case class TraceID(
+ var span: Long,
+ var parentSpan: Option[Long],
+ val host: Int,
+ val vm: String)
+{
+ override def toString = {
+ val spanHex = new RichU64Long(span).toU64HexString
+ val parentSpanHex = parentSpan map (new RichU64Long(_).toU64HexString)
+
+ val spanString = parentSpanHex match {
+ case Some(parentSpanHex) => "%s<:%s".format(spanHex, parentSpanHex)
+ case None => spanHex
+ }
+
+ "%s,%s".format(spanString, vm)
+ }
+}
case class TraceContext(
- var spanID: Long,
- var parentSpanID: Option[Long],
- var transcript: Transcript // an associated transcript
+ var traceID: TraceID,
+ var transcript: Transcript
)
-// Span stuff. Define a thrift struct for carrying this. Blah.
-// transcript: machine identifier (IP address?) subspans, etc? these
-// are usually reconstructed later... it's what we get for logging.
-
object TraceContext {
private[this] val rng = new Random
private[this] val current = new Local[TraceContext]
@@ -35,5 +49,12 @@ object TraceContext {
current().get
}
- def newContext() = TraceContext(rng.nextLong(), None, NullTranscript)
+ def newContext() = {
+ val traceID = TraceID(rng.nextLong(), None, Host(), VMID())
+ TraceContext(traceID, NullTranscript)
+ }
+
+ def reset() {
+ this() = newContext()
+ }
}
View
36 finagle-core/src/main/scala/com/twitter/finagle/tracing/Transcript.scala
@@ -9,19 +9,21 @@ import collection.mutable.ArrayBuffer
import com.twitter.util.Time
case class Record(
- host: Int, // 32-bit IP address
- vmID: String, // virtual machine identifier
- spanID: Long,
- parentSpanID: Option[Long],
+ traceID: TraceID,
timestamp: Time, // (nanosecond granularity)
- message: String // an arbitrary string message
-)
+ message: String) // an arbitrary string message
+{
+ override def toString = "[%s] @ %s: %s".format(traceID, timestamp, message)
+}
trait Transcript extends Iterable[Record] {
- // Log levels?
+ // TODO: support log levels?
def record(message: => String)
def isRecording = true
+ def merge(other: Iterator[Record])
+
+ def print() { foreach { println(_) } }
}
/**
@@ -32,6 +34,7 @@ object Transcript extends Transcript {
def record(message: => String) { TraceContext().transcript.record(message) }
def iterator = TraceContext().transcript.iterator
override def isRecording = TraceContext().transcript.isRecording
+ def merge(other: Iterator[Record]) = TraceContext().transcript.merge(other)
}
/**
@@ -41,20 +44,17 @@ object NullTranscript extends Transcript {
def record(message: => String) {}
def iterator = Iterator.empty
override def isRecording = false
+ def merge(other: Iterator[Record]) {}
}
/**
* Buffers messages to an ArrayBuffer.
*/
-class BufferingTranscript extends Transcript {
+class BufferingTranscript(traceID: TraceID) extends Transcript {
private[this] val buffer = new ArrayBuffer[Record]
def record(message: => String) = synchronized {
- buffer += Record(
- Host(), VMID(),
- TraceContext().spanID, TraceContext().parentSpanID,
- Time.now,
- message)
+ buffer += Record(traceID, Time.now, message)
}
def iterator = buffer.iterator
@@ -62,4 +62,14 @@ class BufferingTranscript extends Transcript {
def clear() = synchronized {
buffer.clear()
}
+
+ def merge(other: Iterator[Record]) = synchronized {
+ // TODO: resolve time drift by causality
+ var combined = buffer ++ other
+ combined = combined sortWith { (a, b) => a.timestamp < b.timestamp }
+ combined = combined distinct
+
+ buffer.clear()
+ buffer.appendAll(combined)
+ }
}
View
49 finagle-core/src/test/scala/com/twitter/finagle/tracing/TranscriptSpec.scala
@@ -0,0 +1,49 @@
+package com.twitter.finagle.tracing
+
+import org.specs.Specification
+
+import com.twitter.conversions.time._
+
+import com.twitter.util.Time
+
+object TrascriptSpec extends Specification {
+ "BufferingTranscript" should {
+ val traceID = TraceID(1L, Some(2L), 0, "myVM")
+
+ "record traceID, current time, and message" in {
+ Time.withCurrentTimeFrozen { timeControl =>
+ val t = new BufferingTranscript(traceID)
+ t.record("hey there")
+
+ val expectedRecord = Record(traceID, Time.now, "hey there")
+
+ t.size must be_==(1)
+ t.head must be_==(expectedRecord)
+ }
+ }
+
+ "merge" in {
+ Time.withCurrentTimeFrozen { timeControl =>
+ val t0 = new BufferingTranscript(traceID)
+ val t1 = new BufferingTranscript(traceID)
+
+ t0.record("1")
+ timeControl.advance(1.second)
+ t1.record("2")
+ timeControl.advance(1.second)
+ t0.record("3")
+
+ t0.merge(t1.iterator)
+ t0 must haveSize(3)
+ val records = t0.toArray
+ records(0).message must be_==("1")
+ records(1).message must be_==("2")
+ records(2).message must be_==("3")
+
+ // Merging again should kill dups:
+ t0.merge(t1.iterator)
+ t0 must haveSize(3)
+ }
+ }
+ }
+}
View
112 finagle-stress/src/main/scala/com/twitter/finagle/demo/Tracing.scala
@@ -0,0 +1,112 @@
+package com.twitter.finagle.demo
+
+import java.util.concurrent.atomic.AtomicInteger
+import java.net.InetSocketAddress
+
+import com.twitter.util.Future
+
+import org.apache.thrift.protocol.TBinaryProtocol
+
+import com.twitter.finagle.builder.{ClientBuilder, ServerBuilder}
+import com.twitter.finagle.thrift.{ThriftServerFramedCodec, ThriftClientFramedCodec}
+import com.twitter.finagle.tracing.{TraceContext, BufferingTranscript}
+
+object Tracing1Service extends Tracing1.ServiceIface {
+ private[this] val transport = ClientBuilder()
+ .hosts("localhost:6002")
+ .codec(ThriftClientFramedCodec())
+ .build()
+
+ private[this] val t2Client =
+ new Tracing2.ServiceToClient(transport, new TBinaryProtocol.Factory())
+
+ def main(args: Array[String]) {
+ ServerBuilder()
+ .codec(ThriftServerFramedCodec())
+ .bindTo(new InetSocketAddress(6001))
+ .build(new Tracing1.Service(this, new TBinaryProtocol.Factory()))
+ }
+
+ def computeSomething(): Future[String] = {
+ println("T1 with trace ID", TraceContext().traceID)
+ TraceContext().transcript.record("hey i'm issuing a call")
+
+ t2Client.computeSomethingElse() map { somethingElse =>
+ "t1: " + somethingElse
+ }
+ }
+}
+
+object Tracing2Service extends Tracing2.ServiceIface {
+ private[this] val transport = ClientBuilder()
+ .hosts("localhost:6003")
+ .codec(ThriftClientFramedCodec())
+ .build()
+
+ private[this] val t3Client =
+ new Tracing3.ServiceToClient(transport, new TBinaryProtocol.Factory())
+
+ def main(args: Array[String]) {
+ ServerBuilder()
+ .codec(ThriftServerFramedCodec())
+ .bindTo(new InetSocketAddress(6002))
+ .build(new Tracing2.Service(this, new TBinaryProtocol.Factory()))
+ }
+
+ def computeSomethingElse(): Future[String] = {
+ println("T2 with trace ID", TraceContext().traceID)
+ TraceContext().transcript.record("hey i'm issuing a call")
+
+
+ for {
+ x <- t3Client.oneMoreThingToCompute()
+ y <- t3Client.oneMoreThingToCompute()
+ } yield {
+ TraceContext().transcript.record(
+ "got my results! (%s and %s), returning".format(x, y))
+ "t2: " + x + y
+ }
+ }
+}
+
+object Tracing3Service extends Tracing3.ServiceIface {
+ private[this] val count = new AtomicInteger(0)
+
+ def main(args: Array[String]) {
+ ServerBuilder()
+ .codec(ThriftServerFramedCodec())
+ .bindTo(new InetSocketAddress(6003))
+ .build(new Tracing3.Service(this, new TBinaryProtocol.Factory()))
+ }
+
+ def oneMoreThingToCompute(): Future[String] = {
+ println("T3 with trace ID", TraceContext().traceID)
+
+ val number = count.incrementAndGet()
+ TraceContext().transcript.record(
+ "(t3) hey i'm issuing a call %s".format(number))
+ Future("t3: %d".format(number))
+ }
+}
+
+object Client {
+ def main(args: Array[String]) {
+ val transport = ClientBuilder()
+ .hosts("localhost:6001")
+ .codec(ThriftClientFramedCodec())
+ .build()
+
+ val client = new Tracing1.ServiceToClient(
+ transport, new TBinaryProtocol.Factory())
+
+ // Turn (debug) tracing on.
+ TraceContext().transcript = new BufferingTranscript(TraceContext().traceID)
+
+ TraceContext().transcript.record("about to start issuing the root request..")
+ val result = client.computeSomething()()
+ println("result", result)
+
+ println("Trace:")
+ TraceContext().transcript.print()
+ }
+}
View
14 finagle-stress/src/main/thrift/tracing.thrift
@@ -0,0 +1,14 @@
+
+namespace java com.twitter.finagle.demo
+
+service Tracing1 {
+ string computeSomething();
+}
+
+service Tracing2 {
+ string computeSomethingElse();
+}
+
+service Tracing3 {
+ string oneMoreThingToCompute();
+}
View
34 finagle-thrift/src/main/scala/com/twitter/finagle/thrift/ThriftClientFramedCodec.scala
@@ -1,5 +1,7 @@
package com.twitter.finagle.thrift
+import collection.JavaConversions._
+
import org.jboss.netty.channel.{
SimpleChannelHandler, Channel, ChannelEvent, ChannelHandlerContext,
SimpleChannelDownstreamHandler, MessageEvent, Channels,
@@ -10,13 +12,14 @@ import org.jboss.netty.handler.codec.oneone.OneToOneEncoder
import org.apache.thrift.protocol.{TBinaryProtocol, TMessage, TMessageType}
import org.apache.thrift.transport.{TMemoryBuffer, TMemoryInputTransport}
-import com.twitter.util.Future
+import com.twitter.conversions.time._
+import com.twitter.util.{Time, Future}
import com.twitter.finagle._
import com.twitter.finagle.util.{Ok, Error, Cancelled, TracingHeader}
import com.twitter.finagle.util.Conversions._
import com.twitter.finagle.channel.ChannelService
-import com.twitter.finagle.tracing.TraceContext
+import com.twitter.finagle.tracing.{TraceID, TraceContext, Record}
/**
* ThriftClientFramedCodec implements a framed thrift transport that
@@ -117,7 +120,8 @@ class ThriftClientTracingFilter extends SimpleFilter[ThriftClientRequest, Array[
service: Service[ThriftClientRequest, Array[Byte]]) =
{
val header = new TracedRequest
- header.setParent_span_id(TraceContext().spanID)
+ header.setParent_span_id(TraceContext().traceID.span)
+ header.setDebug(TraceContext().transcript.isRecording)
val tracedRequest = request.copy(
message = OutputBuffer.messageToArray(header) ++ request.message)
@@ -129,10 +133,28 @@ class ThriftClientTracingFilter extends SimpleFilter[ThriftClientRequest, Array[
reply
} else {
reply map { response =>
- val header = new TracedResponse
- val rest = InputBuffer.peelMessage(response, header)
+ val responseHeader = new TracedResponse
+ val rest = InputBuffer.peelMessage(response, responseHeader)
+
+ if (header.debug && responseHeader.isSetTranscript) {
+ val records = responseHeader.transcript map { thriftRecord =>
+ val traceID = TraceID(
+ thriftRecord.getSpan_id(),
+ {
+ val spanID = thriftRecord.getParent_span_id()
+ if (spanID == 0) None else Some(spanID)
+ },
+ thriftRecord.getHost(),
+ thriftRecord.getVm_id())
+
+ Record(
+ traceID,
+ Time.fromMilliseconds(thriftRecord.getTimestamp_ms()),
+ thriftRecord.getMessage())
+ }
- // TODO: merge.
+ TraceContext().transcript.merge(records.iterator)
+ }
rest
}
View
37 finagle-thrift/src/main/scala/com/twitter/finagle/thrift/ThriftServerFramedCodec.scala
@@ -14,7 +14,7 @@ import org.jboss.netty.channel.SimpleChannelUpstreamHandler
import com.twitter.util.Future
import com.twitter.finagle._
import com.twitter.finagle.util.TracingHeader
-import com.twitter.finagle.tracing.TraceContext
+import com.twitter.finagle.tracing.{BufferingTranscript, TraceContext}
class ThriftServerChannelBufferEncoder extends SimpleChannelDownstreamHandler {
override def writeRequested(ctx: ChannelHandlerContext, e: MessageEvent) = {
@@ -33,8 +33,6 @@ object ThriftServerFramedCodec {
def apply() = new ThriftServerFramedCodec
}
-// TODO: use LeftFoldChannelHandler
-
class ThriftServerTracingFilter
extends SimpleFilter[Array[Byte], Array[Byte]]
{
@@ -51,16 +49,32 @@ class ThriftServerTracingFilter
val header = new TracedRequest
val request_ = InputBuffer.peelMessage(request, header)
- // TODO: Check isset?
- TraceContext().parentSpanID = Some(header.getParent_span_id)
+ TraceContext.reset()
+ TraceContext().traceID.parentSpan = Some(header.getParent_span_id)
+
+ if (header.debug && !TraceContext().transcript.isRecording)
+ TraceContext().transcript = new BufferingTranscript(TraceContext().traceID)
- // has a problem with dups-- but we filter them out.
-
service(request_) map { response =>
// Wrap some trace data.
- val header = new TracedResponse
- val headerBytes = OutputBuffer.messageToArray(header)
- headerBytes ++ response
+ val responseHeader = new TracedResponse
+
+ if (header.debug) {
+ TraceContext().transcript foreach { record =>
+ val thriftRecord = new TranscriptRecord(
+ record.traceID.host,
+ record.traceID.vm,
+ record.traceID.span,
+ record.traceID.parentSpan getOrElse 0,
+ record.timestamp.inMilliseconds,
+ record.message
+ )
+ responseHeader.addToTranscript(thriftRecord)
+ }
+ }
+
+ val responseHeaderBytes = OutputBuffer.messageToArray(responseHeader)
+ responseHeaderBytes ++ response
}
} else {
val buffer = new InputBuffer(request)
@@ -77,7 +91,8 @@ class ThriftServerTracingFilter
new TMessage(ThriftTracing.CanTraceMethodName, TMessageType.REPLY, msg.seqid))
buffer().writeMessageEnd()
- // TODO: parse out options?
+ // Note: currently there are no options, so there's no need
+ // top parse them out.
Future.value(buffer.toArray)
} else {
service(request)
View
11 finagle-thrift/src/test/scala/com/twitter/finagle/thrift/EndToEndSpec.scala
@@ -28,7 +28,8 @@ object EndToEndSpec extends Specification {
def add_one(a: Int, b: Int) = Future.void
def multiply(a: Int, b: Int) = Future { a * b }
def complex_return(someString: String) = Future {
- new SomeStruct(123, TraceContext().parentSpanID.get.toString)
+ TraceContext().transcript.record("hey it's me!")
+ new SomeStruct(123, TraceContext().traceID.parentSpan.get.toString)
}
def someway() = Future.void
}
@@ -52,8 +53,14 @@ object EndToEndSpec extends Specification {
val future = client.multiply(10, 30)
future() must be_==(300)
+ import com.twitter.finagle.tracing.BufferingTranscript
+ TraceContext().transcript = new BufferingTranscript(TraceContext().traceID)
+
client.complex_return("a string")().arg_two must be_==(
- "%s".format(TraceContext().spanID.toString))
+ "%s".format(TraceContext().traceID.span.toString))
+
+ TraceContext().transcript must haveSize(1)
+ TraceContext().transcript.head.message must be_==("hey it's me!")
client.add(1, 2)() must throwA[AnException]
client.add_one(1, 2)() // don't block!
View
7 project/build/Project.scala
@@ -82,8 +82,7 @@ class Project(info: ProjectInfo) extends StandardParentProject(info)
*/
val stressProject = project(
"finagle-stress", "finagle-stress",
-
- new StressProject(_), coreProject, ostrich3Project)
+ new StressProject(_), coreProject, ostrich3Project, thriftProject)
class CoreProject(info: ProjectInfo) extends StandardProject(info)
with SubversionPublisher with AdhocInlines
@@ -144,7 +143,11 @@ class Project(info: ProjectInfo) extends StandardParentProject(info)
class StressProject(info: ProjectInfo) extends StandardProject(info)
with SubversionPublisher with IntegrationSpecs with AdhocInlines
+ with CompileFinagleThrift
{
+ override def compileOrder = CompileOrder.JavaThenScala
+ val thrift = "thrift" % "libthrift" % "0.5.0"
+ val slf4jNop = "org.slf4j" % "slf4j-nop" % "1.5.2" % "provided"
val ostrich3 = "com.twitter" % "ostrich" % "3.0.4"
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.