From 8e3ac5011a75c92d2cebd4a9d7aa78226fcd3d01 Mon Sep 17 00:00:00 2001 From: drpacman Date: Wed, 6 Apr 2016 23:30:14 +0100 Subject: [PATCH 1/6] Ensure sampling decision is propogated to downstream service --- .../tracing/http/TracedSprayPipeline.scala | 31 +++++++++----- .../http/TracedSprayPipelineSpec.scala | 40 ++++++++++++++++++- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala index fa9fab5..6352f72 100644 --- a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala +++ b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala @@ -34,16 +34,27 @@ trait TracedSprayPipeline { def tracedPipeline[T](parent: BaseTracingSupport): SendReceive = { val clientRequest = new TracingSupport {} - trace.createChild(clientRequest, parent).map(metadata => - addHeaders(List( - HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(metadata.traceId)), - HttpHeaders.RawHeader(TracingHeaders.SpanId, SpanMetadata.idToString(metadata.spanId)), - HttpHeaders.RawHeader(TracingHeaders.ParentSpanId, SpanMetadata.idToString(metadata.parentId.get)), - HttpHeaders.RawHeader(TracingHeaders.Sampled, "true") - )) ~> - startTrace(clientRequest) ~> - sendAndReceive ~> - completeTrace(clientRequest)).getOrElse(sendAndReceive) + trace.createChild(clientRequest, parent).map(tracedRequest(clientRequest, _)).getOrElse(untracedRequest(parent.tracingId)) + } + + private[this] def untracedRequest(traceId: Long): SendReceive = { + addHeaders(List( + HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(traceId)), + HttpHeaders.RawHeader(TracingHeaders.Sampled, "false") + )) ~> + sendAndReceive + } + + private[this] def tracedRequest(clientRequest: BaseTracingSupport, metadata: SpanMetadata): SendReceive = { + addHeaders(List( + HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(metadata.traceId)), + HttpHeaders.RawHeader(TracingHeaders.SpanId, SpanMetadata.idToString(metadata.spanId)), + HttpHeaders.RawHeader(TracingHeaders.ParentSpanId, SpanMetadata.idToString(metadata.parentId.get)), + HttpHeaders.RawHeader(TracingHeaders.Sampled, "true") + )) ~> + startTrace(clientRequest) ~> + sendAndReceive ~> + completeTrace(clientRequest) } def startTrace(ts: BaseTracingSupport)(request: HttpRequest): HttpRequest = { diff --git a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala index 484686d..08c16dd 100644 --- a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala +++ b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala @@ -1,9 +1,10 @@ package com.github.levkhomich.akka.tracing.http -import scala.concurrent.Future +import scala.concurrent.{ Promise, Future } import org.specs2.matcher.FutureMatchers import org.specs2.mutable.Specification +import org.specs2.matcher.MatchResult import spray.client.pipelining._ import spray.http._ @@ -43,6 +44,43 @@ class TracedSprayPipelineSpec extends Specification with FutureMatchers with Tra childSpan.parent_id mustEqual parentSpan.id childSpan.id mustNotEqual parentSpan.id } + + "indicate to downstream service trace is not sampled if we are not sampling" in { + val result = Promise[MatchResult[_]]() + val parent = nextRandomMessage + val testPipeline = new TracedSprayPipeline { + val system = self.system + override def sendAndReceive = { + case req: HttpRequest => + result.trySuccess( + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("false")) and + (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) + ) + Future.successful(HttpResponse(StatusCodes.OK, bodyEntity)) + } + } + testPipeline.tracedPipeline[String](parent)(Get("http://test.com")) + result.future.await + } + + "indicate to downstream service that we are sampled along with the trace id if we are sampling" in { + val result = Promise[MatchResult[_]]() + val parent = nextRandomMessage + trace.sample(parent, "test trace", force = true) + val testPipeline = new TracedSprayPipeline { + val system = self.system + override def sendAndReceive = { + case req: HttpRequest => + result.trySuccess( + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("true")) and + (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) + ) + Future.successful(HttpResponse(StatusCodes.OK, bodyEntity)) + } + } + testPipeline.tracedPipeline[String](parent)(Get("http://test.com")) + result.future.await + } } step { From d6e5b613cc64666914014fd020e8d826178ec34b Mon Sep 17 00:00:00 2001 From: drpacman Date: Thu, 7 Apr 2016 00:27:46 +0100 Subject: [PATCH 2/6] Ensure upstream sampling decision is honoured on this service --- .../akka/tracing/play/TracingSettings.scala | 10 +++++++- .../akka/tracing/play/PlayTracingSpec.scala | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala index 207cf7e..441e9c3 100644 --- a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala +++ b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala @@ -36,7 +36,15 @@ trait TracingSettings extends GlobalSettings with PlayControllerTracing { lazy val excludedHeaders = Set.empty[String] protected def sample(request: RequestHeader): Unit = { - trace.sample(request, serviceName) + val upstreamSampling = request.headers.get(TracingHeaders.Sampled).map(_.toBoolean) + upstreamSampling match { + case Some(true) => + trace.sample(request, serviceName, true) + case None => + trace.sample(request, serviceName) + case Some(false) => + // don't sample + } } protected def addHttpAnnotations(request: RequestHeader): Unit = { diff --git a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala index 090c4eb..14e3cf3 100644 --- a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala +++ b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala @@ -70,6 +70,12 @@ class PlayTracingSpec extends PlaySpecification with TracingTestCommons with Moc additionalConfiguration = configuration ) + def disabledLocalSamplingApplication: FakeApplication = FakeApplication( + withRoutes = routes, + withGlobal = Some(new GlobalSettings with TracingSettings), + additionalConfiguration = configuration ++ Map(TracingExtension.AkkaTracingSampleRate -> 0) + ) + "Play tracing" should { "sample requests" in new WithApplication(fakeApplication) { val result = route(FakeRequest("GET", TestPath)).map(Await.result(_, defaultAwaitTimeout.duration)) @@ -155,6 +161,25 @@ class PlayTracingSpec extends PlaySpecification with TracingTestCommons with Moc checkAnnotation(span, TracingExtension.getStackTrace(npe)) } + "ensure upstream 'do not sample' decision is honoured" in new WithApplication(fakeApplication) { + val spanId = Random.nextLong + val result = route(FakeRequest("GET", TestPath + "?key=value", + FakeHeaders(Seq( + TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), + TracingHeaders.Sampled -> Seq("false") + )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) + expectSpans(0) + } + + "ensure upstream 'do sample' decision is honoured" in new WithApplication(disabledLocalSamplingApplication) { + val spanId = Random.nextLong + val result = route(FakeRequest("GET", TestPath + "?key=value", + FakeHeaders(Seq( + TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), + TracingHeaders.Sampled -> Seq("true") + )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) + expectSpans(1) + } } step { From d9b0dc646d694ea6c059865ac7310e0796d6749d Mon Sep 17 00:00:00 2001 From: drpacman Date: Thu, 7 Apr 2016 01:30:31 +0100 Subject: [PATCH 3/6] Support both boolean and binary expressions of Sampled value (Finagle impl and openzipkin spec are contraditory) --- .../akka/tracing/play/TracingSettings.scala | 6 +++--- .../akka/tracing/play/PlayTracingSpec.scala | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala index 441e9c3..28eec06 100644 --- a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala +++ b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala @@ -36,13 +36,13 @@ trait TracingSettings extends GlobalSettings with PlayControllerTracing { lazy val excludedHeaders = Set.empty[String] protected def sample(request: RequestHeader): Unit = { - val upstreamSampling = request.headers.get(TracingHeaders.Sampled).map(_.toBoolean) + val upstreamSampling = request.headers.get(TracingHeaders.Sampled).map(_.toLowerCase) upstreamSampling match { - case Some(true) => + case Some("true") | Some("1") => trace.sample(request, serviceName, true) case None => trace.sample(request, serviceName) - case Some(false) => + case Some(x) => // don't sample } } diff --git a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala index 14e3cf3..5bd236a 100644 --- a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala +++ b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala @@ -171,7 +171,7 @@ class PlayTracingSpec extends PlaySpecification with TracingTestCommons with Moc expectSpans(0) } - "ensure upstream 'do sample' decision is honoured" in new WithApplication(disabledLocalSamplingApplication) { + "ensure upstream 'do sample' decision is honoured" in new WithApplication(fakeApplication) { val spanId = Random.nextLong val result = route(FakeRequest("GET", TestPath + "?key=value", FakeHeaders(Seq( @@ -180,6 +180,16 @@ class PlayTracingSpec extends PlaySpecification with TracingTestCommons with Moc )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) expectSpans(1) } + + "ensure upstream 'do sample' decision is honoured (binary value)" in new WithApplication(fakeApplication) { + val spanId = Random.nextLong + val result = route(FakeRequest("GET", TestPath + "?key=value", + FakeHeaders(Seq( + TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), + TracingHeaders.Sampled -> Seq("1") + )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) + expectSpans(1) + } } step { From c31e51bedfadb7e67b18401208801804fc990a62 Mon Sep 17 00:00:00 2001 From: drpacman Date: Thu, 7 Apr 2016 21:14:38 +0100 Subject: [PATCH 4/6] Handle valid values, with invalid sampling values falling back to local decision --- .../akka/tracing/play/TracingSettings.scala | 8 ++-- .../akka/tracing/play/PlayTracingSpec.scala | 42 +++++++++++-------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala index 28eec06..2ab78fb 100644 --- a/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala +++ b/play/src/main/scala/com/github/levkhomich/akka/tracing/play/TracingSettings.scala @@ -38,12 +38,12 @@ trait TracingSettings extends GlobalSettings with PlayControllerTracing { protected def sample(request: RequestHeader): Unit = { val upstreamSampling = request.headers.get(TracingHeaders.Sampled).map(_.toLowerCase) upstreamSampling match { - case Some("true") | Some("1") => + case Some("0") | Some("false") => + // do not sample + case Some("1") | Some("true") => trace.sample(request, serviceName, true) - case None => + case Some(_) | None => trace.sample(request, serviceName) - case Some(x) => - // don't sample } } diff --git a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala index 5bd236a..2af5597 100644 --- a/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala +++ b/play/src/test/scala/com/github/levkhomich/akka/tracing/play/PlayTracingSpec.scala @@ -161,35 +161,41 @@ class PlayTracingSpec extends PlaySpecification with TracingTestCommons with Moc checkAnnotation(span, TracingExtension.getStackTrace(npe)) } - "ensure upstream 'do not sample' decision is honoured" in new WithApplication(fakeApplication) { - val spanId = Random.nextLong - val result = route(FakeRequest("GET", TestPath + "?key=value", - FakeHeaders(Seq( - TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), - TracingHeaders.Sampled -> Seq("false") - )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) - expectSpans(0) + Seq("0", "false").foreach { value => + s"ensure upstream 'do not sample' decision is honoured for sampled value of $value" in new WithApplication(fakeApplication) { + + val spanId = Random.nextLong + val result = route(FakeRequest("GET", TestPath + "?key=value", + FakeHeaders(Seq( + TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), + TracingHeaders.Sampled -> Seq("false") + )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) + expectSpans(0) + } } - "ensure upstream 'do sample' decision is honoured" in new WithApplication(fakeApplication) { - val spanId = Random.nextLong - val result = route(FakeRequest("GET", TestPath + "?key=value", - FakeHeaders(Seq( - TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), - TracingHeaders.Sampled -> Seq("true") - )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) - expectSpans(1) + Seq("1", "true").foreach { value => + s"ensure upstream 'do sample' decision is honoured for sampled value $value" in new WithApplication(disabledLocalSamplingApplication) { + val spanId = Random.nextLong + val result = route(FakeRequest("GET", TestPath + "?key=value", + FakeHeaders(Seq( + TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), + TracingHeaders.Sampled -> Seq(value) + )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) + expectSpans(1) + } } - "ensure upstream 'do sample' decision is honoured (binary value)" in new WithApplication(fakeApplication) { + "ensure local sampling decision is honoured if invalid sampled valued is supplied" in new WithApplication(fakeApplication) { val spanId = Random.nextLong val result = route(FakeRequest("GET", TestPath + "?key=value", FakeHeaders(Seq( TracingHeaders.TraceId -> Seq(SpanMetadata.idToString(spanId)), - TracingHeaders.Sampled -> Seq("1") + TracingHeaders.Sampled -> Seq("unexpected value") )), AnyContentAsEmpty)).map(Await.result(_, defaultAwaitTimeout.duration)) expectSpans(1) } + } step { From 775fbd697cdf396d90f934c94657b5c9256414b7 Mon Sep 17 00:00:00 2001 From: drpacman Date: Thu, 7 Apr 2016 21:47:32 +0100 Subject: [PATCH 5/6] Use 1 and 0 for Sampled values in keeping with values documented at http://zipkin.io/pages/instrumenting.html --- .../levkhomich/akka/tracing/http/TracedSprayPipeline.scala | 4 ++-- .../akka/tracing/http/TracedSprayPipelineSpec.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala index 6352f72..d0e30cd 100644 --- a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala +++ b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala @@ -40,7 +40,7 @@ trait TracedSprayPipeline { private[this] def untracedRequest(traceId: Long): SendReceive = { addHeaders(List( HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(traceId)), - HttpHeaders.RawHeader(TracingHeaders.Sampled, "false") + HttpHeaders.RawHeader(TracingHeaders.Sampled, "0") )) ~> sendAndReceive } @@ -50,7 +50,7 @@ trait TracedSprayPipeline { HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(metadata.traceId)), HttpHeaders.RawHeader(TracingHeaders.SpanId, SpanMetadata.idToString(metadata.spanId)), HttpHeaders.RawHeader(TracingHeaders.ParentSpanId, SpanMetadata.idToString(metadata.parentId.get)), - HttpHeaders.RawHeader(TracingHeaders.Sampled, "true") + HttpHeaders.RawHeader(TracingHeaders.Sampled, "1") )) ~> startTrace(clientRequest) ~> sendAndReceive ~> diff --git a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala index 08c16dd..4ecee8c 100644 --- a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala +++ b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala @@ -53,7 +53,7 @@ class TracedSprayPipelineSpec extends Specification with FutureMatchers with Tra override def sendAndReceive = { case req: HttpRequest => result.trySuccess( - (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("false")) and + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("0")) and (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) ) Future.successful(HttpResponse(StatusCodes.OK, bodyEntity)) @@ -72,7 +72,7 @@ class TracedSprayPipelineSpec extends Specification with FutureMatchers with Tra override def sendAndReceive = { case req: HttpRequest => result.trySuccess( - (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("true")) and + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("1")) and (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) ) Future.successful(HttpResponse(StatusCodes.OK, bodyEntity)) From 5b7714f7efa98ea88ad8c14d883395852a2cf55a Mon Sep 17 00:00:00 2001 From: drpacman Date: Fri, 8 Apr 2016 16:11:24 +0100 Subject: [PATCH 6/6] Switched back to use of Boolean sampled value, for consistency with Finagle --- .../levkhomich/akka/tracing/http/TracedSprayPipeline.scala | 4 ++-- .../akka/tracing/http/TracedSprayPipelineSpec.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala index d0e30cd..6352f72 100644 --- a/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala +++ b/spray-client/src/main/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipeline.scala @@ -40,7 +40,7 @@ trait TracedSprayPipeline { private[this] def untracedRequest(traceId: Long): SendReceive = { addHeaders(List( HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(traceId)), - HttpHeaders.RawHeader(TracingHeaders.Sampled, "0") + HttpHeaders.RawHeader(TracingHeaders.Sampled, "false") )) ~> sendAndReceive } @@ -50,7 +50,7 @@ trait TracedSprayPipeline { HttpHeaders.RawHeader(TracingHeaders.TraceId, SpanMetadata.idToString(metadata.traceId)), HttpHeaders.RawHeader(TracingHeaders.SpanId, SpanMetadata.idToString(metadata.spanId)), HttpHeaders.RawHeader(TracingHeaders.ParentSpanId, SpanMetadata.idToString(metadata.parentId.get)), - HttpHeaders.RawHeader(TracingHeaders.Sampled, "1") + HttpHeaders.RawHeader(TracingHeaders.Sampled, "true") )) ~> startTrace(clientRequest) ~> sendAndReceive ~> diff --git a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala index 4ecee8c..08c16dd 100644 --- a/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala +++ b/spray-client/src/test/scala/com/github/levkhomich/akka/tracing/http/TracedSprayPipelineSpec.scala @@ -53,7 +53,7 @@ class TracedSprayPipelineSpec extends Specification with FutureMatchers with Tra override def sendAndReceive = { case req: HttpRequest => result.trySuccess( - (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("0")) and + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("false")) and (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) ) Future.successful(HttpResponse(StatusCodes.OK, bodyEntity)) @@ -72,7 +72,7 @@ class TracedSprayPipelineSpec extends Specification with FutureMatchers with Tra override def sendAndReceive = { case req: HttpRequest => result.trySuccess( - (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("1")) and + (req.headers.find(_.name == TracingHeaders.Sampled).map(_.value) must beSome("true")) and (req.headers.find(_.name == TracingHeaders.TraceId).map(_.value) must not beNull) ) Future.successful(HttpResponse(StatusCodes.OK, bodyEntity))