Permalink
Browse files

Top annotations

Add collector and query support for adding "top annotation" data that may be
computed offline

* New collector endpoints
  * `storeTopAnnotations`
  * `storeTopKeyValueAnnotations`
* New query endpoints
  * `getTopAnnotations`
  * `getTopKeyValueAnnotations`

Author: @franklinhu
Fixes #46
URL: #46
  • Loading branch information...
1 parent 8a8497d commit 3e15fe1190f9b05246d1b386368236e4a2bc8310 Franklin Hu committed Jun 26, 2012
Showing with 503 additions and 29 deletions.
  1. +4 −0 zipkin-scribe/config/collector-dev.scala
  2. +28 −0 zipkin-scribe/src/main/scala/com/twitter/zipkin/collector/ScribeCollectorService.scala
  3. +29 −0 zipkin-scribe/src/test/scala/com/twitter/zipkin/collector/ScribeCollectorServiceSpec.scala
  4. +4 −0 zipkin-server/config/query-dev.scala
  5. +25 −0 zipkin-server/src/main/scala/com/twitter/zipkin/config/AggregatesConfig.scala
  6. +36 −0 zipkin-server/src/main/scala/com/twitter/zipkin/config/CassandraAggregatesConfig.scala
  7. +5 −1 zipkin-server/src/main/scala/com/twitter/zipkin/config/ZipkinCollectorConfig.scala
  8. +5 −2 zipkin-server/src/main/scala/com/twitter/zipkin/config/ZipkinQueryConfig.scala
  9. +30 −2 zipkin-server/src/main/scala/com/twitter/zipkin/query/QueryService.scala
  10. +3 −3 zipkin-server/src/main/scala/com/twitter/zipkin/query/ZipkinQuery.scala
  11. +38 −0 zipkin-server/src/main/scala/com/twitter/zipkin/storage/Aggregates.scala
  12. +89 −0 zipkin-server/src/main/scala/com/twitter/zipkin/storage/cassandra/CassandraAggregates.scala
  13. +3 −1 zipkin-server/src/schema/cassandra-schema.txt
  14. +25 −0 zipkin-server/src/test/resources/CassandraAggregatesConfig.scala
  15. +47 −19 zipkin-server/src/test/scala/com/twitter/zipkin/query/QueryServiceSpec.scala
  16. +111 −0 zipkin-server/src/test/scala/com/twitter/zipkin/storage/cassandra/CassandraAggregatesSpec.scala
  17. +4 −0 zipkin-test/src/test/resources/TestCollector.scala
  18. +4 −0 zipkin-test/src/test/resources/TestQuery.scala
  19. +1 −1 zipkin-test/src/test/scala/com/twitter/zipkin/ZipkinSpec.scala
  20. +8 −0 zipkin-thrift/src/main/thrift/zipkinCollector.thrift
  21. +4 −0 zipkin-thrift/src/main/thrift/zipkinQuery.thrift
View
4 zipkin-scribe/config/collector-dev.scala
@@ -54,6 +54,10 @@ new ScribeZipkinCollectorConfig {
def cassandraConfig = _cassandraConfig
}
+ def aggregatesConfig = new CassandraAggregatesConfig {
+ def cassandraConfig = _cassandraConfig
+ }
+
override def adaptiveSamplerConfig = new NullAdaptiveSamplerConfig {}
def zkConfig = new ZooKeeperConfig {
View
28 zipkin-scribe/src/main/scala/com/twitter/zipkin/collector/ScribeCollectorService.scala
@@ -103,6 +103,34 @@ class ScribeCollectorService(config: ZipkinCollectorConfig, val writeQueue: Writ
}
}
+ def storeTopAnnotations(serviceName: String, annotations: Seq[String]): Future[Unit] = {
+ Stats.incr("collector.storeTopAnnotations")
+ log.info("storeTopAnnotations: " + serviceName + "; " + annotations)
+
+ Stats.timeFutureMillis("collector.storeTopAnnotations") {
+ config.aggregates.storeTopAnnotations(serviceName, annotations)
+ } rescue {
+ case e: Exception =>
+ log.error(e, "storeTopAnnotations failed")
+ Stats.incr("collector.storeTopAnnotations")
+ Future.exception(gen.AdjustableRateException(e.toString))
+ }
+ }
+
+ def storeTopKeyValueAnnotations(serviceName: String, annotations: Seq[String]): Future[Unit] = {
+ Stats.incr("collector.storeTopKeyValueAnnotations")
+ log.info("storeTopKeyValueAnnotations: " + serviceName + ";" + annotations)
+
+ Stats.timeFutureMillis("collector.storeTopKeyValueAnnotations") {
+ config.aggregates.storeTopKeyValueAnnotations(serviceName, annotations)
+ } rescue {
+ case e: Exception =>
+ log.error(e, "storeTopKeyValueAnnotations failed")
+ Stats.incr("collector.storeTopKeyValueAnnotations")
+ Future.exception(gen.AdjustableRateException(e.toString))
+ }
+ }
+
@throws(classOf[gen.AdjustableRateException])
def setSampleRate(sampleRate: Double): Future[Unit] = {
Stats.incr("collector.set_sample_rate")
View
29 zipkin-scribe/src/test/scala/com/twitter/zipkin/collector/ScribeCollectorServiceSpec.scala
@@ -23,6 +23,7 @@ import com.twitter.zipkin.config.sampler.AdjustableRateConfig
import com.twitter.zipkin.config.ScribeZipkinCollectorConfig
import com.twitter.zipkin.gen
import com.twitter.zipkin.adapter.ThriftAdapter
+import com.twitter.zipkin.storage.Aggregates
import org.specs.Specification
import org.specs.mock.{ClassMocker, JMocker}
@@ -41,16 +42,19 @@ class ScribeCollectorServiceSpec extends Specification with JMocker with ClassMo
val queue = mock[WriteQueue[Seq[String]]]
val zkSampleRateConfig = mock[AdjustableRateConfig]
+ val mockAggregates = mock[Aggregates]
val config = new ScribeZipkinCollectorConfig {
def writeQueueConfig = null
def zkConfig = null
def indexConfig = null
def storageConfig = null
+ def aggregatesConfig = null
def methodConfig = null
override lazy val writeQueue = queue
override lazy val sampleRateConfig = zkSampleRateConfig
+ override lazy val aggregates = mockAggregates
}
def scribeCollectorService = new ScribeCollectorService(config, config.writeQueue, Set(category)) {
@@ -114,5 +118,30 @@ class ScribeCollectorServiceSpec extends Specification with JMocker with ClassMo
val actual = cs.setSampleRate(sampleRate)
actual() mustEqual expected()
}
+
+ "store aggregates" in {
+ val serviceName = "mockingbird"
+ val annotations = Seq("a" , "b", "c")
+
+ "store top annotations" in {
+ val cs = scribeCollectorService
+
+ expect {
+ one(mockAggregates).storeTopAnnotations(serviceName, annotations)
+ }
+
+ cs.storeTopAnnotations(serviceName, annotations)
+ }
+
+ "store top key value annotations" in {
+ val cs = scribeCollectorService
+
+ expect {
+ one(mockAggregates).storeTopKeyValueAnnotations(serviceName, annotations)
+ }
+
+ cs.storeTopKeyValueAnnotations(serviceName, annotations)
+ }
+ }
}
}
View
4 zipkin-server/config/query-dev.scala
@@ -46,6 +46,10 @@ new ZipkinQueryConfig {
def cassandraConfig = _cassandraConfig
}
+ def aggregatesConfig = new CassandraAggregatesConfig {
+ def cassandraConfig = _cassandraConfig
+ }
+
def zkConfig = new ZooKeeperConfig {
servers = List("localhost:2181")
}
View
25 zipkin-server/src/main/scala/com/twitter/zipkin/config/AggregatesConfig.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2012 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.zipkin.config
+
+import com.twitter.util.Config
+import com.twitter.zipkin.storage.{NullAggregates, Aggregates}
+
+trait AggregatesConfig extends Config[Aggregates]
+
+class NullAggregatesConfig extends AggregatesConfig {
+ def apply(): Aggregates = new NullAggregates
+}
View
36 zipkin-server/src/main/scala/com/twitter/zipkin/config/CassandraAggregatesConfig.scala
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2012 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.zipkin.config
+
+import com.twitter.zipkin.storage.cassandra.CassandraAggregates
+import com.twitter.cassie.codecs.{LongCodec, Utf8Codec}
+import com.twitter.cassie.{ColumnFamily, ReadConsistency, WriteConsistency}
+
+trait CassandraAggregatesConfig extends AggregatesConfig { self =>
+
+ def cassandraConfig: CassandraConfig
+ var topAnnotationsCf: String = "TopAnnotations"
+
+ def apply(): CassandraAggregates = {
+ val _topAnnotations = cassandraConfig.keyspace.columnFamily[String, Long, String](
+ topAnnotationsCf,Utf8Codec, LongCodec, Utf8Codec
+ ).consistency(WriteConsistency.One).consistency(ReadConsistency.One)
+
+ new CassandraAggregates {
+ val topAnnotations: ColumnFamily[String, Long, String] = _topAnnotations
+ }
+ }
+}
View
6 zipkin-server/src/main/scala/com/twitter/zipkin/config/ZipkinCollectorConfig.scala
@@ -15,7 +15,7 @@
*/
package com.twitter.zipkin.config
-import com.twitter.zipkin.storage.{Index, Storage}
+import com.twitter.zipkin.storage.{Aggregates, Index, Storage}
import com.twitter.zipkin.collector.{WriteQueue, ZipkinCollector}
import com.twitter.zipkin.collector.filter.{DefaultClientIndexingFilter, IndexingFilter}
import com.twitter.zipkin.collector.sampler.{AdaptiveSampler, ZooKeeperGlobalSampler, GlobalSampler}
@@ -62,6 +62,10 @@ trait ZipkinCollectorConfig extends ZipkinConfig[ZipkinCollector] {
def indexConfig: IndexConfig
lazy val index: Index = indexConfig.apply()
+ /* Aggregates */
+ def aggregatesConfig: AggregatesConfig
+ lazy val aggregates: Aggregates = aggregatesConfig.apply()
+
/* ZooKeeper */
def zkConfig: ZooKeeperConfig
View
7 zipkin-server/src/main/scala/com/twitter/zipkin/config/ZipkinQueryConfig.scala
@@ -17,7 +17,7 @@ package com.twitter.zipkin.config
import com.twitter.zipkin.query.ZipkinQuery
import com.twitter.zipkin.gen
-import com.twitter.zipkin.storage.{Index, Storage}
+import com.twitter.zipkin.storage.{Aggregates, Index, Storage}
import com.twitter.common.zookeeper.{ServerSetImpl, ZooKeeperClient}
import com.twitter.finagle.zipkin.thrift.ZipkinTracer
import com.twitter.ostrich.admin.RuntimeEnvironment
@@ -42,6 +42,9 @@ trait ZipkinQueryConfig extends ZipkinConfig[ZipkinQuery] {
def indexConfig: IndexConfig
lazy val index: Index = indexConfig.apply()
+ def aggregatesConfig: AggregatesConfig
+ lazy val aggregates: Aggregates = aggregatesConfig.apply()
+
def zkConfig: ZooKeeperConfig
def zkClientConfig = new ZooKeeperClientConfig {
@@ -55,6 +58,6 @@ trait ZipkinQueryConfig extends ZipkinConfig[ZipkinQuery] {
ZipkinTracer(statsReceiver = statsReceiver, sampleRate = 1f)
def apply(runtime: RuntimeEnvironment) = {
- new ZipkinQuery(this, serverSet, storage, index)
+ new ZipkinQuery(this, serverSet, storage, index, aggregates)
}
}
View
32 zipkin-server/src/main/scala/com/twitter/zipkin/query/QueryService.scala
@@ -24,7 +24,7 @@ import com.twitter.finagle.tracing.Trace
import com.twitter.util.Future
import com.twitter.zipkin.gen
import com.twitter.zipkin.query.adjusters.Adjuster
-import com.twitter.zipkin.storage.{TraceIdDuration, Index, Storage}
+import com.twitter.zipkin.storage.{Aggregates, TraceIdDuration, Index, Storage}
import java.nio.ByteBuffer
import org.apache.thrift.TException
import scala.collection.Set
@@ -34,7 +34,7 @@ import scala.collection.Set
* by lookup the information in the index and then fetch the required trace data
* from the storage.
*/
-class QueryService(storage: Storage, index: Index, adjusterMap: Map[gen.Adjust, Adjuster])
+class QueryService(storage: Storage, index: Index, aggregates: Aggregates, adjusterMap: Map[gen.Adjust, Adjuster])
extends gen.ZipkinQuery.FutureIface with Service {
private val log = Logger.get
private var running = false
@@ -339,6 +339,34 @@ class QueryService(storage: Storage, index: Index, adjusterMap: Map[gen.Adjust,
}
}
+ def getTopAnnotations(serviceName: String): Future[Seq[String]] = {
+ checkIfRunning()
+ Stats.getCounter("query.get_top_annotations").incr()
+ log.debug("getTopAnnotations: " + serviceName)
+
+ Stats.timeFutureMillis("query.getTopAnnotations") {
+ aggregates.getTopAnnotations(serviceName) onFailure { e =>
+ log.error(e, "getTopAnnotations failed")
+ Stats.getCounter("query.error_get_top_annotations").incr()
+ Future.exception(gen.QueryException(e.toString))
+ }
+ }
+ }
+
+ def getTopKeyValueAnnotations(serviceName: String): Future[Seq[String]] = {
+ checkIfRunning()
+ Stats.getCounter("query.get_top_key_value_annotations").incr()
+ log.debug("getTopKeyValueAnnotations: " + serviceName)
+
+ Stats.timeFutureMillis("query.getTopKeyValueAnnotations") {
+ aggregates.getTopKeyValueAnnotations(serviceName) onFailure { e =>
+ log.error(e, "getTopKeyValueAnnotations failed")
+ Stats.getCounter("query.error_get_top_key_value_annotations").incr()
+ Future.exception(gen.QueryException(e.toString))
+ }
+ }
+ }
+
private def checkIfRunning() = {
if (!running) {
log.warning("Server not running, throwing exception")
View
6 zipkin-server/src/main/scala/com/twitter/zipkin/query/ZipkinQuery.scala
@@ -18,7 +18,7 @@ package com.twitter.zipkin.query
*/
import com.twitter.logging.Logger
import org.apache.thrift.protocol.TBinaryProtocol
-import com.twitter.zipkin.storage.{Index, Storage}
+import com.twitter.zipkin.storage.{Aggregates, Index, Storage}
import com.twitter.zipkin.gen
import com.twitter.finagle.thrift.ThriftServerFramedCodec
import com.twitter.finagle.zookeeper.ZookeeperServerSetCluster
@@ -29,7 +29,7 @@ import com.twitter.zipkin.config.ZipkinQueryConfig
import com.twitter.common.zookeeper.ServerSet
class ZipkinQuery(
- config: ZipkinQueryConfig, serverSet: ServerSet, storage: Storage, index: Index
+ config: ZipkinQueryConfig, serverSet: ServerSet, storage: Storage, index: Index, aggregates: Aggregates
) extends Service {
val log = Logger.get(getClass.getName)
@@ -41,7 +41,7 @@ class ZipkinQuery(
log.info("Starting query thrift service on addr " + serverAddr)
val cluster = new ZookeeperServerSetCluster(serverSet)
- val queryService = new QueryService(storage, index, config.adjusterMap)
+ val queryService = new QueryService(storage, index, aggregates, config.adjusterMap)
queryService.start()
ServiceTracker.register(queryService)
View
38 zipkin-server/src/main/scala/com/twitter/zipkin/storage/Aggregates.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.zipkin.storage
+
+import com.twitter.util.Future
+
+/**
+ * Storage and retrieval interface for aggregates that may be computed offline and reloaded into
+ * online storage
+ */
+trait Aggregates {
+ def getTopAnnotations(serviceName: String): Future[Seq[String]]
+ def getTopKeyValueAnnotations(serviceName: String): Future[Seq[String]]
+
+ def storeTopAnnotations(serviceName: String, a: Seq[String]): Future[Unit]
+ def storeTopKeyValueAnnotations(serviceName: String, a: Seq[String]): Future[Unit]
+}
+
+class NullAggregates extends Aggregates {
+ def getTopAnnotations(serviceName: String) = Future(Seq.empty[String])
+ def getTopKeyValueAnnotations(serviceName: String) = Future(Seq.empty[String])
+
+ def storeTopAnnotations(serviceName: String, a: Seq[String]): Future[Unit] = Future.Unit
+ def storeTopKeyValueAnnotations(serviceName: String, a: Seq[String]): Future[Unit] = Future.Unit
+}
View
89 zipkin-server/src/main/scala/com/twitter/zipkin/storage/cassandra/CassandraAggregates.scala
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2012 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.zipkin.storage.cassandra
+
+import com.twitter.util.Future
+import com.twitter.zipkin.storage.Aggregates
+import scala.collection.JavaConverters._
+import com.twitter.cassie.{Column, ColumnFamily}
+
+/**
+ * Cassandra backed aggregates store
+ *
+ * For each service, we store top annotations and top key value annotations
+ * for use at query time.
+ *
+ * Top annotations are a sequence of the most popular time-based annotation strings
+ * Top key value annotations are a sequence of the most popular _keys_ among key value annotations
+ */
+trait CassandraAggregates extends Aggregates with Cassandra {
+
+ val topAnnotations: ColumnFamily[String, Long, String]
+
+ val Delimiter: String = ":"
+
+ /**
+ * Get the top annotations for a service name
+ */
+ def getTopAnnotations(serviceName: String): Future[Seq[String]] = {
+ getAnnotations(topAnnotationRowKey(serviceName))
+ }
+
+ /**
+ * Get the top key value annotation keys for a service name
+ */
+ def getTopKeyValueAnnotations(serviceName: String): Future[Seq[String]] = {
+ getAnnotations(topKeyValueRowKey(serviceName))
+ }
+
+ /**
+ * Override the top annotations for a service
+ */
+ def storeTopAnnotations(serviceName: String, annotations: Seq[String]): Future[Unit] = {
+ storeAnnotations(topAnnotationRowKey(serviceName), annotations)
+ }
+
+ /**
+ * Override the top key value annotation keys for a service
+ */
+ def storeTopKeyValueAnnotations(serviceName: String, annotations: Seq[String]): Future[Unit] = {
+ storeAnnotations(topKeyValueRowKey(serviceName), annotations)
+ }
+
+ private[cassandra] def getAnnotations(key: String): Future[Seq[String]] = {
+ topAnnotations.getRow(key).map {
+ _.values().asScala.map { _.value }.toSeq
+ }
+ }
+
+ /** Synchronize these so we don't do concurrent writes from the same box */
+ private[cassandra] def storeAnnotations(key: String, annotations: Seq[String]): Future[Unit] = synchronized {
+ val remove = topAnnotations.removeRow(key)
+ val batch = topAnnotations.batch()
+ annotations.zipWithIndex.foreach { case (annotation: String, index: Int) =>
+ batch.insert(key, new Column[Long, String](index, annotation))
+ }
+ remove()
+ Future.join(Seq(batch.execute()))
+ }
+
+
+ private[cassandra] def topAnnotationRowKey(serviceName: String) =
+ serviceName + Delimiter + "annotation"
+
+ private[cassandra] def topKeyValueRowKey(serviceName: String) =
+ serviceName + Delimiter + "kv"
+}
View
4 zipkin-server/src/schema/cassandra-schema.txt
@@ -11,4 +11,6 @@ create column family ServiceNames;
create column family ServiceSpanNameIndex with comparator = LongType;
create column family ServiceNameIndex with comparator = LongType;
create column family AnnotationsIndex with comparator = LongType;
-create column family DurationIndex with comparator = LongType;
+create column family DurationIndex with comparator = LongType;
+
+create column family TopAnnotations with comparator = LongType;
View
25 zipkin-server/src/test/resources/CassandraAggregatesConfig.scala
@@ -0,0 +1,25 @@
+/*
+* Copyright 2012 Twitter Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+import com.twitter.zipkin.config.{CassandraConfig, CassandraAggregatesConfig}
+
+new CassandraAggregatesConfig {
+ var cassandraConfig = new CassandraConfig {
+ nodes = Set("localhost")
+ mapHosts = false
+ useServerSets = false
+ port = 6579
+ }
+}
View
66 zipkin-server/src/test/scala/com/twitter/zipkin/query/QueryServiceSpec.scala
@@ -24,9 +24,9 @@ import com.twitter.zipkin.gen
import com.twitter.zipkin.common._
import java.nio.ByteBuffer
import com.twitter.util.Future
-import com.twitter.zipkin.storage.{TraceIdDuration, Storage, Index}
import com.twitter.scrooge.BinaryThriftStructSerializer
import com.twitter.zipkin.adapter.ThriftAdapter
+import com.twitter.zipkin.storage.{Aggregates, TraceIdDuration, Storage, Index}
class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val ep1 = Endpoint(123, 123, "service1")
@@ -60,19 +60,19 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"QueryService" should {
"generate exception in getTraceIdsByName if service name is null" in {
- val qs = new QueryService(null, null, null)
+ val qs = new QueryService(null, null, null, null)
qs.start
qs.getTraceIdsBySpanName(null, "span", 101, 100, gen.Order.DurationDesc)() must throwA[gen.QueryException]
}
"throw exception in getTraceIdsByServiceName if service name is null" in {
- val qs = new QueryService(null, null, null)
+ val qs = new QueryService(null, null, null, null)
qs.start
qs.getTraceIdsByServiceName(null, 101, 100, gen.Order.DurationDesc)() must throwA[gen.QueryException]
}
"throw exception in getTraceIdsByAnnotation if annotation is null" in {
- val qs = new QueryService(null, null, null)
+ val qs = new QueryService(null, null, null, null)
qs.start
qs.getTraceIdsByAnnotation(null, null, null, 101, 100, gen.Order.DurationDesc)() must throwA[gen.QueryException]
}
@@ -116,7 +116,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val storage = mock[Storage]
val index = new MockIndex {}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(2, 3, 1)
@@ -130,7 +130,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val storage = mock[Storage]
val index = new MockIndex {}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(2, 3, 1)
@@ -143,7 +143,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val storage = mock[Storage]
val index = new MockIndex {}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(1, 2, 3)
@@ -155,7 +155,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"successfully return the trace summary for a trace id" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val traceId = 123L
@@ -171,7 +171,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"successfully return the trace combo for a trace id" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val traceId = 123L
@@ -191,7 +191,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val index = new MockIndex {
override def mockSpanName = None
}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(2, 3, 1)
@@ -203,7 +203,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"find traces in annotation index by timestamp annotation, fetch from storage" in {
val storage = mock[Storage]
val index = new MockIndex {}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(2, 3, 1)
@@ -217,7 +217,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
val index = new MockIndex {
override def mockValue = Some(ByteBuffer.wrap("value".getBytes))
}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val expected = List(2, 3, 1)
@@ -230,7 +230,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"fetch traces from storage" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
expect {
@@ -245,7 +245,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"fetch timeline from storage" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index,
+ val qs = new QueryService(storage, index, null,
Map(gen.Adjust.Nothing -> NullAdjuster, gen.Adjust.TimeSkew -> new TimeSkewAdjuster()))
qs.start()
@@ -270,7 +270,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"fetch timeline with clock skew from storage, fix skew" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map(gen.Adjust.TimeSkew -> new TimeSkewAdjuster()))
+ val qs = new QueryService(storage, index, null, Map(gen.Adjust.TimeSkew -> new TimeSkewAdjuster()))
qs.start()
// these are real traces, except for timestap that has been chopped down to make easier to spot
@@ -332,7 +332,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"fetch timeline with clock skew from storage, fix skew - the sequel" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map(gen.Adjust.TimeSkew -> new TimeSkewAdjuster()))
+ val qs = new QueryService(storage, index, null, Map(gen.Adjust.TimeSkew -> new TimeSkewAdjuster()))
qs.start()
// these are real traces, except for timestap that has been chopped down to make easier to spot
@@ -408,7 +408,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
override def ids: Seq[Long] = Seq()
override def getTracesDuration(traceIds: Seq[Long]): Future[Seq[TraceIdDuration]] = Future(Seq())
}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
List() mustEqual qs.getTraceIdsBySpanName("service", "methodcall", 100, 50, gen.Order.DurationDesc)()
@@ -420,7 +420,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
override def ids: Seq[Long] = Seq()
override def getTracesDuration(traceIds: Seq[Long]): Future[Seq[TraceIdDuration]] = Future(Seq())
}
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
List() mustEqual qs.getTraceIdsByAnnotation("service", "annotation", null, 100, 50, gen.Order.DurationDesc)()
@@ -429,7 +429,7 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
"return the correct ttl" in {
val storage = mock[Storage]
val index = mock[Index]
- val qs = new QueryService(storage, index, Map())
+ val qs = new QueryService(storage, index, null, Map())
qs.start()
val ttl = 123
@@ -439,5 +439,33 @@ class QueryServiceSpec extends Specification with JMocker with ClassMocker {
ttl mustEqual qs.getDataTimeToLive()()
}
+
+ "retrieve aggregates" in {
+ val aggregates = mock[Aggregates]
+ val serviceName = "mockingbird"
+ val annotations = Seq("a", "b", "c")
+
+ "retrieve top annotations" in {
+ val qs = new QueryService(null, null, aggregates, Map())
+ qs.start()
+
+ expect {
+ one(aggregates).getTopAnnotations(serviceName) willReturn Future.value(annotations)
+ }
+
+ qs.getTopAnnotations(serviceName)() mustEqual annotations
+ }
+
+ "retrieve top key value annotations" in {
+ val qs = new QueryService(null, null, aggregates, Map())
+ qs.start()
+
+ expect {
+ one(aggregates).getTopKeyValueAnnotations(serviceName) willReturn Future.value(annotations)
+ }
+
+ qs.getTopKeyValueAnnotations(serviceName)() mustEqual annotations
+ }
+ }
}
}
View
111 ...-server/src/test/scala/com/twitter/zipkin/storage/cassandra/CassandraAggregatesSpec.scala
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2012 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.zipkin.storage.cassandra
+
+import com.twitter.cassie.{Column, ColumnFamily}
+import com.twitter.cassie.tests.util.FakeCassandra
+import com.twitter.io.TempFile
+import com.twitter.ostrich.admin.RuntimeEnvironment
+import com.twitter.util.{Eval, Future}
+import com.twitter.zipkin.config.CassandraAggregatesConfig
+import org.specs.mock.{ClassMocker, JMocker}
+import org.specs.Specification
+import scala.collection.JavaConverters._
+
+class CassandraAggregatesSpec extends Specification with JMocker with ClassMocker {
+
+ val mockCf = mock[ColumnFamily[String, Long, String]]
+
+ def cassandraAggregates = new CassandraAggregates {
+ val topAnnotations = mockCf
+ }
+
+ def column(name: Long, value: String) = new Column[Long, String](name, value)
+
+ "CassandraAggregates" should {
+ val topAnnsSeq = Seq("finagle.retry", "finagle.timeout", "annotation1", "annotation2")
+ val topAnns = topAnnsSeq.zipWithIndex.map { case (ann, index) =>
+ index.toLong -> column(index, ann)
+ }.toMap.asJava
+
+ "retrieval" in {
+ "getTopAnnotations" in {
+ val agg = cassandraAggregates
+ val serviceName = "mockingbird"
+ val rowKey = agg.topAnnotationRowKey(serviceName)
+
+ expect {
+ one(mockCf).getRow(rowKey) willReturn Future.value(topAnns)
+ }
+
+ agg.getTopAnnotations(serviceName)() mustEqual topAnnsSeq
+ }
+
+ "getTopKeyValueAnnotations" in {
+ val agg = cassandraAggregates
+ val serviceName = "mockingbird"
+ val rowKey = agg.topKeyValueRowKey(serviceName)
+
+ expect {
+ one(mockCf).getRow(rowKey) willReturn Future.value(topAnns)
+ }
+
+ agg.getTopKeyValueAnnotations(serviceName)() mustEqual topAnnsSeq
+ }
+ }
+
+ "storage" in {
+ object FakeServer extends FakeCassandra
+ var agg: CassandraAggregates = null
+ val serviceName = "mockingbird"
+
+ doBefore {
+ FakeServer.start()
+ val test = TempFile.fromResourcePath("/CassandraAggregatesConfig.scala")
+ val env = RuntimeEnvironment(this, Array("-f", test.toString))
+ val config = new Eval().apply[CassandraAggregatesConfig](env.configFile)
+ config.cassandraConfig.port = FakeServer.port.get
+ agg = config.apply()
+ }
+
+ doAfter {
+ agg.close()
+ FakeServer.stop()
+ }
+
+ "storeTopAnnotations" in {
+ agg.storeTopAnnotations(serviceName, topAnnsSeq).apply()
+ agg.getTopAnnotations(serviceName).apply() mustEqual topAnnsSeq
+ }
+
+ "storeTopKeyValueAnnotations" in {
+ agg.storeTopKeyValueAnnotations(serviceName, topAnnsSeq).apply()
+ agg.getTopKeyValueAnnotations(serviceName).apply() mustEqual topAnnsSeq
+ }
+
+ "clobber old entries" in {
+ val anns1 = Seq("a1", "a2", "a3", "a4")
+ val anns2 = Seq("a5", "a6")
+
+ agg.storeTopAnnotations(serviceName, anns1).apply()
+ agg.getTopAnnotations(serviceName).apply() mustEqual anns1
+
+ agg.storeTopAnnotations(serviceName, anns2).apply()
+ agg.getTopAnnotations(serviceName).apply() mustEqual anns2
+ }
+ }
+ }
+}
View
4 zipkin-test/src/test/resources/TestCollector.scala
@@ -57,6 +57,10 @@ new ScribeZipkinCollectorConfig {
def cassandraConfig = _cassandraConfig
}
+ def aggregatesConfig = new CassandraAggregatesConfig {
+ def cassandraConfig = _cassandraConfig
+ }
+
override def adaptiveSamplerConfig = new NullAdaptiveSamplerConfig {}
// sample it all
View
4 zipkin-test/src/test/resources/TestQuery.scala
@@ -46,6 +46,10 @@ new ZipkinQueryConfig {
def cassandraConfig = _cassandraConfig
}
+ def aggregatesConfig = new CassandraAggregatesConfig {
+ def cassandraConfig = _cassandraConfig
+ }
+
def zkConfig = new ZooKeeperConfig {
servers = List("localhost:2181")
}
View
2 zipkin-test/src/test/scala/com/twitter/zipkin/ZipkinSpec.scala
@@ -86,7 +86,7 @@ class ZipkinSpec extends Specification with JMocker with ClassMocker {
collector = new ZipkinCollector(collectorConfig)
collector.start()
- query = new ZipkinQuery(queryConfig, nullServerSetsImpl, queryConfig.storage, queryConfig.index)
+ query = new ZipkinQuery(queryConfig, nullServerSetsImpl, queryConfig.storage, queryConfig.index, queryConfig.aggregates)
query.start()
queryTransport = ClientBuilder()
View
8 zipkin-thrift/src/main/thrift/zipkinCollector.thrift
@@ -20,8 +20,16 @@ exception AdjustableRateException {
1: string msg
}
+exception StoreAggregatesException {
+ 1: string msg
+}
+
service ZipkinCollector extends scribe.scribe {
+ /** Aggregates methods */
+ void storeTopAnnotations(1: string service_name, 2: list<string> annotations) throws (1: StoreAggregatesException e);
+ void storeTopKeyValueAnnotations(1: string service_name, 2: list<string> annotations) throws (1: StoreAggregatesException e);
+
//************** ZK config changes **************
/**
View
4 zipkin-thrift/src/main/thrift/zipkinQuery.thrift
@@ -186,4 +186,8 @@ service ZipkinQuery {
* Get the data ttl. This is the number of seconds we keep the data around before deleting it.
*/
i32 getDataTimeToLive() throws (1: QueryException qe);
+
+ /** Aggregates related */
+ list<string> getTopAnnotations(1: string service_name) throws (1: QueryException qe);
+ list<string> getTopKeyValueAnnotations(1: string service_name) throws (1: QueryException qe);
}

0 comments on commit 3e15fe1

Please sign in to comment.