Permalink
Browse files

finagle|util: Verbosity levels for StatsReceivers

Summary: This change introduces the concept of verbosity levels (go/verbose-metrics) for `StatsReceiver`s.

Each metric created via a stats receiver has a `Verbosity` level attached to it. Distinguishing verbosity
levels for metrics is optional and is up to a concrete implementation. Doing this, however, helps to
separate `Verbosity.Debug` metrics (only helpful in troubleshooting) from their operationally required
counterparts (provide a corresponding degree of visibility into a healthy process) thus
potentially reducing the observability cost.

JIRA Issues: CSL-4564

TBR=true

Differential Revision: https://phabricator.twitter.biz/D70112
  • Loading branch information...
vkostyukov authored and jenkins committed Jul 25, 2017
1 parent 28aec2c commit fa91412d243eae8146465a439d69c78a1caca9c6
@@ -3,7 +3,7 @@ package com.twitter.finagle.stats
import com.twitter.conversions.time._
import com.twitter.finagle.util.DefaultTimer
import com.twitter.util.{Timer, TimerTask}
import java.util.logging.Logger
import java.util.logging.{Level, Logger}
import scala.collection.mutable
class JavaLoggerStatsReceiver(logger: Logger, timer: Timer)
@@ -16,25 +16,30 @@ class JavaLoggerStatsReceiver(logger: Logger, timer: Timer)
// is used for debugging only.
def this(logger: Logger) = this(logger, DefaultTimer)
def stat(name: String*): Stat = new Stat {
def stat(verbosity: Verbosity, name: String*): Stat = new Stat {
def add(value: Float) {
logger.info("%s add %f".format(formatName(name), value))
val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
logger.log(level, "%s add %f".format(formatName(name), value))
}
}
def counter(name: String*): Counter = new Counter {
def incr(delta: Long): Unit = {
logger.info("%s incr %d".format(formatName(name), delta))
def counter(verbosity: Verbosity, name: String*): Counter = new Counter {
def incr(delta: Long) {
val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
logger.log(level, "%s incr %d".format(formatName(name), delta))
}
}
protected[this] def registerGauge(name: Seq[String], f: => Float): Unit = synchronized {
deregisterGauge(name)
protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
synchronized {
deregisterGauge(name)
timerTasks(name) = timer.schedule(10.seconds) {
logger.info("%s %2f".format(formatName(name), f))
val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
timerTasks(name) = timer.schedule(10.seconds) {
logger.log(level, "%s %2f".format(formatName(name), f))
}
}
}
protected[this] def deregisterGauge(name: Seq[String]): Unit = synchronized {
timerTasks.remove(name) foreach { _.cancel() }
@@ -12,7 +12,8 @@ import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
class SummarizingStatsReceiver extends StatsReceiverWithCumulativeGauges {
val repr = this
def repr = this
private[this] val counters = new ConcurrentHashMap[Seq[String], AtomicLong]()
@@ -26,21 +27,22 @@ class SummarizingStatsReceiver extends StatsReceiverWithCumulativeGauges {
private[this] var _gauges = Map[Seq[String], () => Float]()
def gauges: Map[Seq[String], () => Float] = synchronized { _gauges }
def counter(name: String*): Counter = new Counter {
def counter(verbosity: Verbosity, name: String*): Counter = new Counter {
counters.putIfAbsent(name, new AtomicLong(0))
def incr(delta: Long) { counters.get(name).getAndAdd(delta) }
}
def stat(name: String*): Stat = new Stat {
def add(value: Float) = SummarizingStatsReceiver.this.synchronized {
def stat(verbosity: Verbosity, name: String*): Stat = new Stat {
def add(value: Float): Unit = SummarizingStatsReceiver.this.synchronized {
stats.get(name) += value
}
}
// Ignoring gauges for now, but we may consider sampling them.
protected[this] def registerGauge(name: Seq[String], f: => Float): Unit = synchronized {
_gauges += (name -> (() => f))
}
protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
synchronized {
_gauges += (name -> (() => f))
}
protected[this] def deregisterGauge(name: Seq[String]): Unit = synchronized {
_gauges -= name
@@ -135,10 +135,12 @@ private object TrafficDistributorTest {
{
private[this] val underlying = new InMemoryStatsReceiver()
override val repr: AnyRef = this
override def counter(name: String*): ReadableCounter = underlying.counter(name: _*)
override def stat(name: String*): ReadableStat = underlying.stat(name: _*)
override def counter(verbosity: Verbosity, name: String*): ReadableCounter =
underlying.counter(verbosity, name: _*)
override def stat(verbosity: Verbosity, name: String*): ReadableStat =
underlying.stat(verbosity, name: _*)
protected[this] def registerGauge(name: Seq[String], f: => Float): Unit =
protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
underlying.addGauge(name: _*)(f)
protected[this] def deregisterGauge(name: Seq[String]): Unit =
@@ -2,7 +2,7 @@ package com.twitter.finagle.http.filter
import com.twitter.finagle.Service
import com.twitter.finagle.http.{Request, Response}
import com.twitter.finagle.stats.InMemoryStatsReceiver
import com.twitter.finagle.stats.{InMemoryStatsReceiver, Verbosity}
import com.twitter.util.{Await, Duration, Future, Time}
import org.junit.runner.RunWith
import org.scalatest.FunSuite
@@ -48,10 +48,10 @@ class StatsFilterTest extends FunSuite {
}
// Verify that the counters and stats were only created once
verify(receiver).counter("status", "404")
verify(receiver).counter("status", "4XX")
verify(receiver).stat("time", "404")
verify(receiver).stat("time", "4XX")
verify(receiver).stat("response_size")
verify(receiver).counter(Verbosity.Default, "status", "404")
verify(receiver).counter(Verbosity.Default, "status", "4XX")
verify(receiver).stat(Verbosity.Default, "time", "404")
verify(receiver).stat(Verbosity.Default, "time", "4XX")
verify(receiver).stat(Verbosity.Default, "response_size")
}
}
@@ -117,7 +117,7 @@ class MetricsStatsReceiver(
/**
* Create and register a counter inside the underlying Metrics library
*/
def counter(names: String*): Counter = {
def counter(verbosity: Verbosity, names: String*): Counter = {
if (log.isLoggable(Level.TRACE))
log.trace(s"Calling StatsReceiver.counter on $names")
counterRequests.increment()
@@ -140,7 +140,7 @@ class MetricsStatsReceiver(
/**
* Create and register a stat (histogram) inside the underlying Metrics library
*/
def stat(names: String*): Stat = {
def stat(verbosity: Verbosity, names: String*): Stat = {
if (log.isLoggable(Level.TRACE))
log.trace(s"Calling StatsReceiver.stat for $names")
statRequests.increment()
@@ -172,14 +172,14 @@ class MetricsStatsReceiver(
stat
}
override def addGauge(name: String*)(f: => Float): Gauge = {
override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = {
if (log.isLoggable(Level.TRACE))
log.trace(s"Calling StatsReceiver.addGauge for $name")
gaugeRequests.increment()
super.addGauge(name: _*)(f)
super.addGauge(verbosity, name: _*)(f)
}
protected[this] def registerGauge(names: Seq[String], f: => Float) {
protected[this] def registerGauge(verbosity: Verbosity, names: Seq[String], f: => Float) {
val gauge = new AbstractGauge[java.lang.Double](format(names)) {
override def read = new java.lang.Double(f)
}

0 comments on commit fa91412

Please sign in to comment.