finagle|util: Verbosity levels for StatsReceivers

Summary: This change introduces the concept of verbosity levels (go/verbose-metrics) for `StatsReceiver`s.

Each metric created via a stats receiver has a `Verbosity` level attached to it. Distinguishing verbosity
levels for metrics is optional and is up to a concrete implementation. Doing this, however, helps to
separate `Verbosity.Debug` metrics (only helpful in troubleshooting) from their operationally required
counterparts (provide a corresponding degree of visibility into a healthy process) thus
potentially reducing the observability cost.

JIRA Issues: CSL-4564

TBR=true

Differential Revision: https://phabricator.twitter.biz/D70112

finagle-core/src/main/scala/com/twitter/finagle/stats/JavaLoggerStatsReceiver.scala

@@ -3,7 +3,7 @@ package com.twitter.finagle.stats
 import com.twitter.conversions.time._
 import com.twitter.finagle.util.DefaultTimer
 import com.twitter.util.{Timer, TimerTask}
-import java.util.logging.Logger
+import java.util.logging.{Level, Logger}
 import scala.collection.mutable
 
 class JavaLoggerStatsReceiver(logger: Logger, timer: Timer)
@@ -16,25 +16,30 @@ class JavaLoggerStatsReceiver(logger: Logger, timer: Timer)
   // is used for debugging only.
   def this(logger: Logger) = this(logger, DefaultTimer)
 
-  def stat(name: String*): Stat = new Stat {
+  def stat(verbosity: Verbosity, name: String*): Stat = new Stat {
     def add(value: Float) {
-      logger.info("%s add %f".format(formatName(name), value))
+      val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
+      logger.log(level, "%s add %f".format(formatName(name), value))
     }
   }
 
-  def counter(name: String*): Counter = new Counter {
-    def incr(delta: Long): Unit = {
-      logger.info("%s incr %d".format(formatName(name), delta))
+  def counter(verbosity: Verbosity, name: String*): Counter = new Counter {
+    def incr(delta: Long) {
+      val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
+      logger.log(level, "%s incr %d".format(formatName(name), delta))
     }
   }
 
-  protected[this] def registerGauge(name: Seq[String], f: => Float): Unit = synchronized {
-    deregisterGauge(name)
+  protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
+    synchronized {
+      deregisterGauge(name)
 
-    timerTasks(name) = timer.schedule(10.seconds) {
-      logger.info("%s %2f".format(formatName(name), f))
+      val level = if (verbosity == Verbosity.Debug) Level.FINEST else Level.INFO
+
+      timerTasks(name) = timer.schedule(10.seconds) {
+        logger.log(level, "%s %2f".format(formatName(name), f))
+      }
     }
-  }
 
   protected[this] def deregisterGauge(name: Seq[String]): Unit = synchronized {
     timerTasks.remove(name) foreach { _.cancel() }

finagle-core/src/main/scala/com/twitter/finagle/stats/SummarizingStatsReceiver.scala

@@ -12,7 +12,8 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 
 class SummarizingStatsReceiver extends StatsReceiverWithCumulativeGauges {
-  val repr = this
+
+  def repr = this
 
   private[this] val counters = new ConcurrentHashMap[Seq[String], AtomicLong]()
 
@@ -26,21 +27,22 @@ class SummarizingStatsReceiver extends StatsReceiverWithCumulativeGauges {
   private[this] var _gauges = Map[Seq[String], () => Float]()
   def gauges: Map[Seq[String], () => Float] = synchronized { _gauges }
 
-  def counter(name: String*): Counter = new Counter {
+  def counter(verbosity: Verbosity, name: String*): Counter = new Counter {
     counters.putIfAbsent(name, new AtomicLong(0))
     def incr(delta: Long) { counters.get(name).getAndAdd(delta) }
   }
 
-  def stat(name: String*): Stat = new Stat {
-    def add(value: Float) = SummarizingStatsReceiver.this.synchronized {
+  def stat(verbosity: Verbosity, name: String*): Stat = new Stat {
+    def add(value: Float): Unit = SummarizingStatsReceiver.this.synchronized {
       stats.get(name) += value
     }
   }
 
   // Ignoring gauges for now, but we may consider sampling them.
-  protected[this] def registerGauge(name: Seq[String], f: => Float): Unit = synchronized {
-    _gauges += (name -> (() => f))
-  }
+  protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
+    synchronized {
+      _gauges += (name -> (() => f))
+    }
 
   protected[this] def deregisterGauge(name: Seq[String]): Unit = synchronized {
     _gauges -= name

finagle-core/src/test/scala/com/twitter/finagle/loadbalancer/TrafficDistributorTest.scala

@@ -135,10 +135,12 @@ private object TrafficDistributorTest {
   {
     private[this] val underlying = new InMemoryStatsReceiver()
     override val repr: AnyRef = this
-    override def counter(name: String*): ReadableCounter = underlying.counter(name: _*)
-    override def stat(name: String*): ReadableStat = underlying.stat(name: _*)
+    override def counter(verbosity: Verbosity, name: String*): ReadableCounter =
+      underlying.counter(verbosity, name: _*)
+    override def stat(verbosity: Verbosity, name: String*): ReadableStat =
+      underlying.stat(verbosity, name: _*)
 
-    protected[this] def registerGauge(name: Seq[String], f: => Float): Unit =
+    protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit =
       underlying.addGauge(name: _*)(f)
 
     protected[this] def deregisterGauge(name: Seq[String]): Unit =

finagle-http/src/test/scala/com/twitter/finagle/http/filter/StatsFilterTest.scala

@@ -2,7 +2,7 @@ package com.twitter.finagle.http.filter
 
 import com.twitter.finagle.Service
 import com.twitter.finagle.http.{Request, Response}
-import com.twitter.finagle.stats.InMemoryStatsReceiver
+import com.twitter.finagle.stats.{InMemoryStatsReceiver, Verbosity}
 import com.twitter.util.{Await, Duration, Future, Time}
 import org.junit.runner.RunWith
 import org.scalatest.FunSuite
@@ -48,10 +48,10 @@ class StatsFilterTest extends FunSuite {
     }
 
     // Verify that the counters and stats were only created once
-    verify(receiver).counter("status", "404")
-    verify(receiver).counter("status", "4XX")
-    verify(receiver).stat("time", "404")
-    verify(receiver).stat("time", "4XX")
-    verify(receiver).stat("response_size")
+    verify(receiver).counter(Verbosity.Default, "status", "404")
+    verify(receiver).counter(Verbosity.Default, "status", "4XX")
+    verify(receiver).stat(Verbosity.Default, "time", "404")
+    verify(receiver).stat(Verbosity.Default, "time", "4XX")
+    verify(receiver).stat(Verbosity.Default, "response_size")
   }
 }

finagle-stats/src/main/scala/com/twitter/finagle/stats/MetricsStatsReceiver.scala

@@ -117,7 +117,7 @@ class MetricsStatsReceiver(
   /**
    * Create and register a counter inside the underlying Metrics library
    */
-  def counter(names: String*): Counter = {
+  def counter(verbosity: Verbosity, names: String*): Counter = {
     if (log.isLoggable(Level.TRACE))
       log.trace(s"Calling StatsReceiver.counter on $names")
     counterRequests.increment()
@@ -140,7 +140,7 @@ class MetricsStatsReceiver(
   /**
    * Create and register a stat (histogram) inside the underlying Metrics library
    */
-  def stat(names: String*): Stat = {
+  def stat(verbosity: Verbosity, names: String*): Stat = {
     if (log.isLoggable(Level.TRACE))
       log.trace(s"Calling StatsReceiver.stat for $names")
     statRequests.increment()
@@ -172,14 +172,14 @@ class MetricsStatsReceiver(
     stat
   }
 
-  override def addGauge(name: String*)(f: => Float): Gauge = {
+  override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = {
     if (log.isLoggable(Level.TRACE))
       log.trace(s"Calling StatsReceiver.addGauge for $name")
     gaugeRequests.increment()
-    super.addGauge(name: _*)(f)
+    super.addGauge(verbosity, name: _*)(f)
   }
 
-  protected[this] def registerGauge(names: Seq[String], f: => Float) {
+  protected[this] def registerGauge(verbosity: Verbosity, names: Seq[String], f: => Float) {
     val gauge = new AbstractGauge[java.lang.Double](format(names)) {
       override def read = new java.lang.Double(f)
     }