add monitoring for backups

DataDog · Jun 30, 2020 · 9dc0378 · 9dc0378
1 parent 1d79f43
commit 9dc0378
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java
@@ -19,6 +19,7 @@
 public class DispatchingDisruptor implements AutoCloseable {
 
   private final Disruptor<TraceBuffer> disruptor;
+  private final Monitor monitor;
 
   public DispatchingDisruptor(
       int disruptorSize,
@@ -35,6 +36,7 @@ public DispatchingDisruptor(
             // block (and use no resources) until there's a batch of data to dispatch
             new BlockingWaitStrategy());
     disruptor.handleEventsWith(new TraceDispatchingHandler(api, monitor, writer));
+    this.monitor = monitor;
   }
 
   public void start() {
@@ -53,6 +55,7 @@ long beginTransaction() {
       try {
         return disruptor.getRingBuffer().tryNext();
       } catch (InsufficientCapacityException insufficientCapacity) {
+        monitor.onBackedUpTraceBuffer();
         long now = System.currentTimeMillis();
         backoffMillis = Math.min(backoffMillis * 2, 1000);
         if (now > nextLogTime) { // log every 20 seconds

diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java
@@ -32,6 +32,8 @@ public interface Monitor {
 
   void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previousIncomplete);
 
+  void onBackedUpTraceBuffer();
+
   void onSerialize(
       final DDAgentWriter agentWriter, final List<DDSpan> trace, final int serializedSizeInBytes);
 
@@ -114,6 +116,11 @@ public void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previ
       // not recorded
     }
 
+    @Override
+    public void onBackedUpTraceBuffer() {
+      statsd.incrementCounter("trace.buffer.backlog");
+    }
+
     @Override
     public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}
 
@@ -203,6 +210,9 @@ public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}
     public void onScheduleFlush(
         final DDAgentWriter agentWriter, final boolean previousIncomplete) {}
 
+    @Override
+    public void onBackedUpTraceBuffer() {}
+
     @Override
     public void onSerialize(
         final DDAgentWriter agentWriter,