From 9dc0378a16d51f63f0c492867062ef9678b3e996 Mon Sep 17 00:00:00 2001
From: Richard Startin <richard.startin@datadoghq.com>
Date: Tue, 30 Jun 2020 14:35:37 +0100
Subject: [PATCH] add monitoring for backups

---
 .../common/writer/ddagent/DispatchingDisruptor.java    |  3 +++
 .../datadog/trace/common/writer/ddagent/Monitor.java   | 10 ++++++++++
 2 files changed, 13 insertions(+)

diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java
index fecfddecce5..071229eb55a 100644
--- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java
+++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java
@@ -19,6 +19,7 @@
 public class DispatchingDisruptor implements AutoCloseable {
 
   private final Disruptor<TraceBuffer> disruptor;
+  private final Monitor monitor;
 
   public DispatchingDisruptor(
       int disruptorSize,
@@ -35,6 +36,7 @@ public DispatchingDisruptor(
             // block (and use no resources) until there's a batch of data to dispatch
             new BlockingWaitStrategy());
     disruptor.handleEventsWith(new TraceDispatchingHandler(api, monitor, writer));
+    this.monitor = monitor;
   }
 
   public void start() {
@@ -53,6 +55,7 @@ long beginTransaction() {
       try {
         return disruptor.getRingBuffer().tryNext();
       } catch (InsufficientCapacityException insufficientCapacity) {
+        monitor.onBackedUpTraceBuffer();
         long now = System.currentTimeMillis();
         backoffMillis = Math.min(backoffMillis * 2, 1000);
         if (now > nextLogTime) { // log every 20 seconds
diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java
index 34cb1b12d64..e16522479d5 100644
--- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java
+++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java
@@ -32,6 +32,8 @@ public interface Monitor {
 
   void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previousIncomplete);
 
+  void onBackedUpTraceBuffer();
+
   void onSerialize(
       final DDAgentWriter agentWriter, final List<DDSpan> trace, final int serializedSizeInBytes);
 
@@ -114,6 +116,11 @@ public void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previ
       // not recorded
     }
 
+    @Override
+    public void onBackedUpTraceBuffer() {
+      statsd.incrementCounter("trace.buffer.backlog");
+    }
+
     @Override
     public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}
 
@@ -203,6 +210,9 @@ public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}
     public void onScheduleFlush(
         final DDAgentWriter agentWriter, final boolean previousIncomplete) {}
 
+    @Override
+    public void onBackedUpTraceBuffer() {}
+
     @Override
     public void onSerialize(
         final DDAgentWriter agentWriter,