Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

don't block when there are no buffers left #1641

Merged
merged 3 commits into from
Jun 30, 2020
Merged

don't block when there are no buffers left #1641

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d79f43
don't block when there are no buffers left
richardstartin Jun 30, 2020
9dc0378
add monitoring for backups
richardstartin Jun 30, 2020
e2f46a9
add a test for backoff behaviour
richardstartin Jun 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
25 changes: 24 additions & 1 deletion dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DispatchingDisruptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.lmax.disruptor.BlockingWaitStrategy;
import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.EventHandler;
import com.lmax.disruptor.InsufficientCapacityException;
import com.lmax.disruptor.dsl.Disruptor;
import com.lmax.disruptor.dsl.ProducerType;
import datadog.common.exec.DaemonThreadFactory;
Expand All @@ -18,6 +19,7 @@
public class DispatchingDisruptor implements AutoCloseable {

private final Disruptor<TraceBuffer> disruptor;
private final Monitor monitor;

public DispatchingDisruptor(
int disruptorSize,
Expand All @@ -34,6 +36,7 @@ public DispatchingDisruptor(
// block (and use no resources) until there's a batch of data to dispatch
new BlockingWaitStrategy());
disruptor.handleEventsWith(new TraceDispatchingHandler(api, monitor, writer));
this.monitor = monitor;
}

public void start() {
Expand All @@ -46,7 +49,27 @@ public void close() {
}

long beginTransaction() {
return disruptor.getRingBuffer().next();
long backoffMillis = 1;
long nextLogTime = 0;
while (!Thread.currentThread().isInterrupted()) {
try {
return disruptor.getRingBuffer().tryNext();
} catch (InsufficientCapacityException insufficientCapacity) {
monitor.onBackedUpTraceBuffer();
long now = System.currentTimeMillis();
backoffMillis = Math.min(backoffMillis * 2, 1000);
if (now > nextLogTime) { // log every 20 seconds
log.debug("no buffer available, sleeping for {}ms", backoffMillis);
richardstartin marked this conversation as resolved.
Show resolved Hide resolved
nextLogTime = now + 20_000;
}
try {
Thread.sleep(backoffMillis);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
}
return -1L;
}

TraceBuffer getTraceBuffer(long sequence) {
Expand Down
10 changes: 10 additions & 0 deletions dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Monitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public interface Monitor {

void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previousIncomplete);

void onBackedUpTraceBuffer();

void onSerialize(
final DDAgentWriter agentWriter, final List<DDSpan> trace, final int serializedSizeInBytes);

Expand Down Expand Up @@ -114,6 +116,11 @@ public void onScheduleFlush(final DDAgentWriter agentWriter, final boolean previ
// not recorded
}

@Override
public void onBackedUpTraceBuffer() {
statsd.incrementCounter("trace.buffer.backlog");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

}

@Override
public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}

Expand Down Expand Up @@ -203,6 +210,9 @@ public void onFlush(final DDAgentWriter agentWriter, final boolean early) {}
public void onScheduleFlush(
final DDAgentWriter agentWriter, final boolean previousIncomplete) {}

@Override
public void onBackedUpTraceBuffer() {}

@Override
public void onSerialize(
final DDAgentWriter agentWriter,
Expand Down