Add trace_segment_ref_limit_per_span configuration mechanism to avoid OOM

CHANGES.md

@@ -8,6 +8,7 @@ Release Notes.
 
 
 #### Java Agent
+* Add `trace_segment_ref_limit_per_span` configuration mechanism to avoid OOM.
 
 
 #### OAP-Backend

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java

@@ -66,6 +66,11 @@ public static class Agent {
          */
         public static String IGNORE_SUFFIX = ".jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg";
 
+        /**
+         * The max number of TraceSegmentRef in a single span to keep memory cost estimatable.
+         */
+        public static int TRACE_SEGMENT_REF_LIMIT_PER_SPAN = 500;
+
         /**
          * The max number of spans in a single segment. Through this config item, SkyWalking keep your application
          * memory cost estimated.

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/TracingContext.java

@@ -189,7 +189,7 @@ public void inject(AbstractSpan exitSpan, ContextCarrier carrier) {
     public void extract(ContextCarrier carrier) {
         TraceSegmentRef ref = new TraceSegmentRef(carrier);
         this.segment.ref(ref);
-        this.segment.relatedGlobalTraces(new PropagatedTraceId(carrier.getTraceId()));
+        this.segment.relatedGlobalTrace(new PropagatedTraceId(carrier.getTraceId()));
         AbstractSpan span = this.activeSpan();
         if (span instanceof EntrySpan) {
             span.ref(ref);
@@ -229,7 +229,7 @@ public void continued(ContextSnapshot snapshot) {
             TraceSegmentRef segmentRef = new TraceSegmentRef(snapshot);
             this.segment.ref(segmentRef);
             this.activeSpan().ref(segmentRef);
-            this.segment.relatedGlobalTraces(snapshot.getTraceId());
+            this.segment.relatedGlobalTrace(snapshot.getTraceId());
             this.correlationContext.continued(snapshot);
             this.extensionContext.continued(snapshot);
             this.extensionContext.handle(this.activeSpan());
@@ -245,7 +245,7 @@ public String getReadablePrimaryTraceId() {
     }
 
     private DistributedTraceId getPrimaryTraceId() {
-        return segment.getRelatedGlobalTraces().get(0);
+        return segment.getRelatedGlobalTrace();
     }
 
     @Override

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/trace/AbstractTracingSpan.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
+import org.apache.skywalking.apm.agent.core.conf.Config;
 import org.apache.skywalking.apm.agent.core.context.ContextManager;
 import org.apache.skywalking.apm.agent.core.context.TracingContext;
 import org.apache.skywalking.apm.agent.core.context.status.StatusCheckService;
@@ -301,6 +302,12 @@ public void ref(TraceSegmentRef ref) {
         if (refs == null) {
             refs = new LinkedList<>();
         }
+        /*
+         * Provide the OOM protection if the entry span hosts too many references.
+         */
+        if (refs.size() == Config.Agent.TRACE_SEGMENT_REF_LIMIT_PER_SPAN) {
+            return;
+        }
         if (!refs.contains(ref)) {
             refs.add(ref);
         }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/trace/TraceSegment.java

@@ -22,7 +22,6 @@
 import java.util.List;
 import org.apache.skywalking.apm.agent.core.conf.Config;
 import org.apache.skywalking.apm.agent.core.context.ids.DistributedTraceId;
-import org.apache.skywalking.apm.agent.core.context.ids.DistributedTraceIds;
 import org.apache.skywalking.apm.agent.core.context.ids.GlobalIdGenerator;
 import org.apache.skywalking.apm.agent.core.context.ids.NewDistributedTraceId;
 import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject;
@@ -39,13 +38,13 @@ public class TraceSegment {
     private String traceSegmentId;
 
     /**
-     * The refs of parent trace segments, except the primary one. For most RPC call, {@link #refs} contains only one
+     * The refs of parent trace segments, except the primary one. For most RPC call, {@link #ref} contains only one
      * element, but if this segment is a start span of batch process, the segment faces multi parents, at this moment,
-     * we use this {@code #refs} to link them.
+     * we only cache the first parent segment reference.
      * <p>
      * This field will not be serialized. Keeping this field is only for quick accessing.
      */
-    private List<TraceSegmentRef> refs;
+    private TraceSegmentRef ref;
 
     /**
      * The spans belong to this trace segment. They all have finished. All active spans are hold and controlled by
@@ -54,14 +53,11 @@ public class TraceSegment {
     private List<AbstractTracingSpan> spans;
 
     /**
-     * The <code>relatedGlobalTraces</code> represent a set of all related trace. Most time it contains only one
+     * The <code>relatedGlobalTraceId</code> represent the related trace. Most time it related only one
      * element, because only one parent {@link TraceSegment} exists, but, in batch scenario, the num becomes greater
-     * than 1, also meaning multi-parents {@link TraceSegment}. <p> The difference between
-     * <code>relatedGlobalTraces</code> and {@link #refs} is: {@link #refs} targets this {@link TraceSegment}'s direct
-     * parent, <p> and <p> <code>relatedGlobalTraces</code> targets this {@link TraceSegment}'s related call chain, a
-     * call chain contains multi {@link TraceSegment}s, only using {@link #refs} is not enough for analysis and ui.
+     * than 1, also meaning multi-parents {@link TraceSegment}. But we only related the first parent TraceSegment.
      */
-    private DistributedTraceIds relatedGlobalTraces;
+    private DistributedTraceId relatedGlobalTraceId;
 
     private boolean ignore = false;
 
@@ -75,8 +71,7 @@ public class TraceSegment {
     public TraceSegment() {
         this.traceSegmentId = GlobalIdGenerator.generate();
         this.spans = new LinkedList<>();
-        this.relatedGlobalTraces = new DistributedTraceIds();
-        this.relatedGlobalTraces.append(new NewDistributedTraceId());
+        this.relatedGlobalTraceId = new NewDistributedTraceId();
         this.createTime = System.currentTimeMillis();
     }
 
@@ -86,19 +81,18 @@ public TraceSegment() {
      * @param refSegment {@link TraceSegmentRef}
      */
     public void ref(TraceSegmentRef refSegment) {
-        if (refs == null) {
-            refs = new LinkedList<>();
-        }
-        if (!refs.contains(refSegment)) {
-            refs.add(refSegment);
+        if (null == ref) {
+            this.ref = refSegment;
         }
     }
 
     /**
-     * Establish the line between this segment and all relative global trace ids.
+     * Establish the line between this segment and the relative global trace id.
      */
-    public void relatedGlobalTraces(DistributedTraceId distributedTraceId) {
-        relatedGlobalTraces.append(distributedTraceId);
+    public void relatedGlobalTrace(DistributedTraceId distributedTraceId) {
+        if (relatedGlobalTraceId instanceof NewDistributedTraceId) {
+            this.relatedGlobalTraceId = distributedTraceId;
+        }
     }
 
     /**
@@ -121,16 +115,15 @@ public String getTraceSegmentId() {
         return traceSegmentId;
     }
 
-    public boolean hasRef() {
-        return !(refs == null || refs.size() == 0);
-    }
-
-    public List<TraceSegmentRef> getRefs() {
-        return refs;
+    /**
+     * Get the first parent segment reference.
+     */
+    public TraceSegmentRef getRef() {
+        return ref;
     }
 
-    public List<DistributedTraceId> getRelatedGlobalTraces() {
-        return relatedGlobalTraces.getRelatedGlobalTraces();
+    public DistributedTraceId getRelatedGlobalTrace() {
+        return relatedGlobalTraceId;
     }
 
     public boolean isSingleSpanSegment() {
@@ -152,7 +145,7 @@ public void setIgnore(boolean ignore) {
      */
     public SegmentObject transform() {
         SegmentObject.Builder traceSegmentBuilder = SegmentObject.newBuilder();
-        traceSegmentBuilder.setTraceId(getRelatedGlobalTraces().get(0).getId());
+        traceSegmentBuilder.setTraceId(getRelatedGlobalTrace().getId());
         /*
          * Trace Segment
          */
@@ -172,7 +165,7 @@ public SegmentObject transform() {
 
     @Override
     public String toString() {
-        return "TraceSegment{" + "traceSegmentId='" + traceSegmentId + '\'' + ", refs=" + refs + ", spans=" + spans + ", relatedGlobalTraces=" + relatedGlobalTraces + '}';
+        return "TraceSegment{" + "traceSegmentId='" + traceSegmentId + '\'' + ", ref=" + ref + ", spans=" + spans + "}";
     }
 
     public long createTime() {

apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/context/ContextManagerTest.java

@@ -54,6 +54,7 @@
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -90,7 +91,7 @@ public void createSpanWithInvalidateContextCarrier() {
         ContextManager.stopSpan();
 
         TraceSegment actualSegment = tracingData.getTraceSegments().get(0);
-        assertNull(actualSegment.getRefs());
+        assertNull(actualSegment.getRef());
 
         List<AbstractTracingSpan> spanList = SegmentHelper.getSpan(actualSegment);
         assertThat(Objects.requireNonNull(spanList).size(), is(1));
@@ -134,9 +135,9 @@ public void createMultipleEntrySpan() {
         assertThat(tracingData.getTraceSegments().size(), is(1));
 
         TraceSegment actualSegment = tracingData.getTraceSegments().get(0);
-        assertThat(actualSegment.getRefs().size(), is(1));
+        assertNotNull(actualSegment.getRef());
 
-        TraceSegmentRef ref = actualSegment.getRefs().get(0);
+        TraceSegmentRef ref = actualSegment.getRef();
         MatcherAssert.assertThat(TraceSegmentRefHelper.getPeerHost(ref), is("127.0.0.1:8080"));
 
         List<AbstractTracingSpan> spanList = SegmentHelper.getSpan(actualSegment);
@@ -192,7 +193,7 @@ public void createMultipleExitSpan() {
 
         assertThat(tracingData.getTraceSegments().size(), is(1));
         TraceSegment actualSegment = tracingData.getTraceSegments().get(0);
-        assertNull(actualSegment.getRefs());
+        assertNull(actualSegment.getRef());
 
         List<AbstractTracingSpan> spanList = SegmentHelper.getSpan(actualSegment);
         assertThat(Objects.requireNonNull(spanList).size(), is(2));

apm-sniffer/apm-sdk-plugin/dubbo-2.7.x-plugin/src/test/java/org/apache/skywalking/apm/plugin/dubbo/DubboInterceptorTest.java

@@ -194,7 +194,7 @@ private void assertProvider() {
         TraceSegment traceSegment = segmentStorage.getTraceSegments().get(0);
         assertThat(SegmentHelper.getSpans(traceSegment).size(), is(1));
         assertProviderSpan(SegmentHelper.getSpans(traceSegment).get(0));
-        assertTraceSegmentRef(traceSegment.getRefs().get(0));
+        assertTraceSegmentRef(traceSegment.getRef());
     }
 
     private void assertTraceSegmentRef(TraceSegmentRef actual) {

apm-sniffer/apm-sdk-plugin/dubbo-plugin/src/test/java/org/apache/skywalking/apm/plugin/dubbo/DubboInterceptorTest.java

@@ -193,7 +193,7 @@ private void assertProvider() {
         TraceSegment traceSegment = segmentStorage.getTraceSegments().get(0);
         assertThat(SegmentHelper.getSpans(traceSegment).size(), is(1));
         assertProviderSpan(SegmentHelper.getSpans(traceSegment).get(0));
-        assertTraceSegmentRef(traceSegment.getRefs().get(0));
+        assertTraceSegmentRef(traceSegment.getRef());
     }
 
     private void assertTraceSegmentRef(TraceSegmentRef actual) {

apm-sniffer/apm-sdk-plugin/finagle-6.25.x-plugin/src/test/java/org/apache/skywalking/apm/plugin/finagle/ServerTracingFilterInterceptorTest.java

@@ -135,7 +135,7 @@ interface TestFunction {
 
     private void assertServer() {
         TraceSegment traceSegment = segmentStorage.getTraceSegments().get(0);
-        TraceSegmentRef actual = traceSegment.getRefs().get(0);
+        TraceSegmentRef actual = traceSegment.getRef();
         assertThat(SegmentRefHelper.getSpanId(actual), is(3));
         assertThat(SegmentRefHelper.getParentServiceInstance(actual), is("instance"));
         assertThat(SegmentRefHelper.getTraceSegmentId(actual).toString(), is("3.4.5"));

apm-sniffer/apm-sdk-plugin/jetty-plugin/jetty-server-9.x-plugin/src/test/java/org/apache/skywalking/apm/plugin/jetty/v9/server/HandleInterceptorTest.java

@@ -119,7 +119,7 @@ public void testWithSerializedContextData() throws Throwable {
         List<AbstractTracingSpan> spans = SegmentHelper.getSpans(traceSegment);
 
         assertHttpSpan(spans.get(0));
-        assertTraceSegmentRef(traceSegment.getRefs().get(0));
+        assertTraceSegmentRef(traceSegment.getRef());
     }
 
     @Test

apm-sniffer/apm-sdk-plugin/kafka-plugin/src/test/java/org/apache/skywalking/apm/plugin/kafka/CallbackInterceptorTest.java

@@ -33,6 +33,7 @@
 import org.apache.skywalking.apm.agent.test.tools.SegmentStoragePoint;
 import org.apache.skywalking.apm.agent.test.tools.SpanAssert;
 import org.apache.skywalking.apm.agent.test.tools.TracingSegmentRunner;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -122,7 +123,7 @@ public void testCallbackWithoutException() throws Throwable {
 
         assertCallbackSpan(abstractSpans.get(0));
 
-        assertCallbackSegmentRef(traceSegment.getRefs());
+        assertCallbackSegmentRef(traceSegment.getRef());
     }
 
     @Test
@@ -139,7 +140,7 @@ public void testCallbackWithException() throws Throwable {
 
         assertCallbackSpanWithException(abstractSpans.get(0));
 
-        assertCallbackSegmentRef(traceSegment.getRefs());
+        assertCallbackSegmentRef(traceSegment.getRef());
     }
 
     @Test
@@ -170,11 +171,10 @@ private void assertCallbackSpanWithException(AbstractTracingSpan span) {
         assertThat(SpanHelper.getErrorOccurred(span), is(true));
     }
 
-    private void assertCallbackSegmentRef(List<TraceSegmentRef> refs) {
-        assertThat(refs.size(), is(1));
+    private void assertCallbackSegmentRef(TraceSegmentRef traceSegmentRef) {
+        Assert.assertNotNull(traceSegmentRef);
 
-        TraceSegmentRef segmentRef = refs.get(0);
-        SegmentRefAssert.assertSpanId(segmentRef, 1);
+        SegmentRefAssert.assertSpanId(traceSegmentRef, 1);
     }
 
     private void assertCallbackSpan(AbstractTracingSpan span) {

apm-sniffer/apm-sdk-plugin/kafka-plugin/src/test/java/org/apache/skywalking/apm/plugin/kafka/KafkaConsumerInterceptorTest.java

@@ -47,6 +47,7 @@
 
 import static org.apache.skywalking.apm.network.trace.component.ComponentsDefine.KAFKA_CONSUMER;
 import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 
 @RunWith(PowerMockRunner.class)
@@ -125,9 +126,8 @@ public void testConsumerWithMessage() throws Throwable {
         assertThat(traceSegments.size(), is(1));
 
         TraceSegment traceSegment = traceSegments.get(0);
-        List<TraceSegmentRef> refs = traceSegment.getRefs();
-        assertThat(refs.size(), is(1));
-        assertTraceSegmentRef(refs.get(0));
+        assertNotNull(traceSegment.getRef());
+        assertTraceSegmentRef(traceSegment.getRef());
 
         List<AbstractTracingSpan> spans = SegmentHelper.getSpans(traceSegment);
         assertThat(spans.size(), is(1));

apm-sniffer/apm-sdk-plugin/motan-plugin/src/test/java/org/apache/skywalking/apm/plugin/motan/MotanProviderInterceptorTest.java

@@ -50,8 +50,8 @@
 
 import static org.apache.skywalking.apm.agent.test.tools.SpanAssert.assertComponent;
 import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.when;
 
 @RunWith(PowerMockRunner.class)
@@ -100,8 +100,7 @@ public void testInvokerWithoutRefSegment() throws Throwable {
         TraceSegment traceSegment = segmentStorage.getTraceSegments().get(0);
         List<AbstractTracingSpan> spans = SegmentHelper.getSpans(traceSegment);
         assertMotanProviderSpan(spans.get(0));
-        assertTrue(traceSegment.getRefs() == null);
-
+        assertNull(traceSegment.getRef());
     }
 
     @Test
@@ -120,7 +119,7 @@ public void testInvokerWithRefSegment() throws Throwable {
         TraceSegment traceSegment = segmentStorage.getTraceSegments().get(0);
         List<AbstractTracingSpan> spans = SegmentHelper.getSpans(traceSegment);
         assertMotanProviderSpan(spans.get(0));
-        assertRefSegment(traceSegment.getRefs().get(0));
+        assertRefSegment(traceSegment.getRef());
     }
 
     @Test