diff --git a/span-normalizer/helm/templates/span-normalizer-config.yaml b/span-normalizer/helm/templates/span-normalizer-config.yaml index 888fcb1b7..d5fd1e3f1 100644 --- a/span-normalizer/helm/templates/span-normalizer-config.yaml +++ b/span-normalizer/helm/templates/span-normalizer-config.yaml @@ -56,6 +56,10 @@ data: {{- if hasKey .Values.spanNormalizerConfig.processor "spanDropCriterion" }} spanDropCriterion = {{ .Values.spanNormalizerConfig.processor.spanDropCriterion | toJson }} {{- end }} + + {{- if hasKey .Values.spanNormalizerConfig.processor "rootExitSpanDropCriterion" }} + rootExitSpanDropCriterion = {{ .Values.spanNormalizerConfig.processor.rootExitSpanDropCriterion | toJson }} + {{- end }} } {{- end }} diff --git a/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java b/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java index adf65c755..93b62e73d 100644 --- a/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java +++ b/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java @@ -22,12 +22,10 @@ public class JaegerSpanPreProcessor implements Transformer> { + static final String SPANS_COUNTER = "hypertrace.reported.spans"; private static final Logger LOG = LoggerFactory.getLogger(JaegerSpanPreProcessor.class); - private static final ConcurrentMap statusToSpansCounter = new ConcurrentHashMap<>(); - static final String SPANS_COUNTER = "hypertrace.reported.spans"; - private TenantIdHandler tenantIdHandler; private SpanFilter spanFilter; @@ -83,23 +81,23 @@ public KeyValue transform(byte[] key, Span value) { } @VisibleForTesting - PreProcessedSpan preProcessSpan(Span value) { + PreProcessedSpan preProcessSpan(Span span) { Map tags = - value.getTagsList().stream() + span.getTagsList().stream() .collect(Collectors.toMap(t -> t.getKey().toLowerCase(), t -> t, (v1, v2) -> v2)); - Optional maybeTenantId = tenantIdHandler.getAllowedTenantId(value, tags); + Optional maybeTenantId = tenantIdHandler.getAllowedTenantId(span, tags); if (maybeTenantId.isEmpty()) { return null; } String tenantId = maybeTenantId.get(); - if (spanFilter.shouldDropSpan(tags)) { + if (spanFilter.shouldDropSpan(span, tags)) { return null; } - return new PreProcessedSpan(tenantId, value); + return new PreProcessedSpan(tenantId, span); } @Override diff --git a/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java b/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java index d6490e1e7..ac4dffbef 100644 --- a/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java +++ b/span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java @@ -1,7 +1,7 @@ package org.hypertrace.core.spannormalizer.jaeger; import com.typesafe.config.Config; -import io.jaegertracing.api_v2.JaegerSpanInternalModel.KeyValue; +import io.jaegertracing.api_v2.JaegerSpanInternalModel; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -11,12 +11,16 @@ import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.hypertrace.core.span.constants.RawSpanConstants; +import org.hypertrace.core.span.constants.v1.SpanAttribute; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SpanFilter { - private static final Logger LOG = LoggerFactory.getLogger(SpanFilter.class); + private static final String SPAN_KIND_TAG = + RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND); + private static final String SPAN_KIND_CLIENT = "client"; /** * Config key using which a list of criterion can be specified to drop the matching spans. Any * span matching any one of the criterion is dropped. Each criteria is a comma separated list of @@ -28,34 +32,69 @@ public class SpanFilter { */ private static final String SPAN_DROP_CRITERION_CONFIG = "processor.spanDropCriterion"; + public static final String ROOT_SPAN_DROP_CRITERION_CONFIG = + "processor.rootExitSpanDropCriterion"; + private static final String ROOT_SPAN_ALWAYS_DROP = "alwaysDrop"; + private static final String ROOT_SPAN_DROP_EXCLUSIONS = "exclusionsMatchCriterion"; + private static final String COMMA = ","; private static final String COLON = ":"; - private final List>> spanDropCriterion; + private List>> spanDropCriterion = Collections.emptyList(); + private boolean alwaysDropRootSpan = false; + private List>> rootSpanDropExclusionCriterion = Collections.emptyList(); public SpanFilter(Config config) { + if (config.hasPath(SPAN_DROP_CRITERION_CONFIG)) { + List criterion = config.getStringList(SPAN_DROP_CRITERION_CONFIG); + LOG.info("Span drop criterion: {}", criterion); + // Parse the config to see if there is any criteria to drop spans. + this.spanDropCriterion = parseStringList(criterion); + } - List criterion = - config.hasPath(SPAN_DROP_CRITERION_CONFIG) - ? config.getStringList(SPAN_DROP_CRITERION_CONFIG) - : Collections.emptyList(); + if (config.hasPath(ROOT_SPAN_DROP_CRITERION_CONFIG)) { + Config rootSpanDropCriterionConfig = config.getConfig(ROOT_SPAN_DROP_CRITERION_CONFIG); + LOG.info("Root Span drop criterion: {}", rootSpanDropCriterionConfig); + this.alwaysDropRootSpan = + rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_ALWAYS_DROP) + && rootSpanDropCriterionConfig.getBoolean(ROOT_SPAN_ALWAYS_DROP); + List exclusionList = + rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_DROP_EXCLUSIONS) + ? rootSpanDropCriterionConfig.getStringList(ROOT_SPAN_DROP_EXCLUSIONS) + : Collections.emptyList(); + // Parse the config to see if there is any criteria to drop spans. + this.rootSpanDropExclusionCriterion = parseStringList(exclusionList); + } + } - // Parse the config to see if there is any criteria to drop spans. - this.spanDropCriterion = - criterion.stream() - // Split each criteria based on comma - .map(s -> s.split(COMMA)) - .map( - a -> - Arrays.stream(a) - .map(this::convertToPair) - .filter(Objects::nonNull) - .collect(Collectors.toList())) - .collect(Collectors.toList()); + private List>> parseStringList(List stringList) { + return stringList.stream() + // Split each criteria based on comma + .map(s -> s.split(COMMA)) + .map( + a -> + Arrays.stream(a) + .map(this::convertToPair) + .filter(Objects::nonNull) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + } - if (!this.spanDropCriterion.isEmpty()) { - LOG.info("Span drop criterion: {}", this.spanDropCriterion); + /** + * Method to check if the given span attributes match any of the drop criterion. Returns true if + * the span should be dropped, false otherwise. + */ + public boolean shouldDropSpan( + JaegerSpanInternalModel.Span span, Map tags) { + if (anyCriteriaMatch(tags, spanDropCriterion)) { + return true; } + + if (isRootExitSpan(span, tags)) { + boolean anyCriteriaMatch = anyCriteriaMatch(tags, rootSpanDropExclusionCriterion); + return (alwaysDropRootSpan && !anyCriteriaMatch) || (!alwaysDropRootSpan && anyCriteriaMatch); + } + return false; } @Nullable @@ -69,12 +108,10 @@ private Pair convertToPair(String s) { return null; } - /** - * Method to check if the given span attributes match any of the drop criterion. Returns true if - * the span should be dropped, false otherwise. - */ - public boolean shouldDropSpan(Map tags) { - return this.spanDropCriterion.stream() + private boolean anyCriteriaMatch( + Map tags, + List>> criteriaList) { + return criteriaList.stream() .anyMatch( l -> l.stream() @@ -84,4 +121,17 @@ public boolean shouldDropSpan(Map tags) { && StringUtils.equals( tags.get(p.getLeft()).getVStr(), p.getRight()))); } + + private boolean isRootExitSpan( + JaegerSpanInternalModel.Span span, Map tags) { + if (!span.getReferencesList().isEmpty()) { + return false; + } + JaegerSpanInternalModel.KeyValue spanKindKeyValue = tags.get(SPAN_KIND_TAG); + if (null == spanKindKeyValue) { + return false; + } + + return SPAN_KIND_CLIENT.equals(spanKindKeyValue.getVStr()); + } } diff --git a/span-normalizer/span-normalizer/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java b/span-normalizer/span-normalizer/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java index 7bc409810..5c56b72d7 100644 --- a/span-normalizer/span-normalizer/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java +++ b/span-normalizer/span-normalizer/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.Map; import java.util.Random; +import org.hypertrace.core.span.constants.RawSpanConstants; +import org.hypertrace.core.span.constants.v1.SpanAttribute; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -179,6 +181,179 @@ public void testDropSpanWithEmptyCriterion() { Assertions.assertNotNull(preProcessedSpan); } + @Test + public void testDropSpan_RootSpan_EmptyExclusionList() { + String tenantId = "tenant-" + random.nextLong(); + Map configs = new HashMap<>(getCommonConfig()); + configs.putAll( + Map.of( + "processor", + Map.of( + "tenantIdTagKey", "tenant-key", + "rootExitSpanDropCriterion.alwaysDrop", "true"))); + + JaegerSpanPreProcessor jaegerSpanPreProcessor = + new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs)); + Process process = Process.newBuilder().setServiceName("testService").build(); + + // root exit span + Span span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build()) + .addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNull(preProcessedSpan); + + span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build()) + .addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("server") + .build()) + .build(); + preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNotNull(preProcessedSpan); + } + + /** Always drop except when there is a match from the exclusion list */ + @Test + public void testDropSpan_RootSpan_AlwaysDrop_ExclusionList() { + String tenantId = "tenant-" + random.nextLong(); + Map configs = new HashMap<>(getCommonConfig()); + configs.putAll( + Map.of( + "processor", + Map.of( + "tenantIdTagKey", "tenant-key", + "rootExitSpanDropCriterion.alwaysDrop", "true", + "rootExitSpanDropCriterion.exclusionsMatchCriterion", + List.of("foo:bar,k1:v1", "k2:v2")))); + + JaegerSpanPreProcessor jaegerSpanPreProcessor = + new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs)); + Process process = Process.newBuilder().setServiceName("testService").build(); + + // root exit span + Span span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build()) + .addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNotNull(preProcessedSpan); + + span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNotNull(preProcessedSpan); + + span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNull(preProcessedSpan); + } + + /** Always keep except when there is a match from the exclusion list */ + @Test + public void testDropSpan_RootSpan_NotAlwaysDrop_ExclusionList() { + String tenantId = "tenant-" + random.nextLong(); + Map configs = new HashMap<>(getCommonConfig()); + configs.putAll( + Map.of( + "processor", + Map.of( + "tenantIdTagKey", "tenant-key", + "rootExitSpanDropCriterion.alwaysDrop", "false", + "rootExitSpanDropCriterion.exclusionsMatchCriterion", + List.of("foo:bar,k1:v1", "k2:v2")))); + + JaegerSpanPreProcessor jaegerSpanPreProcessor = + new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs)); + Process process = Process.newBuilder().setServiceName("testService").build(); + + // root exit span + Span span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build()) + .addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNull(preProcessedSpan); + + span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNull(preProcessedSpan); + + span = + Span.newBuilder() + .setProcess(process) + .addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build()) + .addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build()) + .addTags( + KeyValue.newBuilder() + .setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND)) + .setVStr("client") + .build()) + .build(); + preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span); + Assertions.assertNotNull(preProcessedSpan); + } + private Map getCommonConfig() { return Map.of( "span.type",