Skip to content

Add support for filtering root spans matching a criteria #231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 23, 2021
Merged

Add support for filtering root spans matching a criteria #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c140246
Add support for filtering root spans matching a criteria
avinashkolluru Jul 22, 2021
484e30c
helm changes
avinashkolluru Jul 22, 2021
c198b08
Add support for dropping/processing root exit spans with exclusion cr…
avinashkolluru Jul 23, 2021
b9ce1aa
Moved all span dropping logic to SpanFilter
avinashkolluru Jul 23, 2021
b5c9ea0
Further cleanup of config parsing
avinashkolluru Jul 23, 2021
0e85b1b
Add test
Jul 23, 2021
80125b6
Merge remote-tracking branch 'origin/main' into root_span_drop_criter…
Jul 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions span-normalizer/helm/templates/span-normalizer-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ data:
{{- if hasKey .Values.spanNormalizerConfig.processor "spanDropCriterion" }}
spanDropCriterion = {{ .Values.spanNormalizerConfig.processor.spanDropCriterion | toJson }}
{{- end }}

{{- if hasKey .Values.spanNormalizerConfig.processor "rootExitSpanDropCriterion" }}
rootExitSpanDropCriterion = {{ .Values.spanNormalizerConfig.processor.rootExitSpanDropCriterion | toJson }}
{{- end }}
}
{{- end }}

Expand Down
14 changes: 6 additions & 8 deletions ...lizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,10 @@
public class JaegerSpanPreProcessor
implements Transformer<byte[], Span, KeyValue<byte[], PreProcessedSpan>> {

static final String SPANS_COUNTER = "hypertrace.reported.spans";
private static final Logger LOG = LoggerFactory.getLogger(JaegerSpanPreProcessor.class);

private static final ConcurrentMap<String, Counter> statusToSpansCounter =
new ConcurrentHashMap<>();
static final String SPANS_COUNTER = "hypertrace.reported.spans";

private TenantIdHandler tenantIdHandler;
private SpanFilter spanFilter;

Expand Down Expand Up @@ -83,23 +81,23 @@ public KeyValue<byte[], PreProcessedSpan> transform(byte[] key, Span value) {
}

@VisibleForTesting
PreProcessedSpan preProcessSpan(Span value) {
PreProcessedSpan preProcessSpan(Span span) {
Map<String, JaegerSpanInternalModel.KeyValue> tags =
value.getTagsList().stream()
span.getTagsList().stream()
.collect(Collectors.toMap(t -> t.getKey().toLowerCase(), t -> t, (v1, v2) -> v2));

Optional<String> maybeTenantId = tenantIdHandler.getAllowedTenantId(value, tags);
Optional<String> maybeTenantId = tenantIdHandler.getAllowedTenantId(span, tags);
if (maybeTenantId.isEmpty()) {
return null;
}

String tenantId = maybeTenantId.get();

if (spanFilter.shouldDropSpan(tags)) {
if (spanFilter.shouldDropSpan(span, tags)) {
return null;
}

return new PreProcessedSpan(tenantId, value);
return new PreProcessedSpan(tenantId, span);
}

@Override
Expand Down
104 changes: 77 additions & 27 deletions ...r/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.hypertrace.core.spannormalizer.jaeger;

import com.typesafe.config.Config;
import io.jaegertracing.api_v2.JaegerSpanInternalModel.KeyValue;
import io.jaegertracing.api_v2.JaegerSpanInternalModel;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand All @@ -11,12 +11,16 @@
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.hypertrace.core.span.constants.RawSpanConstants;
import org.hypertrace.core.span.constants.v1.SpanAttribute;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SpanFilter {

private static final Logger LOG = LoggerFactory.getLogger(SpanFilter.class);
private static final String SPAN_KIND_TAG =
RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND);
private static final String SPAN_KIND_CLIENT = "client";
/**
* Config key using which a list of criterion can be specified to drop the matching spans. Any
* span matching any one of the criterion is dropped. Each criteria is a comma separated list of
Expand All @@ -28,34 +32,69 @@ public class SpanFilter {
*/
private static final String SPAN_DROP_CRITERION_CONFIG = "processor.spanDropCriterion";

public static final String ROOT_SPAN_DROP_CRITERION_CONFIG =
"processor.rootExitSpanDropCriterion";
private static final String ROOT_SPAN_ALWAYS_DROP = "alwaysDrop";
private static final String ROOT_SPAN_DROP_EXCLUSIONS = "exclusionsMatchCriterion";

private static final String COMMA = ",";
private static final String COLON = ":";

private final List<List<Pair<String, String>>> spanDropCriterion;
private List<List<Pair<String, String>>> spanDropCriterion = Collections.emptyList();
private boolean alwaysDropRootSpan = false;
private List<List<Pair<String, String>>> rootSpanDropExclusionCriterion = Collections.emptyList();

public SpanFilter(Config config) {
if (config.hasPath(SPAN_DROP_CRITERION_CONFIG)) {
List<String> criterion = config.getStringList(SPAN_DROP_CRITERION_CONFIG);
LOG.info("Span drop criterion: {}", criterion);
// Parse the config to see if there is any criteria to drop spans.
this.spanDropCriterion = parseStringList(criterion);
}

List<String> criterion =
config.hasPath(SPAN_DROP_CRITERION_CONFIG)
? config.getStringList(SPAN_DROP_CRITERION_CONFIG)
: Collections.emptyList();
if (config.hasPath(ROOT_SPAN_DROP_CRITERION_CONFIG)) {
Config rootSpanDropCriterionConfig = config.getConfig(ROOT_SPAN_DROP_CRITERION_CONFIG);
LOG.info("Root Span drop criterion: {}", rootSpanDropCriterionConfig);
this.alwaysDropRootSpan =
rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_ALWAYS_DROP)
&& rootSpanDropCriterionConfig.getBoolean(ROOT_SPAN_ALWAYS_DROP);
List<String> exclusionList =
rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_DROP_EXCLUSIONS)
? rootSpanDropCriterionConfig.getStringList(ROOT_SPAN_DROP_EXCLUSIONS)
: Collections.emptyList();
// Parse the config to see if there is any criteria to drop spans.
this.rootSpanDropExclusionCriterion = parseStringList(exclusionList);
}
}

// Parse the config to see if there is any criteria to drop spans.
this.spanDropCriterion =
criterion.stream()
// Split each criteria based on comma
.map(s -> s.split(COMMA))
.map(
a ->
Arrays.stream(a)
.map(this::convertToPair)
.filter(Objects::nonNull)
.collect(Collectors.toList()))
.collect(Collectors.toList());
private List<List<Pair<String, String>>> parseStringList(List<String> stringList) {
return stringList.stream()
// Split each criteria based on comma
.map(s -> s.split(COMMA))
.map(
a ->
Arrays.stream(a)
.map(this::convertToPair)
.filter(Objects::nonNull)
.collect(Collectors.toList()))
.collect(Collectors.toList());
}

if (!this.spanDropCriterion.isEmpty()) {
LOG.info("Span drop criterion: {}", this.spanDropCriterion);
/**
* Method to check if the given span attributes match any of the drop criterion. Returns true if
* the span should be dropped, false otherwise.
*/
public boolean shouldDropSpan(
JaegerSpanInternalModel.Span span, Map<String, JaegerSpanInternalModel.KeyValue> tags) {
if (anyCriteriaMatch(tags, spanDropCriterion)) {
return true;
}

if (isRootExitSpan(span, tags)) {
boolean anyCriteriaMatch = anyCriteriaMatch(tags, rootSpanDropExclusionCriterion);
return (alwaysDropRootSpan && !anyCriteriaMatch) || (!alwaysDropRootSpan && anyCriteriaMatch);
}
return false;
}

@Nullable
Expand All @@ -69,12 +108,10 @@ private Pair<String, String> convertToPair(String s) {
return null;
}

/**
* Method to check if the given span attributes match any of the drop criterion. Returns true if
* the span should be dropped, false otherwise.
*/
public boolean shouldDropSpan(Map<String, KeyValue> tags) {
return this.spanDropCriterion.stream()
private boolean anyCriteriaMatch(
Map<String, JaegerSpanInternalModel.KeyValue> tags,
List<List<Pair<String, String>>> criteriaList) {
return criteriaList.stream()
.anyMatch(
l ->
l.stream()
Expand All @@ -84,4 +121,17 @@ public boolean shouldDropSpan(Map<String, KeyValue> tags) {
&& StringUtils.equals(
tags.get(p.getLeft()).getVStr(), p.getRight())));
}

private boolean isRootExitSpan(
JaegerSpanInternalModel.Span span, Map<String, JaegerSpanInternalModel.KeyValue> tags) {
if (!span.getReferencesList().isEmpty()) {
return false;
}
JaegerSpanInternalModel.KeyValue spanKindKeyValue = tags.get(SPAN_KIND_TAG);
if (null == spanKindKeyValue) {
return false;
}

return SPAN_KIND_CLIENT.equals(spanKindKeyValue.getVStr());
}
}
175 changes: 175 additions & 0 deletions ...r/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.hypertrace.core.span.constants.RawSpanConstants;
import org.hypertrace.core.span.constants.v1.SpanAttribute;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -179,6 +181,179 @@ public void testDropSpanWithEmptyCriterion() {
Assertions.assertNotNull(preProcessedSpan);
}

@Test
public void testDropSpan_RootSpan_EmptyExclusionList() {
String tenantId = "tenant-" + random.nextLong();
Map<String, Object> configs = new HashMap<>(getCommonConfig());
configs.putAll(
Map.of(
"processor",
Map.of(
"tenantIdTagKey", "tenant-key",
"rootExitSpanDropCriterion.alwaysDrop", "true")));

JaegerSpanPreProcessor jaegerSpanPreProcessor =
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
Process process = Process.newBuilder().setServiceName("testService").build();

// root exit span
Span span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNull(preProcessedSpan);

span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("server")
.build())
.build();
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNotNull(preProcessedSpan);
}

/** Always drop except when there is a match from the exclusion list */
@Test
public void testDropSpan_RootSpan_AlwaysDrop_ExclusionList() {
String tenantId = "tenant-" + random.nextLong();
Map<String, Object> configs = new HashMap<>(getCommonConfig());
configs.putAll(
Map.of(
"processor",
Map.of(
"tenantIdTagKey", "tenant-key",
"rootExitSpanDropCriterion.alwaysDrop", "true",
"rootExitSpanDropCriterion.exclusionsMatchCriterion",
List.of("foo:bar,k1:v1", "k2:v2"))));

JaegerSpanPreProcessor jaegerSpanPreProcessor =
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
Process process = Process.newBuilder().setServiceName("testService").build();

// root exit span
Span span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
.addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNotNull(preProcessedSpan);

span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNotNull(preProcessedSpan);

span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNull(preProcessedSpan);
}

/** Always keep except when there is a match from the exclusion list */
@Test
public void testDropSpan_RootSpan_NotAlwaysDrop_ExclusionList() {
String tenantId = "tenant-" + random.nextLong();
Map<String, Object> configs = new HashMap<>(getCommonConfig());
configs.putAll(
Map.of(
"processor",
Map.of(
"tenantIdTagKey", "tenant-key",
"rootExitSpanDropCriterion.alwaysDrop", "false",
"rootExitSpanDropCriterion.exclusionsMatchCriterion",
List.of("foo:bar,k1:v1", "k2:v2"))));

JaegerSpanPreProcessor jaegerSpanPreProcessor =
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
Process process = Process.newBuilder().setServiceName("testService").build();

// root exit span
Span span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
.addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNull(preProcessedSpan);

span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNull(preProcessedSpan);

span =
Span.newBuilder()
.setProcess(process)
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
.addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build())
.addTags(
KeyValue.newBuilder()
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
.setVStr("client")
.build())
.build();
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
Assertions.assertNotNull(preProcessedSpan);
}

private Map<String, Object> getCommonConfig() {
return Map.of(
"span.type",
Expand Down