data streams: Use backend parameters for histograms (#7050)

data streams: Use backend parameters for histograms
DataDog · Jun 6, 2024 · 34bb51d · 34bb51d
1 parent 95319d2
commit 34bb51d
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 10 deletions.
diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle
@@ -64,7 +64,7 @@ dependencies {
   implementation deps.moshi
   implementation deps.jctools
 
-  implementation group: 'com.datadoghq', name: 'sketches-java', version: '0.8.2'
+  implementation group: 'com.datadoghq', name: 'sketches-java', version: '0.8.3'
 
   implementation group: 'com.google.re2j', name: 're2j', version: '1.7'
 

diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java
@@ -18,9 +18,9 @@ public StatsGroup(List<String> edgeTags, long hash, long parentHash) {
     this.edgeTags = edgeTags;
     this.hash = hash;
     this.parentHash = parentHash;
-    pathwayLatency = Histograms.newHistogram();
-    edgeLatency = Histograms.newHistogram();
-    payloadSize = Histograms.newHistogram();
+    pathwayLatency = Histograms.newLogHistogram();
+    edgeLatency = Histograms.newLogHistogram();
+    payloadSize = Histograms.newLogHistogram();
   }
 
   public void add(long pathwayLatencyNano, long edgeLatencyNano, long payloadSizeBytes) {

diff --git a/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java b/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java
@@ -3,18 +3,29 @@
 import com.datadoghq.sketch.ddsketch.DDSketch;
 import com.datadoghq.sketch.ddsketch.DDSketches;
 import com.datadoghq.sketch.ddsketch.mapping.BitwiseLinearlyInterpolatedMapping;
+import com.datadoghq.sketch.ddsketch.mapping.LogarithmicMapping;
 import com.datadoghq.sketch.ddsketch.store.CollapsingLowestDenseStore;
 
 public final class Histograms {
 
   private static final BitwiseLinearlyInterpolatedMapping INDEX_MAPPING =
       new BitwiseLinearlyInterpolatedMapping(1.0 / 128.0);
+  // use the same gamma and index offset as the Datadog backend, to avoid doing any conversions in
+  // the backend
+  // that would lead to a loss of precision
+  private static final LogarithmicMapping LOG_INDEX_MAPPING =
+      new LogarithmicMapping(1.015625, 1.8761281912861705);
 
   public static Histogram newHistogram() {
     DDSketch sketch = new DDSketch(INDEX_MAPPING, () -> new CollapsingLowestDenseStore(1024));
     return new Histogram(sketch);
   }
 
+  public static Histogram newLogHistogram() {
+    DDSketch sketch = new DDSketch(LOG_INDEX_MAPPING, () -> new CollapsingLowestDenseStore(1024));
+    return new Histogram(sketch);
+  }
+
   public static Histogram newHistogram(double relativeAccuracy, int maxNumBins) {
     DDSketch sketch = DDSketches.logarithmicCollapsingLowestDense(relativeAccuracy, maxNumBins);
     return new Histogram(sketch);

diff --git a/...re/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy b/...re/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy
@@ -451,8 +451,7 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification {
         edgeTags.size() == 3
         hash == 1
         parentHash == 2
-        pathwayLatency.getMaxValue() >= 10
-        pathwayLatency.getMaxValue() < 10.1
+        Math.abs((pathwayLatency.getMaxValue()-10)/10) < 0.01
       }
     }
 
@@ -467,17 +466,15 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification {
         parentHash == 2
         edgeTags.containsAll(["type:testType", "group:testGroup", "topic:testTopic"])
         edgeTags.size() == 3
-        pathwayLatency.getMaxValue() >= 5
-        pathwayLatency.getMaxValue() < 5.1
+        Math.abs((pathwayLatency.getMaxValue()-5)/5) < 0.01
       }
 
       with(sortedGroups[1]) {
         hash == 3
         parentHash == 4
         edgeTags.containsAll(["type:testType", "group:testGroup", "topic:testTopic2"])
         edgeTags.size() == 3
-        pathwayLatency.getMaxValue() >= 2
-        pathwayLatency.getMaxValue() < 2.1
+        Math.abs((pathwayLatency.getMaxValue()-2)/2) < 0.01
       }
     }