From 34bb51d458661c0dbb9757a87f6d1de1172e7ed4 Mon Sep 17 00:00:00 2001
From: Piotr WOLSKI <piotr.wolski42@gmail.com>
Date: Thu, 6 Jun 2024 16:04:45 -0400
Subject: [PATCH] data streams: Use backend parameters for histograms (#7050)

data streams: Use backend parameters for histograms
---
 dd-trace-core/build.gradle                            |  2 +-
 .../datadog/trace/core/datastreams/StatsGroup.java    |  6 +++---
 .../java/datadog/trace/core/histogram/Histograms.java | 11 +++++++++++
 .../DefaultDataStreamsMonitoringTest.groovy           |  9 +++------
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle
index 785abe0a9d0..b669edea379 100644
--- a/dd-trace-core/build.gradle
+++ b/dd-trace-core/build.gradle
@@ -64,7 +64,7 @@ dependencies {
   implementation deps.moshi
   implementation deps.jctools
 
-  implementation group: 'com.datadoghq', name: 'sketches-java', version: '0.8.2'
+  implementation group: 'com.datadoghq', name: 'sketches-java', version: '0.8.3'
 
   implementation group: 'com.google.re2j', name: 're2j', version: '1.7'
 
diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java
index bbdb7dcc4e2..b5923c36e6d 100644
--- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java
+++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsGroup.java
@@ -18,9 +18,9 @@ public StatsGroup(List<String> edgeTags, long hash, long parentHash) {
     this.edgeTags = edgeTags;
     this.hash = hash;
     this.parentHash = parentHash;
-    pathwayLatency = Histograms.newHistogram();
-    edgeLatency = Histograms.newHistogram();
-    payloadSize = Histograms.newHistogram();
+    pathwayLatency = Histograms.newLogHistogram();
+    edgeLatency = Histograms.newLogHistogram();
+    payloadSize = Histograms.newLogHistogram();
   }
 
   public void add(long pathwayLatencyNano, long edgeLatencyNano, long payloadSizeBytes) {
diff --git a/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java b/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java
index 4dca7634586..3fc465756cc 100644
--- a/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java
+++ b/dd-trace-core/src/main/java/datadog/trace/core/histogram/Histograms.java
@@ -3,18 +3,29 @@
 import com.datadoghq.sketch.ddsketch.DDSketch;
 import com.datadoghq.sketch.ddsketch.DDSketches;
 import com.datadoghq.sketch.ddsketch.mapping.BitwiseLinearlyInterpolatedMapping;
+import com.datadoghq.sketch.ddsketch.mapping.LogarithmicMapping;
 import com.datadoghq.sketch.ddsketch.store.CollapsingLowestDenseStore;
 
 public final class Histograms {
 
   private static final BitwiseLinearlyInterpolatedMapping INDEX_MAPPING =
       new BitwiseLinearlyInterpolatedMapping(1.0 / 128.0);
+  // use the same gamma and index offset as the Datadog backend, to avoid doing any conversions in
+  // the backend
+  // that would lead to a loss of precision
+  private static final LogarithmicMapping LOG_INDEX_MAPPING =
+      new LogarithmicMapping(1.015625, 1.8761281912861705);
 
   public static Histogram newHistogram() {
     DDSketch sketch = new DDSketch(INDEX_MAPPING, () -> new CollapsingLowestDenseStore(1024));
     return new Histogram(sketch);
   }
 
+  public static Histogram newLogHistogram() {
+    DDSketch sketch = new DDSketch(LOG_INDEX_MAPPING, () -> new CollapsingLowestDenseStore(1024));
+    return new Histogram(sketch);
+  }
+
   public static Histogram newHistogram(double relativeAccuracy, int maxNumBins) {
     DDSketch sketch = DDSketches.logarithmicCollapsingLowestDense(relativeAccuracy, maxNumBins);
     return new Histogram(sketch);
diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy
index 6732792e136..c67dbe8f786 100644
--- a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy
+++ b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy
@@ -451,8 +451,7 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification {
         edgeTags.size() == 3
         hash == 1
         parentHash == 2
-        pathwayLatency.getMaxValue() >= 10
-        pathwayLatency.getMaxValue() < 10.1
+        Math.abs((pathwayLatency.getMaxValue()-10)/10) < 0.01
       }
     }
 
@@ -467,8 +466,7 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification {
         parentHash == 2
         edgeTags.containsAll(["type:testType", "group:testGroup", "topic:testTopic"])
         edgeTags.size() == 3
-        pathwayLatency.getMaxValue() >= 5
-        pathwayLatency.getMaxValue() < 5.1
+        Math.abs((pathwayLatency.getMaxValue()-5)/5) < 0.01
       }
 
       with(sortedGroups[1]) {
@@ -476,8 +474,7 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification {
         parentHash == 4
         edgeTags.containsAll(["type:testType", "group:testGroup", "topic:testTopic2"])
         edgeTags.size() == 3
-        pathwayLatency.getMaxValue() >= 2
-        pathwayLatency.getMaxValue() < 2.1
+        Math.abs((pathwayLatency.getMaxValue()-2)/2) < 0.01
       }
     }