ref(statsd): Add metric name as a tag for Sentry errors (#1797)

From time to time, we still get errors from relay dropping metrics due to the channel being full. This commit attempts to add a tag to that Sentry issue, to identify more easily what metrics may be worsening the bottleneck. Next steps TBD, based on conclusions from the errors. This code is likely to be temporary and be removed after we figure out what and how to deal with the errors.
getsentry · Jan 31, 2023 · d873ce2 · d873ce2
1 parent bd18149
commit d873ce2
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@
 
 - Add support for client hints. ([#1752](https://github.com/getsentry/relay/pull/1752))
 
+**Internal**:
+
+- Add metric name as tag on Sentry errors from relay dropping metrics. ([#1797](https://github.com/getsentry/relay/pull/1797))
+
 ## 23.1.1
 
 **Features**:

diff --git a/relay-statsd/src/lib.rs b/relay-statsd/src/lib.rs
@@ -492,81 +492,122 @@ pub trait GaugeMetric {
 macro_rules! metric {
     // counter increment
     (counter($id:expr) += $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
-        $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.count_with_tags(&$crate::CounterMetric::name(&$id), $value)
-                $(.with_tag(stringify!($k), $v))*
-            )
-        })
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::CounterMetric::name(&$id));
+            }, || {
+                $crate::with_client(|client| {
+                    use $crate::_pred::*;
+                    client.send_metric(
+                        client.count_with_tags(&$crate::CounterMetric::name(&$id), $value)
+                        $(.with_tag(stringify!($k), $v))*
+                    )
+                })
+            }
+        )
     };
 
     // counter decrement
     (counter($id:expr) -= $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
-        $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.count_with_tags(&$crate::CounterMetric::name(&$id), -$value)
-                    $(.with_tag(stringify!($k), $v))*
-            )
-        })
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::CounterMetric::name(&$id));
+            }, || {
+                $crate::with_client(|client| {
+                    use $crate::_pred::*;
+                    client.send_metric(
+                        client.count_with_tags(&$crate::CounterMetric::name(&$id), -$value)
+                            $(.with_tag(stringify!($k), $v))*
+                    )
+                })
+            }
+        )
     };
 
     // gauge set
     (gauge($id:expr) = $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
         $crate::with_client(|client| {
             use $crate::_pred::*;
-            client.send_metric(
-                client.gauge_with_tags(&$crate::GaugeMetric::name(&$id), $value)
-                    $(.with_tag(stringify!($k), $v))*
+            relay_log::with_scope(
+                |scope| {
+                    scope.set_tag("key", &$crate::GaugeMetric::name(&$id));
+                }, || {
+                    client.send_metric(
+                        client.gauge_with_tags(&$crate::GaugeMetric::name(&$id), $value)
+                            $(.with_tag(stringify!($k), $v))*
+                    )
+                }
             )
         })
     };
 
     // histogram
     (histogram($id:expr) = $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
-        $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.histogram_with_tags(&$crate::HistogramMetric::name(&$id), $value)
-                    $(.with_tag(stringify!($k), $v))*
-            )
-        })
+        use $crate::_pred::*;
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::HistogramMetric::name(&$id));
+            }, || {
+                $crate::with_client(|client| {
+                    client.send_metric(
+                        client.histogram_with_tags(&$crate::HistogramMetric::name(&$id), $value)
+                            $(.with_tag(stringify!($k), $v))*
+                    )
+                })
+            }
+        )
     };
 
     // sets (count unique occurrences of a value per time interval)
     (set($id:expr) = $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
-        $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.set_with_tags(&$crate::SetMetric::name(&$id), $value)
-                    $(.with_tag(stringify!($k), $v))*
-            )
-        })
+        use $crate::_pred::*;
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::SetMetric::name(&$id));
+            }, || {
+                $crate::with_client(|client| {
+                    client.send_metric(
+                        client.set_with_tags(&$crate::SetMetric::name(&$id), $value)
+                            $(.with_tag(stringify!($k), $v))*
+                    )
+                })
+            }
+        )
     };
 
     // timer value (duration)
     (timer($id:expr) = $value:expr $(, $k:ident = $v:expr)* $(,)?) => {
         $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.time_with_tags(&$crate::TimerMetric::name(&$id), $value)
-                    $(.with_tag(stringify!($k), $v))*
-            )
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::TimerMetric::name(&$id));
+            }, || {
+                use $crate::_pred::*;
+                client.send_metric(
+                    client.time_with_tags(&$crate::TimerMetric::name(&$id), $value)
+                        $(.with_tag(stringify!($k), $v))*
+                )
+            })
         })
     };
 
     // timed block
     (timer($id:expr), $($k:ident = $v:expr,)* $block:block) => {{
         let now = std::time::Instant::now();
         let rv = {$block};
-        $crate::with_client(|client| {
-            use $crate::_pred::*;
-            client.send_metric(
-                client.time_with_tags(&$crate::TimerMetric::name(&$id), now.elapsed())
-                    $(.with_tag(stringify!($k), $v))*
-            )
-        });
+        relay_log::with_scope(
+            |scope| {
+                scope.set_tag("key", &$crate::TimerMetric::name(&$id));
+            }, || {
+                $crate::with_client(|client| {
+                    use $crate::_pred::*;
+                    client.send_metric(
+                        client.time_with_tags(&$crate::TimerMetric::name(&$id), now.elapsed())
+                            $(.with_tag(stringify!($k), $v))*
+                    )
+                });
+            }
+        );
         rv
     }};
 }