Automatically downsample transaction when system is under load (backp…

…ressure) (#3072)

Co-authored-by: Sentry Github Bot <bot+github-bot@sentry.io>
Co-authored-by: Roman Zavarnitsyn <rom4ek93@gmail.com>
Co-authored-by: Markus Hintersteiner <markus.hintersteiner@sentry.io>
Co-authored-by: Lukas Bloder <lukas.bloder@gmail.com>
Co-authored-by: Stefano <stefano.siano@sentry.io>
Co-authored-by: getsentry-bot <bot@sentry.io>
Co-authored-by: getsentry-bot <bot@getsentry.com>
Co-authored-by: Richard Harrah <1672786+ToppleTheNun@users.noreply.github.com>

CHANGELOG.md

@@ -5,6 +5,11 @@
 ### Features
 
 - Support multiple debug-metadata.properties ([#3024](https://github.com/getsentry/sentry-java/pull/3024))
+- Automatically downsample transactions when the system is under load ([#3072](https://github.com/getsentry/sentry-java/pull/3072))
+  - You can opt into this behaviour by setting `enable-backpressure-handling=true`.
+  - We're happy to receive feedback, e.g. [in this GitHub issue](https://github.com/getsentry/sentry-java/issues/2829)
+  - When the system is under load we start reducing the `tracesSampleRate` automatically.
+  - Once the system goes back to healthy, we reset the `tracesSampleRate` to its original value.
 - (Android) Experimental: Provide more detailed cold app start information ([#3057](https://github.com/getsentry/sentry-java/pull/3057))
   - Attaches spans for Application, ContentProvider, and Activities to app-start timings
   - Uses Process.startUptimeMillis to calculate app-start timings

sentry-samples/sentry-samples-spring-boot-jakarta/src/main/resources/application.properties

@@ -13,6 +13,7 @@ sentry.enable-tracing=true
 sentry.ignored-checkins=ignored_monitor_slug_1,ignored_monitor_slug_2
 sentry.debug=true
 sentry.graphql.ignored-error-types=SOME_ERROR,ANOTHER_ERROR
+sentry.enable-backpressure-handling=true
 in-app-includes="io.sentry.samples"
 
 # Uncomment and set to true to enable aot compatibility

sentry-samples/sentry-samples-spring-boot-webflux-jakarta/src/main/resources/application.properties

@@ -9,3 +9,4 @@ sentry.logging.minimum-event-level=info
 sentry.logging.minimum-breadcrumb-level=debug
 sentry.reactive.thread-local-accessor-enabled=true
 sentry.enable-tracing=true
+sentry.enable-backpressure-handling=true

sentry-samples/sentry-samples-spring-boot-webflux/src/main/resources/application.properties

@@ -11,3 +11,4 @@ sentry.enable-tracing=true
 spring.graphql.graphiql.enabled=true
 spring.graphql.websocket.path=/graphql
 spring.graphql.schema.printer.enabled=true
+sentry.enable-backpressure-handling=true

sentry-samples/sentry-samples-spring-boot/src/main/resources/application.properties

@@ -13,6 +13,7 @@ sentry.enable-tracing=true
 sentry.ignored-checkins=ignored_monitor_slug_1,ignored_monitor_slug_2
 sentry.debug=true
 sentry.graphql.ignored-error-types=SOME_ERROR,ANOTHER_ERROR
+sentry.enable-backpressure-handling=true
 in-app-includes="io.sentry.samples"
 
 # Database configuration

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt

@@ -162,7 +162,8 @@ class SentryAutoConfigurationTest {
             "sentry.trace-propagation-targets=localhost,^(http|https)://api\\..*\$",
             "sentry.enabled=false",
             "sentry.send-modules=false",
-            "sentry.ignored-checkins=slug1,slugB"
+            "sentry.ignored-checkins=slug1,slugB",
+            "sentry.enable-backpressure-handling=true"
         ).run {
             val options = it.getBean(SentryProperties::class.java)
             assertThat(options.readTimeoutMillis).isEqualTo(10)
@@ -194,6 +195,7 @@ class SentryAutoConfigurationTest {
             assertThat(options.isEnabled).isEqualTo(false)
             assertThat(options.isSendModules).isEqualTo(false)
             assertThat(options.ignoredCheckIns).containsOnly("slug1", "slugB")
+            assertThat(options.isEnableBackpressureHandling).isEqualTo(true)
         }
     }
 

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/it/SentrySpringIntegrationTest.kt

@@ -210,7 +210,9 @@ class SentrySpringIntegrationTest {
 
 @SpringBootApplication
 open class App {
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt

@@ -162,7 +162,8 @@ class SentryAutoConfigurationTest {
             "sentry.trace-propagation-targets=localhost,^(http|https)://api\\..*\$",
             "sentry.enabled=false",
             "sentry.send-modules=false",
-            "sentry.ignored-checkins=slug1,slugB"
+            "sentry.ignored-checkins=slug1,slugB",
+            "sentry.enable-backpressure-handling=true"
         ).run {
             val options = it.getBean(SentryProperties::class.java)
             assertThat(options.readTimeoutMillis).isEqualTo(10)
@@ -194,6 +195,7 @@ class SentryAutoConfigurationTest {
             assertThat(options.isEnabled).isEqualTo(false)
             assertThat(options.isSendModules).isEqualTo(false)
             assertThat(options.ignoredCheckIns).containsOnly("slug1", "slugB")
+            assertThat(options.isEnableBackpressureHandling).isEqualTo(true)
         }
     }
 

sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/it/SentrySpringIntegrationTest.kt

@@ -210,7 +210,9 @@ class SentrySpringIntegrationTest {
 
 @SpringBootApplication
 open class App {
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-spring-jakarta/src/test/kotlin/io/sentry/spring/jakarta/EnableSentryTest.kt

@@ -6,8 +6,11 @@ import io.sentry.ITransportFactory
 import io.sentry.Integration
 import io.sentry.Sentry
 import io.sentry.SentryOptions
+import io.sentry.transport.ITransport
 import org.assertj.core.api.Assertions.assertThat
+import org.mockito.kotlin.any
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.whenever
 import org.springframework.boot.context.annotation.UserConfigurations
 import org.springframework.boot.test.context.runner.ApplicationContextRunner
 import org.springframework.context.annotation.Bean
@@ -185,7 +188,9 @@ class EnableSentryTest {
     class AppConfigWithCustomTransportFactory {
 
         @Bean
-        fun transport() = mock<ITransportFactory>()
+        fun transport() = mock<ITransportFactory>().also {
+            whenever(it.create(any(), any())).thenReturn(mock<ITransport>())
+        }
     }
 
     @EnableSentry(dsn = "http://key@localhost/proj")

sentry-spring-jakarta/src/test/kotlin/io/sentry/spring/jakarta/webflux/SentryWebfluxIntegrationTest.kt

@@ -145,7 +145,9 @@ class SentryWebfluxIntegrationTest {
 @SpringBootApplication(exclude = [ReactiveSecurityAutoConfiguration::class, SecurityAutoConfiguration::class])
 open class App {
 
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-spring/src/test/kotlin/io/sentry/spring/EnableSentryTest.kt

@@ -6,8 +6,11 @@ import io.sentry.ITransportFactory
 import io.sentry.Integration
 import io.sentry.Sentry
 import io.sentry.SentryOptions
+import io.sentry.transport.ITransport
 import org.assertj.core.api.Assertions.assertThat
+import org.mockito.kotlin.any
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.whenever
 import org.springframework.boot.context.annotation.UserConfigurations
 import org.springframework.boot.test.context.runner.ApplicationContextRunner
 import org.springframework.context.annotation.Bean
@@ -185,7 +188,9 @@ class EnableSentryTest {
     class AppConfigWithCustomTransportFactory {
 
         @Bean
-        fun transport() = mock<ITransportFactory>()
+        fun transport() = mock<ITransportFactory>().also {
+            whenever(it.create(any(), any())).thenReturn(mock<ITransport>())
+        }
     }
 
     @EnableSentry(dsn = "http://key@localhost/proj")

sentry-spring/src/test/kotlin/io/sentry/spring/webflux/SentryWebfluxIntegrationTest.kt

@@ -145,7 +145,9 @@ class SentryWebfluxIntegrationTest {
 @SpringBootApplication(exclude = [ReactiveSecurityAutoConfiguration::class, SecurityAutoConfiguration::class])
 open class App {
 
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-test-support/src/main/kotlin/io/sentry/test/Mocks.kt

@@ -2,19 +2,24 @@
 package io.sentry.test
 
 import io.sentry.ISentryExecutorService
+import io.sentry.backpressure.IBackpressureMonitor
 import org.mockito.kotlin.mock
 import java.util.concurrent.Callable
 import java.util.concurrent.Future
 
 class ImmediateExecutorService : ISentryExecutorService {
     override fun submit(runnable: Runnable): Future<*> {
-        runnable.run()
+        if (runnable !is IBackpressureMonitor) {
+            runnable.run()
+        }
         return mock()
     }
 
     override fun <T> submit(callable: Callable<T>): Future<T> = mock()
     override fun schedule(runnable: Runnable, delayMillis: Long): Future<*> {
-        runnable.run()
+        if (runnable !is IBackpressureMonitor) {
+            runnable.run()
+        }
         return mock<Future<Runnable>>()
     }
     override fun close(timeoutMillis: Long) {}

sentry/api/sentry.api

@@ -336,12 +336,14 @@ public final class io/sentry/ExternalOptions {
 	public fun getTracePropagationTargets ()Ljava/util/List;
 	public fun getTracesSampleRate ()Ljava/lang/Double;
 	public fun getTracingOrigins ()Ljava/util/List;
+	public fun isEnableBackpressureHandling ()Ljava/lang/Boolean;
 	public fun isEnablePrettySerializationOutput ()Ljava/lang/Boolean;
 	public fun isEnabled ()Ljava/lang/Boolean;
 	public fun isSendModules ()Ljava/lang/Boolean;
 	public fun setDebug (Ljava/lang/Boolean;)V
 	public fun setDist (Ljava/lang/String;)V
 	public fun setDsn (Ljava/lang/String;)V
+	public fun setEnableBackpressureHandling (Ljava/lang/Boolean;)V
 	public fun setEnableDeduplication (Ljava/lang/Boolean;)V
 	public fun setEnablePrettySerializationOutput (Ljava/lang/Boolean;)V
 	public fun setEnableTracing (Ljava/lang/Boolean;)V
@@ -435,6 +437,7 @@ public final class io/sentry/Hub : io/sentry/IHub {
 	public fun getTransaction ()Lio/sentry/ITransaction;
 	public fun isCrashedLastRun ()Ljava/lang/Boolean;
 	public fun isEnabled ()Z
+	public fun isHealthy ()Z
 	public fun popScope ()V
 	public fun pushScope ()V
 	public fun removeExtra (Ljava/lang/String;)V
@@ -485,6 +488,7 @@ public final class io/sentry/HubAdapter : io/sentry/IHub {
 	public fun getTransaction ()Lio/sentry/ITransaction;
 	public fun isCrashedLastRun ()Ljava/lang/Boolean;
 	public fun isEnabled ()Z
+	public fun isHealthy ()Z
 	public fun popScope ()V
 	public fun pushScope ()V
 	public fun removeExtra (Ljava/lang/String;)V
@@ -578,6 +582,7 @@ public abstract interface class io/sentry/IHub {
 	public abstract fun getTransaction ()Lio/sentry/ITransaction;
 	public abstract fun isCrashedLastRun ()Ljava/lang/Boolean;
 	public abstract fun isEnabled ()Z
+	public abstract fun isHealthy ()Z
 	public abstract fun popScope ()V
 	public abstract fun pushScope ()V
 	public abstract fun removeExtra (Ljava/lang/String;)V
@@ -718,6 +723,7 @@ public abstract interface class io/sentry/ISentryClient {
 	public abstract fun flush (J)V
 	public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public abstract fun isEnabled ()Z
+	public fun isHealthy ()Z
 }
 
 public abstract interface class io/sentry/ISentryExecutorService {
@@ -1119,6 +1125,7 @@ public final class io/sentry/NoOpHub : io/sentry/IHub {
 	public fun getTransaction ()Lio/sentry/ITransaction;
 	public fun isCrashedLastRun ()Ljava/lang/Boolean;
 	public fun isEnabled ()Z
+	public fun isHealthy ()Z
 	public fun popScope ()V
 	public fun pushScope ()V
 	public fun removeExtra (Ljava/lang/String;)V
@@ -1665,6 +1672,7 @@ public final class io/sentry/Sentry {
 	public static fun init (Ljava/lang/String;)V
 	public static fun isCrashedLastRun ()Ljava/lang/Boolean;
 	public static fun isEnabled ()Z
+	public static fun isHealthy ()Z
 	public static fun popScope ()V
 	public static fun pushScope ()V
 	public static fun removeExtra (Ljava/lang/String;)V
@@ -1781,6 +1789,7 @@ public final class io/sentry/SentryClient : io/sentry/ISentryClient {
 	public fun flush (J)V
 	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun isEnabled ()Z
+	public fun isHealthy ()Z
 }
 
 public final class io/sentry/SentryCrashLastRunState {
@@ -2078,6 +2087,7 @@ public class io/sentry/SentryOptions {
 	public fun addOptionsObserver (Lio/sentry/IOptionsObserver;)V
 	public fun addScopeObserver (Lio/sentry/IScopeObserver;)V
 	public fun addTracingOrigin (Ljava/lang/String;)V
+	public fun getBackpressureMonitor ()Lio/sentry/backpressure/IBackpressureMonitor;
 	public fun getBeforeBreadcrumb ()Lio/sentry/SentryOptions$BeforeBreadcrumbCallback;
 	public fun getBeforeSend ()Lio/sentry/SentryOptions$BeforeSendCallback;
 	public fun getBeforeSendTransaction ()Lio/sentry/SentryOptions$BeforeSendTransactionCallback;
@@ -2156,6 +2166,7 @@ public class io/sentry/SentryOptions {
 	public fun isAttachThreads ()Z
 	public fun isDebug ()Z
 	public fun isEnableAutoSessionTracking ()Z
+	public fun isEnableBackpressureHandling ()Z
 	public fun isEnableDeduplication ()Z
 	public fun isEnableExternalConfiguration ()Z
 	public fun isEnablePrettySerializationOutput ()Z
@@ -2177,6 +2188,7 @@ public class io/sentry/SentryOptions {
 	public fun setAttachServerName (Z)V
 	public fun setAttachStacktrace (Z)V
 	public fun setAttachThreads (Z)V
+	public fun setBackpressureMonitor (Lio/sentry/backpressure/IBackpressureMonitor;)V
 	public fun setBeforeBreadcrumb (Lio/sentry/SentryOptions$BeforeBreadcrumbCallback;)V
 	public fun setBeforeSend (Lio/sentry/SentryOptions$BeforeSendCallback;)V
 	public fun setBeforeSendTransaction (Lio/sentry/SentryOptions$BeforeSendTransactionCallback;)V
@@ -2191,6 +2203,7 @@ public class io/sentry/SentryOptions {
 	public fun setDistinctId (Ljava/lang/String;)V
 	public fun setDsn (Ljava/lang/String;)V
 	public fun setEnableAutoSessionTracking (Z)V
+	public fun setEnableBackpressureHandling (Z)V
 	public fun setEnableDeduplication (Z)V
 	public fun setEnableExternalConfiguration (Z)V
 	public fun setEnablePrettySerializationOutput (Z)V
@@ -2824,6 +2837,24 @@ public final class io/sentry/UserFeedback$JsonKeys {
 	public fun <init> ()V
 }
 
+public final class io/sentry/backpressure/BackpressureMonitor : io/sentry/backpressure/IBackpressureMonitor, java/lang/Runnable {
+	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/IHub;)V
+	public fun getDownsampleFactor ()I
+	public fun run ()V
+	public fun start ()V
+}
+
+public abstract interface class io/sentry/backpressure/IBackpressureMonitor {
+	public abstract fun getDownsampleFactor ()I
+	public abstract fun start ()V
+}
+
+public final class io/sentry/backpressure/NoOpBackpressureMonitor : io/sentry/backpressure/IBackpressureMonitor {
+	public fun getDownsampleFactor ()I
+	public static fun getInstance ()Lio/sentry/backpressure/NoOpBackpressureMonitor;
+	public fun start ()V
+}
+
 public class io/sentry/cache/EnvelopeCache : io/sentry/cache/IEnvelopeCache {
 	public static final field CRASH_MARKER_FILE Ljava/lang/String;
 	public static final field NATIVE_CRASH_MARKER_FILE Ljava/lang/String;
@@ -2925,6 +2956,7 @@ public final class io/sentry/clientreport/ClientReportRecorder : io/sentry/clien
 }
 
 public final class io/sentry/clientreport/DiscardReason : java/lang/Enum {
+	public static final field BACKPRESSURE Lio/sentry/clientreport/DiscardReason;
 	public static final field BEFORE_SEND Lio/sentry/clientreport/DiscardReason;
 	public static final field CACHE_OVERFLOW Lio/sentry/clientreport/DiscardReason;
 	public static final field EVENT_PROCESSOR Lio/sentry/clientreport/DiscardReason;
@@ -4432,6 +4464,7 @@ public final class io/sentry/transport/AsyncHttpTransport : io/sentry/transport/
 	public fun close ()V
 	public fun flush (J)V
 	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
+	public fun isHealthy ()Z
 	public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
 
@@ -4447,6 +4480,7 @@ public abstract interface class io/sentry/transport/ICurrentDateProvider {
 public abstract interface class io/sentry/transport/ITransport : java/io/Closeable {
 	public abstract fun flush (J)V
 	public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
+	public fun isHealthy ()Z
 	public fun send (Lio/sentry/SentryEnvelope;)V
 	public abstract fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
@@ -4481,6 +4515,7 @@ public final class io/sentry/transport/RateLimiter {
 	public fun <init> (Lio/sentry/transport/ICurrentDateProvider;Lio/sentry/SentryOptions;)V
 	public fun filter (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Lio/sentry/SentryEnvelope;
 	public fun isActiveForCategory (Lio/sentry/DataCategory;)Z
+	public fun isAnyRateLimitActive ()Z
 	public fun updateRetryAfterLimits (Ljava/lang/String;Ljava/lang/String;I)V
 }
 

sentry/src/main/java/io/sentry/ExternalOptions.java

@@ -49,6 +49,7 @@ public final class ExternalOptions {
   private @Nullable List<String> ignoredCheckIns;
 
   private @Nullable Boolean sendModules;
+  private @Nullable Boolean enableBackpressureHandling;
 
   @SuppressWarnings("unchecked")
   public static @NotNull ExternalOptions from(
@@ -131,6 +132,9 @@ public final class ExternalOptions {
 
     options.setIgnoredCheckIns(propertiesProvider.getList("ignored-checkins"));
 
+    options.setEnableBackpressureHandling(
+        propertiesProvider.getBooleanProperty("enable-backpressure-handling"));
+
     for (final String ignoredExceptionType :
         propertiesProvider.getList("ignored-exceptions-for-type")) {
       try {
@@ -398,4 +402,14 @@ public void setIgnoredCheckIns(final @Nullable List<String> ignoredCheckIns) {
   public @Nullable List<String> getIgnoredCheckIns() {
     return ignoredCheckIns;
   }
+
+  @ApiStatus.Experimental
+  public void setEnableBackpressureHandling(final @Nullable Boolean enableBackpressureHandling) {
+    this.enableBackpressureHandling = enableBackpressureHandling;
+  }
+
+  @ApiStatus.Experimental
+  public @Nullable Boolean isEnableBackpressureHandling() {
+    return enableBackpressureHandling;
+  }
 }