Skip to content

Commit 8159439

Browse files
maciejwalkowiakmaciejwalkowiak
authored
High throughput transport implementation (#1114)
Added Apache HttpClient 5 based non blocking transport implementation dedicated for high traffic server side applications.
1 parent dfeeb1b commit 8159439

File tree

51 files changed

+1764
-1169
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1764
-1169
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,18 @@
1212
* Fix: Initialize Logback after context refreshes (#1129)
1313
* Ref: Return NoOpTransaction instead of null (#1126)
1414
* Fix: Do not crash when passing null values to @Nullable methods, eg User and Scope
15+
* Ref: `ITransport` implementations are now responsible for executing request in asynchronous or synchronous way (#1118)
16+
* Ref: Add option to set `TransportFactory` instead of `ITransport` on `SentryOptions` (#1124)
17+
* Ref: Simplify ITransport creation in ITransportFactory (#1135)
18+
* Feat: Add non blocking Apache HttpClient 5 based Transport (#1136)
19+
* Enhancement: Autoconfigure Apache HttpClient 5 based Transport in Spring Boot integration (#1143)
1520
* Enhancement: Send user.ip_address = {{auto}} when sendDefaultPii is true (#1015)
1621
* Fix: Resolving dashed properties from external configuration
1722
* Feat: Read `uncaught.handler.enabled` property from the external configuration
1823
* Feat: Resolve servername from the localhost address
1924
* Fix: Consider {{ auto }} as a default ip address (#1015)
2025
* Fix: Set release and environment on Transactions (#1152)
21-
* Fix: Do not set transaction on the scope automatically
26+
* Fix: Do not set transaction on the scope automatically
2227
* Enhancement: Automatically assign span context to captured events (#1156)
2328

2429
# 4.0.0-alpha.2

‎buildSrc/src/main/java/Config.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ object Config {
5757
val springAop = "org.springframework:spring-aop"
5858
val aspectj = "org.aspectj:aspectjweaver"
5959
val servletApi = "javax.servlet:javax.servlet-api"
60+
61+
val apacheHttpClient = "org.apache.httpcomponents.client5:httpclient5:5.0.3"
6062
}
6163

6264
object AnnotationProcessors {

‎sentry-apache-http-client-5/api/sentry-apache-http-client-5.api‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
public final class io/sentry/transport/apache/ApacheHttpClientTransport : io/sentry/transport/ITransport {
2+
public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/RequestDetails;Lorg/apache/hc/client5/http/impl/async/CloseableHttpAsyncClient;Lio/sentry/transport/RateLimiter;)V
3+
public fun close ()V
4+
public fun send (Lio/sentry/SentryEnvelope;Ljava/lang/Object;)V
5+
}
6+
7+
public final class io/sentry/transport/apache/ApacheHttpClientTransportFactory : io/sentry/ITransportFactory {
8+
public fun <init> ()V
9+
public fun <init> (Lorg/apache/hc/core5/util/TimeValue;)V
10+
public fun create (Lio/sentry/SentryOptions;Lio/sentry/RequestDetails;)Lio/sentry/transport/ITransport;
11+
}
12+

‎sentry-apache-http-client-5/build.gradle.kts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
2+
3+
plugins {
4+
`java-library`
5+
kotlin("jvm")
6+
jacoco
7+
id(Config.QualityPlugins.errorProne)
8+
id(Config.QualityPlugins.gradleVersions)
9+
}
10+
11+
configure<JavaPluginConvention> {
12+
sourceCompatibility = JavaVersion.VERSION_1_8
13+
targetCompatibility = JavaVersion.VERSION_1_8
14+
}
15+
16+
tasks.withType<KotlinCompile>().configureEach {
17+
kotlinOptions.jvmTarget = JavaVersion.VERSION_1_8.toString()
18+
kotlinOptions.languageVersion = Config.springKotlinCompatibleLanguageVersion
19+
}
20+
21+
dependencies {
22+
api(project(":sentry"))
23+
api(Config.Libs.apacheHttpClient)
24+
25+
compileOnly(Config.CompileOnly.nopen)
26+
errorprone(Config.CompileOnly.nopenChecker)
27+
errorprone(Config.CompileOnly.errorprone)
28+
errorproneJavac(Config.CompileOnly.errorProneJavac8)
29+
compileOnly(Config.CompileOnly.jetbrainsAnnotations)
30+
31+
// tests
32+
testImplementation(Config.Libs.apacheHttpClient)
33+
testImplementation(project(":sentry-test-support"))
34+
testImplementation(kotlin(Config.kotlinStdLib))
35+
testImplementation(Config.TestLibs.kotlinTestJunit)
36+
testImplementation(Config.TestLibs.mockitoKotlin)
37+
}
38+
39+
configure<SourceSetContainer> {
40+
test {
41+
java.srcDir("src/test/java")
42+
}
43+
}
44+
45+
jacoco {
46+
toolVersion = Config.QualityPlugins.Jacoco.version
47+
}
48+
49+
tasks.jacocoTestReport {
50+
reports {
51+
xml.isEnabled = true
52+
html.isEnabled = false
53+
}
54+
}
55+
56+
tasks {
57+
jacocoTestCoverageVerification {
58+
violationRules {
59+
rule { limit { minimum = Config.QualityPlugins.Jacoco.minimumCoverage } }
60+
}
61+
}
62+
check {
63+
dependsOn(jacocoTestCoverageVerification)
64+
dependsOn(jacocoTestReport)
65+
}
66+
}

‎sentry-apache-http-client-5/src/main/java/io/sentry/transport/apache/ApacheHttpClientTransport.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package io.sentry.transport.apache;
2+
3+
import static io.sentry.SentryLevel.*;
4+
5+
import io.sentry.RequestDetails;
6+
import io.sentry.SentryEnvelope;
7+
import io.sentry.SentryLevel;
8+
import io.sentry.SentryOptions;
9+
import io.sentry.transport.ITransport;
10+
import io.sentry.transport.RateLimiter;
11+
import io.sentry.util.Objects;
12+
import java.io.ByteArrayOutputStream;
13+
import java.io.IOException;
14+
import java.util.Map;
15+
import java.util.concurrent.atomic.AtomicInteger;
16+
import java.util.zip.GZIPOutputStream;
17+
import org.apache.hc.client5.http.async.methods.SimpleHttpRequest;
18+
import org.apache.hc.client5.http.async.methods.SimpleHttpRequests;
19+
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
20+
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
21+
import org.apache.hc.core5.concurrent.FutureCallback;
22+
import org.apache.hc.core5.http.ContentType;
23+
import org.apache.hc.core5.http.Header;
24+
import org.apache.hc.core5.io.CloseMode;
25+
import org.apache.hc.core5.util.TimeValue;
26+
import org.jetbrains.annotations.NotNull;
27+
28+
/**
29+
* {@link ITransport} implementation that executes request asynchronously in a non-blocking manner
30+
* using Apache Http Client 5.
31+
*/
32+
public final class ApacheHttpClientTransport implements ITransport {
33+
private final @NotNull SentryOptions options;
34+
private final @NotNull RequestDetails requestDetails;
35+
private final @NotNull CloseableHttpAsyncClient httpclient;
36+
private final @NotNull RateLimiter rateLimiter;
37+
private final @NotNull AtomicInteger currentlyRunning;
38+
39+
public ApacheHttpClientTransport(
40+
final @NotNull SentryOptions options,
41+
final @NotNull RequestDetails requestDetails,
42+
final @NotNull CloseableHttpAsyncClient httpclient,
43+
final @NotNull RateLimiter rateLimiter) {
44+
this(options, requestDetails, httpclient, rateLimiter, new AtomicInteger());
45+
}
46+
47+
ApacheHttpClientTransport(
48+
final @NotNull SentryOptions options,
49+
final @NotNull RequestDetails requestDetails,
50+
final @NotNull CloseableHttpAsyncClient httpclient,
51+
final @NotNull RateLimiter rateLimiter,
52+
final @NotNull AtomicInteger currentlyRunning) {
53+
this.options = Objects.requireNonNull(options, "options is required");
54+
this.requestDetails = Objects.requireNonNull(requestDetails, "requestDetails is required");
55+
this.httpclient = Objects.requireNonNull(httpclient, "httpclient is required");
56+
this.rateLimiter = Objects.requireNonNull(rateLimiter, "rateLimiter is required");
57+
this.currentlyRunning =
58+
Objects.requireNonNull(currentlyRunning, "currentlyRunning is required");
59+
this.httpclient.start();
60+
}
61+
62+
@Override
63+
@SuppressWarnings("FutureReturnValueIgnored")
64+
public void send(SentryEnvelope envelope, Object hint) throws IOException {
65+
if (isSchedulingAllowed()) {
66+
final SentryEnvelope filteredEnvelope = rateLimiter.filter(envelope, hint);
67+
68+
if (filteredEnvelope != null) {
69+
currentlyRunning.incrementAndGet();
70+
71+
try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
72+
final GZIPOutputStream gzip = new GZIPOutputStream(outputStream)) {
73+
options.getSerializer().serialize(filteredEnvelope, gzip);
74+
75+
final SimpleHttpRequest request =
76+
SimpleHttpRequests.post(requestDetails.getUrl().toString());
77+
request.setBody(
78+
outputStream.toByteArray(), ContentType.create("application/x-sentry-envelope"));
79+
request.setHeader("Content-Encoding", "gzip");
80+
request.setHeader("Accept", "application/json");
81+
82+
for (Map.Entry<String, String> header : requestDetails.getHeaders().entrySet()) {
83+
request.setHeader(header.getKey(), header.getValue());
84+
}
85+
86+
if (options.getLogger().isEnabled(DEBUG)) {
87+
options.getLogger().log(DEBUG, "Currently running %d requests", currentlyRunning.get());
88+
}
89+
90+
httpclient.execute(
91+
request,
92+
new FutureCallback<SimpleHttpResponse>() {
93+
@Override
94+
public void completed(SimpleHttpResponse response) {
95+
currentlyRunning.decrementAndGet();
96+
if (response.getCode() != 200) {
97+
options
98+
.getLogger()
99+
.log(ERROR, "Request failed, API returned %s", response.getCode());
100+
} else {
101+
options.getLogger().log(INFO, "Envelope sent successfully.");
102+
}
103+
final Header retryAfter = response.getFirstHeader("Retry-After");
104+
final Header rateLimits = response.getFirstHeader("X-Sentry-Rate-Limits");
105+
rateLimiter.updateRetryAfterLimits(
106+
rateLimits != null ? rateLimits.getValue() : null,
107+
retryAfter != null ? retryAfter.getValue() : null,
108+
response.getCode());
109+
}
110+
111+
@Override
112+
public void failed(Exception ex) {
113+
currentlyRunning.decrementAndGet();
114+
options.getLogger().log(ERROR, "Error while sending an envelope", ex);
115+
}
116+
117+
@Override
118+
public void cancelled() {
119+
currentlyRunning.decrementAndGet();
120+
options.getLogger().log(WARNING, "Request cancelled");
121+
}
122+
});
123+
} catch (Exception e) {
124+
options.getLogger().log(ERROR, "Error when sending envelope", e);
125+
}
126+
}
127+
} else {
128+
options.getLogger().log(SentryLevel.WARNING, "Submit cancelled");
129+
}
130+
}
131+
132+
@Override
133+
public void close() throws IOException {
134+
options.getLogger().log(DEBUG, "Shutting down");
135+
try {
136+
httpclient.awaitShutdown(TimeValue.ofSeconds(1));
137+
httpclient.close(CloseMode.GRACEFUL);
138+
} catch (InterruptedException e) {
139+
options.getLogger().log(DEBUG, "Thread interrupted while closing the connection.");
140+
Thread.currentThread().interrupt();
141+
}
142+
}
143+
144+
private boolean isSchedulingAllowed() {
145+
return currentlyRunning.get() < options.getMaxQueueSize();
146+
}
147+
}

‎sentry-apache-http-client-5/src/main/java/io/sentry/transport/apache/ApacheHttpClientTransportFactory.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package io.sentry.transport.apache;
2+
3+
import io.sentry.ITransportFactory;
4+
import io.sentry.RequestDetails;
5+
import io.sentry.SentryOptions;
6+
import io.sentry.transport.ITransport;
7+
import io.sentry.transport.RateLimiter;
8+
import io.sentry.util.Objects;
9+
import org.apache.hc.client5.http.impl.DefaultConnectionKeepAliveStrategy;
10+
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
11+
import org.apache.hc.client5.http.impl.async.HttpAsyncClients;
12+
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
13+
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
14+
import org.apache.hc.core5.http.impl.DefaultConnectionReuseStrategy;
15+
import org.apache.hc.core5.pool.PoolConcurrencyPolicy;
16+
import org.apache.hc.core5.util.TimeValue;
17+
import org.jetbrains.annotations.NotNull;
18+
19+
/** Creates {@link ApacheHttpClientTransport}. */
20+
public final class ApacheHttpClientTransportFactory implements ITransportFactory {
21+
private final @NotNull TimeValue connectionTimeToLive;
22+
23+
public ApacheHttpClientTransportFactory() {
24+
this(TimeValue.ofMinutes(1));
25+
}
26+
27+
public ApacheHttpClientTransportFactory(final @NotNull TimeValue connectionTimeToLive) {
28+
this.connectionTimeToLive =
29+
Objects.requireNonNull(connectionTimeToLive, "connectionTimeToLive is required");
30+
}
31+
32+
@Override
33+
public ITransport create(
34+
final @NotNull SentryOptions options, final @NotNull RequestDetails requestDetails) {
35+
Objects.requireNonNull(options, "options is required");
36+
Objects.requireNonNull(requestDetails, "requestDetails is required");
37+
38+
final PoolingAsyncClientConnectionManager connectionManager =
39+
PoolingAsyncClientConnectionManagerBuilder.create()
40+
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
41+
.setConnectionTimeToLive(connectionTimeToLive)
42+
.setMaxConnTotal(options.getMaxQueueSize())
43+
.setMaxConnPerRoute(options.getMaxQueueSize())
44+
.build();
45+
46+
final CloseableHttpAsyncClient httpclient =
47+
HttpAsyncClients.custom()
48+
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
49+
.setConnectionManager(connectionManager)
50+
.setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
51+
.build();
52+
53+
final RateLimiter rateLimiter = new RateLimiter(options.getLogger());
54+
55+
return new ApacheHttpClientTransport(options, requestDetails, httpclient, rateLimiter);
56+
}
57+
}

‎sentry-apache-http-client-5/src/test/kotlin/io/sentry/transport/apache/ApacheHttpClientTransportFactoryTest.kt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package io.sentry.transport.apache
2+
3+
import com.nhaarman.mockitokotlin2.mock
4+
import io.sentry.RequestDetails
5+
import io.sentry.SentryOptions
6+
import kotlin.test.Test
7+
import kotlin.test.assertNotNull
8+
9+
class ApacheHttpClientTransportFactoryTest {
10+
11+
@Test
12+
fun `creates ApacheHttpClientTransport`() {
13+
val factory = ApacheHttpClientTransportFactory()
14+
val options = SentryOptions().apply {
15+
setLogger(mock())
16+
setSerializer(mock())
17+
}
18+
val requestDetails = mock<RequestDetails>()
19+
20+
val transport = factory.create(options, requestDetails)
21+
assertNotNull(transport)
22+
}
23+
}

0 commit comments

Comments
 (0)