Skip to content

Commit e13ad77

Browse files
Ant-hemaseure
Ant-hem
authored and
aseure
committed
adds: GZip compression feature (#591)
[changelog] Adds the possibility to compress POST/PUT requests for SearchClient. For the moment only GZIP compression is available. GZIP compression is enabled by default for SearchClient.
1 parent 1a4c815 commit e13ad77

9 files changed

Lines changed: 126 additions & 30 deletions

File tree

‎algoliasearch-apache/src/main/java/com/algolia/search/ApacheHttpRequester.java‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.algolia.search.models.HttpResponse;
66
import com.algolia.search.util.HttpStatusCodeUtils;
77
import java.io.IOException;
8-
import java.io.InputStream;
98
import java.net.ConnectException;
109
import java.net.SocketTimeoutException;
1110
import java.util.Map;
@@ -18,12 +17,12 @@
1817
import org.apache.http.HttpEntity;
1918
import org.apache.http.client.config.RequestConfig;
2019
import org.apache.http.client.entity.DeflateDecompressingEntity;
21-
import org.apache.http.client.entity.EntityBuilder;
2220
import org.apache.http.client.entity.GzipDecompressingEntity;
2321
import org.apache.http.client.methods.*;
2422
import org.apache.http.concurrent.FutureCallback;
2523
import org.apache.http.conn.ConnectTimeoutException;
2624
import org.apache.http.entity.ContentType;
25+
import org.apache.http.entity.InputStreamEntity;
2726
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
2827
import org.apache.http.impl.nio.client.HttpAsyncClients;
2928
import org.apache.http.util.EntityUtils;
@@ -121,19 +120,19 @@ private HttpRequestBase buildRequest(HttpRequest algoliaRequest) {
121120

122121
case HttpPost.METHOD_NAME:
123122
HttpPost post = new HttpPost(algoliaRequest.getUri().toString());
124-
if (algoliaRequest.getBody() != null) post.setEntity(addEntity(algoliaRequest.getBody()));
123+
if (algoliaRequest.getBody() != null) post.setEntity(addEntity(algoliaRequest));
125124
post.setConfig(buildRequestConfig(algoliaRequest));
126125
return addHeaders(post, algoliaRequest.getHeaders());
127126

128127
case HttpPut.METHOD_NAME:
129128
HttpPut put = new HttpPut(algoliaRequest.getUri().toString());
130-
if (algoliaRequest.getBody() != null) put.setEntity(addEntity(algoliaRequest.getBody()));
129+
if (algoliaRequest.getBody() != null) put.setEntity(addEntity(algoliaRequest));
131130
put.setConfig(buildRequestConfig(algoliaRequest));
132131
return addHeaders(put, algoliaRequest.getHeaders());
133132

134133
case HttpPatch.METHOD_NAME:
135134
HttpPatch patch = new HttpPatch(algoliaRequest.getUri().toString());
136-
if (algoliaRequest.getBody() != null) patch.setEntity(addEntity(algoliaRequest.getBody()));
135+
if (algoliaRequest.getBody() != null) patch.setEntity(addEntity(algoliaRequest));
137136
patch.setConfig(buildRequestConfig(algoliaRequest));
138137
return addHeaders(patch, algoliaRequest.getHeaders());
139138

@@ -153,11 +152,20 @@ private HttpRequestBase addHeaders(
153152
return request;
154153
}
155154

156-
private HttpEntity addEntity(InputStream data) {
157-
return EntityBuilder.create()
158-
.setStream(data)
159-
.setContentType(ContentType.APPLICATION_JSON)
160-
.build();
155+
private HttpEntity addEntity(@Nonnull HttpRequest request) {
156+
try {
157+
InputStreamEntity entity =
158+
new InputStreamEntity(
159+
request.getBody(), request.getBody().available(), ContentType.APPLICATION_JSON);
160+
161+
if (request.canCompress()) {
162+
entity.setContentEncoding(Defaults.CONTENT_ENCODING_GZIP);
163+
}
164+
165+
return entity;
166+
} catch (IOException e) {
167+
throw new AlgoliaRuntimeException("Error while getting body's content length.", e);
168+
}
161169
}
162170

163171
private static HttpEntity handleCompressedEntity(org.apache.http.HttpEntity entity) {

‎algoliasearch-apache/src/test/java/com/algolia/search/IntegrationTestExtension.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.algolia.search;
22

3+
import com.algolia.search.models.common.CompressionType;
34
import com.algolia.search.models.indexing.ActionEnum;
45
import com.algolia.search.models.indexing.BatchOperation;
56
import com.algolia.search.models.indexing.IndicesResponse;
@@ -29,7 +30,12 @@ public class IntegrationTestExtension
2930
public void beforeAll(ExtensionContext context) throws Exception {
3031
checkEnvironmentVariable();
3132
searchClient = DefaultSearchClient.create(ALGOLIA_APPLICATION_ID_1, ALGOLIA_API_KEY_1);
32-
searchClient2 = DefaultSearchClient.create(ALGOLIA_APPLICATION_ID_2, ALGOLIA_API_KEY_2);
33+
// Disabling gzip for client2 because GZip not is not enabled yet on the server
34+
SearchConfig client2Config =
35+
new SearchConfig.Builder(ALGOLIA_APPLICATION_ID_2, ALGOLIA_API_KEY_2)
36+
.setCompressionType(CompressionType.NONE)
37+
.build();
38+
searchClient2 = DefaultSearchClient.create(client2Config);
3339
cleanPreviousIndices();
3440
}
3541

‎algoliasearch-core/src/main/java/com/algolia/search/AnalyticsConfig.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.algolia.search;
22

33
import com.algolia.search.models.common.CallType;
4+
import com.algolia.search.models.common.CompressionType;
45
import java.util.*;
56
import javax.annotation.Nonnull;
67

@@ -16,7 +17,7 @@ public static class Builder extends ConfigBase.Builder<Builder> {
1617
* @param apiKey The API Key
1718
*/
1819
public Builder(@Nonnull String applicationID, @Nonnull String apiKey) {
19-
super(applicationID, apiKey, createDefaultHosts());
20+
super(applicationID, apiKey, createDefaultHosts(), CompressionType.NONE);
2021
}
2122

2223
@Override

‎algoliasearch-core/src/main/java/com/algolia/search/ConfigBase.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.algolia.search;
22

3+
import com.algolia.search.models.common.CompressionType;
34
import com.algolia.search.util.AlgoliaUtils;
45
import java.util.HashMap;
56
import java.util.List;
@@ -23,6 +24,7 @@ public abstract class ConfigBase {
2324
private final Integer connectTimeOut;
2425
private final List<StatefulHost> hosts;
2526
private final ExecutorService executor;
27+
private final CompressionType compressionType;
2628

2729
/** Config base builder to ensure the immutability of the configuration. */
2830
public abstract static class Builder<T extends Builder<T>> {
@@ -36,6 +38,7 @@ public abstract static class Builder<T extends Builder<T>> {
3638
private Integer connectTimeOut;
3739
private List<StatefulHost> hosts;
3840
private ExecutorService executor;
41+
protected CompressionType compressionType;
3942

4043
/**
4144
* Builds a base configuration
@@ -48,13 +51,16 @@ public abstract static class Builder<T extends Builder<T>> {
4851
public Builder(
4952
@Nonnull String applicationID,
5053
@Nonnull String apiKey,
51-
@Nonnull List<StatefulHost> defaultHosts) {
54+
@Nonnull List<StatefulHost> defaultHosts,
55+
@Nonnull CompressionType compressionType) {
56+
5257
this.applicationID = applicationID;
5358
this.apiKey = apiKey;
5459

5560
this.batchSize = 1000;
5661
this.hosts = defaultHosts;
5762
this.connectTimeOut = Defaults.CONNECT_TIMEOUT_MS;
63+
this.compressionType = compressionType;
5864

5965
this.defaultHeaders = new HashMap<>();
6066
this.defaultHeaders.put(Defaults.ALGOLIA_APPLICATION_HEADER, applicationID);
@@ -145,6 +151,7 @@ protected ConfigBase(Builder<?> builder) {
145151
this.applicationID = builder.applicationID;
146152
this.defaultHeaders = builder.defaultHeaders;
147153
this.batchSize = builder.batchSize;
154+
this.compressionType = builder.compressionType;
148155
this.readTimeOut = builder.readTimeOut;
149156
this.writeTimeOut = builder.writeTimeOut;
150157
this.connectTimeOut = builder.connectTimeOut;
@@ -168,6 +175,10 @@ public int getBatchSize() {
168175
return batchSize;
169176
}
170177

178+
public CompressionType getCompressionType() {
179+
return compressionType;
180+
}
181+
171182
public Integer getReadTimeOut() {
172183
return readTimeOut;
173184
}

‎algoliasearch-core/src/main/java/com/algolia/search/HttpTransport.java‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import java.util.concurrent.CompletableFuture;
1818
import java.util.logging.Level;
1919
import java.util.logging.Logger;
20+
import java.util.zip.GZIPOutputStream;
2021
import javax.annotation.Nonnull;
2122

2223
/**
@@ -217,24 +218,38 @@ private <TData> HttpRequest buildRequest(
217218
? requestOptions.getTimeout()
218219
: getTimeOut(callType);
219220

220-
HttpRequest request = new HttpRequest(method, fullPath, headersToSend, timeout);
221+
HttpRequest request =
222+
new HttpRequest(method, fullPath, headersToSend, timeout, config.getCompressionType());
221223

222224
if (data != null) {
223-
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
225+
request.setBody(serializeJSON(data, request));
226+
logRequest(request, data);
227+
}
224228

225-
Defaults.getObjectMapper().writeValue(out, data);
229+
return request;
230+
}
226231

227-
ByteArrayInputStream content = new ByteArrayInputStream(out.toByteArray());
232+
private <TData> InputStream serializeJSON(TData data, HttpRequest request) {
233+
if (request.canCompress()) {
234+
try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
235+
GZIPOutputStream gzipOS = new GZIPOutputStream(bos)) {
228236

229-
logRequest(request, data);
230-
request.setBody(content);
237+
Defaults.getObjectMapper().writeValue(gzipOS, data);
238+
return new ByteArrayInputStream(bos.toByteArray());
231239

232240
} catch (IOException e) {
233241
throw new AlgoliaRuntimeException("Error while serializing the request", e);
234242
}
235-
}
243+
} else {
244+
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
236245

237-
return request;
246+
Defaults.getObjectMapper().writeValue(out, data);
247+
return new ByteArrayInputStream(out.toByteArray());
248+
249+
} catch (IOException e) {
250+
throw new AlgoliaRuntimeException("Error while serializing the request", e);
251+
}
252+
}
238253
}
239254

240255
/**
@@ -323,19 +338,23 @@ private int getTimeOut(CallType callType) {
323338
}
324339
}
325340

326-
private <T> void logRequest(HttpRequest request, T data) throws JsonProcessingException {
341+
private <T> void logRequest(HttpRequest request, T data) {
327342
if (LOGGER.isLoggable(Level.FINEST)) {
328343
LOGGER.finest(
329344
String.format(
330345
"\n Method: %s \n Path: %s \n Headers: %s",
331346
request.getMethod().toString(), request.getMethodPath(), request.getHeaders()));
332347

333-
LOGGER.finest(
334-
String.format(
335-
"Request body: \n %s ",
336-
Defaults.getObjectMapper()
337-
.writerWithDefaultPrettyPrinter()
338-
.writeValueAsString(data)));
348+
try {
349+
LOGGER.finest(
350+
String.format(
351+
"Request body: \n %s ",
352+
Defaults.getObjectMapper()
353+
.writerWithDefaultPrettyPrinter()
354+
.writeValueAsString(data)));
355+
} catch (JsonProcessingException e) {
356+
throw new AlgoliaRuntimeException("Error while serializing the request", e);
357+
}
339358
}
340359
}
341360

‎algoliasearch-core/src/main/java/com/algolia/search/InsightsConfig.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.algolia.search;
22

33
import com.algolia.search.models.common.CallType;
4+
import com.algolia.search.models.common.CompressionType;
45
import com.algolia.search.util.AlgoliaUtils;
56
import java.util.Collections;
67
import java.util.EnumSet;
@@ -29,7 +30,7 @@ public Builder(@Nonnull String applicationID, @Nonnull String apiKey) {
2930
* @param apiKey The API Key
3031
*/
3132
public Builder(@Nonnull String applicationID, @Nonnull String apiKey, @Nonnull String region) {
32-
super(applicationID, apiKey, createDefaultHosts(region));
33+
super(applicationID, apiKey, createDefaultHosts(region), CompressionType.NONE);
3334
}
3435

3536
@Override

‎algoliasearch-core/src/main/java/com/algolia/search/SearchConfig.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.algolia.search;
22

33
import com.algolia.search.models.common.CallType;
4+
import com.algolia.search.models.common.CompressionType;
45
import java.util.*;
56
import java.util.stream.Collectors;
67
import java.util.stream.Stream;
@@ -13,7 +14,7 @@ public static class Builder extends ConfigBase.Builder<Builder> {
1314

1415
/** Builds a {@link SearchConfig} with the default hosts */
1516
public Builder(@Nonnull String applicationID, @Nonnull String apiKey) {
16-
super(applicationID, apiKey, createDefaultHosts(applicationID));
17+
super(applicationID, apiKey, createDefaultHosts(applicationID), CompressionType.GZIP);
1718
}
1819

1920
@Override
@@ -51,6 +52,12 @@ private static List<StatefulHost> createDefaultHosts(@Nonnull String application
5152

5253
return Stream.concat(hosts.stream(), commonHosts.stream()).collect(Collectors.toList());
5354
}
55+
56+
/** Enables compression for the SearchClient. See {@link CompressionType} */
57+
public Builder setCompressionType(@Nonnull CompressionType compressionType) {
58+
this.compressionType = compressionType;
59+
return this;
60+
}
5461
}
5562

5663
private SearchConfig(Builder builder) {

‎algoliasearch-core/src/main/java/com/algolia/search/models/HttpRequest.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,23 @@
11
package com.algolia.search.models;
22

3+
import com.algolia.search.models.common.CompressionType;
34
import java.io.InputStream;
45
import java.net.URL;
56
import java.util.HashMap;
67
import java.util.Map;
78

89
public class HttpRequest {
910

11+
public HttpRequest(
12+
HttpMethod method,
13+
String methodPath,
14+
Map<String, String> headers,
15+
int timeout,
16+
CompressionType compressionType) {
17+
this(method, methodPath, headers, timeout);
18+
this.compressionType = compressionType;
19+
}
20+
1021
public HttpRequest(
1122
HttpMethod method, String methodPath, Map<String, String> headers, int timeout) {
1223
this.method = method;
@@ -69,6 +80,31 @@ public HttpRequest setTimeout(int timeout) {
6980
return this;
7081
}
7182

83+
public CompressionType getCompressionType() {
84+
return compressionType;
85+
}
86+
87+
public HttpRequest setCompressionType(CompressionType compressionType) {
88+
this.compressionType = compressionType;
89+
return this;
90+
}
91+
92+
/**
93+
* Tells if any compression can be enabled for a request or not. Compression is enabled only for
94+
* POST/PUT methods on the Search API (not on Analytics and Insights).
95+
*/
96+
public boolean canCompress() {
97+
if (this.compressionType == null || this.method == null) {
98+
return false;
99+
}
100+
101+
boolean isMethodValid =
102+
this.method.equals(HttpMethod.POST) || this.method.equals(HttpMethod.PUT);
103+
boolean isCompressionEnabled = this.compressionType.equals(CompressionType.GZIP);
104+
105+
return isMethodValid && isCompressionEnabled;
106+
}
107+
72108
public void incrementTimeout(int retryCount) {
73109
this.timeout *= (retryCount + 1);
74110
}
@@ -79,4 +115,5 @@ public void incrementTimeout(int retryCount) {
79115
private Map<String, String> headers;
80116
private InputStream body;
81117
private int timeout;
118+
private CompressionType compressionType;
82119
}

‎algoliasearch-core/src/main/java/com/algolia/search/models/common/CompressionType.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package com.algolia.search.models.common;
2+
3+
public enum CompressionType {
4+
NONE,
5+
GZIP
6+
}

0 commit comments

Comments
 (0)