Skip to content
This repository has been archived by the owner on Jan 15, 2022. It is now read-only.

Add support to request for compressed payloads from the HRavenRestClient #161

Merged
merged 3 commits into from
Apr 10, 2017
Merged

Add support to request for compressed payloads from the HRavenRestClient #161

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2587b5e
Enable gzip compression
Apr 6, 2017
bf21fae
Add compression support by default to hraven client using apache libs
Apr 6, 2017
111167f
Switch to debug
Apr 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions hraven-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,17 @@
<artifactId>commons-lang</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<!-- used in the rest client -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand Down
59 changes: 43 additions & 16 deletions hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.codehaus.jackson.type.TypeReference;

import com.twitter.hraven.util.JSONUtil;
Expand All @@ -20,40 +24,63 @@ class UrlDataLoader<T> {
private TypeReference typeRef;
private int connectTimeout;
private int readTimeout;
private boolean useCompression;

/**
* Constructor.
* Constructor, defaults to using compression (gzip / deflate).
* @param endpointUrl
* @param t TypeReference for json deserialization, should be TypeReference<List<T>>.
* @throws java.io.IOException
*/
public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout)
throws IOException {
this(endpointUrl, t, connectTimeout, readTimeout, true);
}

/**
* Constructor.
* @param endpointUrl
* @param t TypeReference for json deserialization, should be TypeReference<List<T>>.
* @throws java.io.IOException
*/
public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout,
boolean useCompression) throws IOException {
this.endpointURL = endpointUrl;
this.typeRef = t;
this.connectTimeout = connectTimeout;
this.readTimeout = readTimeout;
this.useCompression = useCompression;
}

@SuppressWarnings("unchecked")
public List<T> load() throws IOException {
InputStream input = null;

RequestConfig requestConfig =
RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.setConnectionRequestTimeout(connectTimeout)
.setSocketTimeout(readTimeout).build();
HttpClientBuilder httpClientBuilder =
HttpClientBuilder.create().setDefaultRequestConfig(requestConfig);

if (! useCompression) {
LOG.info("Not using compression!");
httpClientBuilder.disableContentCompression();
} else {
LOG.info("Using compression by default! Trying gzip, deflate");
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's move it to debug.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, chatted with Piyush offline. We can change the default one using compression to debug mode and keep the "not using compression" in info mode

}

CloseableHttpClient httpClient = httpClientBuilder.build();
HttpGet httpGet = new HttpGet(endpointURL);
HttpResponse response = httpClient.execute(httpGet);

try {
URL url = new URL(endpointURL);
URLConnection connection = url.openConnection();
connection.setConnectTimeout(connectTimeout);
connection.setReadTimeout(readTimeout);
input = connection.getInputStream();
input = response.getEntity().getContent();
return (List<T>) JSONUtil.readJson(input, typeRef);
} finally {
if (input != null) {
try {
input.close();
} catch (IOException e) {
LOG.warn(e);
}
}
IOUtils.closeQuietly(input);
IOUtils.closeQuietly(httpClient);
}
}
}