Add compression support by default to hraven client using apache libs

twitter · Apr 6, 2017 · bf21fae · bf21fae
1 parent 2587b5e
commit bf21fae
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 20 deletions.
diff --git a/hraven-core/pom.xml b/hraven-core/pom.xml
@@ -233,6 +233,17 @@
       <artifactId>commons-lang</artifactId>
       <version>2.5</version>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.4</version>
+    </dependency>
+    <!-- used in the rest client -->
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.5.3</version>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

diff --git a/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java b/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java
@@ -2,12 +2,16 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
 import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.codehaus.jackson.type.TypeReference;
 
 import com.twitter.hraven.util.JSONUtil;
@@ -16,8 +20,6 @@ class UrlDataLoader<T> {
 
     private static final Log LOG = LogFactory.getLog(UrlDataLoader.class);
 
-    private static final String ACCEPT_ENCODING = "Accept-Encoding";
-
     private String endpointURL;
     private TypeReference typeRef;
     private int connectTimeout;
@@ -53,25 +55,32 @@ public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, in
     @SuppressWarnings("unchecked")
     public List<T> load() throws IOException {
       InputStream input = null;
+
+      RequestConfig requestConfig =
+          RequestConfig.custom()
+              .setConnectTimeout(connectTimeout)
+              .setConnectionRequestTimeout(connectTimeout)
+              .setSocketTimeout(readTimeout).build();
+      HttpClientBuilder httpClientBuilder =
+          HttpClientBuilder.create().setDefaultRequestConfig(requestConfig);
+
+      if (! useCompression) {
+        LOG.info("Not using compression!");
+        httpClientBuilder.disableContentCompression();
+      } else {
+        LOG.info("Using compression by default! Trying gzip, deflate");
+      }
+
+      CloseableHttpClient httpClient = httpClientBuilder.build();
+      HttpGet httpGet = new HttpGet(endpointURL);
+      HttpResponse response = httpClient.execute(httpGet);
+
       try {
-        URL url = new URL(endpointURL);
-        URLConnection connection = url.openConnection();
-        connection.setConnectTimeout(connectTimeout);
-        connection.setReadTimeout(readTimeout);
-        if (useCompression) {
-          connection.setRequestProperty(ACCEPT_ENCODING, "gzip, deflate");
-        }
-        input = connection.getInputStream();
+        input = response.getEntity().getContent();
         return (List<T>) JSONUtil.readJson(input, typeRef);
       } finally {
-        if (input != null) {
-          try {
-            input.close();
-          } catch (IOException e) {
-            LOG.warn(e);
-          }
-        }
+        IOUtils.closeQuietly(input);
+        IOUtils.closeQuietly(httpClient);
       }
     }
 }
-