Add support to request for compressed payloads from the HRavenRestClient

hraven-core/pom.xml

@@ -233,6 +233,17 @@
       <artifactId>commons-lang</artifactId>
       <version>2.5</version>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.4</version>
+    </dependency>
+    <!-- used in the rest client -->
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.5.3</version>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java

@@ -2,12 +2,16 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
 import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.codehaus.jackson.type.TypeReference;
 
 import com.twitter.hraven.util.JSONUtil;
@@ -20,40 +24,63 @@ class UrlDataLoader<T> {
     private TypeReference typeRef;
     private int connectTimeout;
     private int readTimeout;
+    private boolean useCompression;
 
     /**
-     * Constructor.
+     * Constructor, defaults to using compression (gzip / deflate).
      * @param endpointUrl
      * @param t TypeReference for json deserialization, should be TypeReference<List<T>>.
      * @throws java.io.IOException
      */
     public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout)
         throws IOException {
+      this(endpointUrl, t, connectTimeout, readTimeout, true);
+    }
+
+    /**
+     * Constructor.
+     * @param endpointUrl
+     * @param t TypeReference for json deserialization, should be TypeReference<List<T>>.
+     * @throws java.io.IOException
+    */
+    public UrlDataLoader(String endpointUrl, TypeReference t, int connectTimeout, int readTimeout,
+                         boolean useCompression) throws IOException {
       this.endpointURL = endpointUrl;
       this.typeRef = t;
       this.connectTimeout = connectTimeout;
       this.readTimeout = readTimeout;
+      this.useCompression = useCompression;
     }
 
     @SuppressWarnings("unchecked")
     public List<T> load() throws IOException {
       InputStream input = null;
+
+      RequestConfig requestConfig =
+          RequestConfig.custom()
+              .setConnectTimeout(connectTimeout)
+              .setConnectionRequestTimeout(connectTimeout)
+              .setSocketTimeout(readTimeout).build();
+      HttpClientBuilder httpClientBuilder =
+          HttpClientBuilder.create().setDefaultRequestConfig(requestConfig);
+
+      if (! useCompression) {
+        LOG.info("Not using compression!");
+        httpClientBuilder.disableContentCompression();
+      } else {
+        LOG.debug("Using compression by default! Trying gzip, deflate");
+      }
+
+      CloseableHttpClient httpClient = httpClientBuilder.build();
+      HttpGet httpGet = new HttpGet(endpointURL);
+      HttpResponse response = httpClient.execute(httpGet);
+
       try {
-        URL url = new URL(endpointURL);
-        URLConnection connection = url.openConnection();
-        connection.setConnectTimeout(connectTimeout);
-        connection.setReadTimeout(readTimeout);
-        input = connection.getInputStream();
+        input = response.getEntity().getContent();
         return (List<T>) JSONUtil.readJson(input, typeRef);
       } finally {
-        if (input != null) {
-          try {
-            input.close();
-          } catch (IOException e) {
-            LOG.warn(e);
-          }
-        }
+        IOUtils.closeQuietly(input);
+        IOUtils.closeQuietly(httpClient);
       }
     }
 }
-