Disable transparent ungzip and add decompress interceptor

Parse/src/main/java/com/parse/Parse.java

@@ -596,6 +596,9 @@ private static void initializeParseHttpClientsWithParseNetworkInterceptors() {
 
     // Add interceptors to http clients
     for (ParseHttpClient parseHttpClient : clients) {
+      // We need to add the decompress interceptor before the external interceptors to return
+      // a decompressed response to Parse.
+      parseHttpClient.addInternalInterceptor(new ParseDecompressInterceptor());
       for (ParseNetworkInterceptor interceptor : interceptors) {
         parseHttpClient.addExternalInterceptor(interceptor);
       }

Parse/src/main/java/com/parse/ParseApacheHttpClient.java

@@ -35,7 +35,6 @@
 import org.apache.http.params.BasicHttpParams;
 import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpParams;
-import org.apache.http.params.HttpProtocolParams;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -116,7 +115,9 @@ public ParseApacheHttpClient(int socketOperationTimeout, SSLSessionCache sslSess
     int statusCode = apacheResponse.getStatusLine().getStatusCode();
 
     // Content
-    InputStream content = AndroidHttpClient.getUngzippedContent(apacheResponse.getEntity());
+    InputStream content = disableHttpLibraryAutoDecompress() ?
+        apacheResponse.getEntity().getContent() :
+        AndroidHttpClient.getUngzippedContent(apacheResponse.getEntity());
 
     // Total size
     int totalSize = -1;

Parse/src/main/java/com/parse/ParseDecompressInterceptor.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2015-present, Parse, LLC.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package com.parse;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+/** package */ class ParseDecompressInterceptor implements ParseNetworkInterceptor {
+
+  private static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
+  private static final String CONTENT_LENGTH_HEADER = "Content-Length";
+  private static final String GZIP_ENCODING = "gzip";
+
+  @Override
+  public ParseHttpResponse intercept(Chain chain) throws IOException {
+    ParseHttpRequest request = chain.getRequest();
+    ParseHttpResponse response = chain.proceed(request);
+    // If the response is gziped, we need to decompress the stream and remove the gzip header.
+    if (GZIP_ENCODING.equalsIgnoreCase(response.getHeader(CONTENT_ENCODING_HEADER))) {
+      Map<String, String > newHeaders = new HashMap<>(response.getAllHeaders());
+      newHeaders.remove(CONTENT_ENCODING_HEADER);
+      // Since before we decompress the stream, we can not know the actual length of the stream.
+      // In this situation, we follow the OkHttp library, set the content-length of the response
+      // to -1
+      newHeaders.put(CONTENT_LENGTH_HEADER, "-1");
+      // TODO(mengyan): Add builder constructor based on an existing ParseHttpResponse
+      response = new ParseHttpResponse.Builder()
+          .setTotalSize(-1)
+          .setContentType(response.getContentType())
+          .setHeaders(newHeaders)
+          .setReasonPhase(response.getReasonPhrase())
+          .setStatusCode(response.getStatusCode())
+          .setContent(new GZIPInputStream(response.getContent()))
+          .build();
+    }
+    return response;
+  }
+}
+

Parse/src/main/java/com/parse/ParseHttpClient.java

@@ -32,7 +32,6 @@
   private static final String MAX_CONNECTIONS_PROPERTY_NAME = "http.maxConnections";
   private static final String KEEP_ALIVE_PROPERTY_NAME = "http.keepAlive";
 
-
   public static ParseHttpClient createClient(int socketOperationTimeout,
       SSLSessionCache sslSessionCache) {
     String httpClientLibraryName;
@@ -155,4 +154,14 @@ public ParseHttpResponse proceed(ParseHttpRequest request) throws IOException {
       return executeInternal(request);
     }
   }
+
+  /**
+   * When we find developers use interceptors, since we need expose the raw
+   * response(ungziped response) to interceptors, we need to disable the transparent ungzip.
+   *
+   * @return {@code true} if we should disable the http library level auto decompress.
+   */
+  /* package */ boolean disableHttpLibraryAutoDecompress() {
+    return externalInterceptors != null && externalInterceptors.size() > 0;
+  }
 }

Parse/src/main/java/com/parse/ParseURLConnectionHttpClient.java

@@ -24,6 +24,11 @@
 
 /** package */ class ParseURLConnectionHttpClient extends ParseHttpClient<HttpURLConnection, HttpURLConnection> {
 
+  private static final String ACCEPT_ENCODING_HEADER = "Accept-encoding";
+  private static final String GZIP_ENCODING = "gzip";
+  private static final String CONTENT_LENGTH_HEADER = "Content-Length";
+  private static final String CONTENT_TYPE_HEADER = "Content-Type";
+
   private int socketOperationTimeout;
 
   public ParseURLConnectionHttpClient(int socketOperationTimeout, SSLSessionCache sslSessionCache) {
@@ -71,12 +76,17 @@ public ParseURLConnectionHttpClient(int socketOperationTimeout, SSLSessionCache
       connection.setRequestProperty(entry.getKey(), entry.getValue());
     }
 
+    // When URLConnection is powered by OkHttp, by adding this head, OkHttp will turn off its
+    // transparent decompress which will expose the raw network stream to our interceptors.
+    if (disableHttpLibraryAutoDecompress()) {
+      connection.setRequestProperty(ACCEPT_ENCODING_HEADER, GZIP_ENCODING);
+    }
     // Set body
     ParseHttpBody body = parseRequest.getBody();
     if (body != null) {
       // Content type and content length
-      connection.setRequestProperty("Content-Length", String.valueOf(body.getContentLength()));
-      connection.setRequestProperty("Content-Type", body.getContentType());
+      connection.setRequestProperty(CONTENT_LENGTH_HEADER, String.valueOf(body.getContentLength()));
+      connection.setRequestProperty(CONTENT_TYPE_HEADER, body.getContentType());
       // We need to set this in order to make URLConnection not buffer our request body so that our
       // upload progress callback works.
       connection.setFixedLengthStreamingMode(body.getContentLength());

Parse/src/test/java/com/parse/ParseDecompressInterceptorTest.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2015-present, Parse, LLC.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package com.parse;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricGradleTestRunner;
+import org.robolectric.annotation.Config;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.GZIPOutputStream;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+@RunWith(RobolectricGradleTestRunner.class)
+@Config(constants = BuildConfig.class, sdk = 21)
+public class ParseDecompressInterceptorTest {
+
+  @Test
+  public void testDecompressInterceptorWithNotGZIPResponse() throws Exception {
+    ParseDecompressInterceptor interceptor = new ParseDecompressInterceptor();
+
+    final String responseContent = "content";
+    ParseHttpResponse interceptedResponse =
+        interceptor.intercept(new ParseNetworkInterceptor.Chain() {
+      @Override
+      public ParseHttpRequest getRequest() {
+        // Generate test request
+        return new ParseHttpRequest.Builder()
+            .setUrl("www.parse.com")
+            .setMethod(ParseRequest.Method.GET)
+            .build();
+      }
+
+      @Override
+      public ParseHttpResponse proceed(ParseHttpRequest request) throws IOException {
+        // Generate test response
+        return new ParseHttpResponse.Builder()
+            .setStatusCode(200)
+            .setTotalSize(responseContent.length())
+            .setReasonPhase("Success")
+            .setContentType("text/plain")
+            .setContent(new ByteArrayInputStream(responseContent.getBytes()))
+            .build();
+      }
+    });
+
+    // Verify response is correct
+    assertEquals(200, interceptedResponse.getStatusCode());
+    assertEquals(responseContent.length(), interceptedResponse.getTotalSize());
+    assertEquals("Success", interceptedResponse.getReasonPhrase());
+    assertEquals("text/plain", interceptedResponse.getContentType());
+    byte[] content = ParseIOUtils.toByteArray(interceptedResponse.getContent());
+    assertArrayEquals(responseContent.getBytes(), content);
+  }
+
+  @Test
+  public void testDecompressInterceptorWithGZIPResponse() throws Exception {
+    ParseDecompressInterceptor interceptor = new ParseDecompressInterceptor();
+
+    final String responseContent = "content";
+    ParseHttpResponse interceptedResponse =
+        interceptor.intercept(new ParseNetworkInterceptor.Chain() {
+      @Override
+      public ParseHttpRequest getRequest() {
+        // Generate test request
+        return new ParseHttpRequest.Builder()
+            .setUrl("www.parse.com")
+            .setMethod(ParseRequest.Method.GET)
+            .build();
+      }
+
+      @Override
+      public ParseHttpResponse proceed(ParseHttpRequest request) throws IOException {
+        // Make gzip response content
+        ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
+        GZIPOutputStream gzipOut = new GZIPOutputStream(byteOut);
+        gzipOut.write(responseContent.getBytes());
+        gzipOut.close();
+        // Make gzip encoding headers
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Encoding", "gzip");
+        // Generate test response
+        return new ParseHttpResponse.Builder()
+            .setStatusCode(200)
+            .setTotalSize(byteOut.toByteArray().length)
+            .setReasonPhase("Success")
+            .setContentType("text/plain")
+            .setContent(new ByteArrayInputStream(byteOut.toByteArray()))
+            .setHeaders(headers)
+            .build();
+      }
+    });
+
+    // Verify response is correct
+    assertEquals(200, interceptedResponse.getStatusCode());
+    assertEquals(-1, interceptedResponse.getTotalSize());
+    assertEquals("Success", interceptedResponse.getReasonPhrase());
+    assertEquals("text/plain", interceptedResponse.getContentType());
+    assertNull(interceptedResponse.getHeader("Content-Encoding"));
+    byte[] content = ParseIOUtils.toByteArray(interceptedResponse.getContent());
+    assertArrayEquals(responseContent.getBytes(), content);
+  }
+}

Parse/src/test/java/com/parse/ParseHttpClientTest.java

@@ -13,23 +13,25 @@
 import com.squareup.okhttp.mockwebserver.MockWebServer;
 import com.squareup.okhttp.mockwebserver.RecordedRequest;
 
-import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.robolectric.RobolectricGradleTestRunner;
 import org.robolectric.annotation.Config;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.zip.GZIPOutputStream;
+
+import okio.Buffer;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 @RunWith(RobolectricGradleTestRunner.class)
@@ -42,34 +44,50 @@ public class ParseHttpClientTest {
 
   @Test
   public void testParseApacheHttpClientExecuteWithSuccessResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(200, "OK", "Success",
-        new ParseApacheHttpClient(10000, null));
+    doSingleParseHttpClientExecuteWithResponse(
+        200, "OK", "Success", new ParseApacheHttpClient(10000, null));
   }
 
   @Test
   public void testParseURLConnectionHttpClientExecuteWithSuccessResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(200, "OK", "Success",
-        new ParseApacheHttpClient(10000, null));  }
+    doSingleParseHttpClientExecuteWithResponse(
+        200, "OK", "Success", new ParseURLConnectionHttpClient(10000, null));  }
 
   @Test
   public void testParseOkHttpClientExecuteWithSuccessResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(200, "OK", "Success",
-        new ParseApacheHttpClient(10000, null));  }
+    doSingleParseHttpClientExecuteWithResponse(
+        200, "OK", "Success", new ParseOkHttpClient(10000, null));  }
 
   @Test
   public void testParseApacheHttpClientExecuteWithErrorResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(404, "NOT FOUND", "Error",
-        new ParseApacheHttpClient(10000, null));  }
+    doSingleParseHttpClientExecuteWithResponse(
+        404, "NOT FOUND", "Error", new ParseApacheHttpClient(10000, null));  }
 
   @Test
   public void testParseURLConnectionHttpClientExecuteWithErrorResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(404, "NOT FOUND", "Error",
-        new ParseURLConnectionHttpClient(10000, null));  }
+    doSingleParseHttpClientExecuteWithResponse(
+        404, "NOT FOUND", "Error", new ParseURLConnectionHttpClient(10000, null));  }
 
   @Test
   public void testParseOkHttpClientExecuteWithErrorResponse() throws Exception {
-    doSingleParseHttpClientExecuteWithResponse(404, "NOT FOUND", "Error",
-        new ParseOkHttpClient(10000, null));  }
+    doSingleParseHttpClientExecuteWithResponse(
+        404, "NOT FOUND", "Error", new ParseOkHttpClient(10000, null));  }
+
+  @Test
+  public void testParseApacheHttpClientExecuteWithGzipResponse() throws Exception {
+    doSingleParseHttpClientExecuteWithGzipResponse(
+        200, "OK", "Success", new ParseApacheHttpClient(10000, null));
+  }
+
+  // TODO(mengyan): Add testParseURLConnectionHttpClientExecuteWithGzipResponse, right now we can
+  // not do that since in unit test env, URLConnection does not use OKHttp internally, so there is
+  // no transparent ungzip
+
+  @Test
+  public void testParseOkHttpClientExecuteWithGzipResponse() throws Exception {
+    doSingleParseHttpClientExecuteWithGzipResponse(
+        200, "OK", "Success", new ParseOkHttpClient(10000, null));
+  }
 
   private void doSingleParseHttpClientExecuteWithResponse(int responseCode, String responseStatus,
       String responseContent, ParseHttpClient client) throws Exception {
@@ -141,4 +159,53 @@ private void doSingleParseHttpClientExecuteWithResponse(int responseCode, String
     // Shutdown mock server
     server.shutdown();
   }
+
+  private void doSingleParseHttpClientExecuteWithGzipResponse(
+      int responseCode, String responseStatus, final String responseContent, ParseHttpClient client)
+      throws Exception {
+    MockWebServer server = new MockWebServer();
+
+    // Make mock response
+    Buffer buffer = new Buffer();
+    ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
+    GZIPOutputStream gzipOut = new GZIPOutputStream(byteOut);
+    gzipOut.write(responseContent.getBytes());
+    gzipOut.close();
+    buffer.write(byteOut.toByteArray());
+    MockResponse mockResponse = new MockResponse()
+        .setStatus("HTTP/1.1 " + responseCode + " " + responseStatus)
+        .setBody(buffer)
+        .setHeader("Content-Encoding", "gzip");
+
+    // Start mock server
+    server.enqueue(mockResponse);
+    server.start();
+
+    // We do not need to add Accept-Encoding header manually, httpClient library should do that.
+    String requestUrl = server.getUrl("/").toString();
+    ParseHttpRequest parseRequest = new ParseHttpRequest.Builder()
+        .setUrl(requestUrl)
+        .setMethod(ParseRequest.Method.GET)
+        .build();
+
+    // Execute request
+    ParseHttpResponse parseResponse = client.execute(parseRequest);
+
+    RecordedRequest recordedRequest = server.takeRequest();
+
+    // Verify request method
+    assertEquals(ParseRequest.Method.GET.toString(), recordedRequest.getMethod());
+
+    // Verify request headers
+    Headers recordedHeaders = recordedRequest.getHeaders();
+
+    assertEquals("gzip", recordedHeaders.get("Accept-Encoding"));
+
+    // Verify response body
+    byte[] content = ParseIOUtils.toByteArray(parseResponse.getContent());
+    assertArrayEquals(responseContent.getBytes(), content);
+
+    // Shutdown mock server
+    server.shutdown();
+  }
 }