Support compressed request bodies for JSON

backdrop/write/api.py

@@ -7,6 +7,7 @@
 from backdrop.core.data_set import DataSet
 from backdrop.core.flaskutils import DataSetConverter
 from backdrop.core.repository import DataSetConfigRepository
+from backdrop.write.decompressing_request import DecompressingRequest
 
 from ..core.errors import ParseError, ValidationError
 from ..core import database, log_handler, cache_control
@@ -18,6 +19,9 @@
 
 app = Flask("backdrop.write.api")
 
+# We want to allow clients to compress large JSON documents.
+app.request_class = DecompressingRequest
+
 feature_flags = FeatureFlag(app)
 
 # Configuration

backdrop/write/decompressing_request.py

@@ -0,0 +1,53 @@
+import gzip
+
+from flask import current_app, g
+from flask.wrappers import Request
+from io import BytesIO
+
+
+class DecompressingRequest(Request):
+    """
+    Subclass of flask.wrappers.Request which supports requests made with
+    compressed requests bodies.
+
+    See http://tools.ietf.org/html/rfc2616#section-14.11
+    """
+
+    def get_data(self, cache=True, as_text=False, parse_form_data=False):
+        """
+        Override the get_data method to check whether the request entity
+        was compressed. If it was, transparently decompress it and carry on.
+
+        This is somewhat inefficient in that the entire entity is loaded
+        into memory, rather than decorating the existing stream.
+        """
+        content_encoding = self.headers.get('content-encoding', '').lower()
+
+        if (not g.get('_has_decompressed_entity', False)
+                and 'gzip' in content_encoding):
+
+            # Decompress the stream
+            bytes = super(DecompressingRequest, self).get_data(cache,
+                                                               as_text,
+                                                               parse_form_data)
+
+            current_app.logger.debug("Got gzipped stream of length <%d>" %
+                                     len(bytes))
+
+            gzipped_content = BytesIO(bytes)
+
+            decompressed_content = gzip.GzipFile(mode='rb',
+                                                 fileobj=gzipped_content)
+
+            data = decompressed_content.read().decode('utf-8')
+
+            current_app.logger.debug("Got JSON of length <%s>" % (len(data)))
+
+            # Store it on the _cached_data attribute for future reads
+            self._cached_data = data
+
+            g._has_decompressed_entity = True
+
+        return super(DecompressingRequest, self).get_data(cache,
+                                                          as_text,
+                                                          parse_form_data)

features/fixtures/large.json

@@ -0,0 +1,3 @@
+{
+  "key":"some large value here that will really benefit from being compressed, thus being a lot more efficient over the network"
+}

features/steps/write_api.py

@@ -1,8 +1,10 @@
+import gzip
 import os
 
 from behave import given, when, then
 from dateutil import parser
 from hamcrest import assert_that, has_length, equal_to
+from io import BytesIO
 
 
 FIXTURE_PATH = os.path.join(os.path.dirname(__file__), '..', 'fixtures')
@@ -35,6 +37,32 @@ def step(context, token):
     context.bearer_token = token
 
 
+@when('I "{http_method}" the compressed request body to the path "{path}"')
+def step(context, http_method, path):
+    assert http_method in ('POST', 'PUT'), "Only support POST, PUT"
+    http_function = {
+        'POST': context.client.post,
+        'PUT': context.client.put
+    }[http_method]
+
+    def compress_data(io):
+        bio = BytesIO()
+        f = gzip.GzipFile(filename='', mode='wb', fileobj=bio)
+        f.write(io)
+        f.close()
+        return bio.getvalue()
+
+    headers = _make_headers_from_context(context)
+    headers.append(('Content-Encoding', u'gzip'))
+
+    context.response = http_function(
+        path,
+        data=compress_data(context.data_to_post),
+        content_type="application/json",
+        headers=headers,
+    )
+
+
 @when('I {http_method} to the specific path "{path}"')
 def step(context, http_method, path):
     assert http_method in ('POST', 'PUT'), "Only support POST, PUT"

features/write_api/write_api.feature

@@ -83,3 +83,13 @@ Feature: the performance platform write api
          when I POST to the specific path "/data/group/type"
          then I should get back a status of "400"
           and I should get back the message "Expected JSON request body but received zero bytes."
+
+    Scenario: posting large compressed JSON payload to a data-set
+        Given I have the data in "large.json"
+          and I have a data_set named "data_with_times" with settings
+            | key        | value   |
+            | data_group | "group" |
+            | data_type  | "type"  |
+          and I use the bearer token for the data_set
+         when I "POST" the compressed request body to the path "/data/group/type"
+         then I should get back a status of "200"