Merge pull request #1202 from CartoDB/use-gzip-compression-geojson

Use Gzip compression for GeoJSON visualizations

CHANGELOG.md

@@ -5,18 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Changed
+- Optimize local data visualizations size using gzip compression (#1202)
+
 ## [1.0b5] - 2019-11-14
-# Added
+### Added
 - Add isolines_layer helper method (#1135, #1159)
 - Add range_min/max params to the continuous layer helper methods (#1120)
 - Add with_lnglat param to the Isolines service functions (#1134)
 
-# Changed
+### Changed
 - Refactor DO Enrichment into classes (#1127, #1137, #1170, #1196)
 - Improve Credentials UX (#1028, #776, #1022)
 - Optimize local data columns used in the map (#551)
 
-# Fixed
+### Fixed
 - Fix encoding in data upload (#1133, CartoDB/support#2219)
 - Fix dataset.geom_coverage method: (#1153)
 - Allow iterables in the breaks param of color_bins_layer (#1146)

cartoframes/assets/src/bundle.js

@@ -456,8 +456,8 @@ var init = (function () {
     return new carto.source.MVT(layer.data.file, JSON.parse(layer.data.metadata));
   }
 
-  function _decodeJSONData(data) {
-    return JSON.parse(Base64.decode(data.replace(/b\'/, '\'')));
+  function _decodeJSONData(b64Data) {
+    return JSON.parse(pako.inflate(atob(b64Data), { to: 'string' }));
   }
 
   const factory = new SourceFactory();

cartoframes/assets/src/map/SourceFactory.js

@@ -27,6 +27,6 @@ function MVT(layer) {
   return new carto.source.MVT(layer.data.file, JSON.parse(layer.data.metadata));
 }
 
-function _decodeJSONData(data) {
-  return JSON.parse(Base64.decode(data.replace(/b\'/, '\'')));
+function _decodeJSONData(b64Data) {
+  return JSON.parse(pako.inflate(atob(b64Data), { to: 'string' }));
 }

cartoframes/assets/templates/viz/main.html.j2

@@ -22,9 +22,9 @@
   <link href="https://fonts.googleapis.com/css?family=Roboto" rel="stylesheet" type="text/css">
 
   <!-- External libraries -->
-  
-  <!-- base64-js -->
-  <script src='https://libs.cartocdn.com/cartoframes/dependencies/base64.js'></script>
+
+  <!-- pako -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/pako/1.0.10/pako_inflate.min.js"></script>
 
   <!-- html2canvas -->
   {% if is_static %}

cartoframes/utils/utils.py

@@ -4,6 +4,7 @@
 
 import re
 import sys
+import gzip
 import json
 import base64
 import decimal
@@ -20,6 +21,18 @@
 except NameError:
     basestring = str
 
+if sys.version_info < (3, 0):
+    from io import BytesIO
+    from gzip import GzipFile
+
+    def compress(data):
+        buf = BytesIO()
+        with GzipFile(fileobj=buf, mode='wb') as f:
+            f.write(data)
+        return buf.getvalue()
+
+    gzip.compress = compress
+
 
 GEOM_TYPE_POINT = 'point'
 GEOM_TYPE_LINE = 'line'
@@ -282,10 +295,9 @@ def default(self, o):
 
 def encode_geodataframe(data):
     filtered_geometries = _filter_null_geometries(data)
-    data = _set_time_cols_epoc(filtered_geometries).to_json(cls=CustomJSONEncoder)
-    encoded_data = base64.b64encode(data.encode('utf-8')).decode('utf-8')
-
-    return encoded_data
+    data = _set_time_cols_epoc(filtered_geometries).to_json(cls=CustomJSONEncoder, separators=(',', ':'))
+    compressed_data = gzip.compress(data.encode('utf-8'))
+    return base64.b64encode(compressed_data).decode('utf-8')
 
 
 def _filter_null_geometries(data):