Merge e78ef06 into efbee1b

bindings/pydeck/pydeck/bindings/deck.py

@@ -103,6 +103,14 @@ def update(self):
         Intended for use in a Jupyter environment.
         """
         self.deck_widget.json_input = self.to_json()
+        has_binary = False
+        binary_data_sets = []
+        for layer in self.layers:
+            if layer.use_binary_transport:
+                binary_data_sets.extend(layer.get_binary_data())
+                has_binary = True
+        if has_binary:
+            self.deck_widget.data_buffer = binary_data_sets
 
     def to_html(
         self,

bindings/pydeck/pydeck/bindings/json_tools.py

@@ -3,8 +3,14 @@
 """
 import json
 
-# Ignore deck_widget and mapbox_key attributes
-IGNORE_KEYS = ['mapbox_key', 'deck_widget']
+# Attributes to ignore during JSON serialization
+IGNORE_KEYS = [
+    "mapbox_key",
+    "deck_widget",
+    "binary_data_sets",
+    "_binary_data",
+    "_kwargs",
+]
 
 
 def to_camel_case(snake_case):
@@ -20,10 +26,10 @@ def to_camel_case(snake_case):
     str
         Camel-cased (e.g., "camelCased") version of input string
     """
-    output_str = ''
+    output_str = ""
     should_upper_case = False
     for c in snake_case:
-        if c == '_':
+        if c == "_":
             should_upper_case = True
             continue
         output_str = output_str + c.upper() if should_upper_case else output_str + c
@@ -32,7 +38,11 @@ def to_camel_case(snake_case):
 
 
 def lower_first_letter(s):
-    return s[:1].lower() + s[1:] if s else ''
+    return s[:1].lower() + s[1:] if s else ""
+
+
+def camel_and_lower(w):
+    return lower_first_letter(to_camel_case(w))
 
 
 def lower_camel_case_keys(attrs):
@@ -44,9 +54,9 @@ def lower_camel_case_keys(attrs):
         Dictionary for which all the keys should be converted to camel-case
     """
     for snake_key in list(attrs.keys()):
-        if '_' not in snake_key:
+        if "_" not in snake_key:
             continue
-        camel_key = lower_first_letter(to_camel_case(snake_key))
+        camel_key = camel_and_lower(snake_key)
         attrs[camel_key] = attrs.pop(snake_key)
 
 
@@ -68,7 +78,6 @@ def serialize(serializable):
 
 
 class JSONMixin(object):
-
     def __repr__(self):
         """
         Override of string representation method to return a JSON-ified version of the

bindings/pydeck/pydeck/bindings/layer.py

@@ -1,22 +1,22 @@
 import uuid
 
+import numpy as np
+
 from ..data_utils import is_pandas_df
-from .json_tools import JSONMixin
+from .json_tools import JSONMixin, camel_and_lower
 
 
-TYPE_IDENTIFIER = '@@type'
-FUNCTION_IDENTIFIER = '@@='
+TYPE_IDENTIFIER = "@@type"
+FUNCTION_IDENTIFIER = "@@="
 QUOTE_CHARS = {"'", '"', "`"}
 
 
+class BinaryTransportException(Exception):
+    pass
+
+
 class Layer(JSONMixin):
-    def __init__(
-        self,
-        type,
-        data,
-        id=None,
-        **kwargs
-    ):
+    def __init__(self, type, data, id=None, use_binary_transport=None, **kwargs):
         """Configures a deck.gl layer for rendering on a map. Parameters passed
         here will be specific to the particular deck.gl layer that you are choosing to use.
 
@@ -30,10 +30,12 @@ def __init__(
 
         type : str
             Type of layer to render, e.g., `HexagonLayer`
-        id : str
+        id : str, default None
             Unique name for layer
         data : str or list of dict of {str: Any} or pandas.DataFrame
             Either a URL of data to load in or an array of data
+        use_binary_transport : bool, default None
+            Boolean indicating binary data
         **kwargs
             Any of the parameters passable to a deck.gl layer.
 
@@ -75,10 +77,9 @@ def __init__(
         """
         self.type = type
         self.id = id or str(uuid.uuid4())
-        self._data = None
-        self.data = data.to_dict(orient='records') if is_pandas_df(data) else data
 
         # Add any other kwargs to the JSON output
+        self._kwargs = kwargs.copy()
         if kwargs:
             for k, v in kwargs.items():
                 # We assume strings and arrays of strings are identifiers
@@ -89,35 +90,79 @@ def __init__(
 
                 if isinstance(v, str) and v[0] in QUOTE_CHARS and v[0] == v[-1]:
                     # Skip quoted strings
-                    kwargs[k] = v.replace(v[0], '')
+                    kwargs[k] = v.replace(v[0], "")
                 elif isinstance(v, str):
                     # Have @deck.gl/json treat strings values as functions
                     kwargs[k] = FUNCTION_IDENTIFIER + v
 
                 elif isinstance(v, list) and v != [] and isinstance(v[0], str):
                     # Allows the user to pass lists e.g. to specify coordinates
-                    array_as_str = ''
+                    array_as_str = ""
                     for i, identifier in enumerate(v):
                         if i == len(v) - 1:
-                            array_as_str += '{}'.format(identifier)
+                            array_as_str += "{}".format(identifier)
                         else:
-                            array_as_str += '{}, '.format(identifier)
-                    kwargs[k] = '{}[{}]'.format(FUNCTION_IDENTIFIER, array_as_str)
+                            array_as_str += "{}, ".format(identifier)
+                    kwargs[k] = "{}[{}]".format(FUNCTION_IDENTIFIER, array_as_str)
 
             self.__dict__.update(kwargs)
 
+        self._data = None
+        self.use_binary_transport = use_binary_transport
+        self._binary_data = None
+        if not self.use_binary_transport:
+            self.data = data.to_dict(orient="records") if is_pandas_df(data) else data
+        else:
+            self.data = data
+
     @property
     def data(self):
         return self._data
 
     @data.setter
     def data(self, data_set):
-        """Make the data attribute a list no matter the input type"""
-        if is_pandas_df(data_set):
-            self._data = data_set.to_dict(orient='records')
+        """Make the data attribute a list no matter the input type, unless
+        use_binary_transport is specified, which case we circumvent
+        serializing the data to JSON
+        """
+        if self.use_binary_transport:
+            self._binary_data = self._prepare_binary_data(data_set)
+        elif is_pandas_df(data_set):
+            self._data = data_set.to_dict(orient="records")
         else:
             self._data = data_set
 
+    def get_binary_data(self):
+        if not self.use_binary_transport:
+            raise BinaryTransportException(
+                "Layer must be flagged with `use_binary_transport=True`"
+            )
+        return self._binary_data
+
+    def _prepare_binary_data(self, data_set):
+        # Binary format conversion gives a sizable speedup but requires
+        # slightly stricter standards for data input
+        if not is_pandas_df(data_set):
+            raise BinaryTransportException(
+                "Layer data must be a `pandas.DataFrame` type"
+            )
+
+        layer_accessors = self._kwargs
+        inv_map = {v: k for k, v in layer_accessors.items()}
+
+        blobs = []
+        for column in data_set.columns:
+            np_data = np.stack(data_set[column].to_numpy())
+            blobs.append(
+                {
+                    "layer_id": self.id,
+                    "column_name": column,
+                    "accessor": camel_and_lower(inv_map[column]),
+                    "np_data": np_data,
+                }
+            )
+        return blobs
+
     @property
     def type(self):
         return getattr(self, TYPE_IDENTIFIER)

bindings/pydeck/pydeck/data_utils/binary_transfer.py

@@ -0,0 +1,60 @@
+from collections import defaultdict
+
+import numpy as np
+
+# Grafted from
+# https://github.com/maartenbreddels/ipyvolume/blob/d13828dfd8b57739004d5daf7a1d93ad0839ed0f/ipyvolume/serialize.py#L219
+def array_to_binary(ar, obj=None, force_contiguous=True):
+    if ar is None:
+        return None
+    if ar.dtype.kind not in ["u", "i", "f"]:  # ints and floats
+        raise ValueError("unsupported dtype: %s" % (ar.dtype))
+    # WebGL does not support float64, case it here
+    if ar.dtype == np.float64:
+        ar = ar.astype(np.float32)
+    # JS does not support int64
+    if ar.dtype == np.int64:
+        ar = ar.astype(np.int32)
+    # make sure it's contiguous
+    if force_contiguous and not ar.flags["C_CONTIGUOUS"]:
+        ar = np.ascontiguousarray(ar)
+    return {
+        # binary data representation of a numpy matrix
+        "value": memoryview(ar),
+        # dtype convertible to a typed array
+        "dtype": str(ar.dtype),
+        # height of np matrix
+        "length": ar.shape[0],
+        # width of np matrix
+        "size": 1 if len(ar.shape) == 1 else ar.shape[1],
+    }
+
+
+def serialize_columns(data_set_cols, obj=None):
+    if data_set_cols is None:
+        return None
+    layers = defaultdict(dict)
+    # Number of records in data set
+    length = {}
+    for col in data_set_cols:
+        accessor_attribute = array_to_binary(col["np_data"])
+        if length.get(col['layer_id']):
+            length[col['layer_id']] = max(length[col['layer_id']], accessor_attribute['length'])
+        else:
+            length[col['layer_id']] = accessor_attribute['length']
+        # attributes is deck.gl's expected argument name for
+        # binary data transfer
+        if not layers[col['layer_id']].get('attributes'):
+            layers[col['layer_id']]['attributes'] = {}
+        # Add new accessor
+        layers[col['layer_id']]['attributes'][col['accessor']] = {
+            'value': accessor_attribute['value'],
+            'dtype': accessor_attribute['dtype'],
+            'size': accessor_attribute['size']
+        }
+    for layer_key, _ in layers.items():
+        layers[layer_key]['length'] = length[layer_key]
+    return layers
+
+
+data_buffer_serialization = dict(to_json=serialize_columns, from_json=None)

bindings/pydeck/pydeck/io/html.py

@@ -5,7 +5,6 @@
 import webbrowser
 
 import jinja2
-from IPython.display import IFrame
 
 
 TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), './templates/')
@@ -82,6 +81,7 @@ def deck_to_html(
     if open_browser:
         display_html(realpath(f.name))
     if notebook_display:
+        from IPython.display import IFrame  # noqa
         notebook_to_html_path = relpath(f.name)
         display(IFrame(os.path.join('./', notebook_to_html_path), width=iframe_width, height=iframe_height))  # noqa
     return realpath(f.name)

bindings/pydeck/pydeck/widget/widget.py

@@ -5,8 +5,10 @@
 import ipywidgets as widgets
 from traitlets import Any, Bool, Int, Unicode
 
+from ..data_utils.binary_transfer import data_buffer_serialization
 from ._frontend import module_name, module_version
 
+
 @widgets.register
 class DeckGLWidget(widgets.DOMWidget):
     """
@@ -34,16 +36,20 @@ class DeckGLWidget(widgets.DOMWidget):
         js_warning : bool, default False
             Whether the string message from deck.gl should be rendered, defaults to False
     """
-    _model_name = Unicode('DeckGLModel').tag(sync=True)
+
+    _model_name = Unicode("DeckGLModel").tag(sync=True)
     _model_module = Unicode(module_name).tag(sync=True)
     _model_module_version = Unicode(module_version).tag(sync=True)
-    _view_name = Unicode('DeckGLView').tag(sync=True)
+    _view_name = Unicode("DeckGLView").tag(sync=True)
     _view_module = Unicode(module_name).tag(sync=True)
     _view_module_version = Unicode(module_version).tag(sync=True)
-    mapbox_key = Unicode('', allow_none=True).tag(sync=True)
-    json_input = Unicode('').tag(sync=True)
+    mapbox_key = Unicode("", allow_none=True).tag(sync=True)
+    json_input = Unicode("").tag(sync=True)
+    data_buffer = Any(default_value=None, allow_none=True).tag(
+        sync=True, **data_buffer_serialization
+    )
     height = Int(500).tag(sync=True)
-    width = Any('100%').tag(sync=True)
-    selected_data = Unicode('[]').tag(sync=True)
+    width = Any("100%").tag(sync=True)
+    selected_data = Unicode("[]").tag(sync=True)
     tooltip = Any(True).tag(sync=True)
     js_warning = Bool(False).tag(sync=True)

bindings/pydeck/requirements-dev.txt

@@ -13,3 +13,4 @@ requests
 sphinx
 recommonmark
 jupyterlab
+ipython>=5.8.0;python_version<"3.4"

bindings/pydeck/requirements.txt

@@ -1,5 +1,5 @@
 ipykernel>=5.1.2;python_version>="3.4"
-ipython>=5.8.0;python_version<"3.4"
 ipywidgets>=7.0.0
 traitlets>=4.3.2
 Jinja2>=2.10.1
+numpy>=1.16.4