@deck.gl/jupyter-widget and pydeck: Fix binary data bug (#4416)

bindings/pydeck/pydeck/bindings/layer.py

@@ -148,20 +148,24 @@ def _prepare_binary_data(self, data_set):
             )
 
         layer_accessors = self._kwargs
-        inv_map = {v: k for k, v in layer_accessors.items()}
+        inverted_accessor_map = {v: k for k, v in layer_accessors.items() if type(v) not in [list, dict, set]}
 
-        blobs = []
+        binary_transmission = []
+        # Loop through data columns and convert them to numpy arrays
         for column in data_set.columns:
+            # np.stack will take data arrays and conveniently extract the shape
             np_data = np.stack(data_set[column].to_numpy())
-            blobs.append(
+            # Get rid of the accssor so it doesn't appear in the JSON output
+            del self.__dict__[inverted_accessor_map[column]]
+            binary_transmission.append(
                 {
                     "layer_id": self.id,
                     "column_name": column,
-                    "accessor": camel_and_lower(inv_map[column]),
+                    "accessor": camel_and_lower(inverted_accessor_map[column]),
                     "np_data": np_data,
                 }
             )
-        return blobs
+        return binary_transmission
 
     @property
     def type(self):

modules/jupyter-widget/src/binary-transport.js

@@ -48,14 +48,15 @@ function deserializeMatrix(obj, manager) {
   return obj;
 }
 
-function processDataBuffer({dataBuffer, jsonProps}) {
+function processDataBuffer({dataBuffer, convertedJson}) {
   // Takes JSON props and combines them with the binary data buffer
-  for (let i = 0; i < jsonProps.layers.length; i++) {
-    const layerId = jsonProps.layers[i].id;
-    jsonProps.layers[i].data = dataBuffer[layerId];
+  for (let i = 0; i < convertedJson.layers.length; i++) {
+    const layerId = convertedJson.layers[i].id;
+    const layer = convertedJson.layers[i];
     // Replace data on every layer prop
+    convertedJson.layers[i] = layer.clone({data: dataBuffer[layerId]});
   }
-  return jsonProps;
+  return convertedJson;
 }
 
 export {deserializeMatrix, processDataBuffer};

modules/jupyter-widget/src/widget.js

@@ -133,14 +133,13 @@ export class DeckGLView extends DOMWidgetView {
   }
 
   dataBufferChanged() {
-    if (this.model.get('data_buffer')) {
+    if (this.model.get('data_buffer') && this.model.get('json_input')) {
+      const convertedJson = jsonConverter.convert(this.model.get('json_input'));
       const propsWithBinary = processDataBuffer({
         dataBuffer: this.model.get('data_buffer'),
-        jsonProps: this.model.get('json_input')
+        convertedJson
       });
-
-      const convertedJson = jsonConverter.convert(propsWithBinary);
-      this.deck.setProps(convertedJson);
+      this.deck.setProps(propsWithBinary);
     }
   }
 

test/modules/jupyter-widget/binary-transport.spec.js

@@ -30,23 +30,12 @@ const EXPECTED_CONVERSION = {
   }
 };
 
-// Test deck.gl JSON configuration
-const DEMO_JSON_PROPS = {
-  viewport: null,
-  description: 'test json config',
-  layers: [
-    {
-      radius: 100,
-      id: 'layer-id',
-      '@@type': 'ScatterplotLayer'
-    }
-  ]
-};
-
 test('jupyter-widget: binary-transport', t0 => {
   let binaryTransportModule;
+  let widgetCreateDeckModule;
   try {
     binaryTransportModule = require('@deck.gl/jupyter-widget/binary-transport');
+    widgetCreateDeckModule = require('@deck.gl/jupyter-widget/create-deck');
   } catch (error) {
     t0.comment('dist mode, skipping binary-transport tests');
     t0.end();
@@ -73,15 +62,28 @@ test('jupyter-widget: binary-transport', t0 => {
     t.end();
   });
 
+  // Test deck.gl JSON configuration
+  const DEMO_JSON_PROPS = {
+    viewport: null,
+    description: 'Test JSON config, converted into a deck.gl Layer before testing',
+    layers: [
+      {
+        radius: 100,
+        id: 'layer-id',
+        '@@type': 'ScatterplotLayer'
+      }
+    ]
+  };
+
   t0.test('processDataBuffer', t => {
-    const newProps = binaryTransportModule.processDataBuffer({
+    const newDeckProps = binaryTransportModule.processDataBuffer({
       dataBuffer: EXPECTED_CONVERSION,
-      jsonProps: DEMO_JSON_PROPS
+      convertedJson: widgetCreateDeckModule.jsonConverter.convert(DEMO_JSON_PROPS)
     });
 
     t.deepEquals(
+      newDeckProps.layers[0].props.data,
       EXPECTED_CONVERSION['layer-id'],
-      newProps.layers[0].data,
       'should convert buffer input and props to new layers'
     );
     t.end();