[Feature] Abstract Data Container (#1555)

- DataContainerInterface defines how a data container should look like. - RowDataContainer stores data internally as any[][] (the same way the data was stored with allData). - IndexedDataContainer can be used to filter the parent data container. - KeplerTable.allData is substituted with KeplerTable.dataContainer. Most of the places that expected any[][] now expect a dataContainer. - Data literal objects like {data: allData[index], index} now look like {index} - Data container is provided to all accessors as a separate argument: -- in layers: (config.columns) => (dataContainer) => (dataElement) => [] -- in filterValueAccessor: (dataContainer) => (getData, getIndex) => (dataElement) => [] Signed-off-by: Igor Dykhta <dikhta.igor@gmail.com> Signed-off-by: Xun Li <lixun910@gmail.com>
keplergl · Aug 12, 2021 · 7f4a78a · 7f4a78a
1 parent efb9341
commit 7f4a78a
Show file tree

Hide file tree

Showing 78 changed files with 1,761 additions and 653 deletions.
diff --git a/docs/RFC/table-class.md b/docs/RFC/table-class.md
@@ -10,7 +10,7 @@ export type KeplerDataset = {
 
   // fields and data
   fields: KeplerField[];
-  allData: any[][];
+  dataContainer: DataContainer;
 
   allIndexes: number[];
   filteredIndex: number[];
@@ -68,7 +68,7 @@ Missing features
 
 1. `initalize`
 
-to replace utils/dataset-utils `createNewDataEntry` This table method takes a `ProtoTable` and return a `Table` to be saved in kepler state.
+to replace utils/dataset-utils `createNewDataEntry` This table method takes a `ProtoTable` and returns a `Table` to be saved in kepler state.
 
 ```js
 table.initalize(protoTable);

diff --git a/src/components/common/data-table/cell-size.js b/src/components/common/data-table/cell-size.js
@@ -24,11 +24,11 @@ import {parseFieldValue} from 'utils/data-utils';
 const MIN_GHOST_CELL_SIZE = 200;
 
 /**
- * Measure rows and column content to determin min width for each column
+ * Measure rows and column content to determine min width for each column
  * @param {*} param0
  */
 export function renderedSize({
-  text: {rows, column},
+  text: {dataContainer, column},
   type = 'string',
   colIdx,
   numRowsToCalculate = 10,
@@ -50,19 +50,21 @@ export function renderedSize({
   document.body.appendChild(textCanvas);
   const context = textCanvas.getContext('2d');
   context.font = [fontSize, font].join('px ');
+
   let rowsToSample = [...Array(numRowsToCalculate)].map(() =>
-    Math.floor(Math.random() * (rows.length - 1))
+    Math.floor(Math.random() * (dataContainer.numRows() - 1))
   );
 
-  // IF we have less than 10 rows, lets measure all of them
-  if (rows.length <= numRowsToCalculate) {
-    rowsToSample = Array.from(Array(rows.length).keys());
+  // If we have less than 10 rows, lets measure all of them
+  if (dataContainer.numRows() <= numRowsToCalculate) {
+    rowsToSample = Array.from(Array(dataContainer.numRows()).keys());
   }
   const rowWidth = Math.max(
     ...rowsToSample.map(
       rowIdx =>
-        Math.ceil(context.measureText(parseFieldValue(rows[rowIdx][colIdx], type)).width) +
-        cellPadding
+        Math.ceil(
+          context.measureText(parseFieldValue(dataContainer.valueAt(rowIdx, colIdx), type)).width
+        ) + cellPadding
     )
   );
   // header cell only has left padding

diff --git a/src/components/common/data-table/index.js b/src/components/common/data-table/index.js
@@ -232,11 +232,13 @@ const columnWidthFunction = (columns, cellSizeCache, ghost) => ({index}) => {
 /*
  * This is an accessor method used to generalize getting a cell from a data row
  */
-const getRowCell = ({rows, columns, column, colMeta, rowIndex, sortOrder}) => {
+const getRowCell = ({dataContainer, columns, column, colMeta, rowIndex, sortOrder}) => {
   const rowIdx = sortOrder && sortOrder.length ? get(sortOrder, rowIndex) : rowIndex;
   const {type} = colMeta[column];
 
-  return parseFieldValue(get(rows, [rowIdx, columns.indexOf(column)], 'Err'), type);
+  let value = dataContainer.valueAt(rowIdx, columns.indexOf(column));
+  if (value === undefined) value = 'Err';
+  return parseFieldValue(value, type);
 };
 
 export const TableSection = ({
@@ -300,7 +302,7 @@ DataTableFactory.deps = [FieldTokenFactory];
 function DataTableFactory(FieldToken) {
   class DataTable extends Component {
     static defaultProps = {
-      rows: [],
+      dataContainer: null,
       pinnedColumns: [],
       colMeta: {},
       cellSizeCache: {},
@@ -460,16 +462,18 @@ function DataTableFactory(FieldToken) {
     renderDataCell = (columns, isPinned, props) => {
       return cellInfo => {
         const {columnIndex, key, style, rowIndex} = cellInfo;
-        const {rows, colMeta} = props;
+        const {dataContainer, colMeta} = props;
         const column = columns[columnIndex];
         const isGhost = column.ghost;
 
         const rowCell = isGhost ? '' : getRowCell({...props, column, rowIndex});
         const type = isGhost ? null : colMeta[column].type;
 
+        const lastRowIndex = dataContainer ? dataContainer.numRows() - 1 : 0;
+
         const endCell = columnIndex === columns.length - 1;
         const firstCell = columnIndex === 0;
-        const bottomCell = rowIndex === rows.length - 1;
+        const bottomCell = rowIndex === lastRowIndex;
         const alignRight = fieldToAlignRight[type];
 
         const cell = (
@@ -496,7 +500,7 @@ function DataTableFactory(FieldToken) {
     };
 
     render() {
-      const {rows, pinnedColumns, theme = {}, fixedWidth, fixedHeight} = this.props;
+      const {dataContainer, pinnedColumns, theme = {}, fixedWidth, fixedHeight} = this.props;
       const unpinnedColumns = this.unpinnedColumns(this.props);
 
       const {cellSizeCache, moreOptionsColumn, ghost} = this.state;
@@ -521,7 +525,7 @@ function DataTableFactory(FieldToken) {
         cellSizeCache,
         overscanColumnCount,
         overscanRowCount,
-        rowCount: (rows || []).length,
+        rowCount: dataContainer ? dataContainer.numRows() : 0,
         rowHeight
       };
 

diff --git a/src/components/modals/data-table-modal.js b/src/components/modals/data-table-modal.js
@@ -111,14 +111,14 @@ function DataTableModalFactory(DataTable) {
       if (!datasets[dataId]) {
         return {};
       }
-      const {fields, allData} = datasets[dataId];
+      const {fields, dataContainer} = datasets[dataId];
 
       let showCalculate = null;
       if (!this.datasetCellSizeCache[dataId]) {
         showCalculate = true;
       } else if (
         this.datasetCellSizeCache[dataId].fields !== fields ||
-        this.datasetCellSizeCache[dataId].allData !== allData
+        this.datasetCellSizeCache[dataId].dataContainer !== dataContainer
       ) {
         showCalculate = true;
       }
@@ -132,7 +132,7 @@ function DataTableModalFactory(DataTable) {
           ...acc,
           [field.name]: renderedSize({
             text: {
-              rows: allData,
+              dataContainer,
               column: field.name
             },
             colIdx,
@@ -147,7 +147,7 @@ function DataTableModalFactory(DataTable) {
       this.datasetCellSizeCache[dataId] = {
         cellSizeCache,
         fields,
-        allData
+        dataContainer
       };
       return cellSizeCache;
     });
@@ -194,7 +194,7 @@ function DataTableModalFactory(DataTable) {
                 columns={columns}
                 colMeta={colMeta}
                 cellSizeCache={cellSizeCache}
-                rows={activeDataset.allData}
+                dataContainer={activeDataset.dataContainer}
                 pinnedColumns={activeDataset.pinnedColumns}
                 sortOrder={activeDataset.sortOrder}
                 sortColumn={activeDataset.sortColumn}

diff --git a/src/components/modals/export-data-modal.js b/src/components/modals/export-data-modal.js
@@ -54,13 +54,13 @@ const getDataRowCount = (datasets, selectedDataset, filtered, intl) => {
       {fileCount: Object.keys(datasets).length}
     );
   }
-  const {allData, filteredIdxCPU} = selectedData;
+  const {dataContainer, filteredIdxCPU} = selectedData;
 
   if (filtered && !filteredIdxCPU) {
     return '-';
   }
 
-  const rowCount = filtered ? filteredIdxCPU.length : allData.length;
+  const rowCount = filtered ? filteredIdxCPU.length : dataContainer.numRows();
 
   return intl.formatMessage(
     {id: 'modal.exportData.rowCount'},

diff --git a/src/components/side-panel/common/dataset-info.js b/src/components/side-panel/common/dataset-info.js
@@ -36,7 +36,7 @@ export default function DatasetInfoFactory() {
     <StyledDataRowCount className="source-data-rows">
       <FormattedMessage
         id={'datasetInfo.rowCount'}
-        values={{rowCount: numFormat(dataset.allData.length)}}
+        values={{rowCount: numFormat(dataset.dataContainer.numRows())}}
       />
     </StyledDataRowCount>
   );

diff --git a/src/layers/aggregation-layer.js b/src/layers/aggregation-layer.js
@@ -26,15 +26,20 @@ import {HIGHLIGH_COLOR_3D} from 'constants/default-settings';
 
 import {CHANNEL_SCALES, FIELD_OPTS, DEFAULT_AGGREGATION} from 'constants/default-settings';
 
-export const pointPosAccessor = ({lat, lng}) => d => [d.data[lng.fieldIdx], d.data[lat.fieldIdx]];
+export const pointPosAccessor = ({lat, lng}) => dc => d => [
+  dc.valueAt(d.index, lng.fieldIdx),
+  dc.valueAt(d.index, lat.fieldIdx)
+];
 
 export const pointPosResolver = ({lat, lng}) => `${lat.fieldIdx}-${lng.fieldIdx}`;
 
 export const getValueAggrFunc = (field, aggregation) => {
   return points => {
     return field
       ? aggregate(
-          points.map(p => field.valueAccessor(p.data)),
+          points.map(p => {
+            return field.valueAccessor(p);
+          }),
           aggregation
         )
       : points.length;
@@ -52,7 +57,8 @@ export default class AggregationLayer extends Layer {
   constructor(props) {
     super(props);
 
-    this.getPositionAccessor = () => pointPosAccessor(this.config.columns);
+    this.getPositionAccessor = dataContainer =>
+      pointPosAccessor(this.config.columns)(dataContainer);
     this.getColorRange = memoize(getLayerColorRange);
   }
 
@@ -138,7 +144,7 @@ export default class AggregationLayer extends Layer {
   /**
    * Aggregation layer handles visual channel aggregation inside deck.gl layer
    */
-  updateLayerVisualChannel({data, allData}, channel) {
+  updateLayerVisualChannel({data, dataContainer}, channel) {
     this.validateVisualChannel(channel);
   }
 
@@ -209,26 +215,25 @@ export default class AggregationLayer extends Layer {
     return this;
   }
 
-  updateLayerMeta(allData, getPosition) {
+  updateLayerMeta(dataContainer, getPosition) {
     // get bounds from points
-    const bounds = this.getPointsBounds(allData, d => getPosition({data: d}));
+    const bounds = this.getPointsBounds(dataContainer, getPosition);
 
     this.updateMeta({bounds});
   }
 
-  calculateDataAttribute({allData, filteredIndex}, getPosition) {
+  calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
     const data = [];
 
     for (let i = 0; i < filteredIndex.length; i++) {
       const index = filteredIndex[i];
-      const pos = getPosition({data: allData[index]});
+      const pos = getPosition({index});
 
       // if doesn't have point lat or lng, do not add the point
       // deck.gl can't handle position = null
       if (pos.every(Number.isFinite)) {
         data.push({
-          index,
-          data: allData[index]
+          index
         });
       }
     }
@@ -237,8 +242,8 @@ export default class AggregationLayer extends Layer {
   }
 
   formatLayerData(datasets, oldLayerData) {
-    const getPosition = this.getPositionAccessor(); // if (
-    const {gpuFilter} = datasets[this.config.dataId];
+    const {gpuFilter, dataContainer} = datasets[this.config.dataId];
+    const getPosition = this.getPositionAccessor(dataContainer);
 
     const getColorValue = getValueAggrFunc(
       this.config.colorField,
@@ -250,7 +255,8 @@ export default class AggregationLayer extends Layer {
       this.config.visConfig.sizeAggregation
     );
     const hasFilter = Object.values(gpuFilter.filterRange).some(arr => arr.some(v => v !== 0));
-    const getFilterValue = gpuFilter.filterValueAccessor();
+
+    const getFilterValue = gpuFilter.filterValueAccessor(dataContainer)();
     const filterData = hasFilter
       ? getFilterDataFunc(gpuFilter.filterRange, getFilterValue)
       : undefined;

diff --git a/src/layers/arc-layer/arc-layer.js b/src/layers/arc-layer/arc-layer.js
@@ -26,12 +26,12 @@ import {hexToRgb} from 'utils/color-utils';
 import ArcLayerIcon from './arc-layer-icon';
 import {DEFAULT_LAYER_COLOR} from 'constants/default-settings';
 
-export const arcPosAccessor = ({lat0, lng0, lat1, lng1}) => d => [
-  d.data[lng0.fieldIdx],
-  d.data[lat0.fieldIdx],
+export const arcPosAccessor = ({lat0, lng0, lat1, lng1}) => dc => d => [
+  dc.valueAt(d.index, lng0.fieldIdx),
+  dc.valueAt(d.index, lat0.fieldIdx),
   0,
-  d.data[lng1.fieldIdx],
-  d.data[lat1.fieldIdx],
+  dc.valueAt(d.index, lng1.fieldIdx),
+  dc.valueAt(d.index, lat1.fieldIdx),
   0
 ];
 
@@ -56,7 +56,7 @@ export default class ArcLayer extends Layer {
     super(props);
 
     this.registerVisConfig(arcVisConfigs);
-    this.getPositionAccessor = () => arcPosAccessor(this.config.columns);
+    this.getPositionAccessor = dataContainer => arcPosAccessor(this.config.columns)(dataContainer);
   }
 
   get type() {
@@ -127,20 +127,19 @@ export default class ArcLayer extends Layer {
     return {props: [props]};
   }
 
-  calculateDataAttribute({allData, filteredIndex}, getPosition) {
+  calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
     const data = [];
     for (let i = 0; i < filteredIndex.length; i++) {
       const index = filteredIndex[i];
-      const pos = getPosition({data: allData[index]});
+      const pos = getPosition({index});
 
       // if doesn't have point lat or lng, do not add the point
       // deck.gl can't handle position = null
       if (pos.every(Number.isFinite)) {
         data.push({
           index,
           sourcePosition: [pos[0], pos[1], pos[2]],
-          targetPosition: [pos[3], pos[4], pos[5]],
-          data: allData[index]
+          targetPosition: [pos[3], pos[4], pos[5]]
         });
       }
     }
@@ -149,27 +148,27 @@ export default class ArcLayer extends Layer {
   }
 
   formatLayerData(datasets, oldLayerData) {
-    const {gpuFilter} = datasets[this.config.dataId];
+    const {gpuFilter, dataContainer} = datasets[this.config.dataId];
     const {data} = this.updateData(datasets, oldLayerData);
-    const accessors = this.getAttributeAccessors();
+    const accessors = this.getAttributeAccessors({dataContainer});
     return {
       data,
-      getFilterValue: gpuFilter.filterValueAccessor(),
+      getFilterValue: gpuFilter.filterValueAccessor(dataContainer)(),
       ...accessors
     };
   }
   /* eslint-enable complexity */
 
-  updateLayerMeta(allData) {
+  updateLayerMeta(dataContainer) {
     // get bounds from arcs
-    const getPosition = this.getPositionAccessor();
+    const getPosition = this.getPositionAccessor(dataContainer);
 
-    const sBounds = this.getPointsBounds(allData, d => {
-      const pos = getPosition({data: d});
+    const sBounds = this.getPointsBounds(dataContainer, (d, i) => {
+      const pos = getPosition(d);
       return [pos[0], pos[1]];
     });
-    const tBounds = this.getPointsBounds(allData, d => {
-      const pos = getPosition({data: d});
+    const tBounds = this.getPointsBounds(dataContainer, (d, i) => {
+      const pos = getPosition(d);
       return [pos[3], pos[4]];
     });