Skip to content

Commit

Permalink
[Feat] GeoArrow incremental rendering (1) (#2459)
Browse files Browse the repository at this point in the history 
Signed-off-by: Xun Li <lixun910@gmail.com>
  • Loading branch information
@lixun910
lixun910 committed Dec 9, 2023
1 parent aa1c7d1 commit 2024a6d
Show file tree
Hide file tree
Showing 23 changed files with 385 additions and 172 deletions.
20 changes: 20 additions & 0 deletions examples/demo-app/src/cloud-providers/foursquare/foursquare-icon.js
View file
@@ -1,3 +1,23 @@
// Copyright (c) 2023 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import React, {Component} from 'react';
import {Icons} from '@kepler.gl/components';
import PropTypes from 'prop-types';
Expand Down
20 changes: 20 additions & 0 deletions examples/demo-app/src/cloud-providers/foursquare/foursquare-provider.js
View file
@@ -1,3 +1,23 @@
// Copyright (c) 2023 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import FSQIcon from './foursquare-icon';
import {Provider, KEPLER_FORMAT} from '@kepler.gl/cloud-providers';
import {Auth0Client} from '@auth0/auth0-spa-js';
Expand Down
16 changes: 8 additions & 8 deletions package.json
View file
Expand Up @@ -114,7 +114,7 @@
"@hubble.gl/core": "1.2.0-alpha.6",
"@hubble.gl/react": "1.2.0-alpha.6",
"@kepler.gl/components": "3.0.0-alpha.1",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@types/mapbox__geo-viewport": "^0.4.1",
"@typescript-eslint/parser": "^5.27.0",
"eslint-config-developit": "^1.2.0",
Expand Down Expand Up @@ -142,7 +142,7 @@
"@babel/traverse": "^7.12.1",
"@cfaester/enzyme-adapter-react-18": "^0.7.0",
"@deck.gl/test-utils": "^8.9.27",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@luma.gl/test-utils": "^8.5.20",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@probe.gl/env": "^3.5.0",
Expand Down Expand Up @@ -216,12 +216,12 @@
"webpack-stats-plugin": "^0.2.1"
},
"resolutions": {
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/csv": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/json": "^4.0.3",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/csv": "^4.1.0-alpha.2",
"@loaders.gl/gltf": "^4.1.0-alpha.2",
"@loaders.gl/json": "^4.1.0-alpha.2",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@luma.gl/constants": "8.5.20",
"@luma.gl/core": "8.5.20",
"@luma.gl/experimental": "8.5.20",
Expand Down
1 change: 1 addition & 0 deletions src/actions/src/action-types.ts
View file
Expand Up @@ -103,6 +103,7 @@ export const ActionTypes = {
SET_FILTER_PLOT: `${ACTION_PREFIX}SET_FILTER_PLOT`,
LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`,
LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`,
LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`,
LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,
LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,
LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,
Expand Down
14 changes: 14 additions & 0 deletions src/actions/src/vis-state-actions.ts
View file
Expand Up @@ -1240,6 +1240,20 @@ export function loadFileStepSuccess({
};
}

export function loadBatchDataSuccess({
fileName,
fileCache
}: {
fileName: string;
fileCache: FileCacheItem[];
}): Merge<LoadFileStepSuccessAction, {type: typeof ActionTypes.LOAD_BATCH_DATA_SUCCESS}> {
return {
type: ActionTypes.LOAD_BATCH_DATA_SUCCESS,
fileName,
fileCache
};
}

export type LoadFilesErrUpdaterAction = {
fileName: string;
error: any;
Expand Down
10 changes: 5 additions & 5 deletions src/layers/package.json
View file
Expand Up @@ -42,11 +42,11 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.0.3",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/gis": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/wkt": "^4.0.3",
"@loaders.gl/arrow": "^4.1.0-alpha.2",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/gis": "^4.1.0-alpha.2",
"@loaders.gl/gltf": "^4.1.0-alpha.2",
"@loaders.gl/wkt": "^4.1.0-alpha.2",
"@luma.gl/constants": "^8.5.20",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
Expand Down
2 changes: 1 addition & 1 deletion src/layers/src/base-layer.ts
View file
Expand Up @@ -1104,7 +1104,7 @@ class Layer {
const dataUpdateTriggers = this.getDataUpdateTriggers(layerDataset);
const triggerChanged = this.getChangedTriggers(dataUpdateTriggers);

if (triggerChanged && triggerChanged.getMeta) {
if (triggerChanged && (triggerChanged.getMeta || triggerChanged.getData)) {
this.updateLayerMeta(dataContainer, getPosition);
}

Expand Down
81 changes: 49 additions & 32 deletions src/layers/src/geojson-layer/geojson-layer.ts
View file
Expand Up @@ -209,7 +209,7 @@ export default class GeoJsonLayer extends Layer {
dataToFeature: GeojsonDataMaps | BinaryFeatures[] = [];
dataContainer: DataContainerInterface | null = null;
filteredIndex: Uint8ClampedArray | null = null;
filteredIndexTrigger: number[] = [];
filteredIndexTrigger: number[] | null = null;

constructor(props) {
super(props);
Expand Down Expand Up @@ -360,25 +360,32 @@ export default class GeoJsonLayer extends Layer {
return null;
}

calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
// filter geojson/arrow table by values and make a partial copy of the raw table are expensive
// so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering
// in deck.gl layer, see: FilterArrowExtension in @kepler.gl/deckgl-layers
if (!this.filteredIndex) {
this.filteredIndex = new Uint8ClampedArray(dataContainer.numRows());
}
this.filteredIndex.fill(0);
for (let i = 0; i < filteredIndex.length; ++i) {
this.filteredIndex[filteredIndex[i]] = 1;
}

this.filteredIndexTrigger = filteredIndex;
calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) {
if (dataContainer instanceof ArrowDataContainer) {
// filter geojson/arrow table by values and make a partial copy of the raw table are expensive
// so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering
// in deck.gl layer, see: FilterArrowExtension in @kepler.gl/deckgl-layers
if (!this.filteredIndex || this.filteredIndex.length !== dataContainer.numRows()) {
// for incremental data loading, we need to update filteredIndex
this.filteredIndex = new Uint8ClampedArray(dataContainer.numRows());
this.filteredIndex.fill(1);
}

// check if filteredIndex is a range from 0 to numRows if it is, we don't need to update it
const isRange = filteredIndex && filteredIndex.length === dataContainer.numRows();
if (!isRange || this.filteredIndexTrigger !== null) {
this.filteredIndex.fill(0);
for (let i = 0; i < filteredIndex.length; ++i) {
this.filteredIndex[filteredIndex[i]] = 1;
}
this.filteredIndexTrigger = filteredIndex;
}
// for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
return this.dataToFeature;
}

// for geojson, this should work as well and more efficient. But we need to update some test cases e.g. #GeojsonLayer -> formatLayerData
return dataContainer instanceof ArrowDataContainer
? this.dataToFeature
: filteredIndex.map(i => this.dataToFeature[i]).filter(d => d);
return filteredIndex.map(i => this.dataToFeature[i]).filter(d => d);
}

formatLayerData(datasets, oldLayerData) {
Expand Down Expand Up @@ -412,27 +419,37 @@ export default class GeoJsonLayer extends Layer {
}

updateLayerMeta(dataContainer) {
// check datasource is arrow format if dataContainer is arrow data container
this.dataContainer = dataContainer;

const getFeature = this.getPositionAccessor(dataContainer);
const getGeoColumn = geoColumnAccessor(this.config.columns);
const getGeoField = geoFieldAccessor(this.config.columns);

if (this.dataToFeature.length === 0) {
const updateLayerMetaFunc =
dataContainer instanceof ArrowDataContainer
? getGeojsonLayerMetaFromArrow
: getGeojsonLayerMeta;
const {dataToFeature, bounds, fixedRadius, featureTypes} = updateLayerMetaFunc({
dataContainer,
getFeature,
getGeoColumn,
getGeoField
});

this.dataToFeature = dataToFeature;
this.updateMeta({bounds, fixedRadius, featureTypes});
if (dataContainer instanceof ArrowDataContainer) {
// update the latest batch/chunk of geoarrow data when loading data incrementally
if (this.dataToFeature.length < dataContainer.numChunks()) {
const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({
dataContainer,
getGeoColumn,
getGeoField,
chunkIndex: this.dataToFeature.length
});
if (this.dataToFeature.length === 0) {
// not update bounds for every batch, to avoid interrupt user interacts with map while loading the map incrementally
this.updateMeta({bounds, fixedRadius, featureTypes});
}
// @ts-expect-error TODO fix this
this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
}
} else {
if (this.dataToFeature.length === 0) {
const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMeta({
dataContainer,
getFeature
});
this.dataToFeature = dataToFeature;
this.updateMeta({ bounds, fixedRadius, featureTypes });
}
}
}

Expand Down
30 changes: 22 additions & 8 deletions src/layers/src/layer-utils.ts
View file
Expand Up @@ -23,7 +23,11 @@ import {Feature, BBox} from 'geojson';
import {Field, FieldPair} from '@kepler.gl/types';
import {DataContainerInterface} from '@kepler.gl/utils';
import {BinaryFeatures} from '@loaders.gl/schema';
import {getBinaryGeometriesFromArrow, parseGeometryFromArrow} from '@loaders.gl/arrow';
import {
getBinaryGeometriesFromArrow,
parseGeometryFromArrow,
BinaryGeometriesFromArrowOptions
} from '@loaders.gl/arrow';

import {DeckGlGeoTypes} from './geojson-layer/geojson-utils';

Expand Down Expand Up @@ -52,20 +56,32 @@ export type GeojsonLayerMetaProps = {
export function getGeojsonLayerMetaFromArrow({
dataContainer,
getGeoColumn,
getGeoField
getGeoField,
chunkIndex
}: {
dataContainer: DataContainerInterface;
getGeoColumn: (dataContainer: DataContainerInterface) => unknown;
getGeoField: (dataContainer: DataContainerInterface) => Field | null;
chunkIndex?: number;
}): GeojsonLayerMetaProps {
const geoColumn = getGeoColumn(dataContainer) as arrow.Vector;
const arrowField = getGeoField(dataContainer);

const encoding = arrowField?.metadata?.get('ARROW:extension:name');
const options: BinaryGeometriesFromArrowOptions = {
...(chunkIndex !== undefined && chunkIndex >= 0
? {
chunkIndex,
chunkOffset: geoColumn.data[0].length * chunkIndex
}
: {}),
triangulate: true
};
// create binary data from arrow data for GeoJsonLayer
const {binaryGeometries, featureTypes, bounds} = getBinaryGeometriesFromArrow(
geoColumn,
encoding
encoding,
options
);

// since there is no feature.properties.radius, we set fixedRadius to false
Expand Down Expand Up @@ -105,10 +121,7 @@ export function getHoveredObjectFromArrow(
const field = fieldAccessor(dataContainer);
const encoding = field?.metadata?.get('ARROW:extension:name');

const hoveredFeature = parseGeometryFromArrow({
encoding,
data: rawGeometry
});
const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);

const properties = dataContainer.rowAsArray(objectInfo.index).reduce((prev, cur, i) => {
const fieldName = dataContainer?.getField?.(i).name;
Expand All @@ -120,7 +133,8 @@ export function getHoveredObjectFromArrow(

return hoveredFeature
? {
...hoveredFeature,
type: 'Feature',
geometry: hoveredFeature,
properties: {
...properties,
index: objectInfo.index
Expand Down
10 changes: 5 additions & 5 deletions src/processors/package.json
View file
Expand Up @@ -35,11 +35,11 @@
"@kepler.gl/schemas": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.0.3",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/csv": "^4.0.3",
"@loaders.gl/json": "^4.0.3",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/arrow": "^4.1.0-alpha.2",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/csv": "^4.1.0-alpha.2",
"@loaders.gl/json": "^4.1.0-alpha.2",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
"@turf/helpers": "^6.1.4",
Expand Down
8 changes: 6 additions & 2 deletions src/processors/src/file-handler.ts
View file
Expand Up @@ -29,7 +29,7 @@ import {
processKeplerglJSON,
processRowObject
} from './data-processor';
import {generateHashId, isPlainObject} from '@kepler.gl/utils';
import {generateHashId, isPlainObject, generateHashIdFromString} from '@kepler.gl/utils';
import {DATASET_FORMATS} from '@kepler.gl/constants';
import {Loader} from '@loaders.gl/loader-utils';
import {FileCacheItem, ValidKeplerGlMap} from './types';
Expand Down Expand Up @@ -215,10 +215,13 @@ export function processFileData({
fileCache: FileCacheItem[];
}): Promise<FileCacheItem[]> {
return new Promise((resolve, reject) => {
let {data} = content;
let {fileName, data} = content;
let format: string | undefined;
let processor: Function | undefined;

// generate unique id with length of 4 using fileName string
const id = generateHashIdFromString(fileName);

if (isArrowData(data)) {
format = DATASET_FORMATS.arrow;
processor = processArrowTable;
Expand All @@ -241,6 +244,7 @@ export function processFileData({
{
data: result,
info: {
id,
label: content.fileName,
format
}
Expand Down
2 changes: 1 addition & 1 deletion src/reducers/package.json
View file
Expand Up @@ -43,7 +43,7 @@
"@kepler.gl/tasks": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.flattendeep": "^4.4.7",
"@types/lodash.get": "^4.4.6",
Expand Down

0 comments on commit 2024a6d

Please sign in to comment.