Skip to content

Commit

Permalink
[Fix] basemap frozen when incrementally loading GeoArrow (#2474)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lixun910
lixun910 committed Dec 18, 2023
1 parent b290d87 commit 1544e20
Show file tree
Hide file tree
Showing 15 changed files with 131 additions and 163 deletions.
3 changes: 3 additions & 0 deletions examples/webpack.config.local.js
View file
Expand Up @@ -100,6 +100,8 @@ function makeLocalDevConfig(env, EXAMPLE_DIR = LIB_DIR, externals = {}) {
externals['loaders.gl'].forEach(mdl => {
resolveAlias[`@loaders.gl/${mdl}`] = `${EXTERNAL_LOADERS_SRC}/modules/${mdl}/src`;
});
// kepler.gl and loaders.gl need to use same apache-arrow
resolveAlias['apache-arrow'] = resolve(__dirname, '../node_modules/apache-arrow');
}

if (env.hubble_src) {
Expand Down Expand Up @@ -166,6 +168,7 @@ function makeBabelRule(env, exampleDir) {
join(EXTERNAL_DECK_SRC, 'node_modules/@loaders.gl')
]
: []),
...(env.loaders_src ? [join(EXTERNAL_LOADERS_SRC, 'modules')] : []),
...(env.hubble_src ? [join(EXTERNAL_HUBBLE_SRC, 'modules')] : []),
join(exampleDir, 'src'),
SRC_DIR
Expand Down
16 changes: 8 additions & 8 deletions package.json
View file
Expand Up @@ -114,7 +114,7 @@
"@hubble.gl/core": "1.2.0-alpha.6",
"@hubble.gl/react": "1.2.0-alpha.6",
"@kepler.gl/components": "3.0.0-alpha.1",
"@loaders.gl/polyfills": "^4.1.0-alpha.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.4",
"@types/mapbox__geo-viewport": "^0.4.1",
"@typescript-eslint/parser": "^5.27.0",
"eslint-config-developit": "^1.2.0",
Expand Down Expand Up @@ -142,7 +142,7 @@
"@babel/traverse": "^7.12.1",
"@cfaester/enzyme-adapter-react-18": "^0.7.0",
"@deck.gl/test-utils": "^8.9.27",
"@loaders.gl/polyfills": "^4.1.0-alpha.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.4",
"@luma.gl/test-utils": "^8.5.20",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@probe.gl/env": "^3.5.0",
Expand Down Expand Up @@ -216,12 +216,12 @@
"webpack-stats-plugin": "^0.2.1"
},
"resolutions": {
"@loaders.gl/core": "^4.1.0-alpha.3",
"@loaders.gl/csv": "^4.1.0-alpha.3",
"@loaders.gl/gltf": "^4.1.0-alpha.3",
"@loaders.gl/json": "^4.1.0-alpha.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.3",
"@loaders.gl/core": "^4.1.0-alpha.4",
"@loaders.gl/csv": "^4.1.0-alpha.4",
"@loaders.gl/gltf": "^4.1.0-alpha.4",
"@loaders.gl/json": "^4.1.0-alpha.4",
"@loaders.gl/loader-utils": "^4.1.0-alpha.4",
"@loaders.gl/polyfills": "^4.1.0-alpha.4",
"@luma.gl/constants": "8.5.21",
"@luma.gl/core": "8.5.21",
"@luma.gl/experimental": "8.5.21",
Expand Down
14 changes: 0 additions & 14 deletions src/actions/src/vis-state-actions.ts
View file
Expand Up @@ -1240,20 +1240,6 @@ export function loadFileStepSuccess({
};
}

export function loadBatchDataSuccess({
fileName,
fileCache
}: {
fileName: string;
fileCache: FileCacheItem[];
}): Merge<LoadFileStepSuccessAction, {type: typeof ActionTypes.LOAD_BATCH_DATA_SUCCESS}> {
return {
type: ActionTypes.LOAD_BATCH_DATA_SUCCESS,
fileName,
fileCache
};
}

export type LoadFilesErrUpdaterAction = {
fileName: string;
error: any;
Expand Down
11 changes: 6 additions & 5 deletions src/layers/package.json
View file
Expand Up @@ -42,11 +42,12 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.1.0-alpha.3",
"@loaders.gl/core": "^4.1.0-alpha.3",
"@loaders.gl/gis": "^4.1.0-alpha.3",
"@loaders.gl/gltf": "^4.1.0-alpha.3",
"@loaders.gl/wkt": "^4.1.0-alpha.3",
"@loaders.gl/arrow": "^4.1.0-alpha.4",
"@loaders.gl/core": "^4.1.0-alpha.4",
"@loaders.gl/gis": "^4.1.0-alpha.4",
"@loaders.gl/gltf": "^4.1.0-alpha.4",
"@loaders.gl/schema": "^4.1.0-alpha.4",
"@loaders.gl/wkt": "^4.1.0-alpha.4",
"@luma.gl/constants": "^8.5.20",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
Expand Down
9 changes: 2 additions & 7 deletions src/layers/src/geojson-layer/geojson-layer.ts
View file
Expand Up @@ -19,7 +19,6 @@
// THE SOFTWARE.

import * as arrow from 'apache-arrow';
import {BinaryFeatures} from '@loaders.gl/schema';
import {Feature} from 'geojson';
import uniq from 'lodash.uniq';
import {DATA_TYPES} from 'type-analyzer';
Expand Down Expand Up @@ -206,7 +205,7 @@ export default class GeoJsonLayer extends Layer {
declare visConfigSettings: GeoJsonVisConfigSettings;
declare meta: GeoJsonLayerMeta;

dataToFeature: GeojsonDataMaps | BinaryFeatures[] = [];
dataToFeature: GeojsonDataMaps = [];
dataContainer: DataContainerInterface | null = null;
filteredIndex: Uint8ClampedArray | null = null;
filteredIndexTrigger: number[] | null = null;
Expand Down Expand Up @@ -434,11 +433,7 @@ export default class GeoJsonLayer extends Layer {
getGeoField,
chunkIndex: this.dataToFeature.length
});
if (this.dataToFeature.length === 0) {
// not update bounds for every batch, to avoid interrupt user interacts with map while loading the map incrementally
this.updateMeta({bounds, fixedRadius, featureTypes});
}
// @ts-expect-error TODO fix this
this.updateMeta({ bounds, fixedRadius, featureTypes });
this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
}
} else {
Expand Down
7 changes: 4 additions & 3 deletions src/layers/src/geojson-layer/geojson-utils.ts
View file
Expand Up @@ -24,12 +24,13 @@ import bbox from '@turf/bbox';
import {parseSync} from '@loaders.gl/core';
import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
import {binaryToGeometry} from '@loaders.gl/gis';
import {BinaryFeatureCollection} from '@loaders.gl/schema';
import {DataContainerInterface, getSampleData} from '@kepler.gl/utils';

import {GeojsonLayerMetaProps} from '../layer-utils';

export type GetFeature = (d: any) => Feature;
export type GeojsonDataMaps = Array<Feature | null>;
export type GeojsonDataMaps = Array<Feature | BinaryFeatureCollection | null>;

/* eslint-disable */
// TODO: Re-enable eslint when we upgrade to handle enums and type maps
Expand Down Expand Up @@ -84,7 +85,7 @@ export function getGeojsonLayerMeta({
// get bounds from features
const bounds = getGeojsonBounds(dataToFeature);
// if any of the feature has properties.radius set to be true
const fixedRadius = Boolean(dataToFeature.find(d => d && d.properties && d.properties.radius));
const fixedRadius = Boolean(dataToFeature.find(d => d && 'properties' in d && d.properties?.radius));

// keep a record of what type of geometry the collection has
const featureTypes = getGeojsonFeatureTypes(dataToFeature);
Expand Down Expand Up @@ -234,7 +235,7 @@ export function getGeojsonFeatureTypes(allFeatures: GeojsonDataMaps): DeckGlGeoT
const featureTypes: DeckGlGeoTypes = {};
for (let f = 0; f < allFeatures.length; f++) {
const feature = allFeatures[f];
if (feature) {
if (feature && 'geometry' in feature) {
const geoType = featureToDeckGlGeoType[feature.geometry && feature.geometry.type];
if (geoType) {
featureTypes[geoType] = true;
Expand Down
5 changes: 2 additions & 3 deletions src/layers/src/layer-utils.ts
View file
Expand Up @@ -22,14 +22,13 @@ import * as arrow from 'apache-arrow';
import {Feature, BBox} from 'geojson';
import {Field, FieldPair} from '@kepler.gl/types';
import {DataContainerInterface} from '@kepler.gl/utils';
import {BinaryFeatures} from '@loaders.gl/schema';
import {
getBinaryGeometriesFromArrow,
parseGeometryFromArrow,
BinaryGeometriesFromArrowOptions
} from '@loaders.gl/arrow';

import {DeckGlGeoTypes} from './geojson-layer/geojson-utils';
import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';

export function assignPointPairToLayerColumn(pair: FieldPair, hasAlt: boolean) {
const {lat, lng, alt} = pair.pair;
Expand All @@ -47,7 +46,7 @@ export function assignPointPairToLayerColumn(pair: FieldPair, hasAlt: boolean) {
}

export type GeojsonLayerMetaProps = {
dataToFeature: BinaryFeatures[] | Array<Feature | null>;
dataToFeature: GeojsonDataMaps;
featureTypes: DeckGlGeoTypes;
bounds: BBox | null;
fixedRadius: boolean;
Expand Down
10 changes: 5 additions & 5 deletions src/processors/package.json
View file
Expand Up @@ -35,11 +35,11 @@
"@kepler.gl/schemas": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.1.0-alpha.3",
"@loaders.gl/core": "^4.1.0-alpha.3",
"@loaders.gl/csv": "^4.1.0-alpha.3",
"@loaders.gl/json": "^4.1.0-alpha.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.3",
"@loaders.gl/arrow": "^4.1.0-alpha.4",
"@loaders.gl/core": "^4.1.0-alpha.4",
"@loaders.gl/csv": "^4.1.0-alpha.4",
"@loaders.gl/json": "^4.1.0-alpha.4",
"@loaders.gl/loader-utils": "^4.1.0-alpha.4",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
"@turf/helpers": "^6.1.4",
Expand Down
3 changes: 2 additions & 1 deletion src/processors/src/file-handler.ts
View file
Expand Up @@ -48,7 +48,8 @@ const CSV_LOADER_OPTIONS = {
};

const ARROW_LOADER_OPTIONS = {
shape: 'arrow-table'
shape: 'arrow-table',
batchDebounceMs: 10 // time to delay between batches, for incremental loading
};

const JSON_LOADER_OPTIONS = {
Expand Down
2 changes: 1 addition & 1 deletion src/reducers/package.json
View file
Expand Up @@ -43,7 +43,7 @@
"@kepler.gl/tasks": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/loader-utils": "^4.1.0-alpha.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.4",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.flattendeep": "^4.4.7",
"@types/lodash.get": "^4.4.6",
Expand Down
4 changes: 2 additions & 2 deletions src/reducers/src/combined-updaters.ts
View file
Expand Up @@ -164,8 +164,8 @@ export const addDataToMapUpdater = (
const isProgressiveLoading =
Array.isArray(datasets) &&
datasets[0]?.info.format === 'arrow' &&
datasets[0].info.id &&
datasets[0].info.id in state.visState.datasets;
datasets[0]?.info.id &&
datasets[0]?.info.id in state.visState.datasets;
if (isProgressiveLoading) {
return compose_<KeplerGlState>([
pick_('visState')(
Expand Down
20 changes: 2 additions & 18 deletions src/reducers/src/vis-state-updaters.ts
View file
Expand Up @@ -37,7 +37,6 @@ import {
layerTypeChange,
layerVisConfigChange,
layerVisualChannelConfigChange,
loadBatchDataSuccess,
loadFilesErr,
loadFilesSuccess,
loadFileStepSuccess,
Expand Down Expand Up @@ -2277,21 +2276,6 @@ export function parseProgress(prevProgress = {}, progress) {
};
}

export function loadBatchDataSuccessUpdater(
state: VisState,
action: VisStateActions.LoadFileStepSuccessAction
): VisState {
if (!state.fileLoading) {
return state;
}
const {fileCache} = action;
const {onFinish} = state.fileLoading;
return withTask(
state,
DELAY_TASK(200).map(() => onFinish(fileCache))
);
}

/**
* gets called with payload = AsyncGenerator<???>
* @memberof visStateUpdaters
Expand All @@ -2309,10 +2293,10 @@ export const nextFileBatchUpdater = (
});

return withTask(stateWithProgress, [
...(fileName.endsWith('arrow') && accumulated && accumulated.data?.length > 1
...(fileName.endsWith('arrow') && accumulated?.data?.length > 0
? [
PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(
result => loadBatchDataSuccess({fileName, fileCache: result}),
result => loadFilesSuccess(result),
err => loadFilesErr(fileName, err)
)
]
Expand Down
2 changes: 0 additions & 2 deletions src/reducers/src/vis-state.ts
View file
Expand Up @@ -67,8 +67,6 @@ const actionHandler = {

[ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,

[ActionTypes.LOAD_BATCH_DATA_SUCCESS]: visStateUpdaters.loadBatchDataSuccessUpdater,

[ActionTypes.LOAD_FILE_STEP_SUCCESS]: visStateUpdaters.loadFileStepSuccessUpdater,

[ActionTypes.MAP_CLICK]: visStateUpdaters.mapClickUpdater,
Expand Down
2 changes: 1 addition & 1 deletion src/schemas/package.json
View file
Expand Up @@ -35,7 +35,7 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/loader-utils": "^4.1.0-alpha.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.4",
"@types/keymirror": "^0.1.1",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.pick": "^4.4.6",
Expand Down

0 comments on commit 1544e20

Please sign in to comment.