Skip to content

Commit

Permalink
[Feat] support GeoArrow format (#2385)
Browse files Browse the repository at this point in the history
  • Loading branch information
lixun910 committed Nov 9, 2023
1 parent ee6f075 commit d975ea1
Show file tree
Hide file tree
Showing 34 changed files with 1,185 additions and 386 deletions.
6 changes: 6 additions & 0 deletions examples/webpack.config.local.js
Expand Up @@ -146,6 +146,12 @@ function makeLocalDevConfig(env, EXAMPLE_DIR = LIB_DIR, externals = {}) {
use: ['source-map-loader'],
enforce: 'pre',
exclude: [/node_modules\/react-palm/, /node_modules\/react-data-grid/]
},
// for compiling apache-arrow ESM module
{
test: /\.mjs$/,
include: /node_modules\/apache-arrow/,
type: 'javascript/auto'
}
]
},
Expand Down
1 change: 1 addition & 0 deletions jest.setup.js
Expand Up @@ -20,6 +20,7 @@

import '@testing-library/jest-dom';
import * as Utils from '@kepler.gl/utils';
require('@loaders.gl/polyfills');

jest.mock('mapbox-gl/dist/mapbox-gl', () => ({
Map: () => ({})
Expand Down
16 changes: 8 additions & 8 deletions package.json
Expand Up @@ -112,7 +112,7 @@
"@hubble.gl/core": "1.2.0-alpha.6",
"@hubble.gl/react": "1.2.0-alpha.6",
"@kepler.gl/components": "3.0.0-alpha.1",
"@loaders.gl/polyfills": "^4.0.0",
"@loaders.gl/polyfills": "^4.0.3",
"@types/mapbox__geo-viewport": "^0.4.1",
"@typescript-eslint/parser": "^5.27.0",
"eslint-config-developit": "^1.2.0",
Expand Down Expand Up @@ -140,7 +140,7 @@
"@babel/traverse": "^7.12.1",
"@cfaester/enzyme-adapter-react-18": "^0.7.0",
"@deck.gl/test-utils": "^8.9.12",
"@loaders.gl/polyfills": "^4.0.0",
"@loaders.gl/polyfills": "^4.0.3",
"@luma.gl/test-utils": "^8.5.19",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@probe.gl/env": "^3.5.0",
Expand Down Expand Up @@ -214,12 +214,12 @@
"webpack-stats-plugin": "^0.2.1"
},
"resolutions": {
"@loaders.gl/core": "^4.0.0",
"@loaders.gl/csv": "^4.0.0",
"@loaders.gl/gltf": "^4.0.0",
"@loaders.gl/json": "^4.0.0",
"@loaders.gl/loader-utils": "^4.0.0",
"@loaders.gl/polyfills": "^4.0.0",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/csv": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/json": "^4.0.3",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/polyfills": "^4.0.3",
"@luma.gl/constants": "8.5.19",
"@luma.gl/core": "8.5.19",
"@luma.gl/experimental": "8.5.19",
Expand Down
21 changes: 19 additions & 2 deletions src/constants/src/default-settings.ts
Expand Up @@ -430,7 +430,8 @@ export const ALL_FIELD_TYPES = keyMirror({
timestamp: null,
point: null,
array: null,
object: null
object: null,
geoarrow: null
});

// Data Table
Expand Down Expand Up @@ -509,6 +510,10 @@ export const FIELD_TYPE_DISPLAY = {
label: 'geo',
color: BLUE2
},
[ALL_FIELD_TYPES.geoarrow]: {
label: 'geo',
color: BLUE2
},
[ALL_FIELD_TYPES.integer]: {
label: 'int',
color: YELLOW
Expand Down Expand Up @@ -757,6 +762,17 @@ export const FIELD_OPTS = {
tooltip: []
}
},
[ALL_FIELD_TYPES.geoarrow]: {
type: 'geometry',
scale: {
...notSupportedScaleOpts,
...notSupportAggrOpts
},
format: {
legend: d => '...',
tooltip: []
}
},
[ALL_FIELD_TYPES.object]: {
type: 'numerical',
scale: {},
Expand Down Expand Up @@ -1122,7 +1138,8 @@ export const DATASET_FORMATS = keyMirror({
row: null,
geojson: null,
csv: null,
keplergl: null
keplergl: null,
arrow: null
});

export const MAP_CONTROLS = keyMirror({
Expand Down
1 change: 1 addition & 0 deletions src/deckgl-layers/babel.config.js
Expand Up @@ -24,6 +24,7 @@ const PRESETS = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-type
const PLUGINS = [
['@babel/plugin-transform-typescript', {isTSX: true, allowDeclareFields: true}],
'@babel/plugin-transform-modules-commonjs',
'@babel/plugin-proposal-class-properties', // for static class properties
'@babel/plugin-proposal-optional-chaining',
[
'@babel/transform-runtime',
Expand Down
57 changes: 57 additions & 0 deletions src/deckgl-layers/src/deckgl-extensions/filter-arrow-layer.ts
@@ -0,0 +1,57 @@
import {Layer, LayerExtension} from '@deck.gl/core';
import {LayerContext} from '@deck.gl/core/lib/layer';
import GL from '@luma.gl/constants';

import shaderModule from './filter-shader-module';

const VALUE_FILTERED = 1;

const defaultProps = {
getFiltered: {type: 'accessor', value: VALUE_FILTERED}
};

export type FilterArrowExtensionProps = {
getFiltered?: () => number;
};

/**
* FilterArrowExtension - a deck.gl extension to filter arrow layer
*
* A simple extension to filter arrow layer based on the result of CPU filteredIndex,
* so we can avoid filtering on the raw Arrow table and recreating geometry attributes.
* Specifically, an attribute `filtered` is added to the layer to indicate whether the feature has been Filtered
* the shader module is modified to discard the feature if filtered value is 0
* the accessor getFiltered is used to get the value of `filtered` based on the value `filteredIndex` in Arrowlayer
*/
export default class FilterArrowExtension extends LayerExtension {
static defaultProps = defaultProps;
static extensionName = 'FilterArrowExtension';

getShaders(extension: any) {
return {
modules: [shaderModule],
defines: {}
};
}

initializeState(this: Layer<FilterArrowExtensionProps>, context: LayerContext, extension: this) {
const attributeManager = this.getAttributeManager();
if (attributeManager) {
attributeManager.add({
filtered: {
size: 1,
type: GL.FLOAT,
accessor: 'getFiltered',
shaderAttributes: {
filtered: {
divisor: 0
},
instanceFiltered: {
divisor: 1
}
}
}
});
}
}
}
39 changes: 39 additions & 0 deletions src/deckgl-layers/src/deckgl-extensions/filter-shader-module.ts
@@ -0,0 +1,39 @@
import {project} from '@deck.gl/core';

const vs = `
#ifdef NON_INSTANCED_MODEL
#define FILTER_ARROW_ATTRIB filtered
#else
#define FILTER_ARROW_ATTRIB instanceFiltered
#endif
attribute float FILTER_ARROW_ATTRIB;
`;

const fs = ``;

const inject = {
'vs:#decl': `
varying float is_filtered;
`,
'vs:#main-end': `
is_filtered = FILTER_ARROW_ATTRIB;
`,
'fs:#decl': `
varying float is_filtered;
`,
'fs:DECKGL_FILTER_COLOR': `
// abandon the fragments if it is not filtered
if (is_filtered == 0.) {
discard;
}
`
};

export default {
name: 'filter-arrow',
dependencies: [project],
vs: vs,
fs: fs,
inject: inject,
getUniforms: () => {}
};
1 change: 1 addition & 0 deletions src/deckgl-layers/src/index.ts
Expand Up @@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer
export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer';
export {default as EnhancedLineLayer} from './line-layer/line-layer';
export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer';
export {default as FilterArrowExtension} from './deckgl-extensions/filter-arrow-layer';

export * from './layer-utils/shader-utils';

Expand Down
11 changes: 6 additions & 5 deletions src/layers/package.json
Expand Up @@ -42,14 +42,15 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/core": "^4.0.0",
"@loaders.gl/gis": "^4.0.0",
"@loaders.gl/gltf": "^4.0.0",
"@loaders.gl/wkt": "^4.0.0",
"@loaders.gl/arrow": "^4.0.3",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/gis": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/wkt": "^4.0.3",
"@luma.gl/constants": "^8.5.19",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@turf/bbox": "^6.0.1",
"@turf/helpers": "^6.1.4",
"@types/geojson": "^7946.0.7",
Expand Down

0 comments on commit d975ea1

Please sign in to comment.