Skip to content

Commit

Permalink
[Feat] Add support to GeoArrow format
Browse files Browse the repository at this point in the history
[Feat] Add support to GeoArrow format
  • Loading branch information
lixun910 committed Oct 20, 2023
2 parents fdecb05 + 41fb3b1 commit 66f337c
Show file tree
Hide file tree
Showing 33 changed files with 1,802 additions and 43 deletions.
4 changes: 2 additions & 2 deletions src/components/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
"@kepler.gl/actions": "3.0.0-alpha.1",
"@kepler.gl/cloud-providers": "3.0.0-alpha.1",
"@kepler.gl/constants": "3.0.0-alpha.1",
"@kepler.gl/layers": "3.0.0-alpha.1",
"@kepler.gl/effects": "3.0.0-alpha.1",
"@kepler.gl/layers": "3.0.0-alpha.1",
"@kepler.gl/localization": "3.0.0-alpha.1",
"@kepler.gl/processors": "3.0.0-alpha.1",
"@kepler.gl/reducers": "3.0.0-alpha.1",
Expand All @@ -65,8 +65,8 @@
"@types/lodash.throttle": "^4.1.7",
"@types/lodash.uniq": "^4.5.7",
"@types/lodash.uniqby": "^4.7.7",
"@types/react-copy-to-clipboard": "^5.0.2",
"@types/react": "^18.0.28",
"@types/react-copy-to-clipboard": "^5.0.2",
"@types/react-dom": "^18.0.11",
"@types/react-lifecycles-compat": "^3.0.1",
"@types/react-map-gl": "^6.1.3",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,10 @@ export default function LayerConfiguratorFactory(
);
}

_renderGeoarrowLayerConfig(props) {
return this._renderGeojsonLayerConfig(props);
}

_render3DLayerConfig({layer, visConfiguratorProps}) {
return (
<Fragment>
Expand Down
23 changes: 21 additions & 2 deletions src/constants/src/default-settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,8 @@ export const ALL_FIELD_TYPES = keyMirror({
timestamp: null,
point: null,
array: null,
object: null
object: null,
geoarrow: null
});

// Data Table
Expand Down Expand Up @@ -496,6 +497,8 @@ const GREEN2 = '74, 165, 150';
const RED = '237, 88, 106';
const ORANGE = '231, 110, 58';

export const ARROW_GEO_METADATA_KEY = 'geo';

export const FIELD_TYPE_DISPLAY = {
[ALL_FIELD_TYPES.boolean]: {
label: 'bool',
Expand All @@ -509,6 +512,10 @@ export const FIELD_TYPE_DISPLAY = {
label: 'geo',
color: BLUE2
},
[ALL_FIELD_TYPES.geoarrow]: {
label: 'geo',
color: BLUE2
},
[ALL_FIELD_TYPES.integer]: {
label: 'int',
color: YELLOW
Expand Down Expand Up @@ -743,6 +750,17 @@ export const FIELD_OPTS = {
tooltip: []
}
},
[ALL_FIELD_TYPES.geoarrow]: {
type: 'geometry',
scale: {
...notSupportedScaleOpts,
...notSupportAggrOpts
},
format: {
legend: d => '...',
tooltip: []
}
},
[ALL_FIELD_TYPES.object]: {
type: 'numerical',
scale: {},
Expand Down Expand Up @@ -1108,7 +1126,8 @@ export const DATASET_FORMATS = keyMirror({
row: null,
geojson: null,
csv: null,
keplergl: null
keplergl: null,
arrow: null
});

export const MAP_CONTROLS = keyMirror({
Expand Down
1 change: 1 addition & 0 deletions src/constants/src/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,7 @@ export const LAYER_TYPES = keyMirror({
grid: null,
hexagon: null,
geojson: null,
geoarrow: null,
cluster: null,
icon: null,
heatmap: null,
Expand Down
54 changes: 54 additions & 0 deletions src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import {Layer, LayerExtension} from '@deck.gl/core';
import {LayerContext} from '@deck.gl/core/lib/layer';
import GL from '@luma.gl/constants';

import shaderModule from './filter-shader-module';

const VALUE_FILTERED = 1;

const defaultProps = {
getFiltered: {type: 'accessor', value: VALUE_FILTERED}
};

export type FilterArrowExtensionProps = {
getFiltered?: () => number;
};

// TODO: this should be in deck.gl extensions
// A simple extension to filter arrow layer based on the result of CPU filteredIndex,
// so we can avoid filtering on the raw Arrow table and recreating geometry attributes.
// Specifically, an attribute `filtered` is added to the layer to indicate whether the feature has been Filtered
// the shader module is modified to discard the feature if filtered value is 0
// the accessor getFiltered is used to get the value of `filtered` based on eht value `filteredIndex` in Arrowlayer
export default class FilterArrowExtension extends LayerExtension {
static defaultProps = defaultProps;
static extensionName = 'FilterArrowExtension';

getShaders(extension: any) {
return {
modules: [shaderModule],
defines: {}
};
}

initializeState(this: Layer<FilterArrowExtensionProps>, context: LayerContext, extension: this) {
const attributeManager = this.getAttributeManager();
if (attributeManager) {
attributeManager.add({
filtered: {
size: 1,
type: GL.FLOAT,
accessor: 'getFiltered',
shaderAttributes: {
filtered: {
divisor: 0
},
instanceFiltered: {
divisor: 1
}
}
}
});
}
}
}
39 changes: 39 additions & 0 deletions src/deckgl-layers/src/geojson-layer/filter-shader-module.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import {project} from '@deck.gl/core';

const vs = `
#ifdef NON_INSTANCED_MODEL
#define FILTER_ARROW_ATTRIB filtered
#else
#define FILTER_ARROW_ATTRIB instanceFiltered
#endif
attribute float FILTER_ARROW_ATTRIB;
`;

const fs = ``;

const inject = {
'vs:#decl': `
varying float is_filtered;
`,
'vs:#main-end': `
is_filtered = FILTER_ARROW_ATTRIB;
`,
'fs:#decl': `
varying float is_filtered;
`,
'fs:DECKGL_FILTER_COLOR': `
// abandon the fragments if it is not filtered
if (is_filtered == 0.) {
discard;
}
`
};

export default {
name: 'filter-arrow',
dependencies: [project],
vs: vs,
fs: fs,
inject: inject,
getUniforms: () => {}
};
1 change: 1 addition & 0 deletions src/deckgl-layers/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer
export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer';
export {default as EnhancedLineLayer} from './line-layer/line-layer';
export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer';
export {default as FilterArrowExtension} from './geojson-layer/filter-arrow-layer';

export * from './layer-utils/shader-utils';

Expand Down
8 changes: 4 additions & 4 deletions src/effects/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@
"umd"
],
"dependencies": {
"suncalc": "^1.9.0",
"@deck.gl/core": "^8.9.12",
"@luma.gl/shadertools": "^8.5.19",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@kepler.gl/constants": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1"
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@luma.gl/shadertools": "^8.5.19",
"suncalc": "^1.9.0"
},
"nyc": {
"sourceMap": false,
Expand Down
2 changes: 1 addition & 1 deletion src/layers/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@
"@loaders.gl/wkt": "^3.4.14",
"@luma.gl/constants": "^8.5.19",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@turf/bbox": "^6.0.1",
"@turf/helpers": "^6.1.4",
"@types/geojson": "^7946.0.7",
Expand Down

0 comments on commit 66f337c

Please sign in to comment.