diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md index 7dcc81e189..2d688dc0cd 100644 --- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md +++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md @@ -67,7 +67,7 @@ kepler.gl will auto detect layer, if the column names follows certain naming con |**Polygon**| A column content contains `geojson` data types. Acceptable formats include [Well-Known Text](http://www.postgis.net/docs/ST_AsText.html) e.g. `POLYGON ((-74.158 40.835, -74.148 40.830, -74.151 40.832, -74.158 40.835))` and [GeoJSON Geometry](https://tools.ietf.org/html/rfc7946#appendix-A). e.g. `{"type":"LineString","coordinates":[[100.0, 0.0],[101.0, 1.0]]}` #### 3. Embed Geometries in CSV -Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string. +Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string. ##### `GeoJSON` String Use the geometry of a Feature, which includes type and coordinates. It should be a JSON formatted string, with the `"` corrected escaped. More info on [String escape in csv](https://gpdb.docs.pivotal.io/43250/admin_guide/load/topics/g-escaping-in-csv-formatted-files.html) @@ -79,7 +79,7 @@ id,geometry ``` ##### `WKT`String -[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form. +[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form. Example data.csv with WKT ```txt @@ -226,4 +226,4 @@ To add additional datasets to your map: [Back to table of contents](../README.md) -[kepler.gl-data-github]: https://github.com/uber-web/kepler.gl-data +[kepler.gl-data-github]: https://github.com/keplergl/kepler.gl-data diff --git a/examples/demo-app/package.json b/examples/demo-app/package.json index 0eb8219021..2a04a507a5 100644 --- a/examples/demo-app/package.json +++ b/examples/demo-app/package.json @@ -18,6 +18,10 @@ "@kepler.gl/processors": "^3.0.0-alpha.1", "@kepler.gl/reducers": "^3.0.0-alpha.1", "@kepler.gl/styles": "^3.0.0-alpha.1", + "@loaders.gl/arrow": "^4.1.0-alpha.4", + "@loaders.gl/core": "^4.1.0-alpha.4", + "@loaders.gl/csv": "^4.1.0-alpha.4", + "@loaders.gl/json": "^4.1.0-alpha.4", "d3-request": "^1.0.6", "dropbox": "^4.0.12", "global": "^4.3.0", diff --git a/examples/demo-app/src/actions.js b/examples/demo-app/src/actions.js index b6bd7548c6..62e0e149a6 100644 --- a/examples/demo-app/src/actions.js +++ b/examples/demo-app/src/actions.js @@ -21,6 +21,10 @@ import {push} from 'react-router-redux'; import {request, text as requestText, json as requestJson} from 'd3-request'; import {loadFiles, toggleModal} from '@kepler.gl/actions'; +import {load} from '@loaders.gl/core'; +import {CSVLoader} from '@loaders.gl/csv'; +import {ArrowLoader} from '@loaders.gl/arrow'; +import {_GeoJSONLoader as GeoJSONLoader} from '@loaders.gl/json'; import { LOADING_SAMPLE_ERROR_MESSAGE, @@ -183,8 +187,8 @@ function loadRemoteRawData(url) { /** * * @param {Object} options - * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/data.csv - * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/config.json + * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/data.csv + * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/config.json * @param {string} [options.id] the id used as dataset unique identifier, e.g. earthquakes * @param {string} [options.label] the label used to describe the new dataset, e.g. California Earthquakes * @param {string} [options.queryType] the type of query to execute to load data/config, e.g. sample @@ -290,19 +294,19 @@ function loadRemoteData(url) { } // Load data - return new Promise((resolve, reject) => { - requestMethod(url, (error, result) => { - if (error) { - reject(error); - return; - } - const responseError = detectResponseError(result); - if (responseError) { - reject(responseError); - return; - } - resolve(result); - }); + return new Promise(resolve => { + const loaders = [CSVLoader, ArrowLoader, GeoJSONLoader]; + const loadOptions = { + arrow: { + shape: 'arrow-table' + }, + csv: { + shape: 'object-row-table' + }, + metadata: true + }; + const data = load(url, loaders, loadOptions); + resolve(data); }); } diff --git a/examples/demo-app/src/components/map-control/map-control.js b/examples/demo-app/src/components/map-control/map-control.js index e38bf0e38e..fdd6b20c31 100644 --- a/examples/demo-app/src/components/map-control/map-control.js +++ b/examples/demo-app/src/components/map-control/map-control.js @@ -113,8 +113,8 @@ const LinkRenderer = props => { ); }; -// convert https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/nyctrips/config.json -// to https://github.com/uber-web/kepler.gl-data/blob/master/movement_pittsburgh/config.json +// convert https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/nyctrips/config.json +// to https://github.com/keplergl/kepler.gl-data/blob/master/movement_pittsburgh/config.json function getURL(url) { return url ? url diff --git a/examples/demo-app/src/constants/default-settings.js b/examples/demo-app/src/constants/default-settings.js index 8b4e0ebb97..1f22da82ce 100644 --- a/examples/demo-app/src/constants/default-settings.js +++ b/examples/demo-app/src/constants/default-settings.js @@ -23,7 +23,7 @@ and segments both use queryRunner */ import keyMirror from 'keymirror'; export const ASSETS_URL = 'https://d1a3f4spazzrp4.cloudfront.net/kepler.gl/'; -export const DATA_URL = 'https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/'; +export const DATA_URL = 'https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/'; export const MAP_URI = 'demo/map?mapUrl='; /* * If you want to add more samples, feel free to edit the json file on github kepler.gl data repo diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.js index a7673ea93a..b572ab13ce 100644 --- a/examples/demo-app/src/reducers/index.js +++ b/examples/demo-app/src/reducers/index.js @@ -22,7 +22,7 @@ import {combineReducers} from 'redux'; import {handleActions} from 'redux-actions'; import keplerGlReducer, {combinedUpdaters, uiStateUpdaters} from '@kepler.gl/reducers'; -import {processGeojson, processCsvData} from '@kepler.gl/processors'; +import {processGeojson, processRowObject, processArrowTable} from '@kepler.gl/processors'; import KeplerGlSchema from '@kepler.gl/schemas'; import {EXPORT_MAP_FORMATS} from '@kepler.gl/constants'; @@ -107,10 +107,12 @@ export const loadRemoteResourceSuccess = (state, action) => { // TODO: replace generate with a different function const datasetId = action.options.id || generateHashId(6); const {dataUrl} = action.options; - let processorMethod = processCsvData; + let processorMethod = processRowObject; // TODO: create helper to determine file ext eligibility if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) { processorMethod = processGeojson; + } else if (dataUrl.includes('.arrow')) { + processorMethod = processArrowTable; } const datasets = { diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index d95bc4b7ca..f38dee3981 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -359,7 +359,7 @@ export default class GeoJsonLayer extends Layer { return null; } - calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) { + calculateDataAttribute({dataContainer, filteredIndex}, getPosition) { if (dataContainer instanceof ArrowDataContainer) { // filter geojson/arrow table by values and make a partial copy of the raw table are expensive // so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering @@ -379,7 +379,7 @@ export default class GeoJsonLayer extends Layer { } this.filteredIndexTrigger = filteredIndex; } - // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU + // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU return this.dataToFeature; } @@ -427,13 +427,15 @@ export default class GeoJsonLayer extends Layer { if (dataContainer instanceof ArrowDataContainer) { // update the latest batch/chunk of geoarrow data when loading data incrementally if (this.dataToFeature.length < dataContainer.numChunks()) { + // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches + const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1; const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({ dataContainer, getGeoColumn, getGeoField, - chunkIndex: this.dataToFeature.length + ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null) }); - this.updateMeta({ bounds, fixedRadius, featureTypes }); + this.updateMeta({bounds, fixedRadius, featureTypes}); this.dataToFeature = [...this.dataToFeature, ...dataToFeature]; } } else { @@ -443,7 +445,7 @@ export default class GeoJsonLayer extends Layer { getFeature }); this.dataToFeature = dataToFeature; - this.updateMeta({ bounds, fixedRadius, featureTypes }); + this.updateMeta({bounds, fixedRadius, featureTypes}); } } } diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index 311df6946f..176370b8b1 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -22,6 +22,7 @@ import * as arrow from 'apache-arrow'; import {csvParseRows} from 'd3-dsv'; import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer'; import normalize from '@mapbox/geojson-normalize'; +import {ArrowTable} from '@loaders.gl/schema'; import {ALL_FIELD_TYPES, DATASET_FORMATS, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants'; import {ProcessorResult, Field} from '@kepler.gl/types'; import { @@ -393,12 +394,22 @@ export function processKeplerglDataset( } /** - * Parse a arrow table with geometry columns and return a dataset + * Parse arrow table and return a dataset + * + * @param arrowTable ArrowTable to parse, see loaders.gl/schema + * @returns dataset containing `fields` and `rows` or null + */ +export function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null { + return processArrowBatches(arrowTable.data.batches); +} + +/** + * Parse arrow batches returned from parseInBatches() * * @param arrowTable the arrow table to parse * @returns dataset containing `fields` and `rows` or null */ -export function processArrowTable(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null { +export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null { if (arrowBatches.length === 0) { return null; } @@ -442,6 +453,7 @@ export const Processors: { processGeojson: typeof processGeojson; processCsvData: typeof processCsvData; processArrowTable: typeof processArrowTable; + processArrowBatches: typeof processArrowBatches; processRowObject: typeof processRowObject; processKeplerglJSON: typeof processKeplerglJSON; processKeplerglDataset: typeof processKeplerglDataset; @@ -452,6 +464,7 @@ export const Processors: { processGeojson, processCsvData, processArrowTable, + processArrowBatches, processRowObject, processKeplerglJSON, processKeplerglDataset, diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts index 72f3275834..15652c1817 100644 --- a/src/processors/src/file-handler.ts +++ b/src/processors/src/file-handler.ts @@ -24,7 +24,7 @@ import {JSONLoader, _JSONPath} from '@loaders.gl/json'; import {CSVLoader} from '@loaders.gl/csv'; import {ArrowLoader} from '@loaders.gl/arrow'; import { - processArrowTable, + processArrowBatches, processGeojson, processKeplerglJSON, processRowObject @@ -225,7 +225,7 @@ export function processFileData({ if (isArrowData(data)) { format = DATASET_FORMATS.arrow; - processor = processArrowTable; + processor = processArrowBatches; } else if (isKeplerGlMap(data)) { format = DATASET_FORMATS.keplergl; processor = processKeplerglJSON;