[Demo] Add GeoArrow sample dataset (#2483)

docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md

@@ -67,7 +67,7 @@ kepler.gl will auto detect layer, if the column names follows certain naming con
 |**Polygon**| A column content contains `geojson` data types. Acceptable formats include [Well-Known Text](http://www.postgis.net/docs/ST_AsText.html) e.g. `POLYGON ((-74.158 40.835, -74.148 40.830, -74.151 40.832, -74.158 40.835))` and [GeoJSON Geometry](https://tools.ietf.org/html/rfc7946#appendix-A). e.g. `{"type":"LineString","coordinates":[[100.0, 0.0],[101.0, 1.0]]}`
 
 #### 3. Embed Geometries in CSV
-Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string. 
+Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string.
 
 ##### `GeoJSON` String
 Use the geometry of a Feature, which includes type and coordinates. It should be a JSON formatted string, with the `"` corrected escaped. More info on [String escape in csv](https://gpdb.docs.pivotal.io/43250/admin_guide/load/topics/g-escaping-in-csv-formatted-files.html)
@@ -79,7 +79,7 @@ id,geometry
 ```
 
 ##### `WKT`String
-[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form. 
+[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form.
 
 Example data.csv with WKT
 ```txt
@@ -226,4 +226,4 @@ To add additional datasets to your map:
 [Back to table of contents](../README.md)
 
 
-[kepler.gl-data-github]: https://github.com/uber-web/kepler.gl-data
+[kepler.gl-data-github]: https://github.com/keplergl/kepler.gl-data

examples/demo-app/package.json

@@ -18,6 +18,10 @@
     "@kepler.gl/processors": "^3.0.0-alpha.1",
     "@kepler.gl/reducers": "^3.0.0-alpha.1",
     "@kepler.gl/styles": "^3.0.0-alpha.1",
+    "@loaders.gl/arrow": "^4.1.0-alpha.4",
+    "@loaders.gl/core": "^4.1.0-alpha.4",
+    "@loaders.gl/csv": "^4.1.0-alpha.4",
+    "@loaders.gl/json": "^4.1.0-alpha.4",
     "d3-request": "^1.0.6",
     "dropbox": "^4.0.12",
     "global": "^4.3.0",

examples/demo-app/src/actions.js

@@ -21,6 +21,10 @@
 import {push} from 'react-router-redux';
 import {request, text as requestText, json as requestJson} from 'd3-request';
 import {loadFiles, toggleModal} from '@kepler.gl/actions';
+import {load} from '@loaders.gl/core';
+import {CSVLoader} from '@loaders.gl/csv';
+import {ArrowLoader} from '@loaders.gl/arrow';
+import {_GeoJSONLoader as GeoJSONLoader} from '@loaders.gl/json';
 
 import {
   LOADING_SAMPLE_ERROR_MESSAGE,
@@ -183,8 +187,8 @@ function loadRemoteRawData(url) {
 /**
  *
  * @param {Object} options
- * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/data.csv
- * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/config.json
+ * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/data.csv
+ * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/config.json
  * @param {string} [options.id] the id used as dataset unique identifier, e.g. earthquakes
  * @param {string} [options.label] the label used to describe the new dataset, e.g. California Earthquakes
  * @param {string} [options.queryType] the type of query to execute to load data/config, e.g. sample
@@ -290,19 +294,19 @@ function loadRemoteData(url) {
   }
 
   // Load data
-  return new Promise((resolve, reject) => {
-    requestMethod(url, (error, result) => {
-      if (error) {
-        reject(error);
-        return;
-      }
-      const responseError = detectResponseError(result);
-      if (responseError) {
-        reject(responseError);
-        return;
-      }
-      resolve(result);
-    });
+  return new Promise(resolve => {
+    const loaders = [CSVLoader, ArrowLoader, GeoJSONLoader];
+    const loadOptions = {
+      arrow: {
+        shape: 'arrow-table'
+      },
+      csv: {
+        shape: 'object-row-table'
+      },
+      metadata: true
+    };
+    const data = load(url, loaders, loadOptions);
+    resolve(data);
   });
 }
 

examples/demo-app/src/components/map-control/map-control.js

@@ -113,8 +113,8 @@ const LinkRenderer = props => {
   );
 };
 
-// convert https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/nyctrips/config.json
-// to https://github.com/uber-web/kepler.gl-data/blob/master/movement_pittsburgh/config.json
+// convert https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/nyctrips/config.json
+// to https://github.com/keplergl/kepler.gl-data/blob/master/movement_pittsburgh/config.json
 function getURL(url) {
   return url
     ? url

examples/demo-app/src/constants/default-settings.js

@@ -23,7 +23,7 @@ and segments both use queryRunner */
 import keyMirror from 'keymirror';
 
 export const ASSETS_URL = 'https://d1a3f4spazzrp4.cloudfront.net/kepler.gl/';
-export const DATA_URL = 'https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/';
+export const DATA_URL = 'https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/';
 export const MAP_URI = 'demo/map?mapUrl=';
 /*
  * If you want to add more samples, feel free to edit the json file on github kepler.gl data repo

examples/demo-app/src/reducers/index.js

@@ -22,7 +22,7 @@ import {combineReducers} from 'redux';
 import {handleActions} from 'redux-actions';
 
 import keplerGlReducer, {combinedUpdaters, uiStateUpdaters} from '@kepler.gl/reducers';
-import {processGeojson, processCsvData} from '@kepler.gl/processors';
+import {processGeojson, processRowObject, processArrowTable} from '@kepler.gl/processors';
 import KeplerGlSchema from '@kepler.gl/schemas';
 import {EXPORT_MAP_FORMATS} from '@kepler.gl/constants';
 
@@ -107,10 +107,12 @@ export const loadRemoteResourceSuccess = (state, action) => {
   // TODO: replace generate with a different function
   const datasetId = action.options.id || generateHashId(6);
   const {dataUrl} = action.options;
-  let processorMethod = processCsvData;
+  let processorMethod = processRowObject;
   // TODO: create helper to determine file ext eligibility
   if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
     processorMethod = processGeojson;
+  } else if (dataUrl.includes('.arrow')) {
+    processorMethod = processArrowTable;
   }
 
   const datasets = {

src/layers/src/geojson-layer/geojson-layer.ts

@@ -359,7 +359,7 @@ export default class GeoJsonLayer extends Layer {
     return null;
   }
 
-  calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) {
+  calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
     if (dataContainer instanceof ArrowDataContainer) {
       // filter geojson/arrow table by values and make a partial copy of the raw table are expensive
       // so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering
@@ -379,7 +379,7 @@ export default class GeoJsonLayer extends Layer {
         }
         this.filteredIndexTrigger = filteredIndex;
       }
-    // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
+      // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
       return this.dataToFeature;
     }
 
@@ -427,13 +427,15 @@ export default class GeoJsonLayer extends Layer {
     if (dataContainer instanceof ArrowDataContainer) {
       // update the latest batch/chunk of geoarrow data when loading data incrementally
       if (this.dataToFeature.length < dataContainer.numChunks()) {
+        // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches
+        const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1;
         const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({
           dataContainer,
           getGeoColumn,
           getGeoField,
-          chunkIndex: this.dataToFeature.length
+          ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null)
         });
-        this.updateMeta({ bounds, fixedRadius, featureTypes });
+        this.updateMeta({bounds, fixedRadius, featureTypes});
         this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
       }
     } else {
@@ -443,7 +445,7 @@ export default class GeoJsonLayer extends Layer {
           getFeature
         });
         this.dataToFeature = dataToFeature;
-        this.updateMeta({ bounds, fixedRadius, featureTypes });
+        this.updateMeta({bounds, fixedRadius, featureTypes});
       }
     }
   }

src/processors/src/data-processor.ts

@@ -22,6 +22,7 @@ import * as arrow from 'apache-arrow';
 import {csvParseRows} from 'd3-dsv';
 import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
 import normalize from '@mapbox/geojson-normalize';
+import {ArrowTable} from '@loaders.gl/schema';
 import {ALL_FIELD_TYPES, DATASET_FORMATS, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';
 import {ProcessorResult, Field} from '@kepler.gl/types';
 import {
@@ -393,12 +394,22 @@ export function processKeplerglDataset(
 }
 
 /**
- * Parse a arrow table with geometry columns and return a dataset
+ * Parse arrow table and return a dataset
+ *
+ * @param arrowTable ArrowTable to parse, see loaders.gl/schema
+ * @returns dataset containing `fields` and `rows` or null
+ */
+export function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null {
+  return processArrowBatches(arrowTable.data.batches);
+}
+
+/**
+ * Parse arrow batches returned from parseInBatches()
  *
  * @param arrowTable the arrow table to parse
  * @returns dataset containing `fields` and `rows` or null
  */
-export function processArrowTable(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
+export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
   if (arrowBatches.length === 0) {
     return null;
   }
@@ -442,6 +453,7 @@ export const Processors: {
   processGeojson: typeof processGeojson;
   processCsvData: typeof processCsvData;
   processArrowTable: typeof processArrowTable;
+  processArrowBatches: typeof processArrowBatches;
   processRowObject: typeof processRowObject;
   processKeplerglJSON: typeof processKeplerglJSON;
   processKeplerglDataset: typeof processKeplerglDataset;
@@ -452,6 +464,7 @@ export const Processors: {
   processGeojson,
   processCsvData,
   processArrowTable,
+  processArrowBatches,
   processRowObject,
   processKeplerglJSON,
   processKeplerglDataset,

src/processors/src/file-handler.ts

@@ -24,7 +24,7 @@ import {JSONLoader, _JSONPath} from '@loaders.gl/json';
 import {CSVLoader} from '@loaders.gl/csv';
 import {ArrowLoader} from '@loaders.gl/arrow';
 import {
-  processArrowTable,
+  processArrowBatches,
   processGeojson,
   processKeplerglJSON,
   processRowObject
@@ -225,7 +225,7 @@ export function processFileData({
 
     if (isArrowData(data)) {
       format = DATASET_FORMATS.arrow;
-      processor = processArrowTable;
+      processor = processArrowBatches;
     } else if (isKeplerGlMap(data)) {
       format = DATASET_FORMATS.keplergl;
       processor = processKeplerglJSON;