[Enhancement] pass schema to processKeplerGlDataset (#1885)

* merge upstream
keplergl · Jul 28, 2022 · 1f5e26c · 1f5e26c
1 parent 156f898
commit 1f5e26c
Show file tree

Hide file tree

Showing 7 changed files with 253 additions and 179 deletions.
diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts
@@ -665,6 +665,7 @@ function findNonEmptyRowsAtField(rows: unknown[][], fieldIdx: number, total: num
  * Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
  * The json object should contain `datasets` and `config`.
  * @param rawData
+ * @param schema
  * @returns datasets and config `{datasets: {}, config: {}}`
  * @public
  * @example
@@ -673,20 +674,24 @@ function findNonEmptyRowsAtField(rows: unknown[][], fieldIdx: number, total: num
  *
  * dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
  */
-export function processKeplerglJSON(rawData: SavedMap): LoadedMap | null {
-  return rawData ? KeplerGlSchema.load(rawData.datasets, rawData.config) : null;
+export function processKeplerglJSON(rawData: SavedMap, schema = KeplerGlSchema): LoadedMap | null {
+  return rawData ? schema.load(rawData.datasets, rawData.config) : null;
 }
 
 /**
  * Parse a single or an array of datasets saved using kepler.gl schema
  * @param rawData
+ * @param schema
  */
-export function processKeplerglDataset(rawData: unknown): ParsedDataset | ParsedDataset[] | null {
+export function processKeplerglDataset(
+  rawData: object | object[],
+  schema = KeplerGlSchema
+): ParsedDataset | ParsedDataset[] | null {
   if (!rawData) {
     return null;
   }
 
-  const results = KeplerGlSchema.parseSavedData(toArray(rawData));
+  const results = schema.parseSavedData(toArray(rawData));
   if (!results) {
     return null;
   }

diff --git a/src/reducers/vis-state-merger.ts b/src/reducers/vis-state-merger.ts
@@ -34,7 +34,7 @@ import {KeplerTable} from '../utils';
 import {ParsedConfig, ParsedLayer} from 'schemas';
 import {Layer, LayerColumns, LayerColumn} from '@kepler.gl/layers';
 import {TooltipInfo} from './vis-state-updaters';
-import {SavedInteractionConfig} from 'schemas/schema-manager';
+import {SavedInteractionConfig} from 'schemas';
 
 export type Merger = {
   merge: (state: VisState, config: any, fromConfig?: boolean) => VisState;
@@ -98,11 +98,15 @@ export function createLayerFromConfig(state: VisState, layerConfig: any): Layer
 }
 
 export function serializeLayer(newLayer): ParsedLayer {
-  const savedVisState = visStateSchema[CURRENT_VERSION].save({
-    layers: [newLayer],
-    layerOrder: [0]
-  }).visState;
-  const loadedLayer = visStateSchema[CURRENT_VERSION].load(savedVisState).visState.layers[0];
+  const savedVisState = visStateSchema[CURRENT_VERSION].save(
+    // @ts-expect-error not all expected properties are provided
+    {
+      layers: [newLayer],
+      layerOrder: [0]
+    }
+  ).visState;
+  const loadedLayer = visStateSchema[CURRENT_VERSION].load(savedVisState).visState?.layers?.[0];
+  // @ts-expect-error
   return loadedLayer;
 }
 
@@ -239,7 +243,6 @@ export function mergeInteractions(
         }
       }
 
-      // @ts-expect-error
       merged[key] = {
         ...state.interactionConfig[key],
         enabled: Boolean(enabled),

diff --git a/src/schemas/dataset-schema.ts b/src/schemas/dataset-schema.ts
@@ -21,24 +21,64 @@
 import pick from 'lodash.pick';
 import {console as globalConsole} from 'global/window';
 
+import {ProtoDataset} from 'actions';
+import {RGBColor} from 'types';
 import {VERSIONS} from './versions';
 import Schema from './schema';
 import {getFieldsFromData, getSampleForTypeAnalyze} from '@kepler.gl/processors';
 
+export type SavedField = {
+  name: string;
+  type: string;
+  format?: string;
+  analyzerType?: string;
+};
+
+export type ParsedField = {
+  name: string;
+  type: string;
+  format: string;
+  analyzerType: string;
+};
+
+export type SavedDatasetV1 = {
+  version: 'v1';
+  data: {
+    id: string;
+    label: string;
+    color: RGBColor;
+    allData: any[][];
+    fields: SavedField[];
+  };
+};
+
+export type ParsedDataset = {
+  data: {
+    fields: ParsedField[];
+    rows: any[][];
+  };
+  info: {
+    id?: string;
+    label?: string;
+    color?: RGBColor;
+  };
+};
+
 // version v0
-const fieldPropertiesV0 = {
+export const fieldPropertiesV0 = {
   name: null,
   type: null
 };
 
-const fieldPropertiesV1 = {
+export const fieldPropertiesV1 = {
   name: null,
   type: null,
   format: null,
-  analyzerType: null
+  analyzerType: null,
+  metadata: null
 };
 
-class FieldSchema extends Schema {
+export class FieldSchema extends Schema {
   save(fields) {
     return {
       [this.key]: fields.map(f => this.savePropertiesOrApplySchema(f)[this.key])
@@ -49,7 +89,7 @@ class FieldSchema extends Schema {
   }
 }
 
-const propertiesV0 = {
+export const propertiesV0 = {
   id: null,
   label: null,
   color: null,
@@ -61,7 +101,7 @@ const propertiesV0 = {
   })
 };
 
-const propertiesV1 = {
+export const propertiesV1 = {
   ...propertiesV0,
   fields: new FieldSchema({
     key: 'fields',
@@ -70,10 +110,10 @@ const propertiesV1 = {
   })
 };
 
-class DatasetSchema extends Schema {
+export class DatasetSchema extends Schema {
   key = 'dataset';
 
-  save(dataset) {
+  save(dataset): SavedDatasetV1['data'] {
     const datasetFlattened = dataset.dataContainer
       ? {
           ...dataset,
@@ -83,7 +123,7 @@ class DatasetSchema extends Schema {
 
     return this.savePropertiesOrApplySchema(datasetFlattened)[this.key];
   }
-  load(dataset) {
+  load(dataset: SavedDatasetV1['data']): ProtoDataset {
     const {fields, allData} = dataset;
     let updatedFields = fields;
 
@@ -125,7 +165,7 @@ class DatasetSchema extends Schema {
   }
 }
 
-const datasetSchema = {
+export const datasetSchema = {
   [VERSIONS.v0]: new DatasetSchema({
     key: 'dataset',
     version: VERSIONS.v0,

diff --git a/src/schemas/index.ts b/src/schemas/index.ts
@@ -29,14 +29,14 @@ export {
 // eslint-disable-next-line prettier/prettier
 export type {
   ParsedConfig,
-  ParsedDataset,
-  ParsedLayer,
-  ParsedFilter,
   SavedConfigV1,
-  SavedDatasetV1,
   SavedMap
 } from './schema-manager';
 export {CURRENT_VERSION, VERSIONS} from './versions';
+export type {
+  ParsedLayer,
+  ParsedFilter,
+} from './vis-state-schema';
 export {
   visStateSchemaV1,
   FilterSchemaV0,
@@ -47,7 +47,19 @@ export {
   filterPropsV1,
   default as visStateSchema
 } from './vis-state-schema';
-export {default as datasetSchema} from './dataset-schema';
+export type {
+  SavedField,
+  ParsedField,
+  SavedDatasetV1,
+  ParsedDataset
+} from './dataset-schema';
+export {
+  default as datasetSchema,
+  DatasetSchema,
+  fieldPropertiesV1,
+  propertiesV1 as datasetPropertiesV1
+} from './dataset-schema';
+export * from './vis-state-schema';
 export {default as mapStyleSchema} from './map-style-schema';
 export {default as mapStateSchema} from './map-state-schema';
 export {default as Schema} from './schema';