Merge pull request #371 from Kanaries/fix-filter-ks-0607

fix(datasource): set filters & datetime filters

packages/rath-client/public/locales/en-US.json

@@ -628,6 +628,7 @@
         "months": ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"],
         "shortMonths": ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
     },
+    "date_format": "{Y}-{m}-{d}({w}) {H}:{M}:{S}",
     "login": {
         "configKeys": {
             "basic": "Basic",

packages/rath-client/public/locales/zh-CN.json

@@ -604,6 +604,7 @@
         "months": ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"],
         "shortMonths": ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"]
     },
+    "date_format": "{Y}年{m}{d}日 {w} {H}:{M}:{S}",
     "login": {
         "configKeys": {
             "basic": "基础信息",

packages/rath-client/src/components/fieldFilter/index.tsx

@@ -9,7 +9,7 @@ import { IFilter } from '../../interfaces';
 import { useGlobalStore } from '../../store';
 import RangeSelection from './rangeSelection';
 import SetSelection from './setSelection';
-import { getOriginalRange } from './originalRange';
+import { getOriginalDateTimeRange, getOriginalRange } from './originalRange';
 
 interface FieldFilterProps {
     fid: string;
@@ -22,6 +22,7 @@ const FieldFilter: React.FC<FieldFilterProps> = props => {
     const { dataSourceStore } = useGlobalStore();
 
     const meta = dataSourceStore.fieldMetas.find(fm => fm.fid === fid);
+    const filterInUse = dataSourceStore.filters.find(f => f.fid === fid);
 
     const { rawDataStorage, rawDataMetaInfo } = dataSourceStore;
 
@@ -48,17 +49,22 @@ const FieldFilter: React.FC<FieldFilterProps> = props => {
 
     const [fieldRange, setFieldRange] = useState<[number, number]>([0, 0])
     const filterType = filter.type;
-    useEffect(() => {
+    const resetRange = useCallback(() => {
         if (rawDataMetaInfo.versionCode === -1) {
             setFieldRange([0, 0]);
         } else if (filterType !== 'range') {
             setFieldRange([0, 0]);
+        } else if (meta?.semanticType === 'temporal') {
+            getOriginalDateTimeRange(rawDataStorage, fid).then(r => {
+                setFieldRange(r)
+            });
         } else {
             getOriginalRange(rawDataStorage, fid).then(r => {
                 setFieldRange(r)
             })
         }
-    }, [fid, filterType, rawDataStorage, rawDataMetaInfo.versionCode])
+    }, [fid, meta?.semanticType, filterType, rawDataStorage, rawDataMetaInfo.versionCode])
+    useEffect(resetRange, [resetRange])
 
 
     const selection = useMemo(() => {
@@ -85,6 +91,11 @@ const FieldFilter: React.FC<FieldFilterProps> = props => {
         setShowFilterConfig(false);
     }, [filter, meta?.distribution, dataSourceStore, selection])
 
+    const resetFilter = useCallback(() => {
+        resetRange();
+        dataSourceStore.removeFilter(fid);
+    }, [fid, dataSourceStore, resetRange])
+
     const toggleShowFilter = useCallback(() => {
         setShowFilterConfig(v => !v);
     }, [])
@@ -158,7 +169,7 @@ const FieldFilter: React.FC<FieldFilterProps> = props => {
                     />
                 </div>
                 {
-                    filter.type === 'set' && meta && <SetSelection
+                    filter.type === 'set' && meta && !filterInUse && <SetSelection
                         dist={toJS(meta.distribution)}
                         selection={selection}
                     />
@@ -169,15 +180,22 @@ const FieldFilter: React.FC<FieldFilterProps> = props => {
                         left={filter.range[0]}
                         right={filter.range[1]}
                         onValueChange={onRangeValueChange}
+                        type={meta.semanticType === 'temporal' ? 'time' : 'number'}
                     />
                 }
-                <Stack horizontal>
+                <Stack horizontal tokens={{ childrenGap: '1em' }}>
+                    {
+                        filterInUse && <DefaultButton
+                            onClick={resetFilter}
+                            text={intl.get('dataSource.filter.reset') || 'Reset'}
+                        />
+                    }
                     <PrimaryButton
                         text={intl.get('dataSource.filter.submit')}
+                        disabled={filter.type === 'set' && Boolean(filterInUse)}
                         onClick={submitFilter}
                     />
                     <DefaultButton
-                        style={{ marginLeft: '1em' }}
                         text={intl.get('dataSource.filter.cancel')}
                         onClick={toggleShowFilter}
                     />

packages/rath-client/src/components/fieldFilter/originalRange.ts

@@ -29,3 +29,23 @@ export async function getOriginalRange(dataStorage: IteratorStorage, colKey: str
     }
     return [_min, _max].every(Number.isFinite) ? [_min, _max] : [0, 0];
 }
+
+export async function getOriginalDateTimeRange(dataStorage: IteratorStorage, colKey: string): Promise<[number, number]> {
+    const rows = await dataStorage.getAll();
+    const values: number[] = [];
+    for (let row of rows) {
+        const val = new Date(row[colKey]).getTime();
+        if (Number.isFinite(val)) { // we only process valid date time
+            values.push(val);
+        }
+    }
+    if (values.length === 0) return [0, 0];
+    let _min = Infinity;
+    let _max = -Infinity;
+    for (const v of values) {
+        if (Number.isNaN(v)) continue;
+        if (v > _max) _max = v;
+        if (v < _min) _min = v;
+    }
+    return [_min, _max].every(Number.isFinite) ? [_min, _max] : [0, 0];
+}

packages/rath-client/src/components/fieldFilter/rangeSelection.tsx

@@ -1,14 +1,41 @@
+import intl from 'react-intl-universal';
 import { Slider } from '@fluentui/react';
-import React from 'react';
+import React, { useCallback } from 'react';
 
 interface RangeSelectionProps {
     range: [number, number];
     left: number;
     right: number;
     onValueChange: (range: [number, number]) => void;
+    type: 'number' | 'time';
 }
+
+const DateTimeValueLabelStyle = {
+    // monospace
+    fontFamily: 'Courier New',
+    // narrowed
+    letterSpacing: '-0.05em',
+    transform: 'scaleX(0.95)',
+    marginInline: '-1%',
+} as const;
+
 const RangeSelection: React.FC<RangeSelectionProps> = (props) => {
-    const { range, left, right, onValueChange } = props;
+    const { range, left, right, onValueChange, type } = props;
+
+    const formatter = useCallback((v: number) => {
+        if (type === 'time') {
+            return intl.get('date_format', {
+                Y: `${new Date(v).getFullYear()}`.padStart(4, ' '),
+                m: intl.get(`time_format.shortMonths.${new Date(v).getDay()}`),
+                d: `${new Date(v).getDate()}`.padStart(2, '0'),
+                w: intl.get(`time_format.shortDays.${new Date(v).getDay()}`),
+                H: `${new Date(v).getHours()}`.padStart(2, '0'),
+                M: `${new Date(v).getMinutes()}`.padStart(2, '0'),
+                S: `${new Date(v).getSeconds()}`.padStart(2, '0'),
+            });
+        }
+        return `${v}`;
+    }, [type]);
 
     return (
         <Slider
@@ -17,6 +44,7 @@ const RangeSelection: React.FC<RangeSelectionProps> = (props) => {
             max={range[1]}
             value={right}
             lowerValue={left}
+            valueFormat={formatter}
             ranged
             onChange={(_v, r) => {
                 r && onValueChange(r);
@@ -33,10 +61,12 @@ const RangeSelection: React.FC<RangeSelectionProps> = (props) => {
                 },
                 slideBox: {
                     flex: 1,
+                    minWidth: '120px',
                 },
                 valueLabel: {
                     minWidth: '40px',
                     width: 'unset',
+                    ...(type === 'time' ? DateTimeValueLabelStyle : {}),
                 },
             }}
         />

packages/rath-client/src/components/fieldPill/filterCreationPill.tsx

@@ -175,6 +175,7 @@ const FilterCreationPill: React.FC<FilterCreationPillProps> = (props) => {
                                     left={filter.range[0]}
                                     right={filter.range[1]}
                                     onValueChange={onRangeValueChange}
+                                    type={curField.semanticType === 'temporal' ? 'time' : 'number'}
                                 />
                             )}
                             <Stack.Item>

packages/rath-client/src/services/base.ts

@@ -150,12 +150,14 @@ export type FilterServiceProps = {
     computationMode: 'inline';
     dataSource: IRow[];
     extData: Map<string, ICol<any>>;
+    fields: readonly IRawField[];
     filters: IFilter[];
 } | {
     computationMode: 'offline';
     dataStorage: IteratorStorage;
     resultStorage: IteratorStorage;
     extData: Map<string, ICol<any>>;
+    fields: readonly IRawField[];
     filters: IFilter[];
 }
 /**

packages/rath-client/src/store/causalStore/datasetStore.ts

@@ -82,6 +82,7 @@ export default class CausalDatasetStore {
                     dataSource: dataSourceStore.cleanedData,
                     extData: new Map<string, ICol<any>>(),
                     filters: toJS(filters) as IFilter[],
+                    fields: this.allFields,
                 }).then(r => {
                     return this.filteredData.setAll(r.rows);
                 }).then(() => {

packages/rath-client/src/store/dataSourceStore.ts

@@ -170,6 +170,7 @@ export class DataSourceStore {
                     dataStorage: this.rawDataStorage,
                     resultStorage: this.filteredDataStorage,
                     extData: toJS(extData),
+                    fields: toJS(this.mutFields),
                     filters: toJS(filters)
                 }).then(r => {
                     return this.filteredDataStorage.syncMetaInfoFromStorage();
@@ -444,6 +445,13 @@ export class DataSourceStore {
         }
         this.filters = [...this.filters]
     }
+    public removeFilter (fid: string) {
+        const filterIndex = this.filters.findIndex(f => f.fid === fid);
+        if (filterIndex > -1) {
+            this.filters.splice(filterIndex, 1);
+        }
+        this.filters = [...this.filters]
+    }
     public async createBatchFilterByQts (fieldIdList: string[], qts: [number, number][]) {
         const { rawDataStorage } = this;
         const data = await rawDataStorage.getAll();

packages/rath-client/src/workers/engine/filter.ts

@@ -1,11 +1,20 @@
-import { ICol, IFilter, IRow } from "../../interfaces";
+import { ICol, IFilter, IRawField, IRow } from "../../interfaces";
+
+const resolveNumericValue = (row: IRow, f: IRawField): number => {
+    const val = row[f.fid];
+    if (f.semanticType === 'temporal') {
+        return new Date(val).getTime();
+    }
+    return typeof val === 'number' ? val : Number(val);
+};
 
 /**
  * @deprecated
  * TODO: [refactor] remove this
  * kyusho, 7 days ago   (November 22nd, 2022 11:46 PM) 
  */
 export function applyFilters (dataSource: IRow[],
+    fields: IRawField[],
     extFields: Map<string, ICol<any>>,
     filters: IFilter[]) {
     const ans: IRow[] = [];
@@ -21,7 +30,12 @@ export function applyFilters (dataSource: IRow[],
     for (let i = 0; i < dataSource.length; i++) {
         const row = dataSource[i];
         let keep = effectFilters.every(f => {
-            if (f.type === 'range') return f.range[0] <= row[f.fid] && row[f.fid] <= f.range[1];
+            if (f.type === 'range') {
+                const field = fields.find(field => field.fid === f.fid);
+                if (!field) return false;
+                const val = resolveNumericValue(row, field);
+                return f.range[0] <= val && val <= f.range[1];
+            }
             if (f.type === 'set') return f.values.includes(row[f.fid]);
             return false;
         })

packages/rath-client/src/workers/filterData.worker.js

@@ -6,10 +6,10 @@ import { timer } from './timer';
 
 const filterService = (e) => {
     try {
-        const { dataSource, dataStorage, computationMode, extData, filters, resultStorage } = e.data;
+        const { dataSource, dataStorage, computationMode, extData, filters, resultStorage, fields } = e.data;
 
         if (computationMode === 'inline') {
-            const res = applyFilters(dataSource, extData, filters);
+            const res = applyFilters(dataSource, fields, extData, filters);
             self.postMessage({
                 success: true,
                 data: {
@@ -21,7 +21,7 @@ const filterService = (e) => {
             const storage = new IteratorStorage(dataStorage)
             const resSto = new IteratorStorage(resultStorage)
             storage.getAll().then(data => {
-                return applyFilters(data, extData, filters);
+                return applyFilters(data, fields, extData, filters);
             }).then(data => {
                 return resSto.setAll(data);
             }).then(() => {