From a101ed1b62c2fbfec2712f64e08192a4852bce9d Mon Sep 17 00:00:00 2001 From: ghiscoding Date: Mon, 8 Feb 2021 23:29:49 -0500 Subject: [PATCH] feat(perf): huge filtering speed improvements --- .../src/examples/example03.ts | 2 +- .../__tests__/stringFilterCondition.spec.ts | 4 +- .../executeMappedCondition.ts | 206 ++++++++++++------ .../numberFilterCondition.ts | 21 +- .../stringFilterCondition.ts | 2 +- packages/common/src/filters/filters.index.ts | 2 +- .../src/interfaces/columnFilters.interface.ts | 4 +- .../interfaces/filterCondition.interface.ts | 3 +- .../filterConditionOption.interface.ts | 21 +- packages/common/src/interfaces/index.ts | 1 + .../searchColumnFilter.interface.ts | 35 +++ .../common/src/services/filter.service.ts | 204 ++++++++++------- .../components/slick-vanilla-grid-bundle.ts | 1 + 13 files changed, 337 insertions(+), 169 deletions(-) create mode 100644 packages/common/src/interfaces/searchColumnFilter.interface.ts diff --git a/examples/webpack-demo-vanilla-bundle/src/examples/example03.ts b/examples/webpack-demo-vanilla-bundle/src/examples/example03.ts index 4470cd2cc..340c7f18a 100644 --- a/examples/webpack-demo-vanilla-bundle/src/examples/example03.ts +++ b/examples/webpack-demo-vanilla-bundle/src/examples/example03.ts @@ -181,7 +181,7 @@ export class Example3 { // formatter: Formatters.dateIso, type: FieldType.date, outputType: FieldType.dateIso, formatter: Formatters.dateIso, - filterable: true, filter: { model: Filters.compoundDate }, + filterable: true, filter: { model: Filters.dateRange }, grouping: { getter: 'finish', formatter: (g) => `Finish: ${g.value} (${g.count} items)`, diff --git a/packages/common/src/filter-conditions/__tests__/stringFilterCondition.spec.ts b/packages/common/src/filter-conditions/__tests__/stringFilterCondition.spec.ts index f68e7a57c..54a87ce09 100644 --- a/packages/common/src/filter-conditions/__tests__/stringFilterCondition.spec.ts +++ b/packages/common/src/filter-conditions/__tests__/stringFilterCondition.spec.ts @@ -82,8 +82,8 @@ describe('stringFilterCondition method', () => { expect(output).toBe(true); }); - it('should return True when input value provided starts with same substring and the operator is empty string & option "cellValueLastChar" is asterisk (*)', () => { - const options = { dataKey: '', operator: '', cellValueLastChar: '*', cellValue: 'abbostford', fieldType: FieldType.string, searchTerms: ['abb'] } as FilterConditionOption; + it('should return True when input value provided starts with same substring and the operator is empty string & option "searchInputLastChar" is asterisk (*)', () => { + const options = { dataKey: '', operator: '', searchInputLastChar: '*', cellValue: 'abbostford', fieldType: FieldType.string, searchTerms: ['abb'] } as FilterConditionOption; const output = stringFilterCondition(options); expect(output).toBe(true); }); diff --git a/packages/common/src/filter-conditions/executeMappedCondition.ts b/packages/common/src/filter-conditions/executeMappedCondition.ts index b282c3cdc..c7d0b1d8b 100644 --- a/packages/common/src/filter-conditions/executeMappedCondition.ts +++ b/packages/common/src/filter-conditions/executeMappedCondition.ts @@ -4,7 +4,7 @@ import { numberFilterCondition } from './numberFilterCondition'; import { objectFilterCondition } from './objectFilterCondition'; import { stringFilterCondition } from './stringFilterCondition'; -import { FieldType, OperatorType } from '../enums/index'; +import { FieldType, OperatorType, SearchTerm } from '../enums/index'; import { FilterCondition, FilterConditionOption } from '../interfaces/index'; import { mapMomentDateFormatWithFieldType } from './../services/utilities'; import { testFilterCondition } from './filterUtilities'; @@ -12,17 +12,144 @@ import * as moment_ from 'moment-mini'; const moment = moment_['default'] || moment_; // patch to fix rollup "moment has no default export" issue, document here https://github.com/rollup/rollup/issues/670 -export const executeMappedCondition: FilterCondition = (options: FilterConditionOption) => { +export type GeneralFieldType = 'boolean' | 'date' | 'number' | 'object' | 'text'; + +export const executeMappedCondition: FilterCondition = (options: FilterConditionOption, parsedSearchTerms: SearchTerm[]) => { // when using a multi-select ('IN' operator) we will not use the field type but instead go directly with a collection search const operator = options && options.operator && options.operator.toUpperCase(); if (operator === 'IN' || operator === 'NIN' || operator === 'IN_CONTAINS' || operator === 'NIN_CONTAINS') { - return collectionSearchFilterCondition(options); + return collectionSearchFilterCondition(options, parsedSearchTerms); } + const generalType = getGeneralTypeByFieldType(options.fieldType); + + // execute the mapped type, or default to String condition check - switch (options.fieldType) { + switch (generalType) { + case 'boolean': + return booleanFilterCondition(options, parsedSearchTerms); + case 'date': + return executeAssociatedDateCondition(options, ...parsedSearchTerms); + case 'number': + return numberFilterCondition(options, ...parsedSearchTerms as number[]); + case 'object': + return objectFilterCondition(options, parsedSearchTerms); + case 'text': + default: + return stringFilterCondition(options, parsedSearchTerms); + } +}; + +/** + * Execute Date filter condition and use correct date format depending on it's field type (or filterSearchType when that is provided) + * @param options + */ +function executeAssociatedDateCondition(options: FilterConditionOption, ...parsedSearchDates: any[]): boolean { + const filterSearchType = options && (options.filterSearchType || options.fieldType) || FieldType.dateIso; + const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType); + const [searchDate1, searchDate2] = parsedSearchDates; + + // cell value in moment format + const dateCell = moment(options.cellValue, FORMAT, true); + + // return when cell value is not a valid date + if ((!searchDate1 && !searchDate2) || !dateCell.isValid()) { + return false; + } + + // when comparing with Dates only (without time), we need to disregard the time portion, we can do so by setting our time to start at midnight + // ref, see https://stackoverflow.com/a/19699447/1212166 + const dateCellTimestamp = FORMAT.toLowerCase().includes('h') ? dateCell.valueOf() : dateCell.clone().startOf('day').valueOf(); + + // having 2 search dates, we assume that it's a date range filtering and we'll compare against both dates + if (searchDate1 && searchDate2) { + const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive; + const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), dateCellTimestamp, searchDate1.valueOf()); + const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), dateCellTimestamp, searchDate2.valueOf()); + return (resultCondition1 && resultCondition2); + } + + // comparing against a single search date + const dateSearchTimestamp1 = FORMAT.toLowerCase().includes('h') ? searchDate1.valueOf() : searchDate1.clone().startOf('day').valueOf(); + return testFilterCondition(options.operator || '==', dateCellTimestamp, dateSearchTimestamp1); +} + +export function getParsedSearchTermsByFieldType(inputSearchTerms: SearchTerm[] | undefined, inputFilterSearchType: typeof FieldType[keyof typeof FieldType]): SearchTerm[] | undefined { + const generalType = getGeneralTypeByFieldType(inputFilterSearchType); + + switch (generalType) { + case 'date': + return getParsedSearchDates(inputSearchTerms, inputFilterSearchType); + case 'number': + return getParsedSearchNumbers(inputSearchTerms); + } + return undefined; +} + +function getParsedSearchDates(inputSearchTerms: SearchTerm[] | undefined, inputFilterSearchType: typeof FieldType[keyof typeof FieldType]): SearchTerm[] | undefined { + const searchTerms = Array.isArray(inputSearchTerms) && inputSearchTerms || []; + const filterSearchType = inputFilterSearchType || FieldType.dateIso; + const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType); + + const parsedSearchValues: any[] = []; + + if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) { + const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..'); + const searchValue1 = (Array.isArray(searchValues) && searchValues[0] || '') as Date | string; + const searchValue2 = (Array.isArray(searchValues) && searchValues[1] || '') as Date | string; + const searchDate1 = moment(searchValue1, FORMAT, true); + const searchDate2 = moment(searchValue2, FORMAT, true); + + // return if any of the 2 values are invalid dates + if (!searchDate1.isValid() || !searchDate2.isValid()) { + return undefined; + } + parsedSearchValues.push(searchDate1, searchDate2); + } else { + // return if the search term is an invalid date + const searchDate1 = moment(searchTerms[0] as Date | string, FORMAT, true); + if (!searchDate1.isValid()) { + return undefined; + } + parsedSearchValues.push(searchDate1); + } + return parsedSearchValues; +} + +function getParsedSearchNumbers(inputSearchTerms: SearchTerm[] | undefined): number[] | undefined { + const searchTerms = Array.isArray(inputSearchTerms) && inputSearchTerms || []; + const parsedSearchValues: number[] = []; + let searchValue1; + let searchValue2; + + if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) { + const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..'); + searchValue1 = parseFloat(Array.isArray(searchValues) ? (searchValues[0] + '') : ''); + searchValue2 = parseFloat(Array.isArray(searchValues) ? (searchValues[1] + '') : ''); + } else { + searchValue1 = parseFloat(searchTerms[0] + ''); + } + + if (searchValue1 !== undefined && searchValue2 !== undefined) { + parsedSearchValues.push(searchValue1, searchValue2); + } else if (searchValue1 !== undefined) { + parsedSearchValues.push(searchValue1); + } else { + return undefined; + } + return parsedSearchValues; +} + +/** + * From a more specific field type, let's return a simple and more general type (boolean, date, number, object, text) + * @param fieldType - specific field type + * @returns generalType - general field type + */ +function getGeneralTypeByFieldType(fieldType: typeof FieldType[keyof typeof FieldType]): GeneralFieldType { + // return general field type + switch (fieldType) { case FieldType.boolean: - return booleanFilterCondition(options); + return 'boolean'; case FieldType.date: case FieldType.dateIso: case FieldType.dateUtc: @@ -49,77 +176,18 @@ export const executeMappedCondition: FilterCondition = (options: FilterCondition case FieldType.dateTimeUsShort: case FieldType.dateTimeUsShortAmPm: case FieldType.dateTimeUsShortAM_PM: - return executeAssociatedDateCondition(options); + return 'date'; case FieldType.integer: case FieldType.float: case FieldType.number: - return numberFilterCondition(options); + return 'number'; case FieldType.object: - return objectFilterCondition(options); + return 'object'; case FieldType.string: case FieldType.text: case FieldType.password: case FieldType.readonly: default: - return stringFilterCondition(options); - } -}; - -/** - * Execute Date filter condition and use correct date format depending on it's field type (or filterSearchType when that is provided) - * @param options - */ -function executeAssociatedDateCondition(options: FilterConditionOption): boolean { - const filterSearchType = options && (options.filterSearchType || options.fieldType) || FieldType.dateIso; - const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType); - const searchTerms = Array.isArray(options.searchTerms) && options.searchTerms || []; - - let isRangeSearch = false; - let dateSearch1: any; - let dateSearch2: any; - - // return when cell value is not a valid date - if (searchTerms.length === 0 || searchTerms[0] === '' || searchTerms[0] === null || !moment(options.cellValue, FORMAT, true).isValid()) { - return false; + return 'text'; } - - // cell value in moment format - const dateCell = moment(options.cellValue, FORMAT, true); - - if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) { - isRangeSearch = true; - const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..'); - const searchValue1 = (Array.isArray(searchValues) && searchValues[0] || '') as Date | string; - const searchValue2 = (Array.isArray(searchValues) && searchValues[1] || '') as Date | string; - const searchTerm1 = moment(searchValue1, FORMAT, true); - const searchTerm2 = moment(searchValue2, FORMAT, true); - - // return if any of the 2 values are invalid dates - if (!moment(searchTerm1, FORMAT, true).isValid() || !moment(searchTerm2, FORMAT, true).isValid()) { - return false; - } - dateSearch1 = moment(searchTerm1, FORMAT, true); - dateSearch2 = moment(searchTerm2, FORMAT, true); - } else { - // return if the search term is an invalid date - if (!moment(searchTerms[0] as Date | string, FORMAT, true).isValid()) { - return false; - } - dateSearch1 = moment(searchTerms[0] as Date | string, FORMAT, true); - } - - // when comparing with Dates only (without time), we need to disregard the time portion, we can do so by setting our time to start at midnight - // ref, see https://stackoverflow.com/a/19699447/1212166 - const dateCellTimestamp = FORMAT.toLowerCase().includes('h') ? parseInt(dateCell.format('X'), 10) : parseInt(dateCell.clone().startOf('day').format('X'), 10); - - // run the filter condition with date in Unix Timestamp format - if (isRangeSearch) { - const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive; - const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), dateCellTimestamp, parseInt(dateSearch1.format('X'), 10)); - const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), dateCellTimestamp, parseInt(dateSearch2.format('X'), 10)); - return (resultCondition1 && resultCondition2); - } - - const dateSearchTimestamp1 = FORMAT.toLowerCase().includes('h') ? parseInt(dateSearch1.format('X'), 10) : parseInt(dateSearch1.clone().startOf('day').format('X'), 10); - return testFilterCondition(options.operator || '==', dateCellTimestamp, dateSearchTimestamp1); -} +} \ No newline at end of file diff --git a/packages/common/src/filter-conditions/numberFilterCondition.ts b/packages/common/src/filter-conditions/numberFilterCondition.ts index 059638873..e56a5aeb9 100644 --- a/packages/common/src/filter-conditions/numberFilterCondition.ts +++ b/packages/common/src/filter-conditions/numberFilterCondition.ts @@ -2,29 +2,16 @@ import { OperatorType } from '../enums/index'; import { FilterCondition, FilterConditionOption } from '../interfaces/index'; import { testFilterCondition } from './filterUtilities'; -export const numberFilterCondition: FilterCondition = (options: FilterConditionOption) => { +export const numberFilterCondition: FilterCondition = (options: FilterConditionOption, ...parsedSearchValues: number[]) => { const cellValue = parseFloat(options.cellValue); - const searchTerms = Array.isArray(options.searchTerms) && options.searchTerms || [0]; - - let isRangeSearch = false; - let searchValue1; - let searchValue2; - - if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) { - isRangeSearch = true; - const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..'); - searchValue1 = parseFloat(Array.isArray(searchValues) ? (searchValues[0] + '') : ''); - searchValue2 = parseFloat(Array.isArray(searchValues) ? (searchValues[1] + '') : ''); - } else { - searchValue1 = parseFloat(searchTerms[0] + ''); - } + const [searchValue1, searchValue2] = parsedSearchValues; if (!searchValue1 && !options.operator) { return true; } - if (isRangeSearch) { - const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive; + if (searchValue1 !== undefined && searchValue2 !== undefined) { + const isInclusive = options?.operator === OperatorType.rangeInclusive; const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), cellValue, searchValue1); const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), cellValue, searchValue2); return (resultCondition1 && resultCondition2); diff --git a/packages/common/src/filter-conditions/stringFilterCondition.ts b/packages/common/src/filter-conditions/stringFilterCondition.ts index 7685f1c99..612d4e654 100644 --- a/packages/common/src/filter-conditions/stringFilterCondition.ts +++ b/packages/common/src/filter-conditions/stringFilterCondition.ts @@ -15,7 +15,7 @@ export const stringFilterCondition: FilterCondition = (options: FilterConditionO if (options.operator === '*' || options.operator === OperatorType.endsWith) { return cellValue.endsWith(searchTerm); - } else if ((options.operator === '' && options.cellValueLastChar === '*') || options.operator === OperatorType.startsWith) { + } else if ((options.operator === '' && options.searchInputLastChar === '*') || options.operator === OperatorType.startsWith) { return cellValue.startsWith(searchTerm); } else if (options.operator === '' || options.operator === OperatorType.contains) { return (cellValue.indexOf(searchTerm) > -1); diff --git a/packages/common/src/filters/filters.index.ts b/packages/common/src/filters/filters.index.ts index 12adae23c..0d9452c1e 100644 --- a/packages/common/src/filters/filters.index.ts +++ b/packages/common/src/filters/filters.index.ts @@ -40,7 +40,7 @@ export const Filters = { /** Range Date Filter (uses the Flactpickr Date picker with range option) */ dateRange: DateRangeFilter, - /** Alias to inputText, input type text filter */ + /** Alias to inputText, input type text filter (this is the default filter when no type is provided) */ input: InputFilter, /** diff --git a/packages/common/src/interfaces/columnFilters.interface.ts b/packages/common/src/interfaces/columnFilters.interface.ts index 866609af6..9e91e3b80 100644 --- a/packages/common/src/interfaces/columnFilters.interface.ts +++ b/packages/common/src/interfaces/columnFilters.interface.ts @@ -1,5 +1,5 @@ -import { ColumnFilter } from './columnFilter.interface'; +import { SearchColumnFilter } from './searchColumnFilter.interface'; export interface ColumnFilters { - [key: string]: ColumnFilter; + [key: string]: SearchColumnFilter; } diff --git a/packages/common/src/interfaces/filterCondition.interface.ts b/packages/common/src/interfaces/filterCondition.interface.ts index 3d8c61d37..cfeca1b9d 100644 --- a/packages/common/src/interfaces/filterCondition.interface.ts +++ b/packages/common/src/interfaces/filterCondition.interface.ts @@ -1,4 +1,5 @@ +import { SearchTerm } from '../enums/searchTerm.type'; import { FilterConditionOption } from './filterConditionOption.interface'; -export type FilterCondition = (options: FilterConditionOption) => boolean; +export type FilterCondition = (options: FilterConditionOption, parsedSearchTerms?: SearchTerm | SearchTerm[]) => boolean; diff --git a/packages/common/src/interfaces/filterConditionOption.interface.ts b/packages/common/src/interfaces/filterConditionOption.interface.ts index 8018f03c5..efdb7c3a0 100644 --- a/packages/common/src/interfaces/filterConditionOption.interface.ts +++ b/packages/common/src/interfaces/filterConditionOption.interface.ts @@ -1,11 +1,30 @@ import { FieldType, OperatorString, SearchTerm } from '../enums/index'; export interface FilterConditionOption { + /** optional object data key */ dataKey?: string; + + /** filter operator */ operator: OperatorString; + + /** cell value */ cellValue: any; - cellValueLastChar?: string; + + /** last character of the cell value, which is helpful to know if we are dealing with "*" that would be mean startsWith */ + searchInputLastChar?: string; + + /** column field type */ fieldType: typeof FieldType[keyof typeof FieldType]; + + /** filter search field type */ filterSearchType?: typeof FieldType[keyof typeof FieldType]; + + /** + * Parsed Search Terms is similar to SearchTerms but is already parsed in the correct format, + * for example on a date field the searchTerms might be in string format but their respective parsedSearchTerms will be of type Date + */ + parsedSearchTerms?: SearchTerm[] | undefined; + + /** Search Terms provided by the user */ searchTerms?: SearchTerm[] | undefined; } diff --git a/packages/common/src/interfaces/index.ts b/packages/common/src/interfaces/index.ts index 29ff22fd4..a90afebd2 100644 --- a/packages/common/src/interfaces/index.ts +++ b/packages/common/src/interfaces/index.ts @@ -114,6 +114,7 @@ export * from './rowDetailView.interface'; export * from './rowDetailViewOption.interface'; export * from './rowMoveManager.interface'; export * from './rowMoveManagerOption.interface'; +export * from './searchColumnFilter.interface'; export * from './selectOption.interface'; export * from './servicePagination.interface'; export * from './singleColumnSort.interface'; diff --git a/packages/common/src/interfaces/searchColumnFilter.interface.ts b/packages/common/src/interfaces/searchColumnFilter.interface.ts new file mode 100644 index 000000000..bf8db67c1 --- /dev/null +++ b/packages/common/src/interfaces/searchColumnFilter.interface.ts @@ -0,0 +1,35 @@ +import { FieldType, OperatorString, OperatorType, } from '../enums/index'; +import { Column, } from './index'; +import { SearchTerm } from '../enums/searchTerm.type'; + +export interface SearchColumnFilter { + /** Column ID */ + columnId: string; + + /** Column Definition */ + columnDef: Column; + + /** + * Parsed Search Terms is similar to SearchTerms but is already parsed in the correct format, + * for example on a date field the searchTerms might be in string format but their respective parsedSearchTerms will be of type Date + */ + parsedSearchTerms: SearchTerm[]; + + /** Search terms to preload (collection), please note it is better to use the "presets" grid option which is more powerful. */ + searchTerms: SearchTerm[]; + + /** Operator to use when filtering (>, >=, EQ, IN, ...) */ + operator?: OperatorType | OperatorString; + + /** + * Useful when you want to display a certain field to the UI, but you want to use another field to query when Filtering/Sorting. + * Please note that it has higher precendence over the "field" property. + */ + queryField?: string; + + /** Last search input character when it is identified as "*" representing startsWith */ + searchInputLastChar?: string; + + /** What is the Field Type that can be used by the Filter (as precedence over the "type" set the column definition) */ + type: typeof FieldType[keyof typeof FieldType]; +} diff --git a/packages/common/src/services/filter.service.ts b/packages/common/src/services/filter.service.ts index c434aaeb6..81f7a8d6d 100644 --- a/packages/common/src/services/filter.service.ts +++ b/packages/common/src/services/filter.service.ts @@ -1,7 +1,7 @@ import * as isequal_ from 'lodash.isequal'; const isequal = isequal_['default'] || isequal_; // patch to fix rollup to work -import { FilterConditions } from './../filter-conditions/index'; +import { FilterConditions, getParsedSearchTermsByFieldType } from './../filter-conditions/index'; import { FilterFactory } from './../filters/filterFactory'; import { EmitterType, @@ -13,7 +13,6 @@ import { } from '../enums/index'; import { Column, - ColumnFilter, ColumnFilters, CurrentFilter, SlickDataView, @@ -24,6 +23,7 @@ import { FilterConditionOption, GetSlickEventType, GridOption, + SearchColumnFilter, SlickEvent, SlickEventData, SlickEventHandler, @@ -31,7 +31,7 @@ import { SlickNamespace, } from './../interfaces/index'; import { executeBackendCallback, refreshBackendDataset } from './backend-utilities'; -import { debounce, getDescendantProperty, mapOperatorByFieldType } from './utilities'; +import { debounce, deepCopy, getDescendantProperty, mapOperatorByFieldType } from './utilities'; import { PubSubService } from '../services/pubSub.service'; import { SharedService } from './shared.service'; @@ -48,7 +48,8 @@ interface OnSearchChangeEvent { columnDef: Column; columnFilters: ColumnFilters; operator: OperatorType | OperatorString | undefined; - searchTerms: any[] | undefined; + parsedSearchTerms?: SearchTerm[] | undefined; + searchTerms: SearchTerm[] | undefined; grid: SlickGrid; } @@ -191,17 +192,14 @@ export class FilterService { if (this._onSearchChange) { const onSearchChangeHandler = this._onSearchChange; (this._eventHandler as SlickEventHandler>).subscribe(this._onSearchChange, (_e, args) => { - const isGridWithTreeData = this._gridOptions?.enableTreeData ?? false; - // When using Tree Data, we need to do it in 2 steps // step 1. we need to prefilter (search) the data prior, the result will be an array of IDs which are the node(s) and their parent nodes when necessary. // step 2. calling the DataView.refresh() is what triggers the final filtering, with "customLocalFilter()" which will decide which rows should persist - if (isGridWithTreeData) { + if (this._gridOptions?.enableTreeData === true) { this._tmpPreFilteredData = this.preFilterTreeData(this._dataView.getItems(), this._columnFilters); } - const columnId = args.columnId; - if (columnId !== null) { + if (args.columnId !== null) { this._dataView.refresh(); } @@ -221,10 +219,10 @@ export class FilterService { clearFilterByColumnId(event: Event, columnId: number | string) { // get current column filter before clearing, this allow us to know if the filter was empty prior to calling the clear filter - const currentColumnFilters = Object.keys(this._columnFilters) as ColumnFilter[]; - let currentColFilter: ColumnFilter | undefined; - if (Array.isArray(currentColumnFilters)) { - currentColFilter = currentColumnFilters.find((name) => name === columnId); + const currentFilterColumnIds = Object.keys(this._columnFilters); + let currentColFilter: string | undefined; + if (Array.isArray(currentFilterColumnIds)) { + currentColFilter = currentFilterColumnIds.find(name => name === `${columnId}`); } // find the filter object and call its clear method with true (the argument tells the method it was called by a clear filter) @@ -315,13 +313,15 @@ export class FilterService { } else { if (typeof columnFilters === 'object') { for (const columnId of Object.keys(columnFilters)) { - const columnFilter = columnFilters[columnId] as ColumnFilter; - const conditionOptions = this.getFilterConditionOptionsOrBoolean(item, columnFilter, columnId, grid); + const columnFilter = columnFilters[columnId] as SearchColumnFilter; + const conditionOptions = this.getFilterConditionOptionsOrBoolean(item, columnFilter, grid); + if (typeof conditionOptions === 'boolean') { return conditionOptions; } - if (!FilterConditions.executeMappedCondition(conditionOptions as FilterConditionOption)) { + const parsedSearchTerms = columnFilter?.parsedSearchTerms ?? []; + if (!FilterConditions.executeMappedCondition(conditionOptions as FilterConditionOption, parsedSearchTerms)) { return false; } } @@ -332,16 +332,68 @@ export class FilterService { return true; } - getFilterConditionOptionsOrBoolean(item: any, columnFilter: ColumnFilter, columnId: string | number, grid: SlickGrid): FilterConditionOption | boolean { + getSearchInputConditions(inputSearchTerms: SearchTerm[], columnFilter: Omit): Omit { + const searchValues: SearchTerm[] = deepCopy(inputSearchTerms) || []; + let fieldSearchValue = (Array.isArray(searchValues) && searchValues.length === 1) ? searchValues[0] : ''; + const columnDef = columnFilter.columnDef; + const fieldType = columnDef.filter?.type ?? columnDef.type ?? FieldType.string; + + let matches = null; + if (fieldType !== FieldType.object) { + fieldSearchValue = '' + fieldSearchValue; // make sure it's a string + matches = fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])?([\*]?)$/); // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*) + } + + let operator = matches?.[1] || columnFilter.operator; + const searchTerm = matches?.[2] || ''; + const inputLastChar = matches?.[3] || (operator === '*z' ? '*' : ''); + + if (Array.isArray(searchValues) && searchValues.length > 1) { + fieldSearchValue = searchValues.join(','); + } else if (typeof fieldSearchValue === 'string') { + // escaping the search value + fieldSearchValue = fieldSearchValue.replace(`'`, `''`); // escape single quotes by doubling them + if (operator === '*' || operator === 'a*' || operator === '*z' || inputLastChar === '*') { + operator = (operator === '*' || operator === '*z') ? OperatorType.endsWith : OperatorType.startsWith; + } + } + + // if search value has a regex match we will only keep the value without the operator + // in this case we need to overwrite the returned search values to truncate operator from the string search + if (Array.isArray(matches) && matches.length >= 1 && (Array.isArray(searchValues) && searchValues.length === 1)) { + searchValues[0] = searchTerm; + } + + // filter search terms should always be string type (even though we permit the end user to input numbers) + // so make sure each term are strings, if user has some default search terms, we will cast them to string + if (searchValues && Array.isArray(searchValues) && fieldType !== FieldType.object) { + for (let k = 0, ln = searchValues.length; k < ln; k++) { + // make sure all search terms are strings + searchValues[k] = ((searchValues[k] === undefined || searchValues[k] === null) ? '' : searchValues[k]) + ''; + } + } + + // return { searchTerms: searchValues, operator, fieldType }; + return { + dataKey: columnDef.dataKey, + fieldType, + searchTerms: searchValues, + operator: operator as OperatorString, + searchInputLastChar: inputLastChar, + filterSearchType: columnDef.filterSearchType + } as FilterConditionOption; + } + + // TODO find a better and more suited fn name to represent this being called on every cell value filter + getFilterConditionOptionsOrBoolean(item: any, columnFilter: SearchColumnFilter, grid: SlickGrid): FilterConditionOption | boolean { + const columnDef = columnFilter.columnDef; + const columnId = columnFilter.columnId; let columnIndex = grid.getColumnIndex(columnId) as number; - let columnDef = grid.getColumns()[columnIndex] as Column; - const dataView = grid.getData() as SlickDataView; // it might be a hidden column, if so it won't be part of the getColumns (because it could be hidden via setColumns()) // when that happens we can try to get the column definition from all defined columns if (!columnDef && this.sharedService && Array.isArray(this.sharedService.allColumns)) { - columnIndex = this.sharedService.allColumns.findIndex((col) => col.field === columnId); - columnDef = this.sharedService.allColumns[columnIndex]; + columnIndex = this.sharedService.allColumns.findIndex(col => col.field === columnId); } // if we still don't have a column definition then we should return then row anyway (true) @@ -357,13 +409,11 @@ export class FilterService { } } - const dataKey = columnDef.dataKey; let queryFieldName = columnDef.filter?.queryField || columnDef.queryFieldFilter || columnDef.queryField || columnDef.field || ''; if (typeof columnDef.queryFieldNameGetterFn === 'function') { queryFieldName = columnDef.queryFieldNameGetterFn(item); } const fieldType = columnDef.filter?.type ?? columnDef.type ?? FieldType.string; - const filterSearchType = (columnDef.filterSearchType) ? columnDef.filterSearchType : null; let cellValue = item[queryFieldName]; // when item is a complex object (dot "." notation), we need to filter the value contained in the object tree @@ -371,43 +421,14 @@ export class FilterService { cellValue = getDescendantProperty(item, queryFieldName); } - // if we find searchTerms use them but make a deep copy so that we don't affect original array - // we might have to overwrite the value(s) locally that are returned - // e.g: we don't want to operator within the search value, since it will fail filter condition check trigger afterward - const searchValues: SearchTerm[] = (columnFilter && columnFilter.searchTerms) ? $.extend(true, [], columnFilter.searchTerms) : []; - let fieldSearchValue = (Array.isArray(searchValues) && searchValues.length === 1) ? searchValues[0] : ''; - - let matches = null; - if (fieldType !== FieldType.object) { - fieldSearchValue = '' + fieldSearchValue; // make sure it's a string - matches = fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])([\*]?)$/); // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*) - } - - let operator = columnFilter.operator || ((matches) ? matches[1] : ''); - const searchTerm = (!!matches) ? matches[2] : ''; - const lastValueChar = (!!matches) ? matches[3] : (operator === '*z' ? '*' : ''); - - if (searchValues && searchValues.length > 1) { - fieldSearchValue = searchValues.join(','); - } else if (typeof fieldSearchValue === 'string') { - // escaping the search value - fieldSearchValue = fieldSearchValue.replace(`'`, `''`); // escape single quotes by doubling them - if (operator === '*' || operator === 'a*' || operator === '*z' || lastValueChar === '*') { - operator = (operator === '*' || operator === '*z') ? OperatorType.endsWith : OperatorType.startsWith; - } - } + const operator = columnFilter.operator; + const searchValues = columnFilter.searchTerms || []; // no need to query if search value is empty or if the search value is in fact equal to the operator - if (searchTerm === '' && (!searchValues || (Array.isArray(searchValues) && (searchValues.length === 0 || searchValues.length === 1 && operator === searchValues[0])))) { + if (!searchValues || (Array.isArray(searchValues) && (searchValues.length === 0 || searchValues.length === 1 && operator === searchValues[0]))) { return true; } - // if search value has a regex match we will only keep the value without the operator - // in this case we need to overwrite the returned search values to truncate operator from the string search - if (Array.isArray(matches) && matches.length >= 1 && (Array.isArray(searchValues) && searchValues.length === 1)) { - searchValues[0] = searchTerm; - } - // filter search terms should always be string type (even though we permit the end user to input numbers) // so make sure each term are strings, if user has some default search terms, we will cast them to string if (searchValues && Array.isArray(searchValues) && fieldType !== FieldType.object) { @@ -418,7 +439,8 @@ export class FilterService { } // when using localization (i18n), we should use the formatter output to search as the new cell value - if (columnDef && columnDef.params && columnDef.params.useFormatterOuputToFilter) { + if (columnDef?.params?.useFormatterOuputToFilter === true) { + const dataView = grid.getData() as SlickDataView; const idPropName = this._gridOptions.datasetIdPropertyName || 'id'; const rowIndex = (dataView && typeof dataView.getIdxById === 'function') ? dataView.getIdxById(item[idPropName]) : 0; cellValue = (columnDef && typeof columnDef.formatter === 'function') ? columnDef.formatter(rowIndex || 0, columnIndex, cellValue, columnDef, item, this._grid) : ''; @@ -429,15 +451,14 @@ export class FilterService { cellValue = cellValue.toString(); } - const currentCellValue = cellValue; return { - dataKey, + dataKey: columnDef.dataKey, fieldType, searchTerms: searchValues, - cellValue: currentCellValue, + cellValue, operator: operator as OperatorString, - cellValueLastChar: lastValueChar, - filterSearchType + searchInputLastChar: columnFilter.searchInputLastChar, + filterSearchType: columnDef.filterSearchType, } as FilterConditionOption; } @@ -463,17 +484,33 @@ export class FilterService { delete treeObj[inputArray[i][dataViewIdIdentifier]].__used; } + // loop through all column filters and execute filter condition(s) + for (const columnId of Object.keys(columnFilters)) { + const columnFilter = columnFilters[columnId] as SearchColumnFilter; + const searchValues: SearchTerm[] = columnFilter?.searchTerms ? deepCopy(columnFilter.searchTerms) : []; + + const inputSearchConditions = this.getSearchInputConditions(searchValues, columnFilter); + + const columnDef = columnFilter.columnDef; + const fieldType = columnDef?.filter?.type ?? columnDef?.type ?? FieldType.string; + const parsedSearchTerms = getParsedSearchTermsByFieldType(inputSearchConditions.searchTerms, fieldType); + if (parsedSearchTerms) { + columnFilter.parsedSearchTerms = parsedSearchTerms; + } + } + for (let i = 0; i < inputArray.length; i++) { const item = inputArray[i]; let matchFilter = true; // valid until proven otherwise // loop through all column filters and execute filter condition(s) for (const columnId of Object.keys(columnFilters)) { - const columnFilter = columnFilters[columnId] as ColumnFilter; - const conditionOptionResult = this.getFilterConditionOptionsOrBoolean(item, columnFilter, columnId, this._grid); + const columnFilter = columnFilters[columnId] as SearchColumnFilter; + const conditionOptionResult = this.getFilterConditionOptionsOrBoolean(item, columnFilter, this._grid); if (conditionOptionResult) { - const conditionResult = (typeof conditionOptionResult === 'boolean') ? conditionOptionResult : FilterConditions.executeMappedCondition(conditionOptionResult as FilterConditionOption); + const parsedSearchTerms = columnFilter?.parsedSearchTerms ?? []; + const conditionResult = (typeof conditionOptionResult === 'boolean') ? conditionOptionResult : FilterConditions.executeMappedCondition(conditionOptionResult as FilterConditionOption, parsedSearchTerms); if (conditionResult) { // don't return true since we still need to check other keys in columnFilters continue; @@ -519,7 +556,7 @@ export class FilterService { const columnFilter = this._columnFilters[colId]; const filter = { columnId: colId || '' } as CurrentFilter; - if (columnFilter && columnFilter.searchTerms) { + if (columnFilter?.searchTerms) { filter.searchTerms = columnFilter.searchTerms; } if (columnFilter.operator) { @@ -581,7 +618,7 @@ export class FilterService { } // query backend, except when it's called by a ClearFilters then we won't - if (args && args.shouldTriggerQuery) { + if (args?.shouldTriggerQuery) { // call the service to get a query back if (debounceTypingDelay > 0) { debounce(() => { @@ -614,7 +651,7 @@ export class FilterService { // from each presets, we will find the associated columnDef and apply the preset searchTerms & operator if there is const columnPreset = filters.find((presetFilter: CurrentFilter) => presetFilter.columnId === columnDef.id); - if (columnPreset && columnPreset.searchTerms && Array.isArray(columnPreset.searchTerms)) { + if (columnPreset && Array.isArray(columnPreset?.searchTerms)) { columnDef.filter = columnDef.filter || {}; columnDef.filter.operator = columnPreset.operator || columnDef.filter.operator || ''; columnDef.filter.searchTerms = columnPreset.searchTerms; @@ -811,7 +848,7 @@ export class FilterService { /** * Callback method that is called and executed by the individual Filter (DOM element), - * for example when user type in a word to search (which uses InputFilter), this Filter will execute the callback from an input change event. + * for example when user starts typing chars on a search input (which uses InputFilter), this Filter will execute the callback from an input change event. */ protected callbackSearchEvent(event: SlickEventData | undefined, args: FilterCallbackArg) { if (args) { @@ -824,6 +861,7 @@ export class FilterService { const hasSearchTerms = searchTerms && Array.isArray(searchTerms); const termsCount = hasSearchTerms && searchTerms && searchTerms.length; const oldColumnFilters = { ...this._columnFilters }; + let parsedSearchTerms: SearchTerm[] | undefined; if (columnDef && columnId) { if (!hasSearchTerms || termsCount === 0 || (termsCount === 1 && Array.isArray(searchTerms) && searchTerms[0] === '')) { @@ -831,20 +869,32 @@ export class FilterService { // without doing this, it would leave an incorrect state of the previous column filters when filtering on another column delete this._columnFilters[columnId]; } else { - const colId = '' + columnId as string; - const colFilter: ColumnFilter = { + const colId = `${columnId}`; + const colFilter: Omit = { columnId: colId, columnDef, - searchTerms, + parsedSearchTerms: [], + type: fieldType }; - colFilter.operator = operator || mapOperatorByFieldType(fieldType); - this._columnFilters[colId] = colFilter; + const inputSearchConditions = this.getSearchInputConditions(searchTerms, colFilter); + colFilter.operator = operator || inputSearchConditions.operator || mapOperatorByFieldType(fieldType); + parsedSearchTerms = getParsedSearchTermsByFieldType(inputSearchConditions.searchTerms, fieldType); + if (parsedSearchTerms) { + colFilter.parsedSearchTerms = parsedSearchTerms; + } + + // use searchTerms only coming from the input search result because original terms might include extra operator symbols within their string + // and the input search result would be correctly stripped them from input result and assigned to the appropriate operator + // for example we might have: { searchTerms: ['*doe'] } and that should be reassigned to: { operator: EndsWith, searchTerms: 'doe' } + (colFilter as SearchColumnFilter).searchTerms = inputSearchConditions.searchTerms; + this._columnFilters[colId] = colFilter as SearchColumnFilter; } } // event might have been created as a CustomEvent (e.g. CompoundDateFilter), without being a valid Slick.EventData, // if so we will create a new Slick.EventData and merge it with that CustomEvent to avoid having SlickGrid errors const eventData = (event && typeof event.isPropagationStopped !== 'function') ? $.extend({}, new Slick.EventData(), event) : event; + console.time('searching'); // trigger an event only if Filters changed or if ENTER key was pressed const eventKey = event?.key; @@ -858,6 +908,7 @@ export class FilterService { columnFilters: this._columnFilters, operator: operator || mapOperatorByFieldType(fieldType), searchTerms, + parsedSearchTerms, grid: this._grid }, eventData); } @@ -904,12 +955,17 @@ export class FilterService { } protected updateColumnFilters(searchTerms: SearchTerm[] | undefined, columnDef: any, operator?: OperatorType | OperatorString) { + const fieldType = columnDef.filter?.type ?? columnDef.type ?? FieldType.string; + const parsedSearchTerms = getParsedSearchTermsByFieldType(searchTerms, fieldType); + if (searchTerms && columnDef) { this._columnFilters[columnDef.id] = { columnId: columnDef.id, columnDef, searchTerms, - operator + operator, + parsedSearchTerms, + type: fieldType, }; } } diff --git a/packages/vanilla-bundle/src/components/slick-vanilla-grid-bundle.ts b/packages/vanilla-bundle/src/components/slick-vanilla-grid-bundle.ts index e2bb1d84b..1c67c5cc4 100644 --- a/packages/vanilla-bundle/src/components/slick-vanilla-grid-bundle.ts +++ b/packages/vanilla-bundle/src/components/slick-vanilla-grid-bundle.ts @@ -862,6 +862,7 @@ export class SlickVanillaGridBundle { (this._eventHandler as SlickEventHandler>).subscribe(onRowsOrCountChangedHandler, (_e, args) => { grid.invalidate(); this.handleOnItemCountChanged(args.currentRowCount || 0); + console.timeEnd('searching'); }); // when filtering data with local dataset, we need to update each row else it will not always show correctly in the UI