feat(perf): huge filtering speed improvements

examples/webpack-demo-vanilla-bundle/src/examples/example03.ts

@@ -181,7 +181,7 @@ export class Example3 {
         // formatter: Formatters.dateIso,
         type: FieldType.date, outputType: FieldType.dateIso,
         formatter: Formatters.dateIso,
-        filterable: true, filter: { model: Filters.compoundDate },
+        filterable: true, filter: { model: Filters.dateRange },
         grouping: {
           getter: 'finish',
           formatter: (g) => `Finish: ${g.value} <span style="color:green">(${g.count} items)</span>`,

packages/common/src/filter-conditions/__tests__/stringFilterCondition.spec.ts

@@ -82,8 +82,8 @@ describe('stringFilterCondition method', () => {
     expect(output).toBe(true);
   });
 
-  it('should return True when input value provided starts with same substring and the operator is empty string & option "cellValueLastChar" is asterisk (*)', () => {
-    const options = { dataKey: '', operator: '', cellValueLastChar: '*', cellValue: 'abbostford', fieldType: FieldType.string, searchTerms: ['abb'] } as FilterConditionOption;
+  it('should return True when input value provided starts with same substring and the operator is empty string & option "searchInputLastChar" is asterisk (*)', () => {
+    const options = { dataKey: '', operator: '', searchInputLastChar: '*', cellValue: 'abbostford', fieldType: FieldType.string, searchTerms: ['abb'] } as FilterConditionOption;
     const output = stringFilterCondition(options);
     expect(output).toBe(true);
   });

packages/common/src/filter-conditions/executeMappedCondition.ts

@@ -4,25 +4,152 @@ import { numberFilterCondition } from './numberFilterCondition';
 import { objectFilterCondition } from './objectFilterCondition';
 import { stringFilterCondition } from './stringFilterCondition';
 
-import { FieldType, OperatorType } from '../enums/index';
+import { FieldType, OperatorType, SearchTerm } from '../enums/index';
 import { FilterCondition, FilterConditionOption } from '../interfaces/index';
 import { mapMomentDateFormatWithFieldType } from './../services/utilities';
 import { testFilterCondition } from './filterUtilities';
 import * as moment_ from 'moment-mini';
 
 const moment = moment_['default'] || moment_; // patch to fix rollup "moment has no default export" issue, document here https://github.com/rollup/rollup/issues/670
 
-export const executeMappedCondition: FilterCondition = (options: FilterConditionOption) => {
+export type GeneralFieldType = 'boolean' | 'date' | 'number' | 'object' | 'text';
+
+export const executeMappedCondition: FilterCondition = (options: FilterConditionOption, parsedSearchTerms: SearchTerm[]) => {
   // when using a multi-select ('IN' operator) we will not use the field type but instead go directly with a collection search
   const operator = options && options.operator && options.operator.toUpperCase();
   if (operator === 'IN' || operator === 'NIN' || operator === 'IN_CONTAINS' || operator === 'NIN_CONTAINS') {
-    return collectionSearchFilterCondition(options);
+    return collectionSearchFilterCondition(options, parsedSearchTerms);
   }
 
+  const generalType = getGeneralTypeByFieldType(options.fieldType);
+
+
   // execute the mapped type, or default to String condition check
-  switch (options.fieldType) {
+  switch (generalType) {
+    case 'boolean':
+      return booleanFilterCondition(options, parsedSearchTerms);
+    case 'date':
+      return executeAssociatedDateCondition(options, ...parsedSearchTerms);
+    case 'number':
+      return numberFilterCondition(options, ...parsedSearchTerms as number[]);
+    case 'object':
+      return objectFilterCondition(options, parsedSearchTerms);
+    case 'text':
+    default:
+      return stringFilterCondition(options, parsedSearchTerms);
+  }
+};
+
+/**
+ * Execute Date filter condition and use correct date format depending on it's field type (or filterSearchType when that is provided)
+ * @param options
+ */
+function executeAssociatedDateCondition(options: FilterConditionOption, ...parsedSearchDates: any[]): boolean {
+  const filterSearchType = options && (options.filterSearchType || options.fieldType) || FieldType.dateIso;
+  const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType);
+  const [searchDate1, searchDate2] = parsedSearchDates;
+
+  // cell value in moment format
+  const dateCell = moment(options.cellValue, FORMAT, true);
+
+  // return when cell value is not a valid date
+  if ((!searchDate1 && !searchDate2) || !dateCell.isValid()) {
+    return false;
+  }
+
+  // when comparing with Dates only (without time), we need to disregard the time portion, we can do so by setting our time to start at midnight
+  // ref, see https://stackoverflow.com/a/19699447/1212166
+  const dateCellTimestamp = FORMAT.toLowerCase().includes('h') ? dateCell.valueOf() : dateCell.clone().startOf('day').valueOf();
+
+  // having 2 search dates, we assume that it's a date range filtering and we'll compare against both dates
+  if (searchDate1 && searchDate2) {
+    const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive;
+    const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), dateCellTimestamp, searchDate1.valueOf());
+    const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), dateCellTimestamp, searchDate2.valueOf());
+    return (resultCondition1 && resultCondition2);
+  }
+
+  // comparing against a single search date
+  const dateSearchTimestamp1 = FORMAT.toLowerCase().includes('h') ? searchDate1.valueOf() : searchDate1.clone().startOf('day').valueOf();
+  return testFilterCondition(options.operator || '==', dateCellTimestamp, dateSearchTimestamp1);
+}
+
+export function getParsedSearchTermsByFieldType(inputSearchTerms: SearchTerm[] | undefined, inputFilterSearchType: typeof FieldType[keyof typeof FieldType]): SearchTerm[] | undefined {
+  const generalType = getGeneralTypeByFieldType(inputFilterSearchType);
+
+  switch (generalType) {
+    case 'date':
+      return getParsedSearchDates(inputSearchTerms, inputFilterSearchType);
+    case 'number':
+      return getParsedSearchNumbers(inputSearchTerms);
+  }
+  return undefined;
+}
+
+function getParsedSearchDates(inputSearchTerms: SearchTerm[] | undefined, inputFilterSearchType: typeof FieldType[keyof typeof FieldType]): SearchTerm[] | undefined {
+  const searchTerms = Array.isArray(inputSearchTerms) && inputSearchTerms || [];
+  const filterSearchType = inputFilterSearchType || FieldType.dateIso;
+  const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType);
+
+  const parsedSearchValues: any[] = [];
+
+  if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) {
+    const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..');
+    const searchValue1 = (Array.isArray(searchValues) && searchValues[0] || '') as Date | string;
+    const searchValue2 = (Array.isArray(searchValues) && searchValues[1] || '') as Date | string;
+    const searchDate1 = moment(searchValue1, FORMAT, true);
+    const searchDate2 = moment(searchValue2, FORMAT, true);
+
+    // return if any of the 2 values are invalid dates
+    if (!searchDate1.isValid() || !searchDate2.isValid()) {
+      return undefined;
+    }
+    parsedSearchValues.push(searchDate1, searchDate2);
+  } else {
+    // return if the search term is an invalid date
+    const searchDate1 = moment(searchTerms[0] as Date | string, FORMAT, true);
+    if (!searchDate1.isValid()) {
+      return undefined;
+    }
+    parsedSearchValues.push(searchDate1);
+  }
+  return parsedSearchValues;
+}
+
+function getParsedSearchNumbers(inputSearchTerms: SearchTerm[] | undefined): number[] | undefined {
+  const searchTerms = Array.isArray(inputSearchTerms) && inputSearchTerms || [];
+  const parsedSearchValues: number[] = [];
+  let searchValue1;
+  let searchValue2;
+
+  if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) {
+    const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..');
+    searchValue1 = parseFloat(Array.isArray(searchValues) ? (searchValues[0] + '') : '');
+    searchValue2 = parseFloat(Array.isArray(searchValues) ? (searchValues[1] + '') : '');
+  } else {
+    searchValue1 = parseFloat(searchTerms[0] + '');
+  }
+
+  if (searchValue1 !== undefined && searchValue2 !== undefined) {
+    parsedSearchValues.push(searchValue1, searchValue2);
+  } else if (searchValue1 !== undefined) {
+    parsedSearchValues.push(searchValue1);
+  } else {
+    return undefined;
+  }
+  return parsedSearchValues;
+}
+
+/**
+ * From a more specific field type, let's return a simple and more general type (boolean, date, number, object, text)
+ * @param fieldType - specific field type
+ * @returns generalType - general field type
+ */
+function getGeneralTypeByFieldType(fieldType: typeof FieldType[keyof typeof FieldType]): GeneralFieldType {
+  // return general field type
+  switch (fieldType) {
     case FieldType.boolean:
-      return booleanFilterCondition(options);
+      return 'boolean';
     case FieldType.date:
     case FieldType.dateIso:
     case FieldType.dateUtc:
@@ -49,77 +176,18 @@ export const executeMappedCondition: FilterCondition = (options: FilterCondition
     case FieldType.dateTimeUsShort:
     case FieldType.dateTimeUsShortAmPm:
     case FieldType.dateTimeUsShortAM_PM:
-      return executeAssociatedDateCondition(options);
+      return 'date';
     case FieldType.integer:
     case FieldType.float:
     case FieldType.number:
-      return numberFilterCondition(options);
+      return 'number';
     case FieldType.object:
-      return objectFilterCondition(options);
+      return 'object';
     case FieldType.string:
     case FieldType.text:
     case FieldType.password:
     case FieldType.readonly:
     default:
-      return stringFilterCondition(options);
-  }
-};
-
-/**
- * Execute Date filter condition and use correct date format depending on it's field type (or filterSearchType when that is provided)
- * @param options
- */
-function executeAssociatedDateCondition(options: FilterConditionOption): boolean {
-  const filterSearchType = options && (options.filterSearchType || options.fieldType) || FieldType.dateIso;
-  const FORMAT = mapMomentDateFormatWithFieldType(filterSearchType);
-  const searchTerms = Array.isArray(options.searchTerms) && options.searchTerms || [];
-
-  let isRangeSearch = false;
-  let dateSearch1: any;
-  let dateSearch2: any;
-
-  // return when cell value is not a valid date
-  if (searchTerms.length === 0 || searchTerms[0] === '' || searchTerms[0] === null || !moment(options.cellValue, FORMAT, true).isValid()) {
-    return false;
+      return 'text';
   }
-
-  // cell value in moment format
-  const dateCell = moment(options.cellValue, FORMAT, true);
-
-  if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) {
-    isRangeSearch = true;
-    const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..');
-    const searchValue1 = (Array.isArray(searchValues) && searchValues[0] || '') as Date | string;
-    const searchValue2 = (Array.isArray(searchValues) && searchValues[1] || '') as Date | string;
-    const searchTerm1 = moment(searchValue1, FORMAT, true);
-    const searchTerm2 = moment(searchValue2, FORMAT, true);
-
-    // return if any of the 2 values are invalid dates
-    if (!moment(searchTerm1, FORMAT, true).isValid() || !moment(searchTerm2, FORMAT, true).isValid()) {
-      return false;
-    }
-    dateSearch1 = moment(searchTerm1, FORMAT, true);
-    dateSearch2 = moment(searchTerm2, FORMAT, true);
-  } else {
-    // return if the search term is an invalid date
-    if (!moment(searchTerms[0] as Date | string, FORMAT, true).isValid()) {
-      return false;
-    }
-    dateSearch1 = moment(searchTerms[0] as Date | string, FORMAT, true);
-  }
-
-  // when comparing with Dates only (without time), we need to disregard the time portion, we can do so by setting our time to start at midnight
-  // ref, see https://stackoverflow.com/a/19699447/1212166
-  const dateCellTimestamp = FORMAT.toLowerCase().includes('h') ? parseInt(dateCell.format('X'), 10) : parseInt(dateCell.clone().startOf('day').format('X'), 10);
-
-  // run the filter condition with date in Unix Timestamp format
-  if (isRangeSearch) {
-    const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive;
-    const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), dateCellTimestamp, parseInt(dateSearch1.format('X'), 10));
-    const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), dateCellTimestamp, parseInt(dateSearch2.format('X'), 10));
-    return (resultCondition1 && resultCondition2);
-  }
-
-  const dateSearchTimestamp1 = FORMAT.toLowerCase().includes('h') ? parseInt(dateSearch1.format('X'), 10) : parseInt(dateSearch1.clone().startOf('day').format('X'), 10);
-  return testFilterCondition(options.operator || '==', dateCellTimestamp, dateSearchTimestamp1);
-}
+}

packages/common/src/filter-conditions/numberFilterCondition.ts

@@ -2,29 +2,16 @@ import { OperatorType } from '../enums/index';
 import { FilterCondition, FilterConditionOption } from '../interfaces/index';
 import { testFilterCondition } from './filterUtilities';
 
-export const numberFilterCondition: FilterCondition = (options: FilterConditionOption) => {
+export const numberFilterCondition: FilterCondition = (options: FilterConditionOption, ...parsedSearchValues: number[]) => {
   const cellValue = parseFloat(options.cellValue);
-  const searchTerms = Array.isArray(options.searchTerms) && options.searchTerms || [0];
-
-  let isRangeSearch = false;
-  let searchValue1;
-  let searchValue2;
-
-  if (searchTerms.length === 2 || (typeof searchTerms[0] === 'string' && (searchTerms[0] as string).indexOf('..') > 0)) {
-    isRangeSearch = true;
-    const searchValues = (searchTerms.length === 2) ? searchTerms : (searchTerms[0] as string).split('..');
-    searchValue1 = parseFloat(Array.isArray(searchValues) ? (searchValues[0] + '') : '');
-    searchValue2 = parseFloat(Array.isArray(searchValues) ? (searchValues[1] + '') : '');
-  } else {
-    searchValue1 = parseFloat(searchTerms[0] + '');
-  }
+  const [searchValue1, searchValue2] = parsedSearchValues;
 
   if (!searchValue1 && !options.operator) {
     return true;
   }
 
-  if (isRangeSearch) {
-    const isInclusive = options.operator && options.operator === OperatorType.rangeInclusive;
+  if (searchValue1 !== undefined && searchValue2 !== undefined) {
+    const isInclusive = options?.operator === OperatorType.rangeInclusive;
     const resultCondition1 = testFilterCondition((isInclusive ? '>=' : '>'), cellValue, searchValue1);
     const resultCondition2 = testFilterCondition((isInclusive ? '<=' : '<'), cellValue, searchValue2);
     return (resultCondition1 && resultCondition2);

packages/common/src/filter-conditions/stringFilterCondition.ts

@@ -15,7 +15,7 @@ export const stringFilterCondition: FilterCondition = (options: FilterConditionO
 
   if (options.operator === '*' || options.operator === OperatorType.endsWith) {
     return cellValue.endsWith(searchTerm);
-  } else if ((options.operator === '' && options.cellValueLastChar === '*') || options.operator === OperatorType.startsWith) {
+  } else if ((options.operator === '' && options.searchInputLastChar === '*') || options.operator === OperatorType.startsWith) {
     return cellValue.startsWith(searchTerm);
   } else if (options.operator === '' || options.operator === OperatorType.contains) {
     return (cellValue.indexOf(searchTerm) > -1);

packages/common/src/filters/filters.index.ts

@@ -40,7 +40,7 @@ export const Filters = {
   /** Range Date Filter (uses the Flactpickr Date picker with range option) */
   dateRange: DateRangeFilter,
 
-  /** Alias to inputText, input type text filter */
+  /** Alias to inputText, input type text filter (this is the default filter when no type is provided) */
   input: InputFilter,
 
   /**

packages/common/src/interfaces/columnFilters.interface.ts

@@ -1,5 +1,5 @@
-import { ColumnFilter } from './columnFilter.interface';
+import { SearchColumnFilter } from './searchColumnFilter.interface';
 
 export interface ColumnFilters {
-  [key: string]: ColumnFilter;
+  [key: string]: SearchColumnFilter;
 }

packages/common/src/interfaces/filterCondition.interface.ts

@@ -1,4 +1,5 @@
+import { SearchTerm } from '../enums/searchTerm.type';
 import { FilterConditionOption } from './filterConditionOption.interface';
 
 
-export type FilterCondition = (options: FilterConditionOption) => boolean;
+export type FilterCondition = (options: FilterConditionOption, parsedSearchTerms?: SearchTerm | SearchTerm[]) => boolean;

packages/common/src/interfaces/filterConditionOption.interface.ts

@@ -1,11 +1,30 @@
 import { FieldType, OperatorString, SearchTerm } from '../enums/index';
 
 export interface FilterConditionOption {
+  /** optional object data key */
   dataKey?: string;
+
+  /** filter operator */
   operator: OperatorString;
+
+  /** cell value */
   cellValue: any;
-  cellValueLastChar?: string;
+
+  /** last character of the cell value, which is helpful to know if we are dealing with "*" that would be mean startsWith */
+  searchInputLastChar?: string;
+
+  /** column field type */
   fieldType: typeof FieldType[keyof typeof FieldType];
+
+  /** filter search field type */
   filterSearchType?: typeof FieldType[keyof typeof FieldType];
+
+  /**
+   * Parsed Search Terms is similar to SearchTerms but is already parsed in the correct format,
+   * for example on a date field the searchTerms might be in string format but their respective parsedSearchTerms will be of type Date
+   */
+  parsedSearchTerms?: SearchTerm[] | undefined;
+
+  /** Search Terms provided by the user */
   searchTerms?: SearchTerm[] | undefined;
 }

packages/common/src/interfaces/index.ts

@@ -114,6 +114,7 @@ export * from './rowDetailView.interface';
 export * from './rowDetailViewOption.interface';
 export * from './rowMoveManager.interface';
 export * from './rowMoveManagerOption.interface';
+export * from './searchColumnFilter.interface';
 export * from './selectOption.interface';
 export * from './servicePagination.interface';
 export * from './singleColumnSort.interface';