better compare value types

src/model/BooleansColumn.ts

@@ -6,6 +6,7 @@ import {ICategoricalColorMappingFunction, DEFAULT_COLOR_FUNCTION, restoreColorMa
 import ValueColumn, {dataLoaded} from './ValueColumn';
 import Column, {labelChanged, metaDataChanged, dirty, dirtyHeader, dirtyValues, rendererTypeChanged, groupRendererChanged, summaryRendererChanged, visibilityChanged, widthChanged, dirtyCaches} from './Column';
 import {IEventListener} from '../internal/AEventDispatcher';
+import {chooseUIntByDataLength} from '../provider/sort';
 
 
 export declare type IBooleansColumnDesc = IArrayColumnDesc<boolean>;
@@ -46,6 +47,10 @@ export default class BooleansColumn extends ArrayColumn<boolean> implements ISet
     return v.reduce((a, b) => a + (b ? 1 : 0), 0);
   }
 
+  toCompareValueType() {
+    return chooseUIntByDataLength(this.dataLength);
+  }
+
   getCategories(row: IDataRow) {
     const categories = this.categories;
     return super.getValues(row).map((v, i) => {

src/model/Column.ts

@@ -543,7 +543,7 @@ export default class Column extends AEventDispatcher {
   }
 
   toCompareValueType(): ECompareValueType | ECompareValueType[] {
-    return ECompareValueType.UINT;
+    return ECompareValueType.UINT8;
   }
 
   /**

src/model/DateColumn.ts

@@ -158,6 +158,10 @@ export default class DateColumn extends ValueColumn<Date> implements IDateColumn
     return v.getTime();
   }
 
+  toCompareValueType() {
+    return ECompareValueType.INT32;
+  }
+
   getDateGrouper() {
     return Object.assign({}, this.currentGrouper);
   }
@@ -192,7 +196,7 @@ export default class DateColumn extends ValueColumn<Date> implements IDateColumn
   }
 
   toCompareGroupValueType() {
-    return ECompareValueType.UINT;
+    return ECompareValueType.INT32;
   }
 }
 

src/model/DatesColumn.ts

@@ -9,6 +9,7 @@ import {IDataRow} from './interfaces';
 import {isMissingValue} from './missing';
 import {IEventListener} from '../internal/AEventDispatcher';
 import DateColumn from './DateColumn';
+import {chooseUIntByDataLength} from '../provider/sort';
 
 export enum EDateSort {
   min = 'min',
@@ -152,7 +153,7 @@ export default class DatesColumn extends ArrayColumn<Date | null> implements IDa
   }
 
   toCompareValueType() {
-    return [ECompareValueType.UINT, ECompareValueType.UINT];
+    return [chooseUIntByDataLength(this.dataLength), ECompareValueType.INT32];
   }
 
   isFiltered() {

src/model/GroupColumn.ts

@@ -87,6 +87,6 @@ export default class GroupColumn extends Column {
   }
 
   toCompareGroupValueType() {
-    return this.groupSortMethod === 'count' ? ECompareValueType.UINT : ECompareValueType.STRING;
+    return this.groupSortMethod === 'count' ? ECompareValueType.COUNT : ECompareValueType.STRING;
   }
 }

src/model/ICategoricalColumn.ts

@@ -149,7 +149,7 @@ export function toGroupCompareCategoryValue(rows: ISequence<IDataRow>, col: ICat
   return [mostFrequent.cat.value, mostFrequent.count];
 }
 
-export const COMPARE_GROUP_CATEGORY_VALUE_TYPES = [ECompareValueType.FLOAT, ECompareValueType.UINT];
+export const COMPARE_GROUP_CATEGORY_VALUE_TYPES = [ECompareValueType.FLOAT, ECompareValueType.COUNT];
 
 /** @internal */
 function compareCategory(a: ICategory | null, b: ICategory | null) {

src/model/NestedColumn.ts

@@ -1,6 +1,6 @@
 import {IDataRow} from './interfaces';
 import MultiLevelCompositeColumn from './MultiLevelCompositeColumn';
-import {ICompareValue, ECompareValueType} from './Column';
+import {concat} from '../internal';
 
 /**
  * factory for creating a description creating a mean column
@@ -18,13 +18,11 @@ export function createNestedDesc(label: string = 'Nested') {
 export default class NestedColumn extends MultiLevelCompositeColumn {
 
   toCompareValue(row: IDataRow) {
-    const r: ICompareValue[] = [];
-    return r.concat(...this.children.map((d) => d.toCompareValue(row)));
+    return concat(this.children.map((d) => d.toCompareValue(row)));
   }
 
   toCompareValueType() {
-    const r: ECompareValueType[] = [];
-    return r.concat(...this.children.map((d) => d.toCompareValueType()));
+    return concat(this.children.map((d) => d.toCompareValueType()));
   }
 
   getLabel(row: IDataRow) {

src/model/Ranking.ts

@@ -9,7 +9,7 @@ import NumberColumn, {filterChanged} from './NumberColumn';
 import CompositeColumn from './CompositeColumn';
 import {IEventListener} from '../internal/AEventDispatcher';
 import {IRankingDump} from '../provider/interfaces';
-import {chooseByLength} from '../provider/sort';
+import {createIndexArray} from '../provider/sort';
 import {ISequence} from '../internal/interable';
 
 export interface ISortCriteria {
@@ -763,7 +763,7 @@ function toOrder(groups: IOrderedGroup[]) {
       return groups[0].order;
     default:
       const total = groups.reduce((a, b) => a + b.order.length, 0);
-      const r = chooseByLength(total);
+      const r = createIndexArray(total);
       let shift = 0;
       for (const g of groups) {
         r.set(g.order, shift);

src/model/SetColumn.ts

@@ -7,6 +7,7 @@ import {IDataRow} from './interfaces';
 import ValueColumn, {IValueColumnDesc, dataLoaded} from './ValueColumn';
 import {IEventListener} from '../internal/AEventDispatcher';
 import {ICategoricalColorMappingFunction, DEFAULT_COLOR_FUNCTION, restoreColorMapping} from './CategoricalColorMappingFunction';
+import {chooseUIntByDataLength} from '../provider/sort';
 
 export interface ISetDesc extends ICategoricalDesc {
   separator?: string;
@@ -224,6 +225,6 @@ export default class SetColumn extends ValueColumn<string[]> implements IArrayCo
   }
 
   toCompareValueType() {
-    return [ECompareValueType.UINT].concat(this.categories.map(() => ECompareValueType.BINARY));
+    return [chooseUIntByDataLength(this.categories.length)].concat(this.categories.map(() => ECompareValueType.BINARY));
   }
 }

src/provider/LocalDataProvider.ts

@@ -4,7 +4,7 @@ import Ranking, {EDirtyReason} from '../model/Ranking';
 import ACommonDataProvider from './ACommonDataProvider';
 import ADataProvider from './ADataProvider';
 import {IDataProviderOptions} from './interfaces';
-import {chooseByLength, CompareLookup, ISortWorker, local, sortComplex, WorkerSortWorker} from './sort';
+import {createIndexArray, CompareLookup, ISortWorker, local, sortComplex, WorkerSortWorker} from './sort';
 import {DirectRenderTasks, IRenderTaskExectutor, ScheduleRenderTasks} from './tasks';
 
 
@@ -195,7 +195,7 @@ export default class LocalDataProvider extends ACommonDataProvider {
   private noSorting() {
     // initial no sorting required just index mapping
     const l = this._data.length;
-    const order = chooseByLength(l);
+    const order = createIndexArray(l);
     const index2pos = order.slice();
     for (let i = 0; i < l; ++i) {
       order[i] = i;
@@ -263,7 +263,7 @@ export default class LocalDataProvider extends ACommonDataProvider {
   }
 
   private index2pos(groups: IOrderedGroup[], maxDataIndex: number) {
-    const index2pos = chooseByLength(maxDataIndex + 1);
+    const index2pos = createIndexArray(maxDataIndex + 1);
     let offset = 1;
     for (const g of groups) {
       // tslint:disable-next-line

src/provider/sort.ts

@@ -5,17 +5,38 @@ import {IPoorManWorkerScope, toFunctionBody, createWorkerCodeBlob} from './worke
 import {ISequence} from '../internal/interable';
 
 export enum ECompareValueType {
-  FLOAT = 0,
-  BINARY = 1,
-  UINT = 2,
-  STRING = 3,
-  FLOAT_ASC = 4,
+  BINARY,
+  COUNT, // count max to the number of rows
+  UINT8,
+  UINT16,
+  UINT32,
+  INT8,
+  INT16,
+  INT32,
+  FLOAT,
+  FLOAT_ASC,
+  DOUBLE,
+  DOUBLE_ASC,
+  STRING
+}
+
+export function chooseUIntByDataLength(dataLength?: number | null) {
+  if (dataLength == null || typeof dataLength !== 'number' && !isNaN(dataLength)) {
+    return ECompareValueType.UINT32; // worst case
+  }
+  if (length <= 255) {
+    return ECompareValueType.UINT8;
+  }
+  if (length <= 65535) {
+    return ECompareValueType.UINT16;
+  }
+  return ECompareValueType.UINT32;
 }
 
 /**
  * @internal
  */
-export function chooseByLength(length: number) {
+export function createIndexArray(length: number) {
   if (length <= 255) {
     return new Uint8Array(length);
   }
@@ -25,7 +46,7 @@ export function chooseByLength(length: number) {
   return new Uint32Array(length);
 }
 
-function fromByLength(arr: ISequence<number>) {
+function toIndexArray(arr: ISequence<number>): UIntTypedArray {
   const l = arr.length;
   if (l <= 255) {
     return Uint8Array.from(arr);
@@ -37,9 +58,13 @@ function fromByLength(arr: ISequence<number>) {
 }
 
 
-const missingBinary = FIRST_IS_MISSING > 0 ? 255 : 0;
+const missingUInt8 = FIRST_IS_MISSING > 0 ? 255 : 0;
+const missingBinary = missingUInt8;
 const missingUInt16 = FIRST_IS_MISSING > 0 ? 65535 : 0; // max or 0
 const missingUInt32 = FIRST_IS_MISSING > 0 ? 4294967295 : 0; // max or 0
+const missingInt8 = FIRST_IS_MISSING > 0 ? 127 : -128; // max or min
+const missingInt16 = FIRST_IS_MISSING > 0 ? 32767 : -32768; // max or min
+const missingInt32 = FIRST_IS_MISSING > 0 ? 2147483647 : -2147483648; // max or min
 const missingFloat = FIRST_IS_NAN > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
 const missingFloatAsc = FIRST_IS_MISSING > 0 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
 const missingString = FIRST_IS_MISSING > 0 ? '\uffff' : '\u0000'; // first or last character
@@ -55,32 +80,46 @@ function chooseMissingByLength(length: number) {
   return missingUInt32;
 }
 
-declare type ILookUpArray = UIntTypedArray | string[] | Float32Array;
+declare type ILookUpArray = Uint8Array | Uint16Array | Uint32Array | Int8Array | Int16Array | Int32Array | string[] | Float32Array | Float64Array;
 
 
 function toCompareLookUp(rawLength: number, type: ECompareValueType): ILookUpArray {
   switch (type) {
+    case ECompareValueType.COUNT:
+      return createIndexArray(rawLength + 1);
     case ECompareValueType.BINARY:
+    case ECompareValueType.UINT8:
       return new Uint8Array(rawLength);
-    case ECompareValueType.UINT:
-      return chooseByLength(rawLength + 1);
+    case ECompareValueType.UINT16:
+      return new Uint16Array(rawLength);
+    case ECompareValueType.UINT32:
+      return new Uint32Array(rawLength);
+    case ECompareValueType.INT8:
+      return new Int8Array(rawLength);
+    case ECompareValueType.INT16:
+      return new Int16Array(rawLength);
+    case ECompareValueType.INT32:
+      return new Int32Array(rawLength);
     case ECompareValueType.STRING:
       return <string[]>[];
     case ECompareValueType.FLOAT_ASC:
     case ECompareValueType.FLOAT:
       return new Float32Array(rawLength);
+    case ECompareValueType.DOUBLE_ASC:
+    case ECompareValueType.DOUBLE:
+      return new Float64Array(rawLength);
   }
 }
 
 
 export class CompareLookup {
   private readonly lookups: ILookUpArray[];
-  private readonly missingUInt: number; // since length dependent
+  private readonly missingCount: number; // since length dependent
 
   constructor(rawLength: number, private readonly comparators: {asc: boolean, v: ECompareValueType}[]) {
     this.lookups = comparators.map((d) => toCompareLookUp(rawLength, d.v));
 
-    this.missingUInt = chooseMissingByLength(rawLength + 1); // + 1 for the value shift to have 0 as start
+    this.missingCount = chooseMissingByLength(rawLength + 1); // + 1 for the value shift to have 0 as start
   }
 
   get sortOrders() {
@@ -102,18 +141,38 @@ export class CompareLookup {
         case ECompareValueType.BINARY: // just 0 or 1 -> convert to 0=-Ininity 1 2 255=+Infinity
           lookup[index] = v == null || isNaN(<number>v) ? missingBinary : (<number>v) + 1;
           break;
-        case ECompareValueType.UINT: // uint32
-          lookup[index] = v == null || isNaN(<number>v) ? this.missingUInt : (<number>v) + 1;
+        case ECompareValueType.COUNT: // uint32
+          lookup[index] = v == null || isNaN(<number>v) ? this.missingCount : (<number>v) + 1;
           break;
-        case ECompareValueType.FLOAT_ASC:
-          lookup[index] = v == null || isNaN(<number>v) ? missingFloatAsc : v;
+        case ECompareValueType.UINT8: // shift by one to have 0 for -Inf
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt8 : (<number>v) + 1;
+          break;
+        case ECompareValueType.UINT16: // shift by one to have 0 for -Inf
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt16 : (<number>v) + 1;
+          break;
+        case ECompareValueType.UINT32: // shift by one to have 0 for -Inf
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt32 : (<number>v) + 1;
+          break;
+        case ECompareValueType.INT8:
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt8 : (<number>v);
+          break;
+        case ECompareValueType.INT16:
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt16 : (<number>v);
+          break;
+        case ECompareValueType.INT32:
+          lookup[index] = v == null || isNaN(<number>v) ? missingInt32 : (<number>v);
           break;
         case ECompareValueType.STRING:
           lookup[index] = v == null || v === '' ? missingString : v;
           break;
         case ECompareValueType.FLOAT:
+        case ECompareValueType.DOUBLE:
           lookup[index] = v == null || isNaN(<number>v) ? missingFloat : v;
           break;
+        case ECompareValueType.FLOAT_ASC:
+        case ECompareValueType.DOUBLE_ASC:
+          lookup[index] = v == null || isNaN(<number>v) ? missingFloatAsc : v;
+          break;
       }
     }
   }
@@ -175,7 +234,7 @@ export function sortComplex(indices: UIntTypedArray | number[], comparators: {as
 
 
 function sort(indices: number[], _singleCall: boolean, lookups?: CompareLookup) {
-  const order = fromByLength(indices);
+  const order = toIndexArray(indices);
   if (lookups) {
     sortComplex(order, lookups.sortOrders);
   }
@@ -234,7 +293,7 @@ export class WorkerSortWorker implements ISortWorker {
   private cleanUpWorkerTimer: number = -1;
 
   private readonly workerBlob = createWorkerCodeBlob([
-    chooseByLength.toString(),
+    createIndexArray.toString(),
     asc.toString(),
     desc.toString(),
     sortComplex.toString(),
@@ -298,7 +357,7 @@ export class WorkerSortWorker implements ISortWorker {
         resolve(r.order);
       };
 
-      const indexArray = fromByLength(indices);
+      const indexArray = toIndexArray(indices);
 
       const toTransfer = [indexArray.buffer];