Pu/vectorization (#673)

* parsing of matrices

* size prediction

* evaluation, parsing, unparsing

* tests

* doc

* linter

* compiles

* linter

* small refactor

* kind of works

* cleanup

* linter

* extra config options

* tests

* config

* Update CHANGELOG.md

* docs

CHANGELOG.md

@@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added ARRAY_CONSTRAIN function. (#661)
 - Added casting to scalars from non-range arrays. (#663)
 - Added support for range interpolation. (#665)
+- Added parsing of arrays in formulas (together with respective config options for separators). (#671)
+- Added support for vectorization of scalar functions. (#673)
 - Added support for time in JS `Date()` objects on the input. (#648)
 - Added validation of API argument types for simple types. (#654)
 - Added named expression handling to engine factories. (#680)

docs/guide/built-in-functions.md

@@ -216,7 +216,7 @@ Total number of functions: **{{ $page.functionsCount }}**
 | IFNA | Returns the value if the cell does not contains the #N/A (value not available) error value, or the alternative value if it does. | IFNA(Value; Alternate_value) |
 | IFERROR | Returns the value if the cell does not contains an error value, or the alternative value if it does. | IFERROR(Value; Alternate_value) |
 | NOT | Complements (inverts) a logical value. | NOT(Logicalvalue) |
-| SWITCH | For each value on the input array, tests its value against cases and returns corresponding value. | SWITCH(Array; Case1, Value1[, Case2, Value2[..., Case_n, Value_n[, Default]]]) |
+| SWITCH | Evaluates a list of arguments, consisting of an expression followed by a value. | SWITCH(Expression1, Value1[, Expression2, Value2[..., Expression_n, Value_n]]) |
 | OR | Returns TRUE if at least one argument is TRUE. | OR(Logicalvalue1; Logicalvalue2 ...Logicalvalue30) |
 | TRUE | The logical value is set to TRUE. | TRUE() |
 | XOR | Returns true if an odd number of arguments evaluates to TRUE. | XOR(Logicalvalue1; Logicalvalue2 ...Logicalvalue30) |

docs/guide/custom-functions.md

@@ -57,6 +57,14 @@ class CountHF extends FunctionPlugin {
   };
 }
 ```
+
+Similarly, there are other useful properties. 
+`isDependentOnSheetStructureChange` marks functions that need to be recalculated with 
+each change of the shape of the engine sheets. 
+`doesNotNeedArgumentsToBeComputed` marks functions that treat references or ranges in their arguments
+as arguments that do not create dependency. Other arguments are properly evaluated.
+`arrayFunction` denotes functions that enable array arithmetic in its arguments and nested expressions.
+`vectorizationForbidden` when set prevents function from ever being vectorized (however, it is up to implementation of a function to properly handle vectorization).
 ## Aliases
 
 Aliases are available since the <Badge text="v0.4.0"  vertical="middle"/> version.

src/MatrixSize.ts

@@ -4,10 +4,13 @@
  */
 
 import {AbsoluteCellRange} from './AbsoluteCellRange'
-import {SimpleCellAddress} from './Cell'
+import {CellError, ErrorType, SimpleCellAddress} from './Cell'
 import {Config} from './Config'
+import {ErrorMessage} from './error-message'
 import {FunctionRegistry} from './interpreter/FunctionRegistry'
 import {InterpreterState} from './interpreter/InterpreterState'
+import {ArgumentTypes} from './interpreter/plugin/FunctionPlugin'
+import {SimpleRangeValue} from './interpreter/SimpleRangeValue'
 import {Ast, AstNodeType} from './parser'
 
 export class MatrixSize {
@@ -169,7 +172,31 @@ export class MatrixSizePredictor {
             return new MatrixSize(width, height)
           }
           default: {
-            return new MatrixSize(1, 1)
+            if(metadata === undefined || metadata.expandRanges || !state.arraysFlag || metadata.vectorizationForbidden || metadata.parameters === undefined) {
+              return new MatrixSize(1, 1)
+            }
+            const argumentDefinitions = [...metadata.parameters]
+            if (metadata.repeatLastArgs === undefined && argumentDefinitions.length < subChecks.length) {
+              return MatrixSize.error()
+            }
+            if (metadata.repeatLastArgs !== undefined && argumentDefinitions.length < subChecks.length &&
+              (subChecks.length - argumentDefinitions.length) % metadata.repeatLastArgs !== 0) {
+              return MatrixSize.error()
+            }
+
+            while(argumentDefinitions.length < subChecks.length) {
+              argumentDefinitions.push(...argumentDefinitions.slice(argumentDefinitions.length-metadata.repeatLastArgs!))
+            }
+
+            let maxWidth = 1
+            let maxHeight = 1
+            for(let i=0;i<subChecks.length;i++) {
+              if(argumentDefinitions[i].argumentType !== ArgumentTypes.RANGE && argumentDefinitions[i].argumentType !== ArgumentTypes.ANY) {
+                maxHeight = Math.max(maxHeight, subChecks[i].height)
+                maxWidth = Math.max(maxWidth, subChecks[i].width)
+              }
+            }
+            return new MatrixSize(maxWidth, maxHeight)
           }
         }
       }

src/interpreter/plugin/AbsPlugin.ts

@@ -5,7 +5,7 @@
 
 import {ProcedureAst} from '../../parser'
 import {InterpreterState} from '../InterpreterState'
-import {InternalScalarValue} from '../InterpreterValue'
+import {InternalScalarValue, InterpreterValue} from '../InterpreterValue'
 import {ArgumentTypes, FunctionPlugin, FunctionPluginTypecheck} from './FunctionPlugin'
 
 export class AbsPlugin extends FunctionPlugin implements FunctionPluginTypecheck<AbsPlugin>{
@@ -18,7 +18,7 @@ export class AbsPlugin extends FunctionPlugin implements FunctionPluginTypecheck
     },
   }
 
-  public abs(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public abs(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('ABS'), Math.abs)
   }
 }

src/interpreter/plugin/ArrayPlugin.ts

@@ -28,6 +28,7 @@ export class ArrayPlugin extends FunctionPlugin implements FunctionPluginTypeche
         {argumentType: ArgumentTypes.INTEGER, minValue: 1},
         {argumentType: ArgumentTypes.INTEGER, minValue: 1},
       ],
+      vectorizationForbidden: true,
     },
     'FILTER': {
       method: 'filter',
@@ -40,12 +41,12 @@ export class ArrayPlugin extends FunctionPlugin implements FunctionPluginTypeche
     }
   }
 
-  public arrayformula(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public arrayformula(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('ARRAYFORMULA'), (value) => value)
   }
 
   public arrayconstrain(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
-    return this.runMatrixFunction(ast.args, state, this.metadata('ARRAY_CONSTRAIN'), (range: SimpleRangeValue, numRows: number, numCols: number) => {
+    return this.runFunction(ast.args, state, this.metadata('ARRAY_CONSTRAIN'), (range: SimpleRangeValue, numRows: number, numCols: number) => {
       numRows = Math.min(numRows, range.height())
       numCols = Math.min(numCols, range.width())
       const data: InternalScalarValue[][] = range.data
@@ -58,7 +59,7 @@ export class ArrayPlugin extends FunctionPlugin implements FunctionPluginTypeche
   }
 
   public filter(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
-    return this.runMatrixFunction(ast.args, state, this.metadata('FILTER'), (rangeVals: SimpleRangeValue, ...rangeFilters: SimpleRangeValue[]) => {
+    return this.runFunction(ast.args, state, this.metadata('FILTER'), (rangeVals: SimpleRangeValue, ...rangeFilters: SimpleRangeValue[]) => {
       for(const filter of rangeFilters) {
         if (rangeVals.width() !== filter.width() || rangeVals.height() !== filter.height()) {
           return new CellError(ErrorType.NA, ErrorMessage.EqualLength)

src/interpreter/plugin/BitShiftPlugin.ts

@@ -7,7 +7,7 @@ import {CellError, ErrorType} from '../../Cell'
 import {ErrorMessage} from '../../error-message'
 import {ProcedureAst} from '../../parser'
 import {InterpreterState} from '../InterpreterState'
-import {InternalScalarValue} from '../InterpreterValue'
+import {InternalScalarValue, InterpreterValue} from '../InterpreterValue'
 import {ArgumentTypes, FunctionPlugin, FunctionPluginTypecheck} from './FunctionPlugin'
 
 const MAX_48BIT_INTEGER = 281474976710655
@@ -32,11 +32,11 @@ export class BitShiftPlugin extends FunctionPlugin implements FunctionPluginType
     },
   }
 
-  public bitlshift(ast: ProcedureAst, state: InterpreterState): InternalScalarValue  {
+  public bitlshift(ast: ProcedureAst, state: InterpreterState): InterpreterValue  {
     return this.runFunction(ast.args, state, this.metadata('BITLSHIFT'), shiftLeft)
   }
 
-  public bitrshift(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public bitrshift(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('BITRSHIFT'), shiftRight)
   }
 }

src/interpreter/plugin/BitwiseLogicOperationsPlugin.ts

@@ -5,7 +5,7 @@
 
 import {ProcedureAst} from '../../parser'
 import {InterpreterState} from '../InterpreterState'
-import {InternalScalarValue} from '../InterpreterValue'
+import {InternalScalarValue, InterpreterValue} from '../InterpreterValue'
 import {ArgumentTypes, FunctionPlugin, FunctionPluginTypecheck} from './FunctionPlugin'
 
 export class BitwiseLogicOperationsPlugin extends FunctionPlugin implements FunctionPluginTypecheck<BitwiseLogicOperationsPlugin>{
@@ -33,19 +33,19 @@ export class BitwiseLogicOperationsPlugin extends FunctionPlugin implements Func
     },
   }
 
-  public bitand(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public bitand(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('BITAND'),
       (left: number, right: number) => left & right
     )
   }
 
-  public bitor(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public bitor(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('BITOR'),
       (left: number, right: number) => left | right
     )
   }
 
-  public bitxor(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public bitxor(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('BITXOR'),
       (left: number, right: number) => left ^ right
     )

src/interpreter/plugin/BooleanPlugin.ts

@@ -65,11 +65,11 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
     'SWITCH': {
       method: 'switch',
       parameters: [
-          {argumentType: ArgumentTypes.RANGE},
+          {argumentType: ArgumentTypes.NOERROR},
           {argumentType: ArgumentTypes.SCALAR, passSubtype: true},
           {argumentType: ArgumentTypes.SCALAR, passSubtype: true},
         ],
-        repeatLastArgs: 1,
+      repeatLastArgs: 1,
     },
     'IFERROR': {
       method: 'iferror',
@@ -103,7 +103,7 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
    * @param ast
    * @param state
    */
-  public literalTrue(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public literalTrue(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('TRUE'), () => true)
   }
 
@@ -115,7 +115,7 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
    * @param ast
    * @param state
    */
-  public literalFalse(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public literalFalse(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('FALSE'), () => false)
   }
 
@@ -141,7 +141,7 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
    * @param ast
    * @param state
    */
-  public and(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public and(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('AND'),
       (...args) => !args.some((arg: boolean) => !arg)
     )
@@ -155,17 +155,17 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
    * @param ast
    * @param state
    */
-  public or(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public or(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('OR'),
       (...args) => args.some((arg: boolean) => arg)
     )
   }
 
-  public not(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public not(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('NOT'), (arg) => !arg)
   }
 
-  public xor(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public xor(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('XOR'), (...args) => {
       let cnt = 0
       args.forEach((arg: boolean) => {
@@ -177,37 +177,27 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
     })
   }
 
-  public switch(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
-    return this.runMatrixFunction(ast.args, state, this.metadata('SWITCH'), (selectorArr: SimpleRangeValue, ...args) => {
+  public switch(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
+    return this.runFunction(ast.args, state, this.metadata('SWITCH'), (selector, ...args) => {
       const n = args.length
-      const ret: InternalScalarValue[][] = []
-      for(const row of selectorArr.data) {
-        const newrow: InternalScalarValue[] = row.map( (selector) => {
-          let i = 0
-          if(selector instanceof CellError) {
-            return selector
-          }
-          for (; i + 1 < n; i += 2) {
-            if (args[i] instanceof CellError) {
-              continue
-            }
-            if (this.interpreter.arithmeticHelper.eq(selector, args[i] as InternalNoErrorScalarValue)) {
-              return args[i + 1]
-            }
-          }
-          if (i < n) {
-            return args[i]
-          } else {
-            return new CellError(ErrorType.NA, ErrorMessage.NoDefault)
-          }
-        })
-        ret.push(newrow)
+      let i = 0
+      for (; i + 1 < n; i += 2) {
+        if (args[i] instanceof CellError) {
+          continue
+        }
+        if (this.interpreter.arithmeticHelper.eq(selector, args[i] as InternalNoErrorScalarValue)) {
+          return args[i + 1]
+        }
+      }
+      if (i < n) {
+        return args[i]
+      } else {
+        return new CellError(ErrorType.NA, ErrorMessage.NoDefault)
       }
-      return SimpleRangeValue.onlyValues(ret)
     })
   }
 
-  public iferror(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public iferror(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('IFERROR'), (arg1: InternalScalarValue, arg2: InternalScalarValue) => {
       if (arg1 instanceof CellError) {
         return arg2
@@ -217,7 +207,7 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
     })
   }
 
-  public ifna(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public ifna(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('IFNA'), (arg1: InternalScalarValue, arg2: InternalScalarValue) => {
       if (arg1 instanceof CellError && arg1.type === ErrorType.NA) {
         return arg2
@@ -227,7 +217,7 @@ export class BooleanPlugin extends FunctionPlugin implements FunctionPluginTypec
     })
   }
 
-  public choose(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public choose(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('CHOOSE'), (selector, ...args) => {
       if (selector > args.length) {
         return new CellError(ErrorType.NUM, ErrorMessage.Selector)

src/interpreter/plugin/CharPlugin.ts

@@ -7,7 +7,7 @@ import {CellError, ErrorType} from '../../Cell'
 import {ErrorMessage} from '../../error-message'
 import {ProcedureAst} from '../../parser'
 import {InterpreterState} from '../InterpreterState'
-import {InternalScalarValue} from '../InterpreterValue'
+import {InternalScalarValue, InterpreterValue} from '../InterpreterValue'
 import {ArgumentTypes, FunctionPlugin, FunctionPluginTypecheck} from './FunctionPlugin'
 
 export class CharPlugin extends FunctionPlugin implements FunctionPluginTypecheck<CharPlugin>{
@@ -26,7 +26,7 @@ export class CharPlugin extends FunctionPlugin implements FunctionPluginTypechec
     },
   }
 
-  public char(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public char(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('CHAR'), (value: number) => {
       if (value < 1 || value >= 256) {
         return new CellError(ErrorType.VALUE, ErrorMessage.CharacterCodeBounds)
@@ -36,7 +36,7 @@ export class CharPlugin extends FunctionPlugin implements FunctionPluginTypechec
     })
   }
 
-  public unichar(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public unichar(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('CHAR'), (value: number) => {
       if (value < 1 || value >= 1114112) {
         return new CellError(ErrorType.VALUE, ErrorMessage.CharacterCodeBounds)

src/interpreter/plugin/CodePlugin.ts

@@ -7,7 +7,7 @@ import {CellError, ErrorType} from '../../Cell'
 import {ErrorMessage} from '../../error-message'
 import {ProcedureAst} from '../../parser'
 import {InterpreterState} from '../InterpreterState'
-import {InternalScalarValue} from '../InterpreterValue'
+import {InternalScalarValue, InterpreterValue} from '../InterpreterValue'
 import {ArgumentTypes, FunctionPlugin, FunctionPluginTypecheck} from './FunctionPlugin'
 
 export class CodePlugin extends FunctionPlugin implements FunctionPluginTypecheck<CodePlugin>{
@@ -26,7 +26,7 @@ export class CodePlugin extends FunctionPlugin implements FunctionPluginTypechec
     },
   }
 
-  public code(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public code(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('CODE'), (value: string) => {
       if (value.length === 0) {
         return new CellError(ErrorType.VALUE, ErrorMessage.EmptyString)
@@ -35,7 +35,7 @@ export class CodePlugin extends FunctionPlugin implements FunctionPluginTypechec
     })
   }
 
-  public unicode(ast: ProcedureAst, state: InterpreterState): InternalScalarValue {
+  public unicode(ast: ProcedureAst, state: InterpreterState): InterpreterValue {
     return this.runFunction(ast.args, state, this.metadata('UNICODE'), (value: string) => {
       return value.codePointAt(0) ?? new CellError(ErrorType.VALUE, ErrorMessage.EmptyString)
     })