Calculate dependent fields for expressions. (#4099)

* Calculate dependent fields for expressions.

* Find all prefixes when parsing expressions

* Extract expression parsing into its own file. Use for filters as well.

* More tests

src/compile/data/calculate.ts

@@ -5,21 +5,26 @@ import {FieldRefOption} from '../../fielddef';
 import {fieldFilterExpression} from '../../predicate';
 import {isSortArray} from '../../sort';
 import {CalculateTransform} from '../../transform';
-import {duplicate} from '../../util';
+import {duplicate, StringSet} from '../../util';
 import {VgFormulaTransform} from '../../vega.schema';
 import {ModelWithField} from '../model';
 import {DataFlowNode} from './dataflow';
+import {getDependentFields} from './expressions';
 
 /**
  * We don't know what a calculate node depends on so we should never move it beyond anything that produces fields.
  */
 export class CalculateNode extends DataFlowNode {
+  private _dependentFields: StringSet;
+
   public clone() {
     return new CalculateNode(null, duplicate(this.transform));
   }
 
   constructor(parent: DataFlowNode, private transform: CalculateTransform) {
     super(parent);
+
+    this._dependentFields = getDependentFields(this.transform.calculate);
   }
 
   public static parseAllForSortIndex(parent: DataFlowNode, model: ModelWithField) {
@@ -54,6 +59,10 @@ export class CalculateNode extends DataFlowNode {
     return out;
   }
 
+  public dependentFields() {
+    return this._dependentFields;
+  }
+
   public assemble(): VgFormulaTransform {
     return {
       type: 'formula',

src/compile/data/expressions.ts

@@ -0,0 +1,44 @@
+import {parse} from 'vega-expression';
+import {StringSet} from './../../util';
+
+function getName(node: any) {
+  let name: string[] = [];
+
+  if (node.type === 'Identifier') {
+    return [node.name];
+  }
+
+  if (node.type === 'Literal') {
+    return [node.value];
+  }
+
+  if (node.type === 'MemberExpression') {
+    name = name.concat(getName(node.object));
+    name = name.concat(getName(node.property));
+  }
+
+  return name;
+}
+
+function startsWithDatum(node: any): boolean {
+  if (node.object.type === 'MemberExpression') {
+    return startsWithDatum(node.object);
+  }
+  return node.object.name === 'datum';
+}
+
+export function getDependentFields(expression: string) {
+  const ast = parse(expression);
+  const dependents: StringSet = {};
+  ast.visit((node: any) => {
+    if (node.type === 'MemberExpression' && startsWithDatum(node)) {
+      dependents[
+        getName(node)
+          .slice(1)
+          .join('.')
+      ] = true;
+    }
+  });
+
+  return dependents;
+}

src/compile/data/filter.ts

@@ -3,17 +3,26 @@ import {expression, Predicate} from '../../predicate';
 import {duplicate} from '../../util';
 import {VgFilterTransform} from '../../vega.schema';
 import {Model} from '../model';
+import {StringSet} from './../../util';
 import {DataFlowNode} from './dataflow';
+import {getDependentFields} from './expressions';
 
 export class FilterNode extends DataFlowNode {
   private expr: string;
+  private _dependentFields: StringSet;
   public clone() {
     return new FilterNode(null, this.model, duplicate(this.filter));
   }
 
   constructor(parent: DataFlowNode, private readonly model: Model, private filter: LogicalOperand<Predicate>) {
     super(parent);
     this.expr = expression(this.model, this.filter, this);
+
+    this._dependentFields = getDependentFields(this.expr);
+  }
+
+  public dependentFields() {
+    return this._dependentFields;
   }
 
   public assemble(): VgFilterTransform {

test/compile/data/calculate.test.ts

@@ -29,4 +29,16 @@ describe('compile/data/calculate', () => {
       });
     });
   });
+
+  describe('dependentFields and producedFields', () => {
+    it('returns the right fields', () => {
+      const node = new CalculateNode(null, {
+        calculate: 'datum.foo + 2',
+        as: 'bar'
+      });
+
+      expect(node.dependentFields()).toEqual({foo: true});
+      expect(node.producedFields()).toEqual({bar: true});
+    });
+  });
 });

test/compile/data/expressions.test.ts

@@ -0,0 +1,18 @@
+import {getDependentFields} from '../../../src/compile/data/expressions';
+
+describe('compile/data/expressions', () => {
+  describe('getDependentFields', () => {
+    it('calcuates right dependent fields for simple expression', () => {
+      expect(getDependentFields('datum.x + datum.y')).toEqual({x: true, y: true});
+    });
+
+    it('calcuates right dependent fields for compres expression', () => {
+      expect(getDependentFields('toString(datum.x) + 12')).toEqual({x: true});
+    });
+
+    it('calculates right dependent fields for nested field', () => {
+      expect(getDependentFields('datum.x.y')).toEqual({x: true, 'x.y': true});
+      expect(getDependentFields('datum["x.y"]')).toEqual({'x.y': true});
+    });
+  });
+});

test/compile/data/filter.test.ts

@@ -5,6 +5,7 @@ import {ParseNode} from '../../../src/compile/data/formatparse';
 import {parseTransformArray} from '../../../src/compile/data/parse';
 import {Dict} from '../../../src/util';
 import {parseUnitModel} from '../../util';
+import {FilterNode} from './../../../src/compile/data/filter';
 
 describe('compile/data/filter', () => {
   it('should create parse for filtered fields', () => {
@@ -42,4 +43,13 @@ describe('compile/data/filter', () => {
       d: 'number'
     });
   });
+
+  describe('dependentFields and producedFields', () => {
+    it('returns the right fields', () => {
+      const node = new FilterNode(null, null, 'datum.foo > 2');
+
+      expect(node.dependentFields()).toEqual({foo: true});
+      expect(node.producedFields()).toEqual({});
+    });
+  });
 });

typings/vega-expression.d.ts

@@ -0,0 +1,3 @@
+declare module 'vega-expression' {
+  export function parse(expression: string): any;
+}