refactor script compilation to make it easy to add new expressions #41 …

…#53

CONTRIBUTING.md

@@ -7,9 +7,9 @@
   - Proposing new features
   - Becoming a maintainer
 
-## Pull Request Process
+## Pull request process
 
-- [GitHub flow](https://guides.github.com/introduction/flow/index.html) is used
+- [GitHub flow](https://guides.github.com/introduction/flow/index.html) with [GitOps](./img/architecture/gitops.png) is used
 - Your pull requests are actively welcomed.
 - The steps:
   1. Fork the repo and create your branch from master.
@@ -25,4 +25,10 @@
 
 ## License
 
-By contributing, you agree that your contributions will be licensed under its GNU General Public License v3.0.
+By contributing, you agree that your contributions will be licensed under its [GNU General Public License v3.0](./LICENSE).
+
+## Read more
+
+- See [tests](./docs/tests.md) for testing
+- See [extend script](./README.md#extend-scripts) for quick steps to extend scripts
+- See [architecture overview](./README.md#architecture-overview) to deep dive into privacy.sexy codebase

docs/application.md

@@ -2,7 +2,7 @@
 
 - It's mainly responsible for
   - creating and event based [application state](#application-state)
-  - parsing and compiling [application data](#application-data)
+  - [parsing](#parsing) and [compiling](#compiling) [application data](#application-data)
 
 ## Application state
 
@@ -14,9 +14,23 @@
 
 ## Application data
 
-- Compiled to `Application` domain object.
+- Compiled to [`Application`](./../src/domain/Application.ts) domain object.
 - The scripts are defined and controlled in different data files per OS
 - Enables [data-driven programming](https://en.wikipedia.org/wiki/Data-driven_programming) and easier contributions
 - Application data is defined in collection files and
 - 📖 See [Application data | Presentation layer](./presentation.md#application-data) to read how the application data is read by the presentation layer.
 - 📖 See [collection files documentation](./collection-files.md) to read more about how the data files are structured/defined and see [collection yaml files](./../src/application/collections/) to directly check the code.
+
+## Parsing
+
+- Application data is parsed to domain object [`Application.ts`](./../src/domain/Application.ts)
+- Steps
+  1. (Compile time) Load application data from [collection yaml files](./../src/application/collections/) using webpack loader
+  2. (Runtime) Parse and compile application and make it available to presentation layer by [`ApplicationFactory.ts`](./../src/application/ApplicationFactory.ts)
+
+### Compiling
+
+- Parsing the application files includes compiling scripts using [collection file defined functions](./collection-files.md#function)
+- To extend the syntax:
+  1. Add a new parser under [SyntaxParsers](./../src/application/Parser/Script/Compiler/Expressions/SyntaxParsers) where you can look at other parsers to understand more.
+  2. Register your in [CompositeExpressionParser](./../src/application/Parser/Script/Compiler/Expressions/Parser/CompositeExpressionParser.ts)

src/application/Parser/CategoryCollectionParser.ts

@@ -2,19 +2,20 @@ import { Category } from '@/domain/Category';
 import { CollectionData } from 'js-yaml-loader!@/*';
 import { parseCategory } from './CategoryParser';
 import { OperatingSystem } from '@/domain/OperatingSystem';
-import { parseScriptingDefinition } from './ScriptingDefinitionParser';
 import { createEnumParser } from '../Common/Enum';
 import { ICategoryCollection } from '@/domain/ICategoryCollection';
 import { CategoryCollection } from '@/domain/CategoryCollection';
 import { IProjectInformation } from '@/domain/IProjectInformation';
 import { CategoryCollectionParseContext } from './Script/CategoryCollectionParseContext';
+import { ScriptingDefinitionParser } from './ScriptingDefinition/ScriptingDefinitionParser';
 
 export function parseCategoryCollection(
     content: CollectionData,
     info: IProjectInformation,
     osParser = createEnumParser(OperatingSystem)): ICategoryCollection {
     validate(content);
-    const scripting = parseScriptingDefinition(content.scripting, info);
+    const scripting = new ScriptingDefinitionParser()
+        .parse(content.scripting, info);
     const context = new CategoryCollectionParseContext(content.functions, scripting);
     const categories = new Array<Category>();
     for (const action of content.actions) {

src/application/Parser/Script/Compiler/Expressions/Expression/Expression.ts

@@ -0,0 +1,35 @@
+import { ExpressionPosition } from './ExpressionPosition';
+import { ExpressionArguments, IExpression } from './IExpression';
+
+export type ExpressionEvaluator = (args?: ExpressionArguments) => string;
+export class Expression implements IExpression {
+    constructor(
+        public readonly position: ExpressionPosition,
+        public readonly evaluator: ExpressionEvaluator,
+        public readonly parameters: readonly string[] = new Array<string>()) {
+        if (!position) {
+            throw new Error('undefined position');
+        }
+        if (!evaluator) {
+            throw new Error('undefined evaluator');
+        }
+    }
+    public evaluate(args?: ExpressionArguments): string {
+        args = filterUnusedArguments(this.parameters, args);
+        return this.evaluator(args);
+    }
+}
+
+function filterUnusedArguments(
+    parameters: readonly string[], args: ExpressionArguments): ExpressionArguments {
+    let result: ExpressionArguments = {};
+    for (const parameter of Object.keys(args)) {
+        if (parameters.includes(parameter)) {
+            result = {
+                ...result,
+                [parameter]: args[parameter],
+            };
+        }
+    }
+    return result;
+}

src/application/Parser/Script/Compiler/Expressions/Expression/ExpressionPosition.ts

@@ -0,0 +1,15 @@
+export class ExpressionPosition {
+    constructor(
+        public readonly start: number,
+        public readonly end: number) {
+        if (start === end) {
+            throw new Error(`no length (start = end = ${start})`);
+        }
+        if (start > end) {
+            throw Error(`start (${start}) after end (${end})`);
+        }
+        if (start < 0) {
+            throw Error(`negative start position: ${start}`);
+        }
+    }
+}

src/application/Parser/Script/Compiler/Expressions/Expression/IExpression.ts

@@ -0,0 +1,12 @@
+import { ExpressionPosition } from './ExpressionPosition';
+
+export interface IExpression {
+    readonly position: ExpressionPosition;
+    readonly parameters?: readonly string[];
+    evaluate(args?: ExpressionArguments): string;
+}
+
+export interface ExpressionArguments {
+    readonly [parameter: string]: string;
+}
+

src/application/Parser/Script/Compiler/Expressions/ExpressionsCompiler.ts

@@ -1,31 +1,49 @@
 import { IExpressionsCompiler, ParameterValueDictionary } from './IExpressionsCompiler';
-import { generateIlCode, IILCode } from './ILCode';
+import { IExpression } from './Expression/IExpression';
+import { IExpressionParser } from './Parser/IExpressionParser';
+import { CompositeExpressionParser } from './Parser/CompositeExpressionParser';
 
 export class ExpressionsCompiler implements IExpressionsCompiler {
-    public static readonly instance: IExpressionsCompiler = new ExpressionsCompiler();
-    protected constructor() { }
+    public constructor(private readonly extractor: IExpressionParser = new CompositeExpressionParser()) { }
     public compileExpressions(code: string, parameters?: ParameterValueDictionary): string {
-        let intermediateCode = generateIlCode(code);
-        intermediateCode = substituteParameters(intermediateCode, parameters);
-        return intermediateCode.compile();
+        const expressions = this.extractor.findExpressions(code);
+        const requiredParameterNames = expressions.map((e) => e.parameters).filter((p) => p).flat();
+        const uniqueParameterNames = Array.from(new Set(requiredParameterNames));
+        ensureRequiredArgsProvided(uniqueParameterNames, parameters);
+        return compileExpressions(expressions, code, parameters);
     }
 }
 
-function substituteParameters(intermediateCode: IILCode, parameters: ParameterValueDictionary): IILCode {
-    const parameterNames = intermediateCode.getUniqueParameterNames();
-    ensureValuesProvided(parameterNames, parameters);
-    for (const parameterName of parameterNames) {
-        const parameterValue = parameters[parameterName];
-        intermediateCode = intermediateCode.substituteParameter(parameterName, parameterValue);
+function compileExpressions(expressions: IExpression[], code: string, parameters?: ParameterValueDictionary) {
+    let compiledCode = '';
+    expressions = expressions
+        .slice() // copy the array to not mutate the parameter
+        .sort((a, b) => b.position.start - a.position.start);
+    let index = 0;
+    while (index !== code.length) {
+        const nextExpression = expressions.pop();
+        if (nextExpression) {
+            compiledCode += code.substring(index, nextExpression.position.start);
+            const expressionCode = nextExpression.evaluate(parameters);
+            compiledCode += expressionCode;
+            index = nextExpression.position.end;
+        } else {
+            compiledCode += code.substring(index, code.length);
+            break;
+        }
     }
-    return intermediateCode;
+    return compiledCode;
 }
 
-function ensureValuesProvided(names: string[], nameValues: ParameterValueDictionary) {
-    nameValues = nameValues || {};
-    const notProvidedNames = names.filter((name) => !Boolean(nameValues[name]));
-    if (notProvidedNames.length) {
-        throw new Error(`parameter value(s) not provided for: ${printList(notProvidedNames)}`);
+function ensureRequiredArgsProvided(parameters: readonly string[], args: ParameterValueDictionary) {
+    parameters = parameters || [];
+    args = args || {};
+    if (!parameters.length) {
+        return;
+    }
+    const notProvidedParameters = parameters.filter((parameter) => !Boolean(args[parameter]));
+    if (notProvidedParameters.length) {
+        throw new Error(`parameter value(s) not provided for: ${printList(notProvidedParameters)}`);
     }
 }
 

src/application/Parser/Script/Compiler/Expressions/Parser/CompositeExpressionParser.ts

@@ -0,0 +1,23 @@
+import { IExpression } from '../Expression/IExpression';
+import { IExpressionParser } from './IExpressionParser';
+import { ParameterSubstitutionParser } from '../SyntaxParsers/ParameterSubstitutionParser';
+
+const parsers = [
+    new ParameterSubstitutionParser(),
+];
+
+export class CompositeExpressionParser implements IExpressionParser {
+    public constructor(private readonly leafs: readonly IExpressionParser[] = parsers) {
+        if (leafs.some((leaf) => !leaf)) { throw new Error('undefined leaf'); }
+    }
+    public findExpressions(code: string): IExpression[] {
+        const expressions = new Array<IExpression>();
+        for (const parser of this.leafs) {
+            const newExpressions = parser.findExpressions(code);
+            if (newExpressions && newExpressions.length) {
+                expressions.push(...newExpressions);
+            }
+        }
+        return expressions;
+    }
+}

src/application/Parser/Script/Compiler/Expressions/Parser/IExpressionParser.ts

@@ -0,0 +1,5 @@
+import { IExpression } from '../Expression/IExpression';
+
+export interface IExpressionParser {
+    findExpressions(code: string): IExpression[];
+}

src/application/Parser/Script/Compiler/Expressions/Parser/RegexParser.ts

@@ -0,0 +1,35 @@
+import { IExpressionParser } from './IExpressionParser';
+import { ExpressionPosition } from '../Expression/ExpressionPosition';
+import { IExpression } from '../Expression/IExpression';
+import { Expression, ExpressionEvaluator } from '../Expression/Expression';
+
+export abstract class RegexParser implements IExpressionParser {
+    protected abstract readonly regex: RegExp;
+    public findExpressions(code: string): IExpression[] {
+        return Array.from(this.findRegexExpressions(code));
+    }
+
+    protected abstract buildExpression(match: RegExpMatchArray): IPrimitiveExpression;
+
+    private* findRegexExpressions(code: string): Iterable<IExpression> {
+        const matches = Array.from(code.matchAll(this.regex));
+        for (const match of matches) {
+            const startPos = match.index;
+            const endPos = startPos + match[0].length;
+            let position: ExpressionPosition;
+            try {
+                position = new ExpressionPosition(startPos, endPos);
+            } catch (error) {
+                throw new Error(`[${this.constructor.name}] invalid script position: ${error.message}\nRegex ${this.regex}\nCode: ${code}`);
+            }
+            const primitiveExpression = this.buildExpression(match);
+            const expression = new Expression(position, primitiveExpression.evaluator, primitiveExpression.parameters);
+            yield expression;
+        }
+    }
+}
+
+export interface IPrimitiveExpression {
+    evaluator: ExpressionEvaluator;
+    parameters?: readonly string[];
+}

src/application/Parser/Script/Compiler/Expressions/SyntaxParsers/ParameterSubstitutionParser.ts

@@ -0,0 +1,12 @@
+import { RegexParser, IPrimitiveExpression } from '../Parser/RegexParser';
+
+export class ParameterSubstitutionParser extends RegexParser {
+    protected readonly regex = /{{\s*\$\s*([^}| ]+)\s*}}/g;
+    protected buildExpression(match: RegExpMatchArray): IPrimitiveExpression {
+        const parameterName = match[1];
+        return {
+            parameters: [ parameterName ],
+            evaluator: (args) => args[parameterName],
+        };
+    }
+}

src/application/Parser/Script/Compiler/Function/FunctionCompiler.ts

@@ -49,6 +49,7 @@ function ensureValidFunctions(functions: readonly FunctionData[]) {
     ensureNoDuplicatesInParameterNames(functions);
     ensureNoDuplicateCode(functions);
     ensureEitherCallOrCodeIsDefined(functions);
+    ensureExpectedParameterNameTypes(functions);
 }
 
 function printList(list: readonly string[]): string {
@@ -67,6 +68,17 @@ function ensureEitherCallOrCodeIsDefined(holders: readonly InstructionHolder[])
         throw new Error(`neither "code" or "call" is defined in ${printNames(hasEitherCodeOrCall)}`);
     }
 }
+
+function ensureExpectedParameterNameTypes(functions: readonly FunctionData[]) {
+    const unexpectedFunctions = functions.filter((func) => func.parameters && !isArrayOfStrings(func.parameters));
+    if (unexpectedFunctions.length) {
+        throw new Error(`unexpected parameter name type in ${printNames(unexpectedFunctions)}`);
+    }
+    function isArrayOfStrings(value: any): boolean {
+        return Array.isArray(value) && value.every((item) => typeof item === 'string');
+     }
+}
+
 function printNames(holders: readonly InstructionHolder[]) {
     return printList(holders.map((holder) => holder.name));
 }