From bf6976f923d397af9266b5c4a901541e10b895cd Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Tue, 24 May 2022 18:14:11 +0800 Subject: [PATCH 01/12] feat(core): query builder support in template engine - Add builder value visitor to modify AST - Add execute filter to call builder.execute() - Support replace callback while visit nodes' children --- .../core/src/containers/modules/executor.ts | 16 +++- .../src/containers/modules/templateEngine.ts | 4 + .../extensions/filters/execute.ts | 12 +++ .../extensions/filters/index.ts | 1 + .../template-engine/extensions/tags/req.ts | 11 ++- .../lib/template-engine/nunjucksCompiler.ts | 23 +++-- .../lib/template-engine/visitors/astWalker.ts | 77 ++++++++++++++-- .../visitors/builderValueVisitor.ts | 89 +++++++++++++++++++ .../src/lib/template-engine/visitors/index.ts | 1 + .../visitors/parametersVisitor.ts | 5 +- .../core/test/template-engine/builder.spec.ts | 69 ++++++++++++++ .../extensions/tags/req.spec.ts | 15 +++- types/nunjucks.d.ts | 10 ++- 13 files changed, 306 insertions(+), 27 deletions(-) create mode 100644 packages/core/src/lib/template-engine/extensions/filters/execute.ts create mode 100644 packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts create mode 100644 packages/core/test/template-engine/builder.spec.ts diff --git a/packages/core/src/containers/modules/executor.ts b/packages/core/src/containers/modules/executor.ts index 0d0dfe90..2948d182 100644 --- a/packages/core/src/containers/modules/executor.ts +++ b/packages/core/src/containers/modules/executor.ts @@ -1,13 +1,21 @@ import { ContainerModule } from 'inversify'; -import { Executor } from '@vulcan/core/template-engine'; +import { Executor, QueryBuilder } from '@vulcan/core/template-engine'; import { TYPES } from '../types'; +class MockBuilder implements QueryBuilder { + public count() { + return this; + } + + public async value() { + return []; + } +} + export const executorModule = () => new ContainerModule((bind) => { bind(TYPES.Executor).toConstantValue({ // TODO: Mock value - executeQuery: async () => { - return []; - }, + createBuilder: async () => new MockBuilder(), }); }); diff --git a/packages/core/src/containers/modules/templateEngine.ts b/packages/core/src/containers/modules/templateEngine.ts index d34871e9..efe6966c 100644 --- a/packages/core/src/containers/modules/templateEngine.ts +++ b/packages/core/src/containers/modules/templateEngine.ts @@ -14,6 +14,7 @@ import { NunjucksCompiler, Compiler, TemplateEngine, + ExecuteExtension, } from '@vulcan/core/template-engine'; import { ContainerModule, interfaces } from 'inversify'; import { TemplateEngineOptions } from '../../options'; @@ -49,6 +50,9 @@ export const templateEngineModule = (options: ITemplateEngineOptions) => bind(TYPES.CompilerExtension) .to(ReqExtension) .inSingletonScope(); + bind(TYPES.CompilerExtension) + .to(ExecuteExtension) + .inSingletonScope(); // Loader bind(TYPES.CompilerLoader) diff --git a/packages/core/src/lib/template-engine/extensions/filters/execute.ts b/packages/core/src/lib/template-engine/extensions/filters/execute.ts new file mode 100644 index 00000000..1abaf640 --- /dev/null +++ b/packages/core/src/lib/template-engine/extensions/filters/execute.ts @@ -0,0 +1,12 @@ +import { NunjucksFilterExtension } from '../extension'; +import { injectable } from 'inversify'; +import { QueryBuilder } from '../tags'; + +@injectable() +export class ExecuteExtension implements NunjucksFilterExtension { + public name = 'execute'; + public async transform({ value }: { value: any; args: any[] }): Promise { + const builder: QueryBuilder = value; + return builder.value(); + } +} diff --git a/packages/core/src/lib/template-engine/extensions/filters/index.ts b/packages/core/src/lib/template-engine/extensions/filters/index.ts index 4d39d161..f76e6638 100644 --- a/packages/core/src/lib/template-engine/extensions/filters/index.ts +++ b/packages/core/src/lib/template-engine/extensions/filters/index.ts @@ -1 +1,2 @@ export * from './unique'; +export * from './execute'; diff --git a/packages/core/src/lib/template-engine/extensions/tags/req.ts b/packages/core/src/lib/template-engine/extensions/tags/req.ts index 466f6a59..8c378394 100644 --- a/packages/core/src/lib/template-engine/extensions/tags/req.ts +++ b/packages/core/src/lib/template-engine/extensions/tags/req.ts @@ -8,8 +8,13 @@ import { injectable, inject } from 'inversify'; import { TYPES } from '@vulcan/core/containers'; // TODO: temporary interface +export interface QueryBuilder { + count(): QueryBuilder; + value(): Promise; +} + export interface Executor { - executeQuery(query: string): Promise; + createBuilder(query: string): Promise; } @injectable() @@ -69,7 +74,7 @@ export class ReqExtension implements NunjucksTagExtension { .split(/\r?\n/) .filter((line) => line.trim().length > 0) .join('\n'); - const result = await this.executor.executeQuery(query); - context.setVariable(name, result); + const builder = await this.executor.createBuilder(query); + context.setVariable(name, builder); } } diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index e41ccd93..7ef60e39 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -9,7 +9,12 @@ import { } from './extensions'; import * as transformer from 'nunjucks/src/transformer'; import { walkAst } from './visitors/astWalker'; -import { ParametersVisitor, ErrorsVisitor, FiltersVisitor } from './visitors'; +import { + ParametersVisitor, + ErrorsVisitor, + FiltersVisitor, + BuilderValueVisitor, +} from './visitors'; import { inject, injectable, multiInject, optional } from 'inversify'; import { TYPES } from '@vulcan/core/containers'; @@ -31,18 +36,23 @@ export class NunjucksCompiler implements Compiler { } public compile(template: string): CompileResult { - const ast = nunjucks.parser.parse(template, this.env.extensionsList, {}); const compiler = new nunjucks.compiler.Compiler( 'main', this.env.opts.throwOnUndefined || false ); - const metadata = this.getMetadata(ast); - const preProcessedAst = this.preProcess(ast); - compiler.compile(preProcessedAst); + const { ast, metadata } = this.generateAst(template); + compiler.compile(ast); const code = compiler.getCode(); return { compiledData: `(() => {${code}})()`, metadata }; } + public generateAst(template: string) { + const ast = nunjucks.parser.parse(template, this.env.extensionsList, {}); + const metadata = this.getMetadata(ast); + const preProcessedAst = this.preProcess(ast); + return { ast: preProcessedAst, metadata }; + } + public async render( templateName: string, data: T @@ -84,7 +94,8 @@ export class NunjucksCompiler implements Compiler { const parameters = new ParametersVisitor(); const errors = new ErrorsVisitor(); const filters = new FiltersVisitor({ env: this.env }); - walkAst(ast, [parameters, errors, filters]); + const builderValueVisitor = new BuilderValueVisitor(); + walkAst(ast, [parameters, errors, filters, builderValueVisitor]); return { parameters: parameters.getParameters(), errors: errors.getErrors(), diff --git a/packages/core/src/lib/template-engine/visitors/astWalker.ts b/packages/core/src/lib/template-engine/visitors/astWalker.ts index 34137b79..6c085465 100644 --- a/packages/core/src/lib/template-engine/visitors/astWalker.ts +++ b/packages/core/src/lib/template-engine/visitors/astWalker.ts @@ -6,24 +6,87 @@ export const walkAst = ( visitors: Visitor[] ): void => { visitors.forEach((visitor) => visitor.visit(root)); + visitChildren(root, (node) => walkAst(node, visitors)); +}; + +export type VisitChildCallback = ( + node: nunjucks.nodes.Node, + replaceFunc: ReplaceChildFunc +) => void; +export type ReplaceChildFunc = ( + /** Provide the node you want to replace, or null if you want to delete this child */ + replaceNode: + | nunjucks.nodes.NodeList + | nunjucks.nodes.CallExtension + | nunjucks.nodes.Node + | null +) => void; + +export const visitChildren = ( + root: nunjucks.nodes.Node, + callBack: VisitChildCallback +) => { if (root instanceof nunjucks.nodes.NodeList) { - root.children.forEach((node) => { - walkAst(node, visitors); + const indexToRemove: number[] = []; + root.children.forEach((node, index) => { + callBack(node, (replaced) => { + if (replaced) { + root.children[index] = replaced; + } else { + indexToRemove.push(index); + } + }); }); + // Must delete in reverse order + for (let index = indexToRemove.length - 1; index >= 0; index--) { + root.children.splice(indexToRemove[index], 1); + } } else if (root instanceof nunjucks.nodes.CallExtension) { if (root.args) { - walkAst(root.args, visitors); + callBack(root.args, (replaced) => { + if (!replaced) { + root.args = new nunjucks.nodes.NodeList( + root.args.lineno, + root.args.colno + ); + } else if (replaced instanceof nunjucks.nodes.NodeList) { + root.args = replaced; + } else { + root.args = new nunjucks.nodes.NodeList( + root.args.lineno, + root.args.colno + ); + root.args.addChild(replaced); + } + }); } if (root.contentArgs) { - root.contentArgs.forEach((n) => { - walkAst(n, visitors); + const indexToRemove: number[] = []; + root.contentArgs.forEach((node, index) => { + callBack(node, (replaced) => { + if (replaced && root.contentArgs) { + root.contentArgs[index] = replaced; + } else { + indexToRemove.push(index); + } + }); }); + // Must delete in reverse order + for (let index = indexToRemove.length - 1; index >= 0; index--) { + root.contentArgs.splice(indexToRemove[index], 1); + } } } else { - root.iterFields((node) => { + root.iterFields((node, fieldName) => { if (node instanceof nunjucks.nodes.Node) { - walkAst(node, visitors); + callBack(node, (replaced) => { + if (replaced) { + (root as any)[fieldName] = replaced; + } else { + delete (root as any)[fieldName]; + } + }); } }); } diff --git a/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts b/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts new file mode 100644 index 00000000..9a1dd2e6 --- /dev/null +++ b/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts @@ -0,0 +1,89 @@ +import * as nunjucks from 'nunjucks'; +import { ReplaceChildFunc, visitChildren } from './astWalker'; +import { Visitor } from './visitor'; + +const MAX_DEPTH = 100; + +// Replace .value() function of builders with a async filter +export class BuilderValueVisitor implements Visitor { + private builderCreatorExtensionName: string; + private executeCommandName: string; + private executeFilterName: string; + + private variableList = new Set(); + + constructor({ + builderCreatorExtensionName = 'built-in-req', + executeCommandName = 'value', + executeFilterName = 'execute', + }: { + builderCreatorExtensionName?: string; + executeCommandName?: string; + executeFilterName?: string; + } = {}) { + this.builderCreatorExtensionName = builderCreatorExtensionName; + this.executeCommandName = executeCommandName; + this.executeFilterName = executeFilterName; + } + + public visit(node: nunjucks.nodes.Node) { + // Record the variable name if it is a extension node + if ( + node instanceof nunjucks.nodes.CallExtensionAsync && + node.extName === this.builderCreatorExtensionName + ) { + const variable = node.args.children[0] as nunjucks.nodes.Literal; + this.variableList.add(variable.value); + return; + } + + visitChildren(node, this.visitChild.bind(this)); + } + + private visitChild(node: nunjucks.nodes.Node, replace: ReplaceChildFunc) { + if ( + node instanceof nunjucks.nodes.FunCall && + node.name instanceof nunjucks.nodes.LookupVal && + node.name.val.value === this.executeCommandName + ) { + let targetNode: typeof node.name.target | null = node.name.target; + let depth = 0; + while (targetNode) { + depth++; + if (depth > MAX_DEPTH) { + throw new Error('Max depth reached'); + } + if (targetNode instanceof nunjucks.nodes.LookupVal) { + targetNode = targetNode.target; + } else if (targetNode instanceof nunjucks.nodes.FunCall) { + targetNode = targetNode.name; + } else if (targetNode instanceof nunjucks.nodes.Symbol) { + break; + } else { + throw new Error( + `Unexpected node type: ${ + (targetNode as nunjucks.nodes.Node)?.typename + }` + ); + } + } + + // If the target node is a variable from {% req xxx %}, replace it with execute filter + if (this.variableList.has((targetNode as nunjucks.nodes.Symbol).value)) { + const args = new nunjucks.nodes.NodeList(node.lineno, node.colno); + args.addChild(node.name.target); + const filter = new nunjucks.nodes.Filter( + node.lineno, + node.colno, + new nunjucks.nodes.Symbol( + node.lineno, + node.colno, + this.executeFilterName + ), + args + ); + replace(filter); + } + } + } +} diff --git a/packages/core/src/lib/template-engine/visitors/index.ts b/packages/core/src/lib/template-engine/visitors/index.ts index c9cfee8d..fccffacb 100644 --- a/packages/core/src/lib/template-engine/visitors/index.ts +++ b/packages/core/src/lib/template-engine/visitors/index.ts @@ -3,3 +3,4 @@ export * from './parametersVisitor'; export * from './errorsVisitor'; export * from './filtersVisitor'; export * from './astWalker'; +export * from './builderValueVisitor'; diff --git a/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts b/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts index 00ac24b3..2348aa66 100644 --- a/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts +++ b/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts @@ -24,8 +24,7 @@ export class ParametersVisitor implements Visitor { public visit(node: nunjucks.nodes.Node) { if (node instanceof nunjucks.nodes.LookupVal) { let name = node.val.value; - let parent: nunjucks.nodes.LookupVal | nunjucks.nodes.Symbol | null = - node.target; + let parent: typeof node.target | null = node.target; let depth = 0; while (parent) { depth++; @@ -35,6 +34,8 @@ export class ParametersVisitor implements Visitor { if (parent instanceof nunjucks.nodes.LookupVal) { name = parent.val.value + '.' + name; parent = parent.target; + } else if (parent instanceof nunjucks.nodes.FunCall) { + parent = parent.name; } else { if (parent.value === this.lookupParameter) { this.parameters.push({ diff --git a/packages/core/test/template-engine/builder.spec.ts b/packages/core/test/template-engine/builder.spec.ts new file mode 100644 index 00000000..4c37ce9c --- /dev/null +++ b/packages/core/test/template-engine/builder.spec.ts @@ -0,0 +1,69 @@ +import { TYPES } from '@vulcan/core/containers'; +import { + NunjucksCompiler, + InMemoryCodeLoader, + Executor, + ReqExtension, + Compiler, + QueryBuilder, + ExecuteExtension, +} from '@vulcan/core/template-engine'; +import { Container } from 'inversify'; +import * as sinon from 'ts-sinon'; + +let container: Container; +let mockExecutor: sinon.StubbedInstance; +let mockBuilder: sinon.StubbedInstance; + +beforeEach(() => { + container = new Container(); + mockBuilder = sinon.stubInterface(); + mockBuilder.count.returns(mockBuilder); + mockExecutor = sinon.stubInterface(); + mockExecutor.createBuilder.resolves(mockBuilder); + container + .bind(TYPES.CompilerLoader) + .to(InMemoryCodeLoader) + .inSingletonScope(); + container.bind(TYPES.Executor).toConstantValue(mockExecutor); + container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); + container.bind(TYPES.CompilerExtension).to(ReqExtension).inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(ExecuteExtension) + .inSingletonScope(); +}); + +afterEach(() => { + container.unbindAll(); +}); + +it('query builder should be executes when .value() function called', async () => { + // Arrange + const compiler = container.get(TYPES.Compiler); + const loader = container.get(TYPES.CompilerLoader); + const { compiledData } = compiler.compile(` +{% req user %} +select * from public.users limit 1 where id = '{{ params.userId }}'; +{% endreq %} + +{% if user.count().value() == 1 %} +Existed +{% else %} +Not existed +{% endif %} + + `); + mockBuilder.value.resolves(1); + // Action + loader.setSource('test', compiledData); + const query = await compiler.render('test', { + params: { userId: 'user-id' }, + }); + // Assert + expect(mockExecutor.createBuilder.firstCall.args[0]).toBe( + `select * from public.users limit 1 where id = 'user-id';` + ); + expect(mockBuilder.count.calledOnce).toBe(true); + expect(query).toBe('Existed'); +}); diff --git a/packages/core/test/template-engine/extensions/tags/req.spec.ts b/packages/core/test/template-engine/extensions/tags/req.spec.ts index 0c71a0c0..c559e13c 100644 --- a/packages/core/test/template-engine/extensions/tags/req.spec.ts +++ b/packages/core/test/template-engine/extensions/tags/req.spec.ts @@ -5,16 +5,22 @@ import { Executor, ReqExtension, Compiler, + QueryBuilder, + ExecuteExtension, } from '@vulcan/core/template-engine'; import { Container } from 'inversify'; import * as sinon from 'ts-sinon'; let container: Container; let mockExecutor: sinon.StubbedInstance; +let mockBuilder: sinon.StubbedInstance; beforeEach(() => { container = new Container(); + mockBuilder = sinon.stubInterface(); + mockBuilder.value.resolves([{ count: 1 }]); mockExecutor = sinon.stubInterface(); + mockExecutor.createBuilder.resolves(mockBuilder); container .bind(TYPES.CompilerLoader) .to(InMemoryCodeLoader) @@ -22,6 +28,10 @@ beforeEach(() => { container.bind(TYPES.Executor).toConstantValue(mockExecutor); container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); container.bind(TYPES.CompilerExtension).to(ReqExtension).inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(ExecuteExtension) + .inSingletonScope(); }); afterEach(() => { @@ -36,9 +46,8 @@ it('req extension should execute correct query and set variable', async () => { {% req userCount %} select count(*) as count from user where user.id = '{{ params.userId }}'; {% endreq %} -{{ userCount[0].count }} +{{ userCount.value()[0].count }} `); - mockExecutor.executeQuery.resolves([{ count: 1 }]); // Action loader.setSource('test', compiledData); @@ -46,7 +55,7 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; params: { userId: 'user-id' }, }); // Assert - expect(mockExecutor.executeQuery.firstCall.args[0]).toBe( + expect(mockExecutor.createBuilder.firstCall.args[0]).toBe( `select count(*) as count from user where user.id = 'user-id';` ); expect(query).toBe('1'); diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index d8ca3565..e30b1851 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -319,7 +319,10 @@ declare module 'nunjucks' { class CallExtensionAsync extends CallExtension {} class LookupVal extends Node { - target: Literal | Symbol; + target: + | Symbol // a.b + | FunCall // a().b + | LookupVal; // a.b.c val: Value; } @@ -332,7 +335,10 @@ declare module 'nunjucks' { class Symbol extends Value {} class FunCall extends Node { - name: Node; + name: + | Symbol // a() + | LookupVal // a.b() + | FunCall; // a().b() args: NodeList; } From 76fd09454e8a054af0f2c701e7fb1da266e4cf5c Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Tue, 31 May 2022 17:04:15 +0800 Subject: [PATCH 02/12] feat(core): add "main" denotation of req extension add a new argument (the second one) of extension, its value should be "true" or "false", indicating whether this builder is the final builder or not. --- .../template-engine/extensions/tags/req.ts | 85 +++++++++++++------ .../extensions/tags/req.spec.ts | 39 ++++++++- types/nunjucks.d.ts | 27 ++++++ 3 files changed, 125 insertions(+), 26 deletions(-) diff --git a/packages/core/src/lib/template-engine/extensions/tags/req.ts b/packages/core/src/lib/template-engine/extensions/tags/req.ts index 8c378394..04b551d8 100644 --- a/packages/core/src/lib/template-engine/extensions/tags/req.ts +++ b/packages/core/src/lib/template-engine/extensions/tags/req.ts @@ -7,6 +7,8 @@ import * as nunjucks from 'nunjucks'; import { injectable, inject } from 'inversify'; import { TYPES } from '@vulcan/core/containers'; +const FINIAL_BUILDER_NAME = 'FINAL_BUILDER'; + // TODO: temporary interface export interface QueryBuilder { count(): QueryBuilder; @@ -29,37 +31,68 @@ export class ReqExtension implements NunjucksTagExtension { public parse( parser: nunjucks.parser.Parser, - nodes: typeof nunjucks.nodes + nodes: typeof nunjucks.nodes, + lexer: typeof nunjucks.lexer ): NunjucksTagExtensionParseResult { - // get the tag token - const token = parser.nextToken(); - - const args = parser.parseSignature(null, true); - parser.advanceAfterBlockEnd(token.value); - - const requestQuery = parser.parseUntilBlocks('endreq'); - parser.advanceAfterBlockEnd(); + // {% req var (main) %} body {% endreq %} + // consume req tag + const reqToken = parser.nextToken(); + // variable + let nextToken = parser.peekToken(); + if (nextToken.type === lexer.TOKEN_BLOCK_END) { + // {% req %} + parser.fail(`Expected a variable`, nextToken.lineno, nextToken.colno); + } + if (nextToken.type !== lexer.TOKEN_SYMBOL) { + parser.fail( + `Expected a symbol, but got ${nextToken.type}`, + nextToken.lineno, + nextToken.colno + ); + } + const variable = parser.parseExpression(); - const variable = args.children[0]; - if (!variable) { - parser.fail(`Expected a variable`, token.lineno, token.colno); + // main denotation + nextToken = parser.peekToken(); + let mainBuilder = false; + if (nextToken.type !== lexer.TOKEN_BLOCK_END) { + if (nextToken.type !== lexer.TOKEN_SYMBOL || nextToken.value !== 'main') { + parser.fail( + `Expected a symbol "main"`, + nextToken.lineno, + nextToken.colno + ); + } + mainBuilder = true; + // Consume this token (main) + parser.nextToken(); } - if (!(variable instanceof nodes.Symbol)) { + + const endToken = parser.nextToken(); + if (endToken.type !== lexer.TOKEN_BLOCK_END) { parser.fail( - `Expected a symbol, but got ${variable.typename}`, - variable.lineno, - variable.colno + `Expected a block end, but got ${endToken.type}`, + endToken.lineno, + endToken.colno ); } - const variableName = new nodes.Literal( - variable.colno, - variable.lineno, - (variable as nunjucks.nodes.Symbol).value - ); + const requestQuery = parser.parseUntilBlocks('endreq'); + parser.advanceAfterBlockEnd(); - const argsNodeToPass = new nodes.NodeList(args.lineno, args.colno); - argsNodeToPass.addChild(variableName); + const argsNodeToPass = new nodes.NodeList(reqToken.lineno, reqToken.colno); + // variable name + argsNodeToPass.addChild( + new nodes.Literal( + variable.colno, + variable.lineno, + (variable as nunjucks.nodes.Symbol).value + ) + ); + // is main builder + argsNodeToPass.addChild( + new nodes.Literal(variable.colno, variable.lineno, String(mainBuilder)) + ); return { argsNodeList: argsNodeToPass, @@ -69,12 +102,16 @@ export class ReqExtension implements NunjucksTagExtension { public async run({ context, args }: NunjucksTagExtensionRunOptions) { const name: string = args[0]; - const requestQuery: () => string = args[1]; + const requestQuery: () => string = args[2]; const query = requestQuery() .split(/\r?\n/) .filter((line) => line.trim().length > 0) .join('\n'); const builder = await this.executor.createBuilder(query); context.setVariable(name, builder); + + if (Boolean(args[1])) { + context.setVariable(FINIAL_BUILDER_NAME, builder); + } } } diff --git a/packages/core/test/template-engine/extensions/tags/req.spec.ts b/packages/core/test/template-engine/extensions/tags/req.spec.ts index c559e13c..9fda7c55 100644 --- a/packages/core/test/template-engine/extensions/tags/req.spec.ts +++ b/packages/core/test/template-engine/extensions/tags/req.spec.ts @@ -38,7 +38,7 @@ afterEach(() => { container.unbindAll(); }); -it('req extension should execute correct query and set variable', async () => { +it.only('req extension should execute correct query and set variable', async () => { // Arrange const compiler = container.get(TYPES.Compiler); const loader = container.get(TYPES.CompilerLoader); @@ -72,7 +72,7 @@ it('if argument is not a symbol, extension should throw', async () => { select count(*) as count from user where user.id = '{{ params.userId }}'; {% endreq %} `) - ).toThrow(`Expected a symbol, but got Literal`); + ).toThrow(`Expected a symbol, but got string`); }); it('if argument is missing, extension should throw', async () => { @@ -88,3 +88,38 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; `) ).toThrow(`Expected a variable`); }); + +it('if the main denotation is replaces other keywords than "main", extension should throw an error', async () => { + // Arrange + const compiler = container.get(TYPES.Compiler); + + // Action, Assert + expect(() => + compiler.compile(` +{% req user super %} +some statement +{% endreq %} + `) + ).toThrow(`Expected a symbol "main"`); +}); + +it('the main denotation should be parsed into the second args node', async () => { + // Arrange + const compiler = container.get(TYPES.Compiler); + + // Action + const { ast: astWithMainBuilder } = compiler.generateAst( + `{% req user main %} some statement {% endreq %}` + ); + const { ast: astWithoutMainBuilder } = compiler.generateAst( + `{% req user %} some statement {% endreq %}` + ); + + // Assert + expect((astWithMainBuilder as any).children[0].args.children[1].value).toBe( + 'true' + ); + expect( + (astWithoutMainBuilder as any).children[0].args.children[1].value + ).toBe('false'); +}); diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index e30b1851..3c740e09 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -284,6 +284,7 @@ declare module 'nunjucks' { parseUntilBlocks(...blockName: string[]): nodes.NodeList; advanceAfterBlockEnd(): Token; fail(message: string, lineno?: number, colno?: number): void; + parseExpression(): Nodes; } } @@ -353,6 +354,32 @@ declare module 'nunjucks' { namespace lexer { function lexer(src: string, opts: any): any; + const TOKEN_STRING: string; + const TOKEN_WHITESPACE: string; + const TOKEN_DATA: string; + const TOKEN_BLOCK_START: string; + const TOKEN_BLOCK_END: string; + const TOKEN_VARIABLE_START: string; + const TOKEN_VARIABLE_END: string; + const TOKEN_COMMENT: string; + const TOKEN_LEFT_PAREN: string; + const TOKEN_RIGHT_PAREN: string; + const TOKEN_LEFT_BRACKET: string; + const TOKEN_RIGHT_BRACKET: string; + const TOKEN_LEFT_CURLY: string; + const TOKEN_RIGHT_CURLY: string; + const TOKEN_OPERATOR: string; + const TOKEN_COMMA: string; + const TOKEN_COLON: string; + const TOKEN_TILDE: string; + const TOKEN_PIPE: string; + const TOKEN_INT: string; + const TOKEN_FLOAT: string; + const TOKEN_BOOLEAN: string; + const TOKEN_NONE: string; + const TOKEN_SYMBOL: string; + const TOKEN_SPECIAL: string; + const TOKEN_REGEX: string; } interface Token { From e91d652909d03589eed325dcaf0d355fa6a1e21a Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Tue, 31 May 2022 17:46:50 +0800 Subject: [PATCH 03/12] feat(core): ads main builder visitor to validate and wrap builders --- .../core/src/lib/template-engine/compiler.ts | 1 + .../lib/template-engine/nunjucksCompiler.ts | 20 +++++- .../lib/template-engine/visitors/astWalker.ts | 3 + .../src/lib/template-engine/visitors/index.ts | 1 + .../visitors/mainBuilderVisitor.ts | 55 ++++++++++++++ .../lib/template-engine/visitors/visitor.ts | 1 + .../visitors/mainBuilder.spec.ts | 71 +++++++++++++++++++ types/nunjucks.d.ts | 6 +- 8 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts create mode 100644 packages/core/test/template-engine/visitors/mainBuilder.spec.ts diff --git a/packages/core/src/lib/template-engine/compiler.ts b/packages/core/src/lib/template-engine/compiler.ts index 60031182..f8d4c2f1 100644 --- a/packages/core/src/lib/template-engine/compiler.ts +++ b/packages/core/src/lib/template-engine/compiler.ts @@ -30,5 +30,6 @@ export interface Compiler { * @param template The path or identifier of a template source */ compile(template: string): CompileResult; + execute(template: string, data: T): Promise; render(templateName: string, data: T): Promise; } diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index 7ef60e39..85e93a05 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -14,6 +14,7 @@ import { ErrorsVisitor, FiltersVisitor, BuilderValueVisitor, + MainBuilderVisitor, } from './visitors'; import { inject, injectable, multiInject, optional } from 'inversify'; import { TYPES } from '@vulcan/core/containers'; @@ -53,6 +54,16 @@ export class NunjucksCompiler implements Compiler { return { ast: preProcessedAst, metadata }; } + public async execute( + templateName: string, + data: T + ): Promise { + const template = this.env.getTemplate(templateName, true); + + // const query = await this.render(template, data); + // console.log(data); + } + public async render( templateName: string, data: T @@ -95,7 +106,14 @@ export class NunjucksCompiler implements Compiler { const errors = new ErrorsVisitor(); const filters = new FiltersVisitor({ env: this.env }); const builderValueVisitor = new BuilderValueVisitor(); - walkAst(ast, [parameters, errors, filters, builderValueVisitor]); + const mainBuilderVisitor = new MainBuilderVisitor(); + walkAst(ast, [ + parameters, + errors, + filters, + builderValueVisitor, + mainBuilderVisitor, + ]); return { parameters: parameters.getParameters(), errors: errors.getErrors(), diff --git a/packages/core/src/lib/template-engine/visitors/astWalker.ts b/packages/core/src/lib/template-engine/visitors/astWalker.ts index 6c085465..3710cf2e 100644 --- a/packages/core/src/lib/template-engine/visitors/astWalker.ts +++ b/packages/core/src/lib/template-engine/visitors/astWalker.ts @@ -7,6 +7,9 @@ export const walkAst = ( ): void => { visitors.forEach((visitor) => visitor.visit(root)); visitChildren(root, (node) => walkAst(node, visitors)); + if (root instanceof nunjucks.nodes.Root) { + visitors.forEach((visitor) => visitor.finish?.()); + } }; export type VisitChildCallback = ( diff --git a/packages/core/src/lib/template-engine/visitors/index.ts b/packages/core/src/lib/template-engine/visitors/index.ts index fccffacb..b6d6aeda 100644 --- a/packages/core/src/lib/template-engine/visitors/index.ts +++ b/packages/core/src/lib/template-engine/visitors/index.ts @@ -4,3 +4,4 @@ export * from './errorsVisitor'; export * from './filtersVisitor'; export * from './astWalker'; export * from './builderValueVisitor'; +export * from './mainBuilderVisitor'; diff --git a/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts b/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts new file mode 100644 index 00000000..0776ed1e --- /dev/null +++ b/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts @@ -0,0 +1,55 @@ +import { Visitor } from './visitor'; +import * as nunjucks from 'nunjucks'; + +const ReqExtensionName = 'built-in-req'; + +export class MainBuilderVisitor implements Visitor { + private root?: nunjucks.nodes.Root; + private hasMainBuilder = false; + + public visit(node: nunjucks.nodes.Node) { + // save the root + if (node instanceof nunjucks.nodes.Root) { + this.root = node; + } + this.checkMainBuilder(node); + } + + public finish() { + if (!this.hasMainBuilder) { + this.wrapOutputWithBuilder(); + } + } + + private checkMainBuilder(node: nunjucks.nodes.Node) { + const isMainBuilder = + node instanceof nunjucks.nodes.CallExtensionAsync && + node.extName === ReqExtensionName && + (node.args.children[1] as nunjucks.nodes.Literal).value === 'true'; + if (!isMainBuilder) return; + + if (this.hasMainBuilder) { + throw new Error(`Only one main builder is allowed.`); + } + this.hasMainBuilder = true; + } + + private wrapOutputWithBuilder() { + if (!this.root) { + throw new Error('No root node found.'); + } + const originalChildren = this.root.children; + const args = new nunjucks.nodes.NodeList(0, 0); + // variable name + args.addChild(new nunjucks.nodes.Literal(0, 0, '__wrapped__builder')); + // is main builder + args.addChild(new nunjucks.nodes.Literal(0, 0, 'true')); + const builder = new nunjucks.nodes.CallExtensionAsync( + ReqExtensionName, + 'run', + args, + originalChildren + ); + this.root.children = [builder]; + } +} diff --git a/packages/core/src/lib/template-engine/visitors/visitor.ts b/packages/core/src/lib/template-engine/visitors/visitor.ts index 575f382a..d6598b97 100644 --- a/packages/core/src/lib/template-engine/visitors/visitor.ts +++ b/packages/core/src/lib/template-engine/visitors/visitor.ts @@ -2,4 +2,5 @@ import * as nunjucks from 'nunjucks'; export interface Visitor { visit: (node: nunjucks.nodes.Node) => void; + finish?: () => void; } diff --git a/packages/core/test/template-engine/visitors/mainBuilder.spec.ts b/packages/core/test/template-engine/visitors/mainBuilder.spec.ts new file mode 100644 index 00000000..916689ce --- /dev/null +++ b/packages/core/test/template-engine/visitors/mainBuilder.spec.ts @@ -0,0 +1,71 @@ +import { + Executor, + MainBuilderVisitor, + NunjucksTagExtensionWrapper, + ReqExtension, + walkAst, +} from '@vulcan/core/template-engine'; +import * as nunjucks from 'nunjucks'; +import * as sinon from 'ts-sinon'; + +let extensions: nunjucks.Extension[] = []; + +beforeEach(() => { + const mockExecutor = sinon.stubInterface(); + const { transform: reqExtension } = NunjucksTagExtensionWrapper( + new ReqExtension(mockExecutor) + ); + extensions = [reqExtension]; +}); + +it('Should do nothing if there is exactly one main builder', async () => { + // Arrange + const ast = nunjucks.parser.parse( + `some output {% req user main %} select * from users; {% endreq %}`, + extensions, + {} + ); + const visitor = new MainBuilderVisitor(); + // Act + walkAst(ast, [visitor]); + // Arrange + expect(ast.children.length).toBe(2); +}); + +it('Should throw an error if there are tow main builders', async () => { + // Arrange + const ast = nunjucks.parser.parse( + ` + {% req user main %} select * from users; {% endreq %} + {% req user2 main %} select * from users; {% endreq %} + `, + extensions, + {} + ); + const visitor = new MainBuilderVisitor(); + // Act, Arrange + expect(() => walkAst(ast, [visitor])).toThrowError( + `Only one main builder is allowed.` + ); +}); + +it('Should wrap the output in a builder if there is no main builder', async () => { + // Arrange + const ast = nunjucks.parser.parse( + ` +select * from users +where id = {{ params.id }}; + `, + extensions, + {} + ); + const visitor = new MainBuilderVisitor(); + // Act + walkAst(ast, [visitor]); + // Arrange + expect(ast.children.length).toBe(1); + expect(ast.children[0] instanceof nunjucks.nodes.CallExtensionAsync).toBe( + true + ); + expect((ast.children[0] as any).args.children[1].value).toBe('true'); // main builder notation +}); diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index 3c740e09..682c68dd 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -305,9 +305,11 @@ declare module 'nunjucks' { addChild(child: Node): void; } + class Root extends NodeList {} + class CallExtension extends Node { constructor( - ext: object, + ext: object | string, prop: string, args: nodes.NodeList | null, contentArgs: nodes.Node[] | null @@ -350,6 +352,8 @@ declare module 'nunjucks' { body: Node | null; targets: Node[]; } + + class TemplateData extends Literal {} } namespace lexer { From 67181f9e7a6b1e06afabd5800aed79e0b65d75fe Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Thu, 2 Jun 2022 18:15:29 +0800 Subject: [PATCH 04/12] fix(core): update compiler to use new builder --- .../core/src/lib/template-engine/compiler.ts | 1 - .../template-engine/extensions/extension.ts | 31 ++++++-- .../template-engine/extensions/tags/error.ts | 6 +- .../template-engine/extensions/tags/req.ts | 16 +++-- .../lib/template-engine/nunjucksCompiler.ts | 39 +++++------ .../src/lib/template-engine/templateEngine.ts | 6 +- .../visitors/mainBuilderVisitor.ts | 2 +- .../core/test/template-engine/builder.spec.ts | 58 +++------------ .../extensions/filters/unique.spec.ts | 46 +++--------- .../extensions/tags/error.spec.ts | 34 ++------- .../extensions/tags/req.spec.ts | 70 ++++--------------- .../template-engine/nunjuckCompiler.spec.ts | 39 +++-------- .../template-engine/templateEngine.spec.ts | 6 +- .../core/test/template-engine/testCompiler.ts | 56 +++++++++++++++ types/nunjucks.d.ts | 14 ++++ 15 files changed, 179 insertions(+), 245 deletions(-) create mode 100644 packages/core/test/template-engine/testCompiler.ts diff --git a/packages/core/src/lib/template-engine/compiler.ts b/packages/core/src/lib/template-engine/compiler.ts index f8d4c2f1..108ccda2 100644 --- a/packages/core/src/lib/template-engine/compiler.ts +++ b/packages/core/src/lib/template-engine/compiler.ts @@ -31,5 +31,4 @@ export interface Compiler { */ compile(template: string): CompileResult; execute(template: string, data: T): Promise; - render(templateName: string, data: T): Promise; } diff --git a/packages/core/src/lib/template-engine/extensions/extension.ts b/packages/core/src/lib/template-engine/extensions/extension.ts index 6c5600bc..d18670a9 100644 --- a/packages/core/src/lib/template-engine/extensions/extension.ts +++ b/packages/core/src/lib/template-engine/extensions/extension.ts @@ -11,9 +11,14 @@ export interface NunjucksTagExtensionParseResult { contentNodes: nunjucks.nodes.Node[]; } +export type TagExtensionContentArgGetter = () => Promise; + +export type TagExtensionArgTypes = string | number | boolean; + export interface NunjucksTagExtensionRunOptions { context: any; - args: any[]; + args: TagExtensionArgTypes[]; + contentArgs: TagExtensionContentArgGetter[]; } class WrapperTagExtension { @@ -44,14 +49,28 @@ class WrapperTagExtension { ); } - public async __run(...args: any[]) { - const context = args[0]; + public __run(...originalArgs: any[]) { + const context = originalArgs[0]; // Nunjucks use the pass the callback function for async extension at the last argument // https://github.com/mozilla/nunjucks/blob/master/nunjucks/src/compiler.js#L256 - const callback = args[args.length - 1]; - const otherArgs = args.slice(1, args.length - 1); + const callback = originalArgs[originalArgs.length - 1]; + const args = originalArgs + .slice(1, originalArgs.length - 1) + .filter((value) => typeof value !== 'function'); + const contentArgs = originalArgs + .slice(1, originalArgs.length - 1) + .filter((value) => typeof value === 'function') + .map((cb) => () => { + return new Promise((resolve, reject) => { + cb((err: any, result: any) => { + if (err) reject(err); + else resolve(result); + }); + }); + }); + this.extension - .run({ context, args: otherArgs }) + .run({ context, args, contentArgs }) .then((result) => callback(null, result)) .catch((err) => callback(err, null)); } diff --git a/packages/core/src/lib/template-engine/extensions/tags/error.ts b/packages/core/src/lib/template-engine/extensions/tags/error.ts index 3d914e19..0adab33a 100644 --- a/packages/core/src/lib/template-engine/extensions/tags/error.ts +++ b/packages/core/src/lib/template-engine/extensions/tags/error.ts @@ -35,9 +35,9 @@ export class ErrorExtension implements NunjucksTagExtension { } public async run({ args }: NunjucksTagExtensionRunOptions) { - const message: string = args[0]; - const lineno: number = args[1]; - const colno: number = args[2]; + const message = args[0]; + const lineno = args[1]; + const colno = args[2]; throw new Error(`${message} at ${lineno}:${colno}`); } } diff --git a/packages/core/src/lib/template-engine/extensions/tags/req.ts b/packages/core/src/lib/template-engine/extensions/tags/req.ts index 04b551d8..a5b39b34 100644 --- a/packages/core/src/lib/template-engine/extensions/tags/req.ts +++ b/packages/core/src/lib/template-engine/extensions/tags/req.ts @@ -100,10 +100,17 @@ export class ReqExtension implements NunjucksTagExtension { }; } - public async run({ context, args }: NunjucksTagExtensionRunOptions) { - const name: string = args[0]; - const requestQuery: () => string = args[2]; - const query = requestQuery() + public async run({ + context, + args, + contentArgs, + }: NunjucksTagExtensionRunOptions) { + const name = args[0]; + let query = ''; + for (let index = 0; index < contentArgs.length; index++) { + query += await contentArgs[index](); + } + query = query .split(/\r?\n/) .filter((line) => line.trim().length > 0) .join('\n'); @@ -112,6 +119,7 @@ export class ReqExtension implements NunjucksTagExtension { if (Boolean(args[1])) { context.setVariable(FINIAL_BUILDER_NAME, builder); + context.addExport(FINIAL_BUILDER_NAME); } } } diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index 85e93a05..2745de85 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -6,6 +6,7 @@ import { NunjucksCompilerExtension, NunjucksFilterExtensionWrapper, NunjucksTagExtensionWrapper, + QueryBuilder, } from './extensions'; import * as transformer from 'nunjucks/src/transformer'; import { walkAst } from './visitors/astWalker'; @@ -58,29 +59,8 @@ export class NunjucksCompiler implements Compiler { templateName: string, data: T ): Promise { - const template = this.env.getTemplate(templateName, true); - - // const query = await this.render(template, data); - // console.log(data); - } - - public async render( - templateName: string, - data: T - ): Promise { - return new Promise((resolve, reject) => { - this.env.render(templateName, data, (err, res) => { - if (err) return reject(err); - if (!res) return resolve(''); - else - return resolve( - res - .split(/\r?\n/) - .filter((line) => line.trim().length > 0) - .join('\n') - ); - }); - }); + const builder = await this.renderAndGetMainBuilder(templateName, data); + return builder.value(); } public loadExtension(extension: NunjucksCompilerExtension): void { @@ -125,4 +105,17 @@ export class NunjucksCompiler implements Compiler { // Nunjucks'll handle the async filter via pre-process functions return transformer.transform(ast, this.env.asyncFilters); } + + private renderAndGetMainBuilder(templateName: string, data: any) { + const template = this.env.getTemplate(templateName, true); + return new Promise((resolve, reject) => { + template.getExported<{ FINAL_BUILDER: QueryBuilder }>( + data, + (err, res) => { + if (err) return reject(err); + else resolve(res.FINAL_BUILDER); + } + ); + }); + } } diff --git a/packages/core/src/lib/template-engine/templateEngine.ts b/packages/core/src/lib/template-engine/templateEngine.ts index eb6bbc78..e0d1246b 100644 --- a/packages/core/src/lib/template-engine/templateEngine.ts +++ b/packages/core/src/lib/template-engine/templateEngine.ts @@ -46,10 +46,10 @@ export class TemplateEngine { }; } - public async render( + public async execute( templateName: string, data: T - ): Promise { - return this.compiler.render(templateName, data); + ): Promise { + return this.compiler.execute(templateName, data); } } diff --git a/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts b/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts index 0776ed1e..73570c10 100644 --- a/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts +++ b/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts @@ -46,7 +46,7 @@ export class MainBuilderVisitor implements Visitor { args.addChild(new nunjucks.nodes.Literal(0, 0, 'true')); const builder = new nunjucks.nodes.CallExtensionAsync( ReqExtensionName, - 'run', + '__run', args, originalChildren ); diff --git a/packages/core/test/template-engine/builder.spec.ts b/packages/core/test/template-engine/builder.spec.ts index 4c37ce9c..4afa85d5 100644 --- a/packages/core/test/template-engine/builder.spec.ts +++ b/packages/core/test/template-engine/builder.spec.ts @@ -1,69 +1,31 @@ -import { TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - InMemoryCodeLoader, - Executor, - ReqExtension, - Compiler, - QueryBuilder, - ExecuteExtension, -} from '@vulcan/core/template-engine'; -import { Container } from 'inversify'; -import * as sinon from 'ts-sinon'; - -let container: Container; -let mockExecutor: sinon.StubbedInstance; -let mockBuilder: sinon.StubbedInstance; - -beforeEach(() => { - container = new Container(); - mockBuilder = sinon.stubInterface(); - mockBuilder.count.returns(mockBuilder); - mockExecutor = sinon.stubInterface(); - mockExecutor.createBuilder.resolves(mockBuilder); - container - .bind(TYPES.CompilerLoader) - .to(InMemoryCodeLoader) - .inSingletonScope(); - container.bind(TYPES.Executor).toConstantValue(mockExecutor); - container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); - container.bind(TYPES.CompilerExtension).to(ReqExtension).inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(ExecuteExtension) - .inSingletonScope(); -}); - -afterEach(() => { - container.unbindAll(); -}); +import { createTestCompiler } from './testCompiler'; it('query builder should be executes when .value() function called', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader, builder, executor } = createTestCompiler(); const { compiledData } = compiler.compile(` {% req user %} select * from public.users limit 1 where id = '{{ params.userId }}'; {% endreq %} {% if user.count().value() == 1 %} -Existed +select * from public.user where id = '{{ user.id }}'; {% else %} -Not existed +error "User not found" {% endif %} `); - mockBuilder.value.resolves(1); + builder.value.onFirstCall().resolves([1]); + builder.value.onSecondCall().resolves([{ id: 1, name: 'test' }]); // Action loader.setSource('test', compiledData); - const query = await compiler.render('test', { + const finalResult = await compiler.execute('test', { params: { userId: 'user-id' }, }); // Assert - expect(mockExecutor.createBuilder.firstCall.args[0]).toBe( + expect(executor.createBuilder.firstCall.args[0]).toBe( `select * from public.users limit 1 where id = 'user-id';` ); - expect(mockBuilder.count.calledOnce).toBe(true); - expect(query).toBe('Existed'); + expect(builder.count.callCount).toBe(1); + expect(finalResult).toEqual([{ id: 1, name: 'test' }]); }); diff --git a/packages/core/test/template-engine/extensions/filters/unique.spec.ts b/packages/core/test/template-engine/extensions/filters/unique.spec.ts index 38baec3e..ebf234f1 100644 --- a/packages/core/test/template-engine/extensions/filters/unique.spec.ts +++ b/packages/core/test/template-engine/extensions/filters/unique.spec.ts @@ -1,32 +1,8 @@ -import { TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - InMemoryCodeLoader, - UniqueExtension, - Compiler, -} from '@vulcan/core/template-engine'; -import { Container } from 'inversify'; - -let container: Container; - -beforeEach(() => { - container = new Container(); - container - .bind(TYPES.CompilerLoader) - .to(InMemoryCodeLoader) - .inSingletonScope(); - container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); - container.bind(TYPES.CompilerExtension).to(UniqueExtension); -}); - -afterEach(() => { - container.unbindAll(); -}); +import { createTestCompiler } from '../../testCompiler'; it('Extension should return correct values without unique by argument', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader, executor } = createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [1,2,3,4,4] %} @@ -37,15 +13,14 @@ it('Extension should return correct values without unique by argument', async () ); // Action loader.setSource('test', compiledData); - const result = await compiler.render('test', {}); + await compiler.execute('test', {}); // Assert - expect(result).toBe('1\n2\n3\n4'); + expect(executor.createBuilder.firstCall.args[0]).toBe('1\n2\n3\n4'); }); it('Extension should return correct values with unique by keyword argument', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader, executor } = createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [{name: "Tom"}, {name: "Tom"}, {name: "Joy"}] %} @@ -56,15 +31,14 @@ it('Extension should return correct values with unique by keyword argument', asy ); // Action loader.setSource('test', compiledData); - const result = await compiler.render('test', {}); + await compiler.execute('test', {}); // Assert - expect(result).toBe('Tom\nJoy'); + expect(executor.createBuilder.firstCall.args[0]).toBe('Tom\nJoy'); }); it('Extension should return correct values with unique by argument', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader, executor } = createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [{name: "Tom"}, {name: "Tom"}, {name: "Joy"}] %} @@ -75,7 +49,7 @@ it('Extension should return correct values with unique by argument', async () => ); // Action loader.setSource('test', compiledData); - const result = await compiler.render('test', {}); + await compiler.execute('test', {}); // Assert - expect(result).toBe('Tom\nJoy'); + expect(executor.createBuilder.firstCall.args[0]).toBe('Tom\nJoy'); }); diff --git a/packages/core/test/template-engine/extensions/tags/error.spec.ts b/packages/core/test/template-engine/extensions/tags/error.spec.ts index 8f2f30e3..83334d7c 100644 --- a/packages/core/test/template-engine/extensions/tags/error.spec.ts +++ b/packages/core/test/template-engine/extensions/tags/error.spec.ts @@ -1,38 +1,14 @@ -import { TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - InMemoryCodeLoader, - ErrorExtension, - Compiler, -} from '@vulcan/core/template-engine'; -import { Container } from 'inversify'; - -let container: Container; - -beforeEach(() => { - container = new Container(); - container - .bind(TYPES.CompilerLoader) - .to(InMemoryCodeLoader) - .inSingletonScope(); - container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); - container.bind(TYPES.CompilerExtension).to(ErrorExtension); -}); - -afterEach(() => { - container.unbindAll(); -}); +import { createTestCompiler } from '../../testCompiler'; it('Error extension should throw error with error code and the position while rendering', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader } = createTestCompiler(); const { compiledData } = compiler.compile(` {% error "This is an error" %} `); // Action, Assert loader.setSource('test', compiledData); - await expect(compiler.render('test', { name: 'World' })).rejects.toThrowError( - 'This is an error at 1:3' - ); + await expect( + compiler.execute('test', { name: 'World' }) + ).rejects.toThrowError('This is an error at 1:3'); }); diff --git a/packages/core/test/template-engine/extensions/tags/req.spec.ts b/packages/core/test/template-engine/extensions/tags/req.spec.ts index 9fda7c55..d4831412 100644 --- a/packages/core/test/template-engine/extensions/tags/req.spec.ts +++ b/packages/core/test/template-engine/extensions/tags/req.spec.ts @@ -1,69 +1,29 @@ -import { TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - InMemoryCodeLoader, - Executor, - ReqExtension, - Compiler, - QueryBuilder, - ExecuteExtension, -} from '@vulcan/core/template-engine'; -import { Container } from 'inversify'; -import * as sinon from 'ts-sinon'; +import { createTestCompiler } from '../../testCompiler'; -let container: Container; -let mockExecutor: sinon.StubbedInstance; -let mockBuilder: sinon.StubbedInstance; - -beforeEach(() => { - container = new Container(); - mockBuilder = sinon.stubInterface(); - mockBuilder.value.resolves([{ count: 1 }]); - mockExecutor = sinon.stubInterface(); - mockExecutor.createBuilder.resolves(mockBuilder); - container - .bind(TYPES.CompilerLoader) - .to(InMemoryCodeLoader) - .inSingletonScope(); - container.bind(TYPES.Executor).toConstantValue(mockExecutor); - container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); - container.bind(TYPES.CompilerExtension).to(ReqExtension).inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(ExecuteExtension) - .inSingletonScope(); -}); - -afterEach(() => { - container.unbindAll(); -}); - -it.only('req extension should execute correct query and set variable', async () => { +it('req extension should execute correct query and set variable', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); - const loader = container.get(TYPES.CompilerLoader); + const { compiler, loader, builder, executor } = createTestCompiler(); const { compiledData } = compiler.compile(` -{% req userCount %} +{% req userCount main %} select count(*) as count from user where user.id = '{{ params.userId }}'; {% endreq %} -{{ userCount.value()[0].count }} `); - + builder.value.onFirstCall().resolves([{ count: 1 }]); // Action loader.setSource('test', compiledData); - const query = await compiler.render('test', { + const result = await compiler.execute('test', { params: { userId: 'user-id' }, }); // Assert - expect(mockExecutor.createBuilder.firstCall.args[0]).toBe( + expect(executor.createBuilder.firstCall.args[0]).toBe( `select count(*) as count from user where user.id = 'user-id';` ); - expect(query).toBe('1'); + expect(result).toEqual([{ count: 1 }]); }); it('if argument is not a symbol, extension should throw', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action, Assert expect(() => @@ -77,7 +37,7 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; it('if argument is missing, extension should throw', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action, Assert expect(() => @@ -91,7 +51,7 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; it('if the main denotation is replaces other keywords than "main", extension should throw an error', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action, Assert expect(() => @@ -105,21 +65,15 @@ some statement it('the main denotation should be parsed into the second args node', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action const { ast: astWithMainBuilder } = compiler.generateAst( `{% req user main %} some statement {% endreq %}` ); - const { ast: astWithoutMainBuilder } = compiler.generateAst( - `{% req user %} some statement {% endreq %}` - ); // Assert expect((astWithMainBuilder as any).children[0].args.children[1].value).toBe( 'true' ); - expect( - (astWithoutMainBuilder as any).children[0].args.children[1].value - ).toBe('false'); }); diff --git a/packages/core/test/template-engine/nunjuckCompiler.spec.ts b/packages/core/test/template-engine/nunjuckCompiler.spec.ts index 17591007..a6f403fa 100644 --- a/packages/core/test/template-engine/nunjuckCompiler.spec.ts +++ b/packages/core/test/template-engine/nunjuckCompiler.spec.ts @@ -1,30 +1,9 @@ -import { TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - InMemoryCodeLoader, - NunjucksCompilerExtension, - Compiler, -} from '@vulcan/core/template-engine'; -import { Container } from 'inversify'; - -let container: Container; - -beforeEach(() => { - container = new Container(); - container - .bind(TYPES.CompilerLoader) - .to(InMemoryCodeLoader) - .inSingletonScope(); - container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); -}); - -afterEach(() => { - container.unbindAll(); -}); +import { NunjucksCompilerExtension } from '@vulcan/core/template-engine'; +import { createTestCompiler } from './testCompiler'; it('Nunjucks compiler should compile template without error.', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action const compilerCode = compiler.compile('Hello {{ name }}'); @@ -33,23 +12,23 @@ it('Nunjucks compiler should compile template without error.', async () => { expect(compilerCode).toBeTruthy(); }); -it('Nunjucks compiler should load compiled code and render template with it', async () => { +it('Nunjucks compiler should load compiled code and execute rendered template with it', async () => { // Arrange - const loader = container.get(TYPES.CompilerLoader); - const compiler = container.get(TYPES.Compiler); + const { compiler, loader, getCreatedQueries } = createTestCompiler(); const { compiledData } = compiler.compile('Hello {{ name }}!'); // Action loader.setSource('test', compiledData); - const result = await compiler.render('test', { name: 'World' }); + await compiler.execute('test', { name: 'World' }); + const queries = await getCreatedQueries(); // Assert - expect(result).toBe('Hello World!'); + expect(queries[0]).toBe('Hello World!'); }); it('Nunjucks compiler should reject unsupported extensions', async () => { // Arrange - const compiler = container.get(TYPES.Compiler); + const { compiler } = createTestCompiler(); // Action, Assert // extension should have parse and name property expect(() => diff --git a/packages/core/test/template-engine/templateEngine.spec.ts b/packages/core/test/template-engine/templateEngine.spec.ts index 5f0fd5b6..4cce1dd2 100644 --- a/packages/core/test/template-engine/templateEngine.spec.ts +++ b/packages/core/test/template-engine/templateEngine.spec.ts @@ -31,7 +31,7 @@ beforeEach(() => { errors: [], }, }); - stubCompiler.render.resolves('sql-statement'); + stubCompiler.execute.resolves('sql-statement'); const generator = async function* () { yield { @@ -76,9 +76,9 @@ it('Template engine render function should forward correct data to compiler', as }; // Act - const result = await templateEngine.render('template-name', context); + const result = await templateEngine.execute('template-name', context); // Assert - expect(stubCompiler.render.calledWith('template-name', context)).toBe(true); + expect(stubCompiler.execute.calledWith('template-name', context)).toBe(true); expect(result).toBe('sql-statement'); }); diff --git a/packages/core/test/template-engine/testCompiler.ts b/packages/core/test/template-engine/testCompiler.ts new file mode 100644 index 00000000..b7c1f73d --- /dev/null +++ b/packages/core/test/template-engine/testCompiler.ts @@ -0,0 +1,56 @@ +import { TYPES } from '@vulcan/core/containers'; +import { + ErrorExtension, + ExecuteExtension, + Executor, + InMemoryCodeLoader, + NunjucksCompiler, + NunjucksCompilerExtension, + QueryBuilder, + ReqExtension, + UniqueExtension, +} from '@vulcan/core/template-engine'; +import { Container } from 'inversify'; +import * as sinon from 'ts-sinon'; + +export const createTestCompiler = () => { + const container = new Container(); + const stubBuilder = sinon.stubInterface(); + stubBuilder.count.returns(stubBuilder); + const stubExecutor = sinon.stubInterface(); + stubExecutor.createBuilder.resolves(stubBuilder); + + container + .bind(TYPES.CompilerLoader) + .to(InMemoryCodeLoader) + .inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(UniqueExtension) + .inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(ErrorExtension) + .inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(ReqExtension) + .inSingletonScope(); + container + .bind(TYPES.CompilerExtension) + .to(ExecuteExtension) + .inSingletonScope(); + container.bind(TYPES.Executor).toConstantValue(stubExecutor); + + container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); + return { + builder: stubBuilder, + executor: stubExecutor, + compiler: container.get(TYPES.Compiler), + loader: container.get(TYPES.CompilerLoader), + getCreatedQueries: async () => { + const calls = stubExecutor.createBuilder.getCalls(); + return calls.map((call) => call.args[0]); + }, + }; +}; diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index 682c68dd..c78ab7d1 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -6,6 +6,10 @@ declare module 'nunjucks' { err: lib.TemplateError | null, res: T | null ) => void; + export type TemplateExportCallback = ( + err: lib.TemplateError | null, + res: T + ) => void; export type Callback = (err: E | null, res: T | null) => void; export function render(name: string, context?: object): string; @@ -56,6 +60,16 @@ declare module 'nunjucks' { ); render(context?: object): string; render(context?: object, callback?: TemplateCallback): void; + getExported(callback: TemplateExportCallback): void; + getExported( + context: object, + callback: TemplateExportCallback + ): string; + getExported( + context: object, + parentFrame: any, + callback: TemplateExportCallback + ): string; } export function configure(options: ConfigureOptions): Environment; From 797519bbf0d83fcd8c7dc01035199d314e15e5ee Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Mon, 6 Jun 2022 18:31:59 +0800 Subject: [PATCH 05/12] test(core): add more test cases for extensions --- .../core/test/template-engine/builder.spec.ts | 2 +- .../extensions/filters/execute.spec.ts | 60 ++++++ .../extensions/tags/req.spec.ts | 16 +- .../template-engine/templateEngine.spec.ts | 4 +- .../visitors/astWalker.spec.ts | 182 +++++++++++++++++- .../visitors/builderValue.spec.ts | 16 ++ .../visitors/mainBuilder.spec.ts | 16 ++ 7 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 packages/core/test/template-engine/extensions/filters/execute.spec.ts create mode 100644 packages/core/test/template-engine/visitors/builderValue.spec.ts diff --git a/packages/core/test/template-engine/builder.spec.ts b/packages/core/test/template-engine/builder.spec.ts index 4afa85d5..707e997a 100644 --- a/packages/core/test/template-engine/builder.spec.ts +++ b/packages/core/test/template-engine/builder.spec.ts @@ -11,7 +11,7 @@ select * from public.users limit 1 where id = '{{ params.userId }}'; {% if user.count().value() == 1 %} select * from public.user where id = '{{ user.id }}'; {% else %} -error "User not found" +{% error "User not found" %} {% endif %} `); diff --git a/packages/core/test/template-engine/extensions/filters/execute.spec.ts b/packages/core/test/template-engine/extensions/filters/execute.spec.ts new file mode 100644 index 00000000..09e4354a --- /dev/null +++ b/packages/core/test/template-engine/extensions/filters/execute.spec.ts @@ -0,0 +1,60 @@ +import { createTestCompiler } from '../../testCompiler'; + +it('should call the .value() function of builder', async () => { + // Arrange + const { compiler, loader, executor, builder } = createTestCompiler(); + const { compiledData } = compiler.compile( + ` +{% req user %} +select * from users; +{% endreq %} + +select * from group where userId = '{{ user.value()[0].id }}'; +` + ); + builder.value.onFirstCall().resolves([{ id: 'user-id' }]); + // Action + loader.setSource('test', compiledData); + await compiler.execute('test', {}); + // Assert + expect(executor.createBuilder.secondCall.args[0]).toBe( + `select * from group where userId = 'user-id';` + ); +}); + +it('should throw an error if the main builder failed to execute', async () => { + // Arrange + const { compiler, loader, builder } = createTestCompiler(); + const { compiledData } = compiler.compile( + ` +{% req user main %} +select * from users; +{% endreq %} +` + ); + builder.value.onFirstCall().rejects(new Error('something went wrong')); + // Action, Assert + loader.setSource('test', compiledData); + await expect(compiler.execute('test', {})).rejects.toThrow( + 'something went wrong' + ); +}); + +it('should throw an error if one of sub builders failed to execute', async () => { + // Arrange + const { compiler, loader, builder } = createTestCompiler(); + const { compiledData } = compiler.compile( + ` +{% req user %} +select * from users; +{% endreq %} +select * from group where userId = '{{ user.value()[0].id }}'; +` + ); + builder.value.onFirstCall().rejects(new Error('something went wrong')); + // Action, Assert + loader.setSource('test', compiledData); + await expect(compiler.execute('test', {})).rejects.toThrow( + 'something went wrong' + ); +}); diff --git a/packages/core/test/template-engine/extensions/tags/req.spec.ts b/packages/core/test/template-engine/extensions/tags/req.spec.ts index d4831412..a647b5fa 100644 --- a/packages/core/test/template-engine/extensions/tags/req.spec.ts +++ b/packages/core/test/template-engine/extensions/tags/req.spec.ts @@ -1,6 +1,6 @@ import { createTestCompiler } from '../../testCompiler'; -it('req extension should execute correct query and set variable', async () => { +it('req extension should execute correct query and set/export the variable', async () => { // Arrange const { compiler, loader, builder, executor } = createTestCompiler(); const { compiledData } = compiler.compile(` @@ -63,6 +63,20 @@ some statement ).toThrow(`Expected a symbol "main"`); }); +it('if argument have too many elements, extension should throw an error', async () => { + // Arrange + const { compiler } = createTestCompiler(); + + // Action, Assert + expect(() => + compiler.compile(` +{% req user main more %} +select count(*) as count from user where user.id = '{{ params.userId }}'; +{% endreq %} + `) + ).toThrow(`Expected a block end, but got symbol`); +}); + it('the main denotation should be parsed into the second args node', async () => { // Arrange const { compiler } = createTestCompiler(); diff --git a/packages/core/test/template-engine/templateEngine.spec.ts b/packages/core/test/template-engine/templateEngine.spec.ts index 4cce1dd2..40d83949 100644 --- a/packages/core/test/template-engine/templateEngine.spec.ts +++ b/packages/core/test/template-engine/templateEngine.spec.ts @@ -31,7 +31,7 @@ beforeEach(() => { errors: [], }, }); - stubCompiler.execute.resolves('sql-statement'); + stubCompiler.execute.resolves('sql-result'); const generator = async function* () { yield { @@ -80,5 +80,5 @@ it('Template engine render function should forward correct data to compiler', as // Assert expect(stubCompiler.execute.calledWith('template-name', context)).toBe(true); - expect(result).toBe('sql-statement'); + expect(result).toBe('sql-result'); }); diff --git a/packages/core/test/template-engine/visitors/astWalker.spec.ts b/packages/core/test/template-engine/visitors/astWalker.spec.ts index 2c2a9df7..859d0041 100644 --- a/packages/core/test/template-engine/visitors/astWalker.spec.ts +++ b/packages/core/test/template-engine/visitors/astWalker.spec.ts @@ -1,4 +1,4 @@ -import { walkAst } from '@vulcan/core/template-engine'; +import { visitChildren, walkAst } from '@vulcan/core/template-engine'; import * as nunjucks from 'nunjucks'; it('AST walker should traversal all nodes', async () => { @@ -23,3 +23,183 @@ it('AST walker should traversal all nodes', async () => { // Assert expect(visitedNodes).toBe(5); }); + +it('AST visit children function should visit all the child node of NodeList', async () => { + // Arrange + const root = new nunjucks.nodes.NodeList(0, 0); + root.addChild( + new nunjucks.nodes.CallExtension( // 1 + {}, + 'run', + new nunjucks.nodes.NodeList(0, 0), + [new nunjucks.nodes.Literal(0, 0, 'a')] + ) + ); + root.addChild(new nunjucks.nodes.Literal(0, 0, 'b')); // 2 + let visitedNodes = 0; + // Act + visitChildren(root, () => visitedNodes++); + // Assert + expect(visitedNodes).toBe(2); +}); + +it('AST visit children function should visit all the child node of CallExtension', async () => { + // Arrange + const root = new nunjucks.nodes.CallExtensionAsync( + 'test', + 'run', + new nunjucks.nodes.NodeList(0, 0) /* 1 */, + [new nunjucks.nodes.Literal(0, 0, 'a') /* 2 */] + ); + + let visitedNodes = 0; + // Act + visitChildren(root, () => visitedNodes++); + // Assert + expect(visitedNodes).toBe(2); +}); + +it('AST visit children function should visit all the child node of LookupVal', async () => { + // Arrange + const root = new nunjucks.nodes.LookupVal( + 0, + 0, + new nunjucks.nodes.Symbol(0, 0, 'a') /* 1 */ + ); + + let visitedNodes = 0; + // Act + visitChildren(root, () => visitedNodes++); + // Assert + expect(visitedNodes).toBe(1); +}); + +it('AST replace function should work with NodeList', async () => { + // Arrange + const root = new nunjucks.nodes.NodeList(0, 0); + root.addChild( + new nunjucks.nodes.CallExtension( // 1 + {}, + 'run', + new nunjucks.nodes.NodeList(0, 0), + [new nunjucks.nodes.Literal(0, 0, 'a')] + ) + ); + root.addChild(new nunjucks.nodes.Literal(0, 0, 'b')); // 2 + root.addChild(new nunjucks.nodes.Symbol(0, 0, 'c')); // 3 + let visitedNodes = 0; + // Act + visitChildren(root, (_node, replace) => { + if (visitedNodes === 0) { + // Replace the first node + replace(new nunjucks.nodes.Literal(0, 0, 'c')); + } else if (visitedNodes === 1) { + // Delete the second node + replace(null); + } else { + // Replace the third node + replace(new nunjucks.nodes.Literal(0, 0, 'd')); + } + visitedNodes++; + }); + // Assert + expect(root.children[0] instanceof nunjucks.nodes.Literal).toBe(true); + expect(root.children[1] instanceof nunjucks.nodes.Literal).toBe(true); + expect(root.children.length).toBe(2); +}); + +it('AST replace function should work with CallExtension', async () => { + // Arrange + const root = new nunjucks.nodes.CallExtensionAsync( + 'test', + 'run', + new nunjucks.nodes.NodeList(0, 0) /* 1 */, + [ + new nunjucks.nodes.Literal(0, 0, 'a') /* 2 */, + new nunjucks.nodes.Literal(0, 0, 'b') /* 3 */, + new nunjucks.nodes.Literal(0, 0, 'c') /* 4 */, + ] + ); + let visitedNodes = 0; + // Act + visitChildren(root, (_node, replace) => { + if (visitedNodes === 0) { + // Replace the first node + replace(new nunjucks.nodes.NodeList(0, 1)); + } else if (visitedNodes === 1) { + // Replace the second node + replace(new nunjucks.nodes.Symbol(0, 0, 'a-r')); + } else if (visitedNodes === 2) { + // Delete the third node + replace(null); + } + visitedNodes++; + }); + // Assert + expect(root.args.colno).toBe(1); + expect(root.contentArgs![0] instanceof nunjucks.nodes.Symbol).toBe(true); + expect(root.contentArgs![1] instanceof nunjucks.nodes.Literal).toBe(true); + expect(root.contentArgs?.length).toBe(2); +}); + +it('AST replace function should provide fallback for CallExtension if we trying to delete args node', async () => { + // Arrange + const args = new nunjucks.nodes.NodeList(1, 1); + args.addChild(new nunjucks.nodes.Literal(0, 0, 'a')); + const root = new nunjucks.nodes.CallExtensionAsync('test', 'run', args, []); + // Act + visitChildren(root, (node, replace) => { + if (node instanceof nunjucks.nodes.NodeList) { + replace(null); + } + }); + // Assert + expect(root.args.lineno).toBe(1); + expect(root.args.colno).toBe(1); + expect(root.args.children.length).toBe(0); +}); + +it('AST replace function should wrap the result for CallExtension if we trying to replace args with nodes other than NodeList', async () => { + // Arrange + const root = new nunjucks.nodes.CallExtensionAsync( + 'test', + 'run', + new nunjucks.nodes.NodeList(1, 1), + [] + ); + // Act + visitChildren(root, (node, replace) => { + if (node instanceof nunjucks.nodes.NodeList) { + replace(new nunjucks.nodes.Literal(0, 0, 'a')); + } + }); + // Assert + expect(root.args.lineno).toBe(1); + expect(root.args.colno).toBe(1); + expect(root.args.children[0] instanceof nunjucks.nodes.Literal).toBe(true); +}); + +it('AST replace function should work with LookupVal', async () => { + // Arrange + const root = new nunjucks.nodes.LookupVal( + 0, + 0, + new nunjucks.nodes.Literal(0, 0, 'a') /* target 1 */, + new nunjucks.nodes.Value(0, 0, 'b') /* value 2 */ + ); + let visitedNodes = 0; + // Act + visitChildren(root, (_node, replace) => { + if (visitedNodes === 0) { + // Replace the first node + replace(new nunjucks.nodes.Symbol(0, 0, 'c')); + } else if (visitedNodes === 1) { + // Delete the second node + replace(null); + } + visitedNodes++; + }); + // Assert + expect(root.target instanceof nunjucks.nodes.Symbol).toBe(true); + expect(root.val).toBeUndefined(); +}); diff --git a/packages/core/test/template-engine/visitors/builderValue.spec.ts b/packages/core/test/template-engine/visitors/builderValue.spec.ts new file mode 100644 index 00000000..da7d5d24 --- /dev/null +++ b/packages/core/test/template-engine/visitors/builderValue.spec.ts @@ -0,0 +1,16 @@ +import { BuilderValueVisitor, walkAst } from '@vulcan/core/template-engine'; +import * as nunjucks from 'nunjucks'; + +it('Should throw an error if max depth exceeded (100)', async () => { + // Arrange + let queryString = '{{ something.a'; + for (let i = 0; i < 101; i++) { + queryString += '.a'; + } + queryString += '.value() }}'; + const ast = nunjucks.parser.parse(queryString, [], {}); + const visitor = new BuilderValueVisitor(); + // Act, Arrange + + expect(() => walkAst(ast, [visitor])).toThrow('Max depth reached'); +}); diff --git a/packages/core/test/template-engine/visitors/mainBuilder.spec.ts b/packages/core/test/template-engine/visitors/mainBuilder.spec.ts index 916689ce..a17f4057 100644 --- a/packages/core/test/template-engine/visitors/mainBuilder.spec.ts +++ b/packages/core/test/template-engine/visitors/mainBuilder.spec.ts @@ -69,3 +69,19 @@ where id = {{ params.id }}; ); expect((ast.children[0] as any).args.children[1].value).toBe('true'); // main builder notation }); + +it('Should throw an error if there is no root node', async () => { + // Arrange + const ast = nunjucks.parser.parse( + ` +select * from users +where id = {{ params.id }}; + `, + extensions, + {} + ); + const visitor = new MainBuilderVisitor(); + // Act, Arrange + walkAst(ast.children[0], [visitor]); // We start visiting the children of the root node, skipping the root node itself + expect(() => visitor.finish()).toThrow('No root node found.'); +}); From b82aa58016f7e9f96127692bd260ed3c8a79b09a Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Wed, 8 Jun 2022 18:08:14 +0800 Subject: [PATCH 06/12] refactor(core): extensions modulize - Extensions now have two kinds: runtime and compileTime extensions. - Put the extensions with same feature togeter instead of grouping by types. - We only test with a full extension now, instead of test by files. --- packages/core/src/containers/container.ts | 6 +- .../core/src/containers/modules/executor.ts | 6 +- .../src/containers/modules/templateEngine.ts | 44 ++-- packages/core/src/containers/types.ts | 1 + .../custom-error/constants.ts | 1 + .../custom-error/errorTagBuilder.ts | 73 +++++++ .../custom-error/errorTagRunner.ts | 12 ++ .../built-in-extensions/custom-error/index.ts | 4 + .../built-in-extensions/index.ts | 4 + .../query-builder/constants.ts | 5 + .../query-builder/executeBuilder.ts | 6 + .../query-builder/executorRunner.ts | 12 ++ .../query-builder/index.ts | 6 + .../query-builder/reqTagBuilder.ts | 198 ++++++++++++++++++ .../query-builder/reqTagRunner.ts | 43 ++++ .../built-in-extensions/sql-helper/index.ts | 4 + .../sql-helper/uniqueFilterBuilder.ts | 5 + .../sql-helper/uniqueFilterRunner.ts} | 8 +- .../validator/constants.ts | 4 + .../validator/filterChecker.ts | 45 ++++ .../built-in-extensions/validator/index.ts | 4 + .../validator/parametersChecker.ts} | 35 ++-- .../core/src/lib/template-engine/compiler.ts | 6 +- .../helpers.ts} | 14 +- .../template-engine/extension-loader/index.ts | 3 + .../extension-loader/loader.ts | 31 +++ .../extension-loader/models.ts | 139 ++++++++++++ .../template-engine/extensions/extension.ts | 136 ------------ .../extensions/filters/execute.ts | 12 -- .../extensions/filters/index.ts | 2 - .../lib/template-engine/extensions/index.ts | 3 - .../template-engine/extensions/tags/error.ts | 43 ---- .../template-engine/extensions/tags/index.ts | 2 - .../template-engine/extensions/tags/req.ts | 125 ----------- .../core/src/lib/template-engine/index.ts | 3 +- .../lib/template-engine/nunjucksCompiler.ts | 140 ++++++++----- .../visitors/builderValueVisitor.ts | 89 -------- .../template-engine/visitors/errorsVisitor.ts | 49 ----- .../visitors/filtersVisitor.ts | 23 -- .../src/lib/template-engine/visitors/index.ts | 7 - .../visitors/mainBuilderVisitor.ts | 55 ----- .../lib/template-engine/visitors/visitor.ts | 6 - .../core/test/template-engine/builder.spec.ts | 31 --- .../custom-error/custom-error.spec.ts | 61 ++++++ .../query-builder/builder.spec.ts} | 38 ++-- .../query-builder}/execute.spec.ts | 14 +- .../sql-helper}/unique.spec.ts | 6 +- .../validator/filters.spec.ts | 28 +++ .../validator}/parameters.spec.ts | 32 +-- .../test/template-engine/commons/edge.spec.ts | 14 ++ .../helpers.spec.ts} | 6 +- .../extension-loader/loader.spec.ts | 28 +++ .../extensions/tags/error.spec.ts | 14 -- .../template-engine/nunjuckCompiler.spec.ts | 16 +- .../core/test/template-engine/testCompiler.ts | 54 ++--- .../visitors/builderValue.spec.ts | 16 -- .../template-engine/visitors/errors.spec.ts | 82 -------- .../template-engine/visitors/filter.spec.ts | 39 ---- .../visitors/mainBuilder.spec.ts | 87 -------- types/nunjucks.d.ts | 2 +- 60 files changed, 971 insertions(+), 1011 deletions(-) create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/custom-error/constants.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagBuilder.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagRunner.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/custom-error/index.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/index.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/constants.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/executeBuilder.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/index.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterBuilder.ts rename packages/core/src/lib/template-engine/{extensions/filters/unique.ts => built-in-extensions/sql-helper/uniqueFilterRunner.ts} (63%) create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/validator/constants.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/validator/filterChecker.ts create mode 100644 packages/core/src/lib/template-engine/built-in-extensions/validator/index.ts rename packages/core/src/lib/template-engine/{visitors/parametersVisitor.ts => built-in-extensions/validator/parametersChecker.ts} (69%) rename packages/core/src/lib/template-engine/{visitors/astWalker.ts => extension-loader/helpers.ts} (86%) create mode 100644 packages/core/src/lib/template-engine/extension-loader/index.ts create mode 100644 packages/core/src/lib/template-engine/extension-loader/loader.ts create mode 100644 packages/core/src/lib/template-engine/extension-loader/models.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/extension.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/filters/execute.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/filters/index.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/index.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/tags/error.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/tags/index.ts delete mode 100644 packages/core/src/lib/template-engine/extensions/tags/req.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/errorsVisitor.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/filtersVisitor.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/index.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts delete mode 100644 packages/core/src/lib/template-engine/visitors/visitor.ts delete mode 100644 packages/core/test/template-engine/builder.spec.ts create mode 100644 packages/core/test/template-engine/built-in-extensions/custom-error/custom-error.spec.ts rename packages/core/test/template-engine/{extensions/tags/req.spec.ts => built-in-extensions/query-builder/builder.spec.ts} (62%) rename packages/core/test/template-engine/{extensions/filters => built-in-extensions/query-builder}/execute.spec.ts (69%) rename packages/core/test/template-engine/{extensions/filters => built-in-extensions/sql-helper}/unique.spec.ts (87%) create mode 100644 packages/core/test/template-engine/built-in-extensions/validator/filters.spec.ts rename packages/core/test/template-engine/{visitors => built-in-extensions/validator}/parameters.spec.ts (55%) create mode 100644 packages/core/test/template-engine/commons/edge.spec.ts rename packages/core/test/template-engine/{visitors/astWalker.spec.ts => extension-loader/helpers.spec.ts} (96%) create mode 100644 packages/core/test/template-engine/extension-loader/loader.spec.ts delete mode 100644 packages/core/test/template-engine/extensions/tags/error.spec.ts delete mode 100644 packages/core/test/template-engine/visitors/builderValue.spec.ts delete mode 100644 packages/core/test/template-engine/visitors/errors.spec.ts delete mode 100644 packages/core/test/template-engine/visitors/filter.spec.ts delete mode 100644 packages/core/test/template-engine/visitors/mainBuilder.spec.ts diff --git a/packages/core/src/containers/container.ts b/packages/core/src/containers/container.ts index e464c1d5..7811c811 100644 --- a/packages/core/src/containers/container.ts +++ b/packages/core/src/containers/container.ts @@ -14,10 +14,12 @@ export class Container { return this.inversifyContainer.get(type); } - public load(options: ICoreOptions) { + public async load(options: ICoreOptions) { this.inversifyContainer.load(artifactBuilderModule(options.artifact)); this.inversifyContainer.load(executorModule()); - this.inversifyContainer.load(templateEngineModule(options.template)); + await this.inversifyContainer.loadAsync( + templateEngineModule(options.template) + ); this.inversifyContainer.load(validatorModule()); } diff --git a/packages/core/src/containers/modules/executor.ts b/packages/core/src/containers/modules/executor.ts index 2948d182..6e5492dd 100644 --- a/packages/core/src/containers/modules/executor.ts +++ b/packages/core/src/containers/modules/executor.ts @@ -1,5 +1,9 @@ import { ContainerModule } from 'inversify'; -import { Executor, QueryBuilder } from '@vulcan/core/template-engine'; +// TODO: Should replace with a real implementation +import { + QueryBuilder, + Executor, +} from '../../lib/template-engine/built-in-extensions/query-builder/reqTagRunner'; import { TYPES } from '../types'; class MockBuilder implements QueryBuilder { diff --git a/packages/core/src/containers/modules/templateEngine.ts b/packages/core/src/containers/modules/templateEngine.ts index efe6966c..18af5dfe 100644 --- a/packages/core/src/containers/modules/templateEngine.ts +++ b/packages/core/src/containers/modules/templateEngine.ts @@ -4,24 +4,21 @@ import { TemplateProviderType, } from '@vulcan/core/models'; import { - ErrorExtension, FileTemplateProvider, - NunjucksCompilerExtension, - ReqExtension, TemplateProvider, - UniqueExtension, InMemoryCodeLoader, NunjucksCompiler, Compiler, TemplateEngine, - ExecuteExtension, } from '@vulcan/core/template-engine'; -import { ContainerModule, interfaces } from 'inversify'; +import { AsyncContainerModule, interfaces } from 'inversify'; import { TemplateEngineOptions } from '../../options'; import * as nunjucks from 'nunjucks'; +// TODO: fix the path +import { bindExtensions } from '@vulcan/core/template-engine/extension-loader'; export const templateEngineModule = (options: ITemplateEngineOptions) => - new ContainerModule((bind) => { + new AsyncContainerModule(async (bind) => { // Options bind( TYPES.TemplateEngineInputOptions @@ -40,19 +37,23 @@ export const templateEngineModule = (options: ITemplateEngineOptions) => TYPES.Factory_TemplateProvider ).toAutoNamedFactory(TYPES.TemplateProvider); - // Extensions - bind(TYPES.CompilerExtension) - .to(UniqueExtension) - .inSingletonScope(); - bind(TYPES.CompilerExtension) - .to(ErrorExtension) - .inSingletonScope(); - bind(TYPES.CompilerExtension) - .to(ReqExtension) - .inSingletonScope(); - bind(TYPES.CompilerExtension) - .to(ExecuteExtension) - .inSingletonScope(); + // Compiler environment + bind(TYPES.CompilerEnvironment) + .toDynamicValue((context) => { + // We only need loader in runtime + const loader = context.container.get( + TYPES.CompilerLoader + ); + return new nunjucks.Environment(loader); + }) + .inSingletonScope() + .whenTargetNamed('runtime'); + bind(TYPES.CompilerEnvironment) + .toDynamicValue(() => { + return new nunjucks.Environment(); + }) + .inSingletonScope() + .whenTargetNamed('compileTime'); // Loader bind(TYPES.CompilerLoader) @@ -66,4 +67,7 @@ export const templateEngineModule = (options: ITemplateEngineOptions) => bind(TYPES.TemplateEngine) .to(TemplateEngine) .inSingletonScope(); + + // Load Extensions + await bindExtensions(bind); }); diff --git a/packages/core/src/containers/types.ts b/packages/core/src/containers/types.ts index 492f581b..22ae4086 100644 --- a/packages/core/src/containers/types.ts +++ b/packages/core/src/containers/types.ts @@ -12,6 +12,7 @@ export const TYPES = { Factory_TemplateProvider: Symbol.for('Factory_TemplateProvider'), CompilerExtension: Symbol.for('CompilerExtension'), CompilerLoader: Symbol.for('CompilerLoader'), + CompilerEnvironment: Symbol.for('CompilerEnvironment'), Compiler: Symbol.for('Compiler'), TemplateEngine: Symbol.for('TemplateEngine'), TemplateEngineOptions: Symbol.for('TemplateEngineOptions'), diff --git a/packages/core/src/lib/template-engine/built-in-extensions/custom-error/constants.ts b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/constants.ts new file mode 100644 index 00000000..e828b664 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/constants.ts @@ -0,0 +1 @@ +export const METADATA_NAME = 'error.vulcan.com'; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagBuilder.ts new file mode 100644 index 00000000..8330d868 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagBuilder.ts @@ -0,0 +1,73 @@ +import { + OnAstVisit, + ProvideMetadata, + TagBuilder, +} from '../../extension-loader'; +import * as nunjucks from 'nunjucks'; +import { chain } from 'lodash'; +import { METADATA_NAME } from './constants'; + +interface ErrorCode { + code: string; + lineNo: number; + columnNo: number; +} + +export class ErrorTagBuilder + extends TagBuilder + implements OnAstVisit, ProvideMetadata +{ + public tags = ['error']; + private errorCodes: ErrorCode[] = []; + public metadataName = METADATA_NAME; + + public parse(parser: nunjucks.parser.Parser, nodes: typeof nunjucks.nodes) { + // get the tag token + const token = parser.nextToken(); + + const errorMessage = parser.parseSignature(null, true); + parser.advanceAfterBlockEnd(token.value); + + // Add some fake nodes to the AST to indicate error position + errorMessage.addChild( + new nodes.Literal(token.lineno, token.colno, token.lineno) + ); + errorMessage.addChild( + new nodes.Literal(token.lineno, token.colno, token.colno) + ); + + return this.createAsyncExtensionNode(errorMessage, []); + } + + public onVisit(node: nunjucks.nodes.Node) { + if (node instanceof nunjucks.nodes.CallExtension) { + if (node.extName !== this.getName()) return; + + const errorCodeNode = node.args.children[0]; + if (!(errorCodeNode instanceof nunjucks.nodes.Literal)) + throw new Error(`Expected literal, got ${errorCodeNode.typename}`); + + this.errorCodes.push({ + code: errorCodeNode.value, + lineNo: errorCodeNode.lineno, + columnNo: errorCodeNode.colno, + }); + } + } + + public getMetadata() { + return { + errorCodes: chain(this.errorCodes) + .groupBy('code') + .values() + .map((errorCodes) => ({ + code: errorCodes[0].code, + locations: errorCodes.map((errorCode) => ({ + lineNo: errorCode.lineNo, + columnNo: errorCode.columnNo, + })), + })) + .value(), + }; + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagRunner.ts b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagRunner.ts new file mode 100644 index 00000000..e11f9913 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/errorTagRunner.ts @@ -0,0 +1,12 @@ +import { TagRunner, TagRunnerOptions } from '../../extension-loader'; + +export class ErrorTagRunner extends TagRunner { + public tags = ['error']; + + public async run({ args }: TagRunnerOptions) { + const message = args[0]; + const lineno = args[1]; + const colno = args[2]; + throw new Error(`${message} at ${lineno}:${colno}`); + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/custom-error/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/index.ts new file mode 100644 index 00000000..99ee5b51 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/custom-error/index.ts @@ -0,0 +1,4 @@ +import { ErrorTagBuilder } from './errorTagBuilder'; +import { ErrorTagRunner } from './errorTagRunner'; + +export default [ErrorTagBuilder, ErrorTagRunner]; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/index.ts new file mode 100644 index 00000000..1449f292 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/index.ts @@ -0,0 +1,4 @@ +import './query-builder'; +import './custom-error'; +import './sql-helper'; +import './validator'; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/constants.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/constants.ts new file mode 100644 index 00000000..b6939c9a --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/constants.ts @@ -0,0 +1,5 @@ +export const FINIAL_BUILDER_NAME = 'FINAL_BUILDER'; +export const METADATA_NAME = 'builder.vulcan.com'; +export const EXECUTE_COMMAND_NAME = 'value'; +export const EXECUTE_FILTER_NAME = 'execute'; +export const REFERENCE_SEARCH_MAX_DEPTH = 100; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executeBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executeBuilder.ts new file mode 100644 index 00000000..a05f0877 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executeBuilder.ts @@ -0,0 +1,6 @@ +import { FilterBuilder } from '../../extension-loader'; +import { EXECUTE_FILTER_NAME } from './constants'; + +export class ExecutorBuilder extends FilterBuilder { + public filterName = EXECUTE_FILTER_NAME; +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts new file mode 100644 index 00000000..16e621ae --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts @@ -0,0 +1,12 @@ +import { FilterRunner } from '../../extension-loader'; +import { EXECUTE_FILTER_NAME } from './constants'; +import { QueryBuilder } from './reqTagRunner'; + +export class ExecutorRunner extends FilterRunner { + public filterName = EXECUTE_FILTER_NAME; + + public async transform({ value }: { value: any; args: any[] }): Promise { + const builder: QueryBuilder = value; + return builder.value(); + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/index.ts new file mode 100644 index 00000000..7c02c185 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/index.ts @@ -0,0 +1,6 @@ +import { ReqTagBuilder } from './reqTagBuilder'; +import { ReqTagRunner } from './reqTagRunner'; +import { ExecutorRunner } from './executorRunner'; +import { ExecutorBuilder } from './executeBuilder'; + +export default [ReqTagBuilder, ReqTagRunner, ExecutorRunner, ExecutorBuilder]; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts new file mode 100644 index 00000000..79d0bca6 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts @@ -0,0 +1,198 @@ +import { + OnAstVisit, + ProvideMetadata, + ReplaceChildFunc, + TagBuilder, + visitChildren, +} from '../../extension-loader'; +import * as nunjucks from 'nunjucks'; +import { + EXECUTE_COMMAND_NAME, + EXECUTE_FILTER_NAME, + FINIAL_BUILDER_NAME, + METADATA_NAME, + REFERENCE_SEARCH_MAX_DEPTH, +} from './constants'; + +export class ReqTagBuilder + extends TagBuilder + implements OnAstVisit, ProvideMetadata +{ + public tags = ['req']; + public metadataName = METADATA_NAME; + private root?: nunjucks.nodes.Root; + private hasMainBuilder = false; + private variableList = new Set(); + + public parse( + parser: nunjucks.parser.Parser, + nodes: typeof nunjucks.nodes, + lexer: typeof nunjucks.lexer + ) { + // {% req var (main) %} body {% endreq %} + // consume req tag + const reqToken = parser.nextToken(); + // variable + let nextToken = parser.peekToken(); + if (nextToken.type === lexer.TOKEN_BLOCK_END) { + // {% req %} + parser.fail(`Expected a variable`, nextToken.lineno, nextToken.colno); + } + if (nextToken.type !== lexer.TOKEN_SYMBOL) { + parser.fail( + `Expected a symbol, but got ${nextToken.type}`, + nextToken.lineno, + nextToken.colno + ); + } + const variable = parser.parseExpression(); + + // main denotation + nextToken = parser.peekToken(); + let mainBuilder = false; + if (nextToken.type !== lexer.TOKEN_BLOCK_END) { + if (nextToken.type !== lexer.TOKEN_SYMBOL || nextToken.value !== 'main') { + parser.fail( + `Expected a symbol "main"`, + nextToken.lineno, + nextToken.colno + ); + } + mainBuilder = true; + // Consume this token (main) + parser.nextToken(); + } + + const endToken = parser.nextToken(); + if (endToken.type !== lexer.TOKEN_BLOCK_END) { + parser.fail( + `Expected a block end, but got ${endToken.type}`, + endToken.lineno, + endToken.colno + ); + } + + const requestQuery = parser.parseUntilBlocks('endreq'); + parser.advanceAfterBlockEnd(); + + const argsNodeToPass = new nodes.NodeList(reqToken.lineno, reqToken.colno); + // variable name + argsNodeToPass.addChild( + new nodes.Literal( + variable.colno, + variable.lineno, + (variable as nunjucks.nodes.Symbol).value + ) + ); + // is main builder + argsNodeToPass.addChild( + new nodes.Literal(variable.colno, variable.lineno, String(mainBuilder)) + ); + + return this.createAsyncExtensionNode(argsNodeToPass, [requestQuery]); + } + + public onVisit(node: nunjucks.nodes.Node) { + // save the root + if (node instanceof nunjucks.nodes.Root) { + this.root = node; + } else if ( + node instanceof nunjucks.nodes.CallExtensionAsync && + node.extName === this.getName() + ) { + const variable = node.args.children[0] as nunjucks.nodes.Literal; + this.variableList.add(variable.value); + this.checkMainBuilder(node); + } else { + visitChildren(node, this.replaceExecuteFunction.bind(this)); + } + } + + public finish() { + if (!this.hasMainBuilder) { + this.wrapOutputWithBuilder(); + } + } + + public getMetadata() { + return { + finalBuilderName: FINIAL_BUILDER_NAME, + }; + } + + private checkMainBuilder(node: nunjucks.nodes.CallExtensionAsync) { + const isMainBuilder = + node.extName === this.getName() && + (node.args.children[1] as nunjucks.nodes.Literal).value === 'true'; + if (!isMainBuilder) return; + + if (this.hasMainBuilder) { + throw new Error(`Only one main builder is allowed.`); + } + this.hasMainBuilder = true; + } + + private wrapOutputWithBuilder() { + if (!this.root) { + throw new Error('No root node found.'); + } + const originalChildren = this.root.children; + const args = new nunjucks.nodes.NodeList(0, 0); + // variable name + args.addChild(new nunjucks.nodes.Literal(0, 0, '__wrapped__builder')); + // is main builder + args.addChild(new nunjucks.nodes.Literal(0, 0, 'true')); + const builder = this.createAsyncExtensionNode(args, originalChildren); + this.root.children = [builder]; + } + + private replaceExecuteFunction( + node: nunjucks.nodes.Node, + replace: ReplaceChildFunc + ) { + if ( + node instanceof nunjucks.nodes.FunCall && + node.name instanceof nunjucks.nodes.LookupVal && + node.name.val.value === EXECUTE_COMMAND_NAME + ) { + let targetNode: typeof node.name.target | null = node.name.target; + let depth = 0; + while (targetNode) { + depth++; + if (depth > REFERENCE_SEARCH_MAX_DEPTH) { + throw new Error('Max depth reached'); + } + if (targetNode instanceof nunjucks.nodes.LookupVal) { + targetNode = targetNode.target; + } else if (targetNode instanceof nunjucks.nodes.FunCall) { + targetNode = targetNode.name; + } else if (targetNode instanceof nunjucks.nodes.Symbol) { + break; + } else { + throw new Error( + `Unexpected node type: ${ + (targetNode as nunjucks.nodes.Node)?.typename + }` + ); + } + } + + // If the target node is a variable from {% req xxx %}, replace it with execute filter + if (this.variableList.has((targetNode as nunjucks.nodes.Symbol).value)) { + const args = new nunjucks.nodes.NodeList(node.lineno, node.colno); + args.addChild(node.name.target); + const filter = new nunjucks.nodes.Filter( + node.lineno, + node.colno, + new nunjucks.nodes.Symbol( + node.lineno, + node.colno, + EXECUTE_FILTER_NAME + ), + args + ); + replace(filter); + } + } + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts new file mode 100644 index 00000000..49c3d4ee --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts @@ -0,0 +1,43 @@ +import { TYPES } from '@vulcan/core/containers'; +import { inject } from 'inversify'; +import { TagRunnerOptions, TagRunner } from '../../extension-loader'; +import { FINIAL_BUILDER_NAME } from './constants'; + +// TODO: temporary interface +export interface QueryBuilder { + count(): QueryBuilder; + value(): Promise; +} + +export interface Executor { + createBuilder(query: string): Promise; +} + +export class ReqTagRunner extends TagRunner { + public tags = ['req']; + private executor: Executor; + + constructor(@inject(TYPES.Executor) executor: Executor) { + super(); + this.executor = executor; + } + + public async run({ context, args, contentArgs }: TagRunnerOptions) { + const name = args[0]; + let query = ''; + for (let index = 0; index < contentArgs.length; index++) { + query += await contentArgs[index](); + } + query = query + .split(/\r?\n/) + .filter((line) => line.trim().length > 0) + .join('\n'); + const builder = await this.executor.createBuilder(query); + context.setVariable(name, builder); + + if (args[1] === 'true') { + context.setVariable(FINIAL_BUILDER_NAME, builder); + context.addExport(FINIAL_BUILDER_NAME); + } + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts new file mode 100644 index 00000000..8c9b3d76 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/index.ts @@ -0,0 +1,4 @@ +import { UniqueFilterBuilder } from './uniqueFilterBuilder'; +import { UniqueFilterRunner } from './uniqueFilterRunner'; + +export default [UniqueFilterBuilder, UniqueFilterRunner]; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterBuilder.ts new file mode 100644 index 00000000..9af90c8b --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterBuilder.ts @@ -0,0 +1,5 @@ +import { FilterBuilder } from '../../extension-loader'; + +export class UniqueFilterBuilder extends FilterBuilder { + public filterName = 'unique'; +} diff --git a/packages/core/src/lib/template-engine/extensions/filters/unique.ts b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterRunner.ts similarity index 63% rename from packages/core/src/lib/template-engine/extensions/filters/unique.ts rename to packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterRunner.ts index 06637b43..49a2ee7a 100644 --- a/packages/core/src/lib/template-engine/extensions/filters/unique.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterRunner.ts @@ -1,10 +1,8 @@ -import { NunjucksFilterExtension } from '../extension'; import { uniq, uniqBy } from 'lodash'; -import { injectable } from 'inversify'; +import { FilterRunner } from '../../extension-loader'; -@injectable() -export class UniqueExtension implements NunjucksFilterExtension { - public name = 'unique'; +export class UniqueFilterRunner extends FilterRunner { + public filterName = 'unique'; public async transform({ value, args, diff --git a/packages/core/src/lib/template-engine/built-in-extensions/validator/constants.ts b/packages/core/src/lib/template-engine/built-in-extensions/validator/constants.ts new file mode 100644 index 00000000..13b87ab7 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/validator/constants.ts @@ -0,0 +1,4 @@ +export const PARAMETER_METADATA_NAME = 'parameter.vulcan.com'; +export const FILTER_METADATA_NAME = 'filter.vulcan.com'; +export const REFERENCE_SEARCH_MAX_DEPTH = 100; +export const LOOK_UP_PARAMETER = 'params'; diff --git a/packages/core/src/lib/template-engine/built-in-extensions/validator/filterChecker.ts b/packages/core/src/lib/template-engine/built-in-extensions/validator/filterChecker.ts new file mode 100644 index 00000000..e3d64030 --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/validator/filterChecker.ts @@ -0,0 +1,45 @@ +import { TYPES } from '@vulcan/core/containers'; +import { inject, named } from 'inversify'; +import { + CompileTimeExtension, + OnAstVisit, + ProvideMetadata, +} from '../../extension-loader'; +import { FILTER_METADATA_NAME } from './constants'; +import * as nunjucks from 'nunjucks'; + +export class FilterChecker + extends CompileTimeExtension + implements OnAstVisit, ProvideMetadata +{ + public metadataName = FILTER_METADATA_NAME; + private env: nunjucks.Environment; + private filters = new Set(); + + constructor( + @inject(TYPES.CompilerEnvironment) + @named('compileTime') + compileTimeEnv: nunjucks.Environment + ) { + super(); + this.env = compileTimeEnv; + } + + public onVisit(node: nunjucks.nodes.Node) { + if (node instanceof nunjucks.nodes.Filter) { + if ( + node.name instanceof nunjucks.nodes.Symbol || + node.name instanceof nunjucks.nodes.Literal + ) { + // If the node is a filter and has a expected name node, we check whether the filter is loaded. + // If the filter is not loaded, getFilter function will throw an error. + this.env.getFilter(node.name.value); + this.filters.add(node.name.value); + } + } + } + + public getMetadata() { + return Array.from(this.filters); + } +} diff --git a/packages/core/src/lib/template-engine/built-in-extensions/validator/index.ts b/packages/core/src/lib/template-engine/built-in-extensions/validator/index.ts new file mode 100644 index 00000000..18a8056a --- /dev/null +++ b/packages/core/src/lib/template-engine/built-in-extensions/validator/index.ts @@ -0,0 +1,4 @@ +import { ParametersChecker } from './parametersChecker'; +import { FilterChecker } from './filterChecker'; + +export default [ParametersChecker, FilterChecker]; diff --git a/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts b/packages/core/src/lib/template-engine/built-in-extensions/validator/parametersChecker.ts similarity index 69% rename from packages/core/src/lib/template-engine/visitors/parametersVisitor.ts rename to packages/core/src/lib/template-engine/built-in-extensions/validator/parametersChecker.ts index 2348aa66..9e818b56 100644 --- a/packages/core/src/lib/template-engine/visitors/parametersVisitor.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/validator/parametersChecker.ts @@ -1,9 +1,15 @@ import { chain } from 'lodash'; import * as nunjucks from 'nunjucks'; -import { TemplateParameterMetadata } from '../compiler'; -import { Visitor } from './visitor'; - -const MAX_DEPTH = 100; +import { + CompileTimeExtension, + OnAstVisit, + ProvideMetadata, +} from '../../extension-loader'; +import { + LOOK_UP_PARAMETER, + PARAMETER_METADATA_NAME, + REFERENCE_SEARCH_MAX_DEPTH, +} from './constants'; interface Parameter { name: string; @@ -11,24 +17,21 @@ interface Parameter { columnNo: number; } -export class ParametersVisitor implements Visitor { +export class ParametersChecker + extends CompileTimeExtension + implements OnAstVisit, ProvideMetadata +{ + public metadataName = PARAMETER_METADATA_NAME; private parameters: Parameter[] = []; - private lookupParameter: string; - - constructor({ - lookupParameter = 'params', - }: { lookupParameter?: string } = {}) { - this.lookupParameter = lookupParameter; - } - public visit(node: nunjucks.nodes.Node) { + public onVisit(node: nunjucks.nodes.Node): void { if (node instanceof nunjucks.nodes.LookupVal) { let name = node.val.value; let parent: typeof node.target | null = node.target; let depth = 0; while (parent) { depth++; - if (depth > MAX_DEPTH) { + if (depth > REFERENCE_SEARCH_MAX_DEPTH) { throw new Error('Max depth reached'); } if (parent instanceof nunjucks.nodes.LookupVal) { @@ -37,7 +40,7 @@ export class ParametersVisitor implements Visitor { } else if (parent instanceof nunjucks.nodes.FunCall) { parent = parent.name; } else { - if (parent.value === this.lookupParameter) { + if (parent.value === LOOK_UP_PARAMETER) { this.parameters.push({ name, lineNo: node.lineno, @@ -50,7 +53,7 @@ export class ParametersVisitor implements Visitor { } } - public getParameters(): TemplateParameterMetadata[] { + public getMetadata() { return chain(this.parameters) .groupBy('name') .values() diff --git a/packages/core/src/lib/template-engine/compiler.ts b/packages/core/src/lib/template-engine/compiler.ts index 108ccda2..1f79bdc8 100644 --- a/packages/core/src/lib/template-engine/compiler.ts +++ b/packages/core/src/lib/template-engine/compiler.ts @@ -13,11 +13,7 @@ export interface TemplateParameterMetadata { locations: TemplateLocation[]; } -export interface TemplateMetadata { - parameters: TemplateParameterMetadata[]; - errors: TemplateErrorMetadata[]; -} - +export type TemplateMetadata = Record; export interface CompileResult { compiledData: string; metadata: TemplateMetadata; diff --git a/packages/core/src/lib/template-engine/visitors/astWalker.ts b/packages/core/src/lib/template-engine/extension-loader/helpers.ts similarity index 86% rename from packages/core/src/lib/template-engine/visitors/astWalker.ts rename to packages/core/src/lib/template-engine/extension-loader/helpers.ts index 3710cf2e..ad78e1f1 100644 --- a/packages/core/src/lib/template-engine/visitors/astWalker.ts +++ b/packages/core/src/lib/template-engine/extension-loader/helpers.ts @@ -1,11 +1,19 @@ import * as nunjucks from 'nunjucks'; -import { Visitor } from './visitor'; +import { OnAstVisit, ProvideMetadata } from './models'; + +export const generateMetadata = (providers: ProvideMetadata[]) => { + const metadata = providers.reduce((currentMetadata, provider) => { + currentMetadata[provider.metadataName] = provider.getMetadata(); + return currentMetadata; + }, {} as Record); + return metadata; +}; export const walkAst = ( root: nunjucks.nodes.Node, - visitors: Visitor[] + visitors: OnAstVisit[] ): void => { - visitors.forEach((visitor) => visitor.visit(root)); + visitors.forEach((visitor) => visitor.onVisit(root)); visitChildren(root, (node) => walkAst(node, visitors)); if (root instanceof nunjucks.nodes.Root) { visitors.forEach((visitor) => visitor.finish?.()); diff --git a/packages/core/src/lib/template-engine/extension-loader/index.ts b/packages/core/src/lib/template-engine/extension-loader/index.ts new file mode 100644 index 00000000..0d0ec7e6 --- /dev/null +++ b/packages/core/src/lib/template-engine/extension-loader/index.ts @@ -0,0 +1,3 @@ +export * from './models'; +export * from './helpers'; +export * from './loader'; diff --git a/packages/core/src/lib/template-engine/extension-loader/loader.ts b/packages/core/src/lib/template-engine/extension-loader/loader.ts new file mode 100644 index 00000000..b81543d3 --- /dev/null +++ b/packages/core/src/lib/template-engine/extension-loader/loader.ts @@ -0,0 +1,31 @@ +// Import built in extensions to ensure TypeScript compiler includes them. +import '../built-in-extensions'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { flatten } from 'lodash'; +import { interfaces } from 'inversify'; +import { TYPES } from '@vulcan/core/containers'; + +export const importExtensions = async (folder: string) => { + const extensions = await import(folder); + return extensions.default || []; +}; + +export const bindExtensions = async (bind: interfaces.Bind) => { + const builtInExtensionNames = ( + await fs.readdir(path.join(__dirname, '..', 'built-in-extensions')) + ).filter((name) => name !== 'index.ts'); + + const extensions = flatten( + await Promise.all( + builtInExtensionNames.map((name) => + importExtensions( + path.join(__dirname, '..', 'built-in-extensions', name) + ) + ) + ) + ); + extensions.forEach((extension) => { + bind(TYPES.CompilerExtension).to(extension).inSingletonScope(); + }); +}; diff --git a/packages/core/src/lib/template-engine/extension-loader/models.ts b/packages/core/src/lib/template-engine/extension-loader/models.ts new file mode 100644 index 00000000..80276eef --- /dev/null +++ b/packages/core/src/lib/template-engine/extension-loader/models.ts @@ -0,0 +1,139 @@ +import { sortBy } from 'lodash'; +import * as nunjucks from 'nunjucks'; +import { injectable } from 'inversify'; + +export type TagExtensionContentArgGetter = () => Promise; + +export type TagExtensionArgTypes = string | number | boolean; + +export interface TagRunnerOptions { + context: any; + args: TagExtensionArgTypes[]; + contentArgs: TagExtensionContentArgGetter[]; +} + +export type Extension = RuntimeExtension | CompileTimeExtension; + +@injectable() +export abstract class RuntimeExtension {} + +@injectable() +export abstract class CompileTimeExtension {} + +export abstract class TagBuilder extends CompileTimeExtension { + abstract tags: string[]; + abstract parse( + parser: nunjucks.parser.Parser, + nodes: typeof nunjucks.nodes, + lexer: typeof nunjucks.lexer + ): nunjucks.nodes.Node; + + public set __name(_) { + // ignore it + } + + public get __name() { + return this.getName(); + } + + public getName() { + return sortBy(this.tags).join('_'); + } + + protected createAsyncExtensionNode( + /** The arguments if this extension, they'll be render to string and passed to run function */ + argsNodeList: nunjucks.nodes.NodeList, + /** The content (usually the body) of this extension, they'll be passed to run function as render functions */ + contentNodes: nunjucks.nodes.Node[] = [] + ) { + return new nunjucks.nodes.CallExtensionAsync( + this.getName(), + '__run', + argsNodeList, + contentNodes + ); + } +} + +export abstract class TagRunner extends RuntimeExtension { + abstract tags: string[]; + abstract run(options: TagRunnerOptions): Promise; + + public __run(...originalArgs: any[]) { + const context = originalArgs[0]; + const callback = originalArgs[originalArgs.length - 1]; + const args = originalArgs + .slice(1, originalArgs.length - 1) + .filter((value) => typeof value !== 'function'); + const contentArgs = originalArgs + .slice(1, originalArgs.length - 1) + .filter((value) => typeof value === 'function') + .map((cb) => () => { + return new Promise((resolve, reject) => { + cb((err: any, result: any) => { + if (err) reject(err); + else resolve(result); + }); + }); + }); + + this.run({ context, args, contentArgs }) + .then((result) => callback(null, result)) + .catch((err) => callback(err, null)); + } + + public set __name(_) { + // ignore it + } + + public get __name() { + return this.getName(); + } + + public getName() { + return sortBy(this.tags).join('_'); + } +} + +export abstract class FilterBuilder extends CompileTimeExtension { + abstract filterName: string; +} + +export abstract class FilterRunner extends RuntimeExtension { + abstract filterName: string; + abstract transform(options: { + value: V; + args: Record; + }): Promise; + + public __transform(value: any, ...args: any[]) { + const callback = args[args.length - 1]; + const otherArgs = args.slice(0, args.length - 1); + this.transform({ + value, + args: otherArgs, + }) + .then((res) => callback(null, res)) + .catch((err) => callback(err, null)); + } +} + +export const implementedOnAstVisit = (source: any): source is OnAstVisit => { + return !!source.onVisit; +}; + +export interface OnAstVisit { + onVisit(node: nunjucks.nodes.Node): void; + finish?: () => void; +} + +export const implementedProvideMetadata = ( + source: any +): source is ProvideMetadata => { + return !!source.metadataName && !!source.getMetadata; +}; + +export interface ProvideMetadata { + metadataName: string; + getMetadata(): any; +} diff --git a/packages/core/src/lib/template-engine/extensions/extension.ts b/packages/core/src/lib/template-engine/extensions/extension.ts deleted file mode 100644 index d18670a9..00000000 --- a/packages/core/src/lib/template-engine/extensions/extension.ts +++ /dev/null @@ -1,136 +0,0 @@ -import * as nunjucks from 'nunjucks'; - -export type NunjucksCompilerExtension = - | NunjucksTagExtension - | NunjucksFilterExtension; - -export interface NunjucksTagExtensionParseResult { - /** The arguments if this extension, they'll be render to string and passed to run function */ - argsNodeList: nunjucks.nodes.NodeList; - /** The content (usually the body) of this extension, they'll be passed to run function as render functions */ - contentNodes: nunjucks.nodes.Node[]; -} - -export type TagExtensionContentArgGetter = () => Promise; - -export type TagExtensionArgTypes = string | number | boolean; - -export interface NunjucksTagExtensionRunOptions { - context: any; - args: TagExtensionArgTypes[]; - contentArgs: TagExtensionContentArgGetter[]; -} - -class WrapperTagExtension { - // Nunjucks use this value as the name of this extension - public __name: string; - public tags: string[]; - - constructor(private extension: NunjucksTagExtension) { - this.__name = extension.name; - this.tags = extension.tags; - } - - public parse( - parser: nunjucks.parser.Parser, - nodes: typeof nunjucks.nodes, - lexer: typeof nunjucks.lexer - ) { - const { argsNodeList, contentNodes } = this.extension.parse( - parser, - nodes, - lexer - ); - return new nodes.CallExtensionAsync( - this, - '__run', - argsNodeList, - contentNodes - ); - } - - public __run(...originalArgs: any[]) { - const context = originalArgs[0]; - // Nunjucks use the pass the callback function for async extension at the last argument - // https://github.com/mozilla/nunjucks/blob/master/nunjucks/src/compiler.js#L256 - const callback = originalArgs[originalArgs.length - 1]; - const args = originalArgs - .slice(1, originalArgs.length - 1) - .filter((value) => typeof value !== 'function'); - const contentArgs = originalArgs - .slice(1, originalArgs.length - 1) - .filter((value) => typeof value === 'function') - .map((cb) => () => { - return new Promise((resolve, reject) => { - cb((err: any, result: any) => { - if (err) reject(err); - else resolve(result); - }); - }); - }); - - this.extension - .run({ context, args, contentArgs }) - .then((result) => callback(null, result)) - .catch((err) => callback(err, null)); - } -} - -export const NunjucksTagExtensionWrapper = ( - extension: NunjucksTagExtension -) => ({ - name: extension.name, - transform: new WrapperTagExtension(extension), -}); - -export interface NunjucksTagExtension { - name: string; - tags: string[]; - parse( - parser: nunjucks.parser.Parser, - nodes: typeof nunjucks.nodes, - lexer: typeof nunjucks.lexer - ): NunjucksTagExtensionParseResult; - run(options: NunjucksTagExtensionRunOptions): Promise; -} - -export function isTagExtension( - extension: NunjucksCompilerExtension -): extension is NunjucksTagExtension { - return ( - (extension as any).tags !== undefined && - (extension as any).parse !== undefined - ); -} - -export const NunjucksFilterExtensionWrapper = ( - extension: NunjucksFilterExtension -) => { - return { - name: extension.name, - transform: (value: any, ...args: any[]) => { - // Nunjucks use the pass the callback function for async filter at the last argument - // https://github.com/mozilla/nunjucks/blob/master/nunjucks/src/compiler.js#L514 - const callback = args[args.length - 1]; - const otherArgs = args.slice(0, args.length - 1); - extension - .transform({ - value, - args: otherArgs, - }) - .then((res) => callback(null, res)) - .catch((err) => callback(err, null)); - }, - }; -}; - -export interface NunjucksFilterExtension { - name: string; - transform(options: { value: V; args: Record }): Promise; -} - -export function isFilterExtension( - extension: NunjucksCompilerExtension -): extension is NunjucksFilterExtension { - return (extension as any).transform !== undefined; -} diff --git a/packages/core/src/lib/template-engine/extensions/filters/execute.ts b/packages/core/src/lib/template-engine/extensions/filters/execute.ts deleted file mode 100644 index 1abaf640..00000000 --- a/packages/core/src/lib/template-engine/extensions/filters/execute.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { NunjucksFilterExtension } from '../extension'; -import { injectable } from 'inversify'; -import { QueryBuilder } from '../tags'; - -@injectable() -export class ExecuteExtension implements NunjucksFilterExtension { - public name = 'execute'; - public async transform({ value }: { value: any; args: any[] }): Promise { - const builder: QueryBuilder = value; - return builder.value(); - } -} diff --git a/packages/core/src/lib/template-engine/extensions/filters/index.ts b/packages/core/src/lib/template-engine/extensions/filters/index.ts deleted file mode 100644 index f76e6638..00000000 --- a/packages/core/src/lib/template-engine/extensions/filters/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './unique'; -export * from './execute'; diff --git a/packages/core/src/lib/template-engine/extensions/index.ts b/packages/core/src/lib/template-engine/extensions/index.ts deleted file mode 100644 index 2727ea5b..00000000 --- a/packages/core/src/lib/template-engine/extensions/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './extension'; -export * from './tags'; -export * from './filters'; diff --git a/packages/core/src/lib/template-engine/extensions/tags/error.ts b/packages/core/src/lib/template-engine/extensions/tags/error.ts deleted file mode 100644 index 0adab33a..00000000 --- a/packages/core/src/lib/template-engine/extensions/tags/error.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { - NunjucksTagExtension, - NunjucksTagExtensionParseResult, - NunjucksTagExtensionRunOptions, -} from '../extension'; -import * as nunjucks from 'nunjucks'; -import { injectable } from 'inversify'; - -@injectable() -export class ErrorExtension implements NunjucksTagExtension { - public name = 'built-in-error'; - public tags = ['error']; - public parse( - parser: nunjucks.parser.Parser, - nodes: typeof nunjucks.nodes - ): NunjucksTagExtensionParseResult { - // get the tag token - const token = parser.nextToken(); - - const errorMessage = parser.parseSignature(null, true); - parser.advanceAfterBlockEnd(token.value); - - // Add some fake nodes to the AST to indicate error position - errorMessage.addChild( - new nodes.Literal(token.lineno, token.colno, token.lineno) - ); - errorMessage.addChild( - new nodes.Literal(token.lineno, token.colno, token.colno) - ); - - return { - argsNodeList: errorMessage, - contentNodes: [], - }; - } - - public async run({ args }: NunjucksTagExtensionRunOptions) { - const message = args[0]; - const lineno = args[1]; - const colno = args[2]; - throw new Error(`${message} at ${lineno}:${colno}`); - } -} diff --git a/packages/core/src/lib/template-engine/extensions/tags/index.ts b/packages/core/src/lib/template-engine/extensions/tags/index.ts deleted file mode 100644 index 716ec8a1..00000000 --- a/packages/core/src/lib/template-engine/extensions/tags/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './error'; -export * from './req'; diff --git a/packages/core/src/lib/template-engine/extensions/tags/req.ts b/packages/core/src/lib/template-engine/extensions/tags/req.ts deleted file mode 100644 index a5b39b34..00000000 --- a/packages/core/src/lib/template-engine/extensions/tags/req.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { - NunjucksTagExtension, - NunjucksTagExtensionParseResult, - NunjucksTagExtensionRunOptions, -} from '../extension'; -import * as nunjucks from 'nunjucks'; -import { injectable, inject } from 'inversify'; -import { TYPES } from '@vulcan/core/containers'; - -const FINIAL_BUILDER_NAME = 'FINAL_BUILDER'; - -// TODO: temporary interface -export interface QueryBuilder { - count(): QueryBuilder; - value(): Promise; -} - -export interface Executor { - createBuilder(query: string): Promise; -} - -@injectable() -export class ReqExtension implements NunjucksTagExtension { - public name = 'built-in-req'; - public tags = ['req']; - private executor: Executor; - - constructor(@inject(TYPES.Executor) executor: Executor) { - this.executor = executor; - } - - public parse( - parser: nunjucks.parser.Parser, - nodes: typeof nunjucks.nodes, - lexer: typeof nunjucks.lexer - ): NunjucksTagExtensionParseResult { - // {% req var (main) %} body {% endreq %} - // consume req tag - const reqToken = parser.nextToken(); - // variable - let nextToken = parser.peekToken(); - if (nextToken.type === lexer.TOKEN_BLOCK_END) { - // {% req %} - parser.fail(`Expected a variable`, nextToken.lineno, nextToken.colno); - } - if (nextToken.type !== lexer.TOKEN_SYMBOL) { - parser.fail( - `Expected a symbol, but got ${nextToken.type}`, - nextToken.lineno, - nextToken.colno - ); - } - const variable = parser.parseExpression(); - - // main denotation - nextToken = parser.peekToken(); - let mainBuilder = false; - if (nextToken.type !== lexer.TOKEN_BLOCK_END) { - if (nextToken.type !== lexer.TOKEN_SYMBOL || nextToken.value !== 'main') { - parser.fail( - `Expected a symbol "main"`, - nextToken.lineno, - nextToken.colno - ); - } - mainBuilder = true; - // Consume this token (main) - parser.nextToken(); - } - - const endToken = parser.nextToken(); - if (endToken.type !== lexer.TOKEN_BLOCK_END) { - parser.fail( - `Expected a block end, but got ${endToken.type}`, - endToken.lineno, - endToken.colno - ); - } - - const requestQuery = parser.parseUntilBlocks('endreq'); - parser.advanceAfterBlockEnd(); - - const argsNodeToPass = new nodes.NodeList(reqToken.lineno, reqToken.colno); - // variable name - argsNodeToPass.addChild( - new nodes.Literal( - variable.colno, - variable.lineno, - (variable as nunjucks.nodes.Symbol).value - ) - ); - // is main builder - argsNodeToPass.addChild( - new nodes.Literal(variable.colno, variable.lineno, String(mainBuilder)) - ); - - return { - argsNodeList: argsNodeToPass, - contentNodes: [requestQuery], - }; - } - - public async run({ - context, - args, - contentArgs, - }: NunjucksTagExtensionRunOptions) { - const name = args[0]; - let query = ''; - for (let index = 0; index < contentArgs.length; index++) { - query += await contentArgs[index](); - } - query = query - .split(/\r?\n/) - .filter((line) => line.trim().length > 0) - .join('\n'); - const builder = await this.executor.createBuilder(query); - context.setVariable(name, builder); - - if (Boolean(args[1])) { - context.setVariable(FINIAL_BUILDER_NAME, builder); - context.addExport(FINIAL_BUILDER_NAME); - } - } -} diff --git a/packages/core/src/lib/template-engine/index.ts b/packages/core/src/lib/template-engine/index.ts index 635d48ca..c315bab3 100644 --- a/packages/core/src/lib/template-engine/index.ts +++ b/packages/core/src/lib/template-engine/index.ts @@ -1,7 +1,6 @@ export * from './templateEngine'; export * from './compiler'; export * from './template-providers'; -export * from './extensions'; export * from './inMemoryCodeLoader'; export * from './nunjucksCompiler'; -export * from './visitors'; +export * from './extension-loader'; diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index 2745de85..80888cbc 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -1,38 +1,48 @@ import { Compiler, CompileResult } from './compiler'; import * as nunjucks from 'nunjucks'; -import { - isFilterExtension, - isTagExtension, - NunjucksCompilerExtension, - NunjucksFilterExtensionWrapper, - NunjucksTagExtensionWrapper, - QueryBuilder, -} from './extensions'; import * as transformer from 'nunjucks/src/transformer'; -import { walkAst } from './visitors/astWalker'; -import { - ParametersVisitor, - ErrorsVisitor, - FiltersVisitor, - BuilderValueVisitor, - MainBuilderVisitor, -} from './visitors'; -import { inject, injectable, multiInject, optional } from 'inversify'; +import { inject, injectable, multiInject, named, optional } from 'inversify'; import { TYPES } from '@vulcan/core/containers'; +import { + CompileTimeExtension, + Extension, + FilterBuilder, + FilterRunner, + generateMetadata, + implementedOnAstVisit, + implementedProvideMetadata, + OnAstVisit, + ProvideMetadata, + RuntimeExtension, + TagBuilder, + TagRunner, + walkAst, +} from './extension-loader'; +// TODO: Should replace with a real implementation +import { QueryBuilder } from './built-in-extensions/query-builder/reqTagRunner'; @injectable() export class NunjucksCompiler implements Compiler { public name = 'nunjucks'; - private env: nunjucks.Environment; - private extensions: NunjucksCompilerExtension[]; + private runtimeEnv: nunjucks.Environment; + private compileTimeEnv: nunjucks.Environment; + private extensions: Extension[]; + private astVisitors: OnAstVisit[] = []; + private metadataProviders: ProvideMetadata[] = []; constructor( - @inject(TYPES.CompilerLoader) loader: nunjucks.ILoader, @multiInject(TYPES.CompilerExtension) @optional() - extensions: NunjucksCompilerExtension[] = [] + extensions: Extension[] = [], + @inject(TYPES.CompilerEnvironment) + @named('runtime') + runtimeEnv: nunjucks.Environment, + @inject(TYPES.CompilerEnvironment) + @named('compileTime') + compileTimeEnv: nunjucks.Environment ) { - this.env = new nunjucks.Environment(loader); + this.runtimeEnv = runtimeEnv; + this.compileTimeEnv = compileTimeEnv; this.extensions = extensions; this.loadAllExtensions(); } @@ -40,7 +50,7 @@ export class NunjucksCompiler implements Compiler { public compile(template: string): CompileResult { const compiler = new nunjucks.compiler.Compiler( 'main', - this.env.opts.throwOnUndefined || false + this.compileTimeEnv.opts.throwOnUndefined || false ); const { ast, metadata } = this.generateAst(template); compiler.compile(ast); @@ -49,8 +59,13 @@ export class NunjucksCompiler implements Compiler { } public generateAst(template: string) { - const ast = nunjucks.parser.parse(template, this.env.extensionsList, {}); - const metadata = this.getMetadata(ast); + const ast = nunjucks.parser.parse( + template, + this.compileTimeEnv.extensionsList, + {} + ); + this.traverseAST(ast); + const metadata = this.getMetadata(); const preProcessedAst = this.preProcess(ast); return { ast: preProcessedAst, metadata }; } @@ -63,15 +78,15 @@ export class NunjucksCompiler implements Compiler { return builder.value(); } - public loadExtension(extension: NunjucksCompilerExtension): void { - if (isTagExtension(extension)) { - const { name, transform } = NunjucksTagExtensionWrapper(extension); - this.env.addExtension(name, transform); - } else if (isFilterExtension(extension)) { - const { name, transform } = NunjucksFilterExtensionWrapper(extension); - this.env.addFilter(name, transform, true); + public loadExtension(extension: Extension): void { + if (extension instanceof RuntimeExtension) { + this.loadRuntimeExtensions(extension); + } else if (extension instanceof CompileTimeExtension) { + this.loadCompileTimeExtensions(extension); } else { - throw new Error('Unsupported extension'); + throw new Error( + `Extension must be of type RuntimeExtension or CompileTimeExtension` + ); } } @@ -79,35 +94,56 @@ export class NunjucksCompiler implements Compiler { this.extensions.forEach((ext) => this.loadExtension(ext)); } + private loadCompileTimeExtensions(extension: CompileTimeExtension): void { + // Extends + if (extension instanceof TagBuilder) { + this.compileTimeEnv.addExtension(extension.getName(), extension); + } else if (extension instanceof FilterBuilder) { + this.compileTimeEnv.addFilter( + extension.filterName, + () => {}, // We don't need to implement transform function in compile time + true + ); + } + // Implement + if (implementedOnAstVisit(extension)) { + this.astVisitors.push(extension); + } + if (implementedProvideMetadata(extension)) { + this.metadataProviders.push(extension); + } + } + + private loadRuntimeExtensions(extension: RuntimeExtension): void { + if (extension instanceof TagRunner) { + this.runtimeEnv.addExtension(extension.getName(), extension); + } else if (extension instanceof FilterRunner) { + this.runtimeEnv.addFilter( + extension.filterName, + extension.__transform.bind(extension), + true + ); + } + } + + private traverseAST(ast: nunjucks.nodes.Node) { + walkAst(ast, this.astVisitors); + } + /** Get some metadata from the AST tree, e.g. the errors defined by templates. * It'll help use to validate templates, validate schema ...etc. */ - private getMetadata(ast: nunjucks.nodes.Node) { - const parameters = new ParametersVisitor(); - const errors = new ErrorsVisitor(); - const filters = new FiltersVisitor({ env: this.env }); - const builderValueVisitor = new BuilderValueVisitor(); - const mainBuilderVisitor = new MainBuilderVisitor(); - walkAst(ast, [ - parameters, - errors, - filters, - builderValueVisitor, - mainBuilderVisitor, - ]); - return { - parameters: parameters.getParameters(), - errors: errors.getErrors(), - }; + private getMetadata() { + return generateMetadata(this.metadataProviders); } /** Process the AST tree before compiling */ private preProcess(ast: nunjucks.nodes.Node): nunjucks.nodes.Node { // Nunjucks'll handle the async filter via pre-process functions - return transformer.transform(ast, this.env.asyncFilters); + return transformer.transform(ast, this.compileTimeEnv.asyncFilters); } private renderAndGetMainBuilder(templateName: string, data: any) { - const template = this.env.getTemplate(templateName, true); + const template = this.runtimeEnv.getTemplate(templateName, true); return new Promise((resolve, reject) => { template.getExported<{ FINAL_BUILDER: QueryBuilder }>( data, diff --git a/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts b/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts deleted file mode 100644 index 9a1dd2e6..00000000 --- a/packages/core/src/lib/template-engine/visitors/builderValueVisitor.ts +++ /dev/null @@ -1,89 +0,0 @@ -import * as nunjucks from 'nunjucks'; -import { ReplaceChildFunc, visitChildren } from './astWalker'; -import { Visitor } from './visitor'; - -const MAX_DEPTH = 100; - -// Replace .value() function of builders with a async filter -export class BuilderValueVisitor implements Visitor { - private builderCreatorExtensionName: string; - private executeCommandName: string; - private executeFilterName: string; - - private variableList = new Set(); - - constructor({ - builderCreatorExtensionName = 'built-in-req', - executeCommandName = 'value', - executeFilterName = 'execute', - }: { - builderCreatorExtensionName?: string; - executeCommandName?: string; - executeFilterName?: string; - } = {}) { - this.builderCreatorExtensionName = builderCreatorExtensionName; - this.executeCommandName = executeCommandName; - this.executeFilterName = executeFilterName; - } - - public visit(node: nunjucks.nodes.Node) { - // Record the variable name if it is a extension node - if ( - node instanceof nunjucks.nodes.CallExtensionAsync && - node.extName === this.builderCreatorExtensionName - ) { - const variable = node.args.children[0] as nunjucks.nodes.Literal; - this.variableList.add(variable.value); - return; - } - - visitChildren(node, this.visitChild.bind(this)); - } - - private visitChild(node: nunjucks.nodes.Node, replace: ReplaceChildFunc) { - if ( - node instanceof nunjucks.nodes.FunCall && - node.name instanceof nunjucks.nodes.LookupVal && - node.name.val.value === this.executeCommandName - ) { - let targetNode: typeof node.name.target | null = node.name.target; - let depth = 0; - while (targetNode) { - depth++; - if (depth > MAX_DEPTH) { - throw new Error('Max depth reached'); - } - if (targetNode instanceof nunjucks.nodes.LookupVal) { - targetNode = targetNode.target; - } else if (targetNode instanceof nunjucks.nodes.FunCall) { - targetNode = targetNode.name; - } else if (targetNode instanceof nunjucks.nodes.Symbol) { - break; - } else { - throw new Error( - `Unexpected node type: ${ - (targetNode as nunjucks.nodes.Node)?.typename - }` - ); - } - } - - // If the target node is a variable from {% req xxx %}, replace it with execute filter - if (this.variableList.has((targetNode as nunjucks.nodes.Symbol).value)) { - const args = new nunjucks.nodes.NodeList(node.lineno, node.colno); - args.addChild(node.name.target); - const filter = new nunjucks.nodes.Filter( - node.lineno, - node.colno, - new nunjucks.nodes.Symbol( - node.lineno, - node.colno, - this.executeFilterName - ), - args - ); - replace(filter); - } - } - } -} diff --git a/packages/core/src/lib/template-engine/visitors/errorsVisitor.ts b/packages/core/src/lib/template-engine/visitors/errorsVisitor.ts deleted file mode 100644 index fd3ec0f0..00000000 --- a/packages/core/src/lib/template-engine/visitors/errorsVisitor.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { chain } from 'lodash'; -import * as nunjucks from 'nunjucks'; -import { TemplateErrorMetadata } from '../compiler'; -import { Visitor } from './visitor'; - -interface ErrorCode { - code: string; - lineNo: number; - columnNo: number; -} - -export class ErrorsVisitor implements Visitor { - private extensionName: string; - private errorCodes: ErrorCode[] = []; - - constructor({ - extensionName = 'built-in-error', - }: { extensionName?: string } = {}) { - this.extensionName = extensionName; - } - - public visit(node: nunjucks.nodes.Node) { - if (node instanceof nunjucks.nodes.CallExtension) { - if (node.extName !== this.extensionName) return; - const errorCodeNode = node.args.children[0]; - if (!(errorCodeNode instanceof nunjucks.nodes.Literal)) - throw new Error(`Expected literal, got ${errorCodeNode.typename}`); - this.errorCodes.push({ - code: errorCodeNode.value, - lineNo: errorCodeNode.lineno, - columnNo: errorCodeNode.colno, - }); - } - } - - public getErrors(): TemplateErrorMetadata[] { - return chain(this.errorCodes) - .groupBy('code') - .values() - .map((errorCodes) => ({ - code: errorCodes[0].code, - locations: errorCodes.map((errorCode) => ({ - lineNo: errorCode.lineNo, - columnNo: errorCode.columnNo, - })), - })) - .value(); - } -} diff --git a/packages/core/src/lib/template-engine/visitors/filtersVisitor.ts b/packages/core/src/lib/template-engine/visitors/filtersVisitor.ts deleted file mode 100644 index aa9a0110..00000000 --- a/packages/core/src/lib/template-engine/visitors/filtersVisitor.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as nunjucks from 'nunjucks'; -import { Visitor } from './visitor'; - -export class FiltersVisitor implements Visitor { - private env: nunjucks.Environment; - - constructor({ env }: { env: nunjucks.Environment }) { - this.env = env; - } - - public visit(node: nunjucks.nodes.Node) { - if (node instanceof nunjucks.nodes.Filter) { - if ( - node.name instanceof nunjucks.nodes.Symbol || - node.name instanceof nunjucks.nodes.Literal - ) { - // If the node is a filter and has a expected name node, we check whether the filter is loaded. - // If the filter is not loaded, getFilter function will throw an error. - this.env.getFilter(node.name.value); - } - } - } -} diff --git a/packages/core/src/lib/template-engine/visitors/index.ts b/packages/core/src/lib/template-engine/visitors/index.ts deleted file mode 100644 index b6d6aeda..00000000 --- a/packages/core/src/lib/template-engine/visitors/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './visitor'; -export * from './parametersVisitor'; -export * from './errorsVisitor'; -export * from './filtersVisitor'; -export * from './astWalker'; -export * from './builderValueVisitor'; -export * from './mainBuilderVisitor'; diff --git a/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts b/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts deleted file mode 100644 index 73570c10..00000000 --- a/packages/core/src/lib/template-engine/visitors/mainBuilderVisitor.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Visitor } from './visitor'; -import * as nunjucks from 'nunjucks'; - -const ReqExtensionName = 'built-in-req'; - -export class MainBuilderVisitor implements Visitor { - private root?: nunjucks.nodes.Root; - private hasMainBuilder = false; - - public visit(node: nunjucks.nodes.Node) { - // save the root - if (node instanceof nunjucks.nodes.Root) { - this.root = node; - } - this.checkMainBuilder(node); - } - - public finish() { - if (!this.hasMainBuilder) { - this.wrapOutputWithBuilder(); - } - } - - private checkMainBuilder(node: nunjucks.nodes.Node) { - const isMainBuilder = - node instanceof nunjucks.nodes.CallExtensionAsync && - node.extName === ReqExtensionName && - (node.args.children[1] as nunjucks.nodes.Literal).value === 'true'; - if (!isMainBuilder) return; - - if (this.hasMainBuilder) { - throw new Error(`Only one main builder is allowed.`); - } - this.hasMainBuilder = true; - } - - private wrapOutputWithBuilder() { - if (!this.root) { - throw new Error('No root node found.'); - } - const originalChildren = this.root.children; - const args = new nunjucks.nodes.NodeList(0, 0); - // variable name - args.addChild(new nunjucks.nodes.Literal(0, 0, '__wrapped__builder')); - // is main builder - args.addChild(new nunjucks.nodes.Literal(0, 0, 'true')); - const builder = new nunjucks.nodes.CallExtensionAsync( - ReqExtensionName, - '__run', - args, - originalChildren - ); - this.root.children = [builder]; - } -} diff --git a/packages/core/src/lib/template-engine/visitors/visitor.ts b/packages/core/src/lib/template-engine/visitors/visitor.ts deleted file mode 100644 index d6598b97..00000000 --- a/packages/core/src/lib/template-engine/visitors/visitor.ts +++ /dev/null @@ -1,6 +0,0 @@ -import * as nunjucks from 'nunjucks'; - -export interface Visitor { - visit: (node: nunjucks.nodes.Node) => void; - finish?: () => void; -} diff --git a/packages/core/test/template-engine/builder.spec.ts b/packages/core/test/template-engine/builder.spec.ts deleted file mode 100644 index 707e997a..00000000 --- a/packages/core/test/template-engine/builder.spec.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { createTestCompiler } from './testCompiler'; - -it('query builder should be executes when .value() function called', async () => { - // Arrange - const { compiler, loader, builder, executor } = createTestCompiler(); - const { compiledData } = compiler.compile(` -{% req user %} -select * from public.users limit 1 where id = '{{ params.userId }}'; -{% endreq %} - -{% if user.count().value() == 1 %} -select * from public.user where id = '{{ user.id }}'; -{% else %} -{% error "User not found" %} -{% endif %} - - `); - builder.value.onFirstCall().resolves([1]); - builder.value.onSecondCall().resolves([{ id: 1, name: 'test' }]); - // Action - loader.setSource('test', compiledData); - const finalResult = await compiler.execute('test', { - params: { userId: 'user-id' }, - }); - // Assert - expect(executor.createBuilder.firstCall.args[0]).toBe( - `select * from public.users limit 1 where id = 'user-id';` - ); - expect(builder.count.callCount).toBe(1); - expect(finalResult).toEqual([{ id: 1, name: 'test' }]); -}); diff --git a/packages/core/test/template-engine/built-in-extensions/custom-error/custom-error.spec.ts b/packages/core/test/template-engine/built-in-extensions/custom-error/custom-error.spec.ts new file mode 100644 index 00000000..02b6be1e --- /dev/null +++ b/packages/core/test/template-engine/built-in-extensions/custom-error/custom-error.spec.ts @@ -0,0 +1,61 @@ +import { createTestCompiler } from '../../testCompiler'; + +it('Extension should throw custom error with error code and the position while executing', async () => { + // Arrange + const { compiler, loader } = await createTestCompiler(); + const { compiledData } = compiler.compile(` +{% error "This is an error" %} + `); + // Action, Assert + loader.setSource('test', compiledData); + await expect( + compiler.execute('test', { name: 'World' }) + ).rejects.toThrowError('This is an error at 1:3'); +}); + +it('Extension should provide a correct error list', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act + const { metadata } = compiler.compile(` +{% error "ERROR_CODE" %} +{% error "ERROR_CODE_2" %} +{% error "ERROR_CODE_2" %} + `); + + // Assert + expect(metadata['error.vulcan.com'].errorCodes.length).toBe(2); + expect(metadata['error.vulcan.com'].errorCodes).toContainEqual({ + code: 'ERROR_CODE', + locations: [ + { + lineNo: 1, + columnNo: 9, + }, + ], + }); + expect(metadata['error.vulcan.com'].errorCodes).toContainEqual({ + code: 'ERROR_CODE_2', + locations: [ + { + lineNo: 2, + columnNo: 9, + }, + { + lineNo: 3, + columnNo: 9, + }, + ], + }); +}); + +it('If the arguments of the extension are not the same as expected, the extension should throw error', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act, Assert + expect(() => + compiler.compile(` + {% error QAQ %} + `) + ).toThrow(`Expected literal, got Symbol`); +}); diff --git a/packages/core/test/template-engine/extensions/tags/req.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts similarity index 62% rename from packages/core/test/template-engine/extensions/tags/req.spec.ts rename to packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts index a647b5fa..e4b4a073 100644 --- a/packages/core/test/template-engine/extensions/tags/req.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts @@ -1,8 +1,8 @@ import { createTestCompiler } from '../../testCompiler'; -it('req extension should execute correct query and set/export the variable', async () => { +it('Extension should execute correct query and set/export the variable', async () => { // Arrange - const { compiler, loader, builder, executor } = createTestCompiler(); + const { compiler, loader, builder, executor } = await createTestCompiler(); const { compiledData } = compiler.compile(` {% req userCount main %} select count(*) as count from user where user.id = '{{ params.userId }}'; @@ -21,9 +21,9 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; expect(result).toEqual([{ count: 1 }]); }); -it('if argument is not a symbol, extension should throw', async () => { +it('If argument is not a symbol, extension should throw', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action, Assert expect(() => @@ -35,9 +35,9 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; ).toThrow(`Expected a symbol, but got string`); }); -it('if argument is missing, extension should throw', async () => { +it('If argument is missing, extension should throw', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action, Assert expect(() => @@ -49,9 +49,9 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; ).toThrow(`Expected a variable`); }); -it('if the main denotation is replaces other keywords than "main", extension should throw an error', async () => { +it('If the main denotation is replaces other keywords than "main", extension should throw an error', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action, Assert expect(() => @@ -63,9 +63,9 @@ some statement ).toThrow(`Expected a symbol "main"`); }); -it('if argument have too many elements, extension should throw an error', async () => { +it('If argument have too many elements, extension should throw an error', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action, Assert expect(() => @@ -77,9 +77,9 @@ select count(*) as count from user where user.id = '{{ params.userId }}'; ).toThrow(`Expected a block end, but got symbol`); }); -it('the main denotation should be parsed into the second args node', async () => { +it('The main denotation should be parsed into the second args node', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action const { ast: astWithMainBuilder } = compiler.generateAst( @@ -91,3 +91,17 @@ it('the main denotation should be parsed into the second args node', async () => 'true' ); }); + +it('Extension should throw an error if there are tow main builders', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act, Arrange + expect(() => + compiler.compile( + ` + {% req user main %} select * from users; {% endreq %} + {% req user2 main %} select * from users; {% endreq %} + ` + ) + ).toThrowError(`Only one main builder is allowed.`); +}); diff --git a/packages/core/test/template-engine/extensions/filters/execute.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts similarity index 69% rename from packages/core/test/template-engine/extensions/filters/execute.spec.ts rename to packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts index 09e4354a..5fd46003 100644 --- a/packages/core/test/template-engine/extensions/filters/execute.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts @@ -1,15 +1,15 @@ import { createTestCompiler } from '../../testCompiler'; -it('should call the .value() function of builder', async () => { +it('Extension should call the .value() function of builder', async () => { // Arrange - const { compiler, loader, executor, builder } = createTestCompiler(); + const { compiler, loader, executor, builder } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% req user %} select * from users; {% endreq %} -select * from group where userId = '{{ user.value()[0].id }}'; +select * from group where userId = '{{ user.count(3).value()[0].id }}'; ` ); builder.value.onFirstCall().resolves([{ id: 'user-id' }]); @@ -22,9 +22,9 @@ select * from group where userId = '{{ user.value()[0].id }}'; ); }); -it('should throw an error if the main builder failed to execute', async () => { +it('Extension should throw an error if the main builder failed to execute', async () => { // Arrange - const { compiler, loader, builder } = createTestCompiler(); + const { compiler, loader, builder } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% req user main %} @@ -40,9 +40,9 @@ select * from users; ); }); -it('should throw an error if one of sub builders failed to execute', async () => { +it('Extension should throw an error if one of sub builders failed to execute', async () => { // Arrange - const { compiler, loader, builder } = createTestCompiler(); + const { compiler, loader, builder } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% req user %} diff --git a/packages/core/test/template-engine/extensions/filters/unique.spec.ts b/packages/core/test/template-engine/built-in-extensions/sql-helper/unique.spec.ts similarity index 87% rename from packages/core/test/template-engine/extensions/filters/unique.spec.ts rename to packages/core/test/template-engine/built-in-extensions/sql-helper/unique.spec.ts index ebf234f1..4bf181f6 100644 --- a/packages/core/test/template-engine/extensions/filters/unique.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/sql-helper/unique.spec.ts @@ -2,7 +2,7 @@ import { createTestCompiler } from '../../testCompiler'; it('Extension should return correct values without unique by argument', async () => { // Arrange - const { compiler, loader, executor } = createTestCompiler(); + const { compiler, loader, executor } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [1,2,3,4,4] %} @@ -20,7 +20,7 @@ it('Extension should return correct values without unique by argument', async () it('Extension should return correct values with unique by keyword argument', async () => { // Arrange - const { compiler, loader, executor } = createTestCompiler(); + const { compiler, loader, executor } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [{name: "Tom"}, {name: "Tom"}, {name: "Joy"}] %} @@ -38,7 +38,7 @@ it('Extension should return correct values with unique by keyword argument', asy it('Extension should return correct values with unique by argument', async () => { // Arrange - const { compiler, loader, executor } = createTestCompiler(); + const { compiler, loader, executor } = await createTestCompiler(); const { compiledData } = compiler.compile( ` {% set array = [{name: "Tom"}, {name: "Tom"}, {name: "Joy"}] %} diff --git a/packages/core/test/template-engine/built-in-extensions/validator/filters.spec.ts b/packages/core/test/template-engine/built-in-extensions/validator/filters.spec.ts new file mode 100644 index 00000000..449cc5f5 --- /dev/null +++ b/packages/core/test/template-engine/built-in-extensions/validator/filters.spec.ts @@ -0,0 +1,28 @@ +import { createTestCompiler } from '../../testCompiler'; + +it('If we try to use an unloaded filter, extension should throw error', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act, Assert + expect(() => compiler.compile(`{{ 123 | unloadedFilter }}`)).toThrow( + 'filter not found: unloadedFilter' + ); +}); + +it('If we try to use loaded filter, extension should return the correct list', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act + const { metadata } = compiler.compile(`{{ 123 | unique }}`); + // Act, Assert + expect(metadata['filter.vulcan.com'].length).toBe(1); +}); + +it('If we try to use a built-in filter, extension should return the correct list', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act + const { metadata } = compiler.compile(`{{ 123 | abs }}`); + // Act, Assert + expect(metadata['filter.vulcan.com'].length).toBe(1); +}); diff --git a/packages/core/test/template-engine/visitors/parameters.spec.ts b/packages/core/test/template-engine/built-in-extensions/validator/parameters.spec.ts similarity index 55% rename from packages/core/test/template-engine/visitors/parameters.spec.ts rename to packages/core/test/template-engine/built-in-extensions/validator/parameters.spec.ts index e8a23635..2dddc9a1 100644 --- a/packages/core/test/template-engine/visitors/parameters.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/validator/parameters.spec.ts @@ -1,22 +1,16 @@ -import * as nunjucks from 'nunjucks'; -import { walkAst, ParametersVisitor } from '@vulcan/core/template-engine'; +import { createTestCompiler } from '../../testCompiler'; -it('Visitor should return correct parameter', async () => { +it('Extension should return correct parameter list', async () => { // Arrange - const ast = nunjucks.parser.parse( - ` + const { compiler } = await createTestCompiler(); + // Act + const { metadata } = compiler.compile(` {{ params.a }}{{ params.a.b }}{{ other.params.a }} {% if params.c and params.d.e %} {{ params.f.g | capitalize }} {% endif %} - `, - [], - {} - ); - const visitor = new ParametersVisitor({ lookupParameter: 'params' }); - // Act - walkAst(ast, [visitor]); - const parameters = visitor.getParameters(); + `); + const parameters = metadata['parameter.vulcan.com']; // Assert expect(parameters.length).toBe(7); expect(parameters).toContainEqual({ @@ -51,15 +45,3 @@ it('Visitor should return correct parameter', async () => { locations: [{ lineNo: 3, columnNo: 13 }], }); }); - -it('Visitor should throw error when max depth (100) is reached', async () => { - // Arrange - let paramString = 'params'; - for (let i = 0; i < 101; i++) { - paramString += '.a'; - } - const ast = nunjucks.parser.parse(`{{ ${paramString} }}`, [], {}); - const visitor = new ParametersVisitor({ lookupParameter: 'params' }); - // Act, Assert - expect(() => walkAst(ast, [visitor])).toThrow('Max depth reached'); -}); diff --git a/packages/core/test/template-engine/commons/edge.spec.ts b/packages/core/test/template-engine/commons/edge.spec.ts new file mode 100644 index 00000000..8419c14b --- /dev/null +++ b/packages/core/test/template-engine/commons/edge.spec.ts @@ -0,0 +1,14 @@ +import { createTestCompiler } from '../testCompiler'; + +it('Compiler should throw an error if max depth of function call lookup is exceeded (100)', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + let queryString = '{{ something.a'; + for (let i = 0; i < 101; i++) { + queryString += '.a'; + } + queryString += '.value() }}'; + + // Action, Assert + expect(() => compiler.compile(queryString)).toThrow('Max depth reached'); +}); diff --git a/packages/core/test/template-engine/visitors/astWalker.spec.ts b/packages/core/test/template-engine/extension-loader/helpers.spec.ts similarity index 96% rename from packages/core/test/template-engine/visitors/astWalker.spec.ts rename to packages/core/test/template-engine/extension-loader/helpers.spec.ts index 859d0041..f5fbd76d 100644 --- a/packages/core/test/template-engine/visitors/astWalker.spec.ts +++ b/packages/core/test/template-engine/extension-loader/helpers.spec.ts @@ -17,7 +17,7 @@ it('AST walker should traversal all nodes', async () => { // Act walkAst(root, [ { - visit: () => visitedNodes++, + onVisit: () => visitedNodes++, }, ]); // Assert @@ -137,8 +137,8 @@ it('AST replace function should work with CallExtension', async () => { }); // Assert expect(root.args.colno).toBe(1); - expect(root.contentArgs![0] instanceof nunjucks.nodes.Symbol).toBe(true); - expect(root.contentArgs![1] instanceof nunjucks.nodes.Literal).toBe(true); + expect(root.contentArgs?.[0] instanceof nunjucks.nodes.Symbol).toBe(true); + expect(root.contentArgs?.[1] instanceof nunjucks.nodes.Literal).toBe(true); expect(root.contentArgs?.length).toBe(2); }); diff --git a/packages/core/test/template-engine/extension-loader/loader.spec.ts b/packages/core/test/template-engine/extension-loader/loader.spec.ts new file mode 100644 index 00000000..15d8dee0 --- /dev/null +++ b/packages/core/test/template-engine/extension-loader/loader.spec.ts @@ -0,0 +1,28 @@ +import 'reflect-metadata'; + +import { Container, TYPES } from '@vulcan/core/containers'; +import { + NunjucksCompiler, + PersistentStoreType, + SerializerType, + TemplateProviderType, +} from '@vulcan/core'; + +it('test', async () => { + const c = new Container(); + + await c.load({ + artifact: { + provider: PersistentStoreType.LocalFile, + filePath: '', + serializer: SerializerType.JSON, + }, + template: { + provider: TemplateProviderType.LocalFile, + templatePath: '', + }, + }); + const cc = c.get(TYPES.Compiler); + const re = cc.compile(`QQQQQ`); + console.log(re); +}); diff --git a/packages/core/test/template-engine/extensions/tags/error.spec.ts b/packages/core/test/template-engine/extensions/tags/error.spec.ts deleted file mode 100644 index 83334d7c..00000000 --- a/packages/core/test/template-engine/extensions/tags/error.spec.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { createTestCompiler } from '../../testCompiler'; - -it('Error extension should throw error with error code and the position while rendering', async () => { - // Arrange - const { compiler, loader } = createTestCompiler(); - const { compiledData } = compiler.compile(` -{% error "This is an error" %} - `); - // Action, Assert - loader.setSource('test', compiledData); - await expect( - compiler.execute('test', { name: 'World' }) - ).rejects.toThrowError('This is an error at 1:3'); -}); diff --git a/packages/core/test/template-engine/nunjuckCompiler.spec.ts b/packages/core/test/template-engine/nunjuckCompiler.spec.ts index a6f403fa..b0069ce4 100644 --- a/packages/core/test/template-engine/nunjuckCompiler.spec.ts +++ b/packages/core/test/template-engine/nunjuckCompiler.spec.ts @@ -1,9 +1,8 @@ -import { NunjucksCompilerExtension } from '@vulcan/core/template-engine'; import { createTestCompiler } from './testCompiler'; it('Nunjucks compiler should compile template without error.', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action const compilerCode = compiler.compile('Hello {{ name }}'); @@ -14,7 +13,7 @@ it('Nunjucks compiler should compile template without error.', async () => { it('Nunjucks compiler should load compiled code and execute rendered template with it', async () => { // Arrange - const { compiler, loader, getCreatedQueries } = createTestCompiler(); + const { compiler, loader, getCreatedQueries } = await createTestCompiler(); const { compiledData } = compiler.compile('Hello {{ name }}!'); // Action @@ -26,12 +25,11 @@ it('Nunjucks compiler should load compiled code and execute rendered template wi expect(queries[0]).toBe('Hello World!'); }); -it('Nunjucks compiler should reject unsupported extensions', async () => { +it('Nunjucks compiler should reject the extension which has no valid super class', async () => { // Arrange - const { compiler } = createTestCompiler(); + const { compiler } = await createTestCompiler(); // Action, Assert - // extension should have parse and name property - expect(() => - compiler.loadExtension({ tags: ['test'] } as NunjucksCompilerExtension) - ).toThrow('Unsupported extension'); + expect(() => compiler.loadExtension({})).toThrow( + 'Extension must be of type RuntimeExtension or CompileTimeExtension' + ); }); diff --git a/packages/core/test/template-engine/testCompiler.ts b/packages/core/test/template-engine/testCompiler.ts index b7c1f73d..607907c0 100644 --- a/packages/core/test/template-engine/testCompiler.ts +++ b/packages/core/test/template-engine/testCompiler.ts @@ -1,19 +1,19 @@ import { TYPES } from '@vulcan/core/containers'; import { - ErrorExtension, - ExecuteExtension, - Executor, InMemoryCodeLoader, NunjucksCompiler, - NunjucksCompilerExtension, - QueryBuilder, - ReqExtension, - UniqueExtension, } from '@vulcan/core/template-engine'; +import { bindExtensions } from '@vulcan/core/template-engine/extension-loader'; import { Container } from 'inversify'; import * as sinon from 'ts-sinon'; +import * as nunjucks from 'nunjucks'; +// TODO: Should replace with a real implementation +import { + QueryBuilder, + Executor, +} from '@vulcan/core/template-engine/built-in-extensions/query-builder/reqTagRunner'; -export const createTestCompiler = () => { +export const createTestCompiler = async () => { const container = new Container(); const stubBuilder = sinon.stubInterface(); stubBuilder.count.returns(stubBuilder); @@ -24,25 +24,31 @@ export const createTestCompiler = () => { .bind(TYPES.CompilerLoader) .to(InMemoryCodeLoader) .inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(UniqueExtension) - .inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(ErrorExtension) - .inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(ReqExtension) - .inSingletonScope(); - container - .bind(TYPES.CompilerExtension) - .to(ExecuteExtension) - .inSingletonScope(); + await bindExtensions(container.bind.bind(container)); container.bind(TYPES.Executor).toConstantValue(stubExecutor); container.bind(TYPES.Compiler).to(NunjucksCompiler).inSingletonScope(); + + // Compiler environment + container + .bind(TYPES.CompilerEnvironment) + .toDynamicValue((context) => { + // We only need loader in runtime + const loader = context.container.get( + TYPES.CompilerLoader + ); + return new nunjucks.Environment(loader); + }) + .inSingletonScope() + .whenTargetNamed('runtime'); + container + .bind(TYPES.CompilerEnvironment) + .toDynamicValue(() => { + return new nunjucks.Environment(); + }) + .inSingletonScope() + .whenTargetNamed('compileTime'); + return { builder: stubBuilder, executor: stubExecutor, diff --git a/packages/core/test/template-engine/visitors/builderValue.spec.ts b/packages/core/test/template-engine/visitors/builderValue.spec.ts deleted file mode 100644 index da7d5d24..00000000 --- a/packages/core/test/template-engine/visitors/builderValue.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { BuilderValueVisitor, walkAst } from '@vulcan/core/template-engine'; -import * as nunjucks from 'nunjucks'; - -it('Should throw an error if max depth exceeded (100)', async () => { - // Arrange - let queryString = '{{ something.a'; - for (let i = 0; i < 101; i++) { - queryString += '.a'; - } - queryString += '.value() }}'; - const ast = nunjucks.parser.parse(queryString, [], {}); - const visitor = new BuilderValueVisitor(); - // Act, Arrange - - expect(() => walkAst(ast, [visitor])).toThrow('Max depth reached'); -}); diff --git a/packages/core/test/template-engine/visitors/errors.spec.ts b/packages/core/test/template-engine/visitors/errors.spec.ts deleted file mode 100644 index a2c216bc..00000000 --- a/packages/core/test/template-engine/visitors/errors.spec.ts +++ /dev/null @@ -1,82 +0,0 @@ -import * as nunjucks from 'nunjucks'; -import { walkAst, ErrorsVisitor } from '@vulcan/core/template-engine'; -import { - ErrorExtension, - NunjucksTagExtensionWrapper, -} from '@vulcan/core/template-engine/extensions'; - -it('Visitor should return correct error list', async () => { - // Arrange - const env = new nunjucks.Environment(); - const { name, transform } = NunjucksTagExtensionWrapper(new ErrorExtension()); - env.addExtension(name, transform); - const ast = nunjucks.parser.parse( - ` -{% error "ERROR_CODE" %} -{% error "ERROR_CODE_2" %} -{% error "ERROR_CODE_2" %} - `, - env.extensionsList, - {} - ); - const visitor = new ErrorsVisitor({ extensionName: name }); - // Act - walkAst(ast, [visitor]); - const errors = visitor.getErrors(); - // Assert - expect(errors.length).toBe(2); - expect(errors).toContainEqual({ - code: 'ERROR_CODE', - locations: [ - { - lineNo: 1, - columnNo: 9, - }, - ], - }); - expect(errors).toContainEqual({ - code: 'ERROR_CODE_2', - locations: [ - { - lineNo: 2, - columnNo: 9, - }, - { - lineNo: 3, - columnNo: 9, - }, - ], - }); -}); - -it('Visitor should ignore the extension calls which are not belong to error extension', async () => { - // Arrange - const ast = new nunjucks.nodes.CallExtension( - { __name: 'other-ext' }, - 'run', - new nunjucks.nodes.NodeList(0, 0), - [new nunjucks.nodes.Literal(0, 0, 'a')] - ); - const visitor = new ErrorsVisitor({ extensionName: 'error-extension' }); - // Act - walkAst(ast, [visitor]); - const errors = visitor.getErrors(); - // Assert - expect(errors.length).toBe(0); -}); - -it('If the arguments of the extension are not the same as expected, visitor should throw error', async () => { - // Arrange - const args = new nunjucks.nodes.NodeList(0, 0); - args.addChild(new nunjucks.nodes.Symbol(0, 0, 'a')); // Should be a literal - - const ast = new nunjucks.nodes.CallExtension( - { __name: 'error-extension' }, - 'run', - args, - [] - ); - const visitor = new ErrorsVisitor({ extensionName: 'error-extension' }); - // Act, Assert - expect(() => walkAst(ast, [visitor])).toThrow(`Expected literal, got Symbol`); -}); diff --git a/packages/core/test/template-engine/visitors/filter.spec.ts b/packages/core/test/template-engine/visitors/filter.spec.ts deleted file mode 100644 index 483ee3de..00000000 --- a/packages/core/test/template-engine/visitors/filter.spec.ts +++ /dev/null @@ -1,39 +0,0 @@ -import * as nunjucks from 'nunjucks'; -import { walkAst, FiltersVisitor } from '@vulcan/core/template-engine'; - -it('If we try to use an unloaded filter, visitor should throw error', async () => { - // Arrange - const env = new nunjucks.Environment(); - env.addFilter('test', (val) => val); - const ast = nunjucks.parser.parse(`{{ 123 | unloadedFilter }}`, [], {}); - const visitor = new FiltersVisitor({ env }); - // Act, Assert - expect(() => walkAst(ast, [visitor])).toThrow( - 'filter not found: unloadedFilter' - ); -}); - -it('If we try to use loaded filter, visitor should do nothing', async () => { - // Arrange - const env = new nunjucks.Environment(); - env.addFilter('test', (val) => val); - const ast = nunjucks.parser.parse(`{{ 123 | test }}`, [], {}); - const visitor = new FiltersVisitor({ env }); - // Act, Assert - expect(() => walkAst(ast, [visitor])).not.toThrow(); -}); - -it('If we try to visit an unexpected filter node, visitor should do nothing', async () => { - // Arrange - const root = new nunjucks.nodes.Filter( - 0, - 0, - 'test', - new nunjucks.nodes.LookupVal(0, 0, 'a'), // Should be a Symbol or Literal - new nunjucks.nodes.NodeList(0, 0, []) - ); - const env = new nunjucks.Environment(); - const visitor = new FiltersVisitor({ env }); - // Act, Assert - expect(() => walkAst(root, [visitor])).not.toThrow(); -}); diff --git a/packages/core/test/template-engine/visitors/mainBuilder.spec.ts b/packages/core/test/template-engine/visitors/mainBuilder.spec.ts deleted file mode 100644 index a17f4057..00000000 --- a/packages/core/test/template-engine/visitors/mainBuilder.spec.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { - Executor, - MainBuilderVisitor, - NunjucksTagExtensionWrapper, - ReqExtension, - walkAst, -} from '@vulcan/core/template-engine'; -import * as nunjucks from 'nunjucks'; -import * as sinon from 'ts-sinon'; - -let extensions: nunjucks.Extension[] = []; - -beforeEach(() => { - const mockExecutor = sinon.stubInterface(); - const { transform: reqExtension } = NunjucksTagExtensionWrapper( - new ReqExtension(mockExecutor) - ); - extensions = [reqExtension]; -}); - -it('Should do nothing if there is exactly one main builder', async () => { - // Arrange - const ast = nunjucks.parser.parse( - `some output {% req user main %} select * from users; {% endreq %}`, - extensions, - {} - ); - const visitor = new MainBuilderVisitor(); - // Act - walkAst(ast, [visitor]); - // Arrange - expect(ast.children.length).toBe(2); -}); - -it('Should throw an error if there are tow main builders', async () => { - // Arrange - const ast = nunjucks.parser.parse( - ` - {% req user main %} select * from users; {% endreq %} - {% req user2 main %} select * from users; {% endreq %} - `, - extensions, - {} - ); - const visitor = new MainBuilderVisitor(); - // Act, Arrange - expect(() => walkAst(ast, [visitor])).toThrowError( - `Only one main builder is allowed.` - ); -}); - -it('Should wrap the output in a builder if there is no main builder', async () => { - // Arrange - const ast = nunjucks.parser.parse( - ` -select * from users -where id = {{ params.id }}; - `, - extensions, - {} - ); - const visitor = new MainBuilderVisitor(); - // Act - walkAst(ast, [visitor]); - // Arrange - expect(ast.children.length).toBe(1); - expect(ast.children[0] instanceof nunjucks.nodes.CallExtensionAsync).toBe( - true - ); - expect((ast.children[0] as any).args.children[1].value).toBe('true'); // main builder notation -}); - -it('Should throw an error if there is no root node', async () => { - // Arrange - const ast = nunjucks.parser.parse( - ` -select * from users -where id = {{ params.id }}; - `, - extensions, - {} - ); - const visitor = new MainBuilderVisitor(); - // Act, Arrange - walkAst(ast.children[0], [visitor]); // We start visiting the children of the root node, skipping the root node itself - expect(() => visitor.finish()).toThrow('No root node found.'); -}); diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index c78ab7d1..3c2c4b68 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -164,7 +164,7 @@ declare module 'nunjucks' { export interface Extension { tags: string[]; // Parser API is undocumented it is suggested to check the source: https://github.com/mozilla/nunjucks/blob/master/src/parser.js - parse(parser: any, nodes: any, lexer: any): any; + parse?(parser: any, nodes: any, lexer: any): any; } export function installJinjaCompat(): void; From 165fa1f35d5c2d673337a870ad2389812ba902f8 Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Thu, 9 Jun 2022 12:06:11 +0800 Subject: [PATCH 07/12] fix(build): fix dependencies of core package --- packages/build/src/containers/container.ts | 4 +- .../middleware/addMissingErrors.ts | 11 +++- .../middleware/checkParameter.ts | 10 +++- packages/build/src/lib/vulcanBuilder.ts | 2 +- .../middleware/addMissingErrors.spec.ts | 51 ++++++++++--------- .../middleware/checkParameter.spec.ts | 4 +- 6 files changed, 49 insertions(+), 33 deletions(-) diff --git a/packages/build/src/containers/container.ts b/packages/build/src/containers/container.ts index 219b0073..67730cb2 100644 --- a/packages/build/src/containers/container.ts +++ b/packages/build/src/containers/container.ts @@ -10,9 +10,9 @@ export class Container { return this.inversifyContainer.get(type); } - public load(options: IBuildOptions) { + public async load(options: IBuildOptions) { const coreContainer = new CoreContainer(); - coreContainer.load(options); + await coreContainer.load(options); this.inversifyContainer.parent = coreContainer.getInversifyContainer(); this.inversifyContainer.load(schemaParserModule(options.schemaParser)); } diff --git a/packages/build/src/lib/schema-parser/middleware/addMissingErrors.ts b/packages/build/src/lib/schema-parser/middleware/addMissingErrors.ts index 41581760..60a32a80 100644 --- a/packages/build/src/lib/schema-parser/middleware/addMissingErrors.ts +++ b/packages/build/src/lib/schema-parser/middleware/addMissingErrors.ts @@ -1,6 +1,12 @@ import { AllTemplateMetadata, APISchema } from '@vulcan/core'; import { SchemaParserMiddleware } from './middleware'; +interface ErrorCode { + code: string; + lineNo: number; + columnNo: number; +} + // Add error code to definition if it is used in query but not defined in schema export const addMissingErrors = (allMetadata: AllTemplateMetadata): SchemaParserMiddleware => @@ -10,9 +16,10 @@ export const addMissingErrors = const templateName = transformedSchemas.templateSource; const metadata = allMetadata[templateName]; // Skip validation if no metadata found - if (!metadata?.errors) return; + if (!metadata?.['error.vulcan.com']) return; - metadata.errors.forEach((error) => { + const errorCodes: ErrorCode[] = metadata['error.vulcan.com'].errorCodes; + errorCodes.forEach((error) => { if (!transformedSchemas.errors.some((e) => e.code === error.code)) { transformedSchemas.errors.push({ code: error.code, diff --git a/packages/build/src/lib/schema-parser/middleware/checkParameter.ts b/packages/build/src/lib/schema-parser/middleware/checkParameter.ts index 288deff0..d0a55b2a 100644 --- a/packages/build/src/lib/schema-parser/middleware/checkParameter.ts +++ b/packages/build/src/lib/schema-parser/middleware/checkParameter.ts @@ -1,6 +1,12 @@ import { AllTemplateMetadata, APISchema } from '@vulcan/core'; import { SchemaParserMiddleware } from './middleware'; +interface Parameter { + name: string; + lineNo: number; + columnNo: number; +} + export const checkParameter = (allMetadata: AllTemplateMetadata): SchemaParserMiddleware => async (schemas, next) => { @@ -9,9 +15,9 @@ export const checkParameter = const templateName = transformedSchemas.templateSource; const metadata = allMetadata[templateName]; // Skip validation if no metadata found - if (!metadata?.parameters) return; + if (!metadata?.['parameter.vulcan.com']) return; - const parameters = metadata.parameters; + const parameters: Parameter[] = metadata['parameter.vulcan.com']; parameters.forEach((parameter) => { // We only check the first value of nested parameters const name = parameter.name.split('.')[0]; diff --git a/packages/build/src/lib/vulcanBuilder.ts b/packages/build/src/lib/vulcanBuilder.ts index 1972de4d..b395f830 100644 --- a/packages/build/src/lib/vulcanBuilder.ts +++ b/packages/build/src/lib/vulcanBuilder.ts @@ -10,7 +10,7 @@ import { export class VulcanBuilder { public async build(options: IBuildOptions) { const container = new Container(); - container.load(options); + await container.load(options); const schemaParser = container.get(TYPES.SchemaParser); const templateEngine = container.get( CORE_TYPES.TemplateEngine diff --git a/packages/build/test/schema-parser/middleware/addMissingErrors.spec.ts b/packages/build/test/schema-parser/middleware/addMissingErrors.spec.ts index 295ccdee..39eb2815 100644 --- a/packages/build/test/schema-parser/middleware/addMissingErrors.spec.ts +++ b/packages/build/test/schema-parser/middleware/addMissingErrors.spec.ts @@ -12,18 +12,19 @@ it('Should add missing error codes', async () => { }; const metadata: AllTemplateMetadata = { 'some-name': { - parameters: [], - errors: [ - { - code: 'ERROR 1', - locations: [ - { - lineNo: 0, - columnNo: 0, - }, - ], - }, - ], + 'error.vulcan.com': { + errorCodes: [ + { + code: 'ERROR 1', + locations: [ + { + lineNo: 0, + columnNo: 0, + }, + ], + }, + ], + }, }, }; // Act @@ -48,17 +49,19 @@ it('Existed error codes should be kept', async () => { const metadata: AllTemplateMetadata = { 'some-name': { parameters: [], - errors: [ - { - code: 'ERROR 1', - locations: [ - { - lineNo: 0, - columnNo: 0, - }, - ], - }, - ], + 'error.vulcan.com': { + errorCodes: [ + { + code: 'ERROR 1', + locations: [ + { + lineNo: 0, + columnNo: 0, + }, + ], + }, + ], + }, }, }; // Act @@ -82,7 +85,7 @@ it('Should tolerate empty error data', async () => { const metadata: object = { 'some-name': { parameters: [], - errors: null, + 'error.vulcan.com': null, }, }; // Act diff --git a/packages/build/test/schema-parser/middleware/checkParameter.spec.ts b/packages/build/test/schema-parser/middleware/checkParameter.spec.ts index 470713ac..78f40888 100644 --- a/packages/build/test/schema-parser/middleware/checkParameter.spec.ts +++ b/packages/build/test/schema-parser/middleware/checkParameter.spec.ts @@ -50,7 +50,7 @@ it(`Should throw when any parameter hasn't be defined`, async () => { }; const metadata: AllTemplateMetadata = { 'some-name': { - parameters: [ + 'parameter.vulcan.com': [ { name: 'param1', locations: [], @@ -81,7 +81,7 @@ it('Should tolerate empty parameter data', async () => { }; const metadata: object = { 'some-name': { - parameters: null, + 'parameter.vulcan.com': null, errors: [], }, }; From 2c13c1aed1633de3f87096c9ae0f693f6856eeda Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Thu, 9 Jun 2022 14:45:07 +0800 Subject: [PATCH 08/12] feat(core): throw error if different builders use same name --- .../query-builder/reqTagBuilder.ts | 28 +++++++++++++++---- .../query-builder/builder.spec.ts | 16 +++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts index 79d0bca6..e26c8645 100644 --- a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts @@ -14,6 +14,11 @@ import { REFERENCE_SEARCH_MAX_DEPTH, } from './constants'; +interface DeclarationLocation { + lineNo: number; + colNo: number; +} + export class ReqTagBuilder extends TagBuilder implements OnAstVisit, ProvideMetadata @@ -22,7 +27,7 @@ export class ReqTagBuilder public metadataName = METADATA_NAME; private root?: nunjucks.nodes.Root; private hasMainBuilder = false; - private variableList = new Set(); + private variableList = new Map(); public parse( parser: nunjucks.parser.Parser, @@ -79,14 +84,14 @@ export class ReqTagBuilder // variable name argsNodeToPass.addChild( new nodes.Literal( - variable.colno, variable.lineno, + variable.colno, (variable as nunjucks.nodes.Symbol).value ) ); // is main builder argsNodeToPass.addChild( - new nodes.Literal(variable.colno, variable.lineno, String(mainBuilder)) + new nodes.Literal(variable.lineNo, variable.colno, String(mainBuilder)) ); return this.createAsyncExtensionNode(argsNodeToPass, [requestQuery]); @@ -100,8 +105,7 @@ export class ReqTagBuilder node instanceof nunjucks.nodes.CallExtensionAsync && node.extName === this.getName() ) { - const variable = node.args.children[0] as nunjucks.nodes.Literal; - this.variableList.add(variable.value); + this.checkBuilder(node); this.checkMainBuilder(node); } else { visitChildren(node, this.replaceExecuteFunction.bind(this)); @@ -120,6 +124,20 @@ export class ReqTagBuilder }; } + private checkBuilder(node: nunjucks.nodes.CallExtensionAsync) { + const variable = node.args.children[0] as nunjucks.nodes.Literal; + if (this.variableList.has(variable.value)) { + const previousDeclaration = this.variableList.get(variable.value); + throw new Error( + `We can't declare multiple builder with same name. Duplicated name: ${variable.value} (declared at ${previousDeclaration?.lineNo}:${previousDeclaration?.colNo} and ${variable.lineno}:${variable.colno})` + ); + } + this.variableList.set(variable.value, { + lineNo: variable.lineno, + colNo: variable.colno, + }); + } + private checkMainBuilder(node: nunjucks.nodes.CallExtensionAsync) { const isMainBuilder = node.extName === this.getName() && diff --git a/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts index e4b4a073..ea73d959 100644 --- a/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts @@ -105,3 +105,19 @@ it('Extension should throw an error if there are tow main builders', async () => ) ).toThrowError(`Only one main builder is allowed.`); }); + +it('Extension should throw an error if there are multiple builders using same name', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + // Act, Arrange + expect(() => + compiler.compile( + ` +{% req user %} select * from users; {% endreq %} +{% req user %} select * from users; {% endreq %} + ` + ) + ).toThrowError( + `We can't declare multiple builder with same name. Duplicated name: user (declared at 1:7 and 2:7)` + ); +}); From 20a8dce27b5ea858c9a65758fdb0bc3ae673e7f5 Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Wed, 22 Jun 2022 18:10:38 +0800 Subject: [PATCH 09/12] fix(core): delete test scripts and fix container load function calls --- .../core/test/containers/continer.spec.ts | 2 +- .../extension-loader/loader.spec.ts | 28 ------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 packages/core/test/template-engine/extension-loader/loader.spec.ts diff --git a/packages/core/test/containers/continer.spec.ts b/packages/core/test/containers/continer.spec.ts index 61e0f585..5d3b39c2 100644 --- a/packages/core/test/containers/continer.spec.ts +++ b/packages/core/test/containers/continer.spec.ts @@ -16,7 +16,7 @@ it('Container should load options and resolve all dependencies', async () => { fs.unlinkSync(resultPath); } const container = new Container(); - container.load({ + await container.load({ artifact: { provider: PersistentStoreType.LocalFile, filePath: resultPath, diff --git a/packages/core/test/template-engine/extension-loader/loader.spec.ts b/packages/core/test/template-engine/extension-loader/loader.spec.ts deleted file mode 100644 index 15d8dee0..00000000 --- a/packages/core/test/template-engine/extension-loader/loader.spec.ts +++ /dev/null @@ -1,28 +0,0 @@ -import 'reflect-metadata'; - -import { Container, TYPES } from '@vulcan/core/containers'; -import { - NunjucksCompiler, - PersistentStoreType, - SerializerType, - TemplateProviderType, -} from '@vulcan/core'; - -it('test', async () => { - const c = new Container(); - - await c.load({ - artifact: { - provider: PersistentStoreType.LocalFile, - filePath: '', - serializer: SerializerType.JSON, - }, - template: { - provider: TemplateProviderType.LocalFile, - templatePath: '', - }, - }); - const cc = c.get(TYPES.Compiler); - const re = cc.compile(`QQQQQ`); - console.log(re); -}); From 4a27aebc591b08e56b2ac1b4c69d1882c6905a66 Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Thu, 30 Jun 2022 17:27:06 +0800 Subject: [PATCH 10/12] fix(core): fix function name for keep constent --- packages/core/src/lib/template-engine/nunjucksCompiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index 80888cbc..e3a495cd 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -64,7 +64,7 @@ export class NunjucksCompiler implements Compiler { this.compileTimeEnv.extensionsList, {} ); - this.traverseAST(ast); + this.traverseAst(ast); const metadata = this.getMetadata(); const preProcessedAst = this.preProcess(ast); return { ast: preProcessedAst, metadata }; @@ -126,7 +126,7 @@ export class NunjucksCompiler implements Compiler { } } - private traverseAST(ast: nunjucks.nodes.Node) { + private traverseAst(ast: nunjucks.nodes.Node) { walkAst(ast, this.astVisitors); } From 351f58d72a0145f47483f807d248d400b979be6b Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Thu, 30 Jun 2022 17:41:49 +0800 Subject: [PATCH 11/12] fix(core): make parser.fail return never instead of void --- types/nunjucks.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/nunjucks.d.ts b/types/nunjucks.d.ts index 3c2c4b68..b901d800 100644 --- a/types/nunjucks.d.ts +++ b/types/nunjucks.d.ts @@ -297,7 +297,7 @@ declare module 'nunjucks' { advanceAfterBlockEnd(name: string): Token; parseUntilBlocks(...blockName: string[]): nodes.NodeList; advanceAfterBlockEnd(): Token; - fail(message: string, lineno?: number, colno?: number): void; + fail(message: string, lineno?: number, colno?: number): never; parseExpression(): Nodes; } } From 4edd9e96331c5b693f32936266c1e1fb3aaf23ba Mon Sep 17 00:00:00 2001 From: Ivan Tsai Date: Fri, 1 Jul 2022 17:33:35 +0800 Subject: [PATCH 12/12] fix(core): add descriptions and examples for extension options --- .../lib/template-engine/extension-loader/models.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lib/template-engine/extension-loader/models.ts b/packages/core/src/lib/template-engine/extension-loader/models.ts index 80276eef..88157a9c 100644 --- a/packages/core/src/lib/template-engine/extension-loader/models.ts +++ b/packages/core/src/lib/template-engine/extension-loader/models.ts @@ -41,9 +41,16 @@ export abstract class TagBuilder extends CompileTimeExtension { } protected createAsyncExtensionNode( - /** The arguments if this extension, they'll be render to string and passed to run function */ + /** + * The arguments of this extension, they'll be rendered and passed to run function. + * It usually contains the configuration of the extension, e.g. {% req variable %} The variable name of req extension. + * Note that these arguments will be pass to run function directly: Literal('123') => "123", so adding Output nodes causes compiling issues. Output("123") => t += "123" + */ argsNodeList: nunjucks.nodes.NodeList, - /** The content (usually the body) of this extension, they'll be passed to run function as render functions */ + /** The content (usually the body) of this extension, they'll be passed to run function as render functions + * It usually contains the Output of your extension, e.g. {% req variable %} select * from user {% endreq %}, the "select * from user" should be put in this field. + * Note that these nodes will be rendered as the output of template: Output("123") => t = ""; t += "123", so adding nodes with no output like Symbol, Literal ... might cause compiling issues. Literal('123') => t = ""; 123 + */ contentNodes: nunjucks.nodes.Node[] = [] ) { return new nunjucks.nodes.CallExtensionAsync(