diff --git a/labs/playground1/sqls/artist/works.sql b/labs/playground1/sqls/artist/works.sql index 682490b1..c20a9815 100644 --- a/labs/playground1/sqls/artist/works.sql +++ b/labs/playground1/sqls/artist/works.sql @@ -1,3 +1,11 @@ +{% req artist %} + select count(*) as count from "artists" where ConstituentID = {{ context.params.id }} +{% endreq %} + +{% if artist.value()[0].count == 0 %} + {% error "Artist not found" %} +{% endif %} + select * from "artworks" diff --git a/packages/build/test/schema-parser/middleware/responseSampler.spec.ts b/packages/build/test/schema-parser/middleware/responseSampler.spec.ts index b51b24a6..f29d5f6e 100644 --- a/packages/build/test/schema-parser/middleware/responseSampler.spec.ts +++ b/packages/build/test/schema-parser/middleware/responseSampler.spec.ts @@ -1,7 +1,7 @@ import { RawAPISchema } from '@vulcan-sql/build/schema-parser'; import { ResponseSampler } from '@vulcan-sql/build/schema-parser/middleware/responseSampler'; import { FieldDataType, TemplateEngine } from '@vulcan-sql/core'; -import { Stream } from 'stream'; +import { Readable } from 'stream'; import * as sinon from 'ts-sinon'; it('Should create response definition when example parameter is provided', async () => { @@ -19,7 +19,7 @@ it('Should create response definition when example parameter is provided', async { name: 'id', type: 'string' }, { name: 'age', type: 'number' }, ], - getData: () => new Stream(), + getData: () => new Readable(), }); const responseSampler = new ResponseSampler(stubTemplateEngine); // Act @@ -44,7 +44,7 @@ it('Should create response definition when example parameter is a empty object', { name: 'id', type: 'string' }, { name: 'age', type: 'number' }, ], - getData: () => new Stream(), + getData: () => new Readable(), }); const responseSampler = new ResponseSampler(stubTemplateEngine); // Act @@ -68,7 +68,7 @@ it('Should not create response definition when example parameter is not provided { name: 'id', type: 'string' }, { name: 'age', type: 'number' }, ], - getData: () => new Stream(), + getData: () => new Readable(), }); const responseSampler = new ResponseSampler(stubTemplateEngine); // Act @@ -97,7 +97,7 @@ it('Should append response definition when there are some existed definitions', { name: 'age', type: 'number' }, { name: 'name', type: 'boolean' }, ], - getData: () => new Stream(), + getData: () => new Readable(), }); const responseSampler = new ResponseSampler(stubTemplateEngine); // Act diff --git a/packages/core/src/lib/data-query/builder/dataQueryBuilder.ts b/packages/core/src/lib/data-query/builder/dataQueryBuilder.ts index 784ea16c..79ac5cba 100644 --- a/packages/core/src/lib/data-query/builder/dataQueryBuilder.ts +++ b/packages/core/src/lib/data-query/builder/dataQueryBuilder.ts @@ -2,6 +2,7 @@ import { DataSource, Pagination, BindParameters, + DataResult, } from '@vulcan-sql/core/models'; import * as uuid from 'uuid'; @@ -403,7 +404,7 @@ export interface IDataQueryBuilder { take(size: number, move: number): IDataQueryBuilder; // paginate paginate(pagination: Pagination): void; - value(): Promise; + value(): Promise; clone(): IDataQueryBuilder; } diff --git a/packages/core/src/lib/data-source/pg.ts b/packages/core/src/lib/data-source/pg.ts index 2f95effe..d13c0d0e 100644 --- a/packages/core/src/lib/data-source/pg.ts +++ b/packages/core/src/lib/data-source/pg.ts @@ -1,4 +1,4 @@ -import { Stream } from 'stream'; +import { Readable } from 'stream'; import { DataResult, DataSource, @@ -18,7 +18,7 @@ export class PGDataSource extends DataSource { return []; }, getData: () => { - return new Stream(); + return new Readable(); }, }; } diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorBuilder.ts index ff6d81b5..8b52579c 100644 --- a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorBuilder.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorBuilder.ts @@ -2,9 +2,36 @@ import { FilterBuilder, VulcanInternalExtension, } from '@vulcan-sql/core/models'; -import { EXECUTE_FILTER_NAME } from './constants'; +import { EXECUTE_COMMAND_NAME, EXECUTE_FILTER_NAME } from './constants'; +import * as nunjucks from 'nunjucks'; +import { ReplaceChildFunc, visitChildren } from '../../extension-utils'; @VulcanInternalExtension() export class ExecutorBuilder extends FilterBuilder { public filterName = EXECUTE_FILTER_NAME; + + public override onVisit(node: nunjucks.nodes.Node) { + visitChildren(node, this.replaceExecuteFunction.bind(this)); + } + + private replaceExecuteFunction( + node: nunjucks.nodes.Node, + replace: ReplaceChildFunc + ) { + if ( + node instanceof nunjucks.nodes.FunCall && + node.name instanceof nunjucks.nodes.LookupVal && + node.name.val.value === EXECUTE_COMMAND_NAME + ) { + const args = new nunjucks.nodes.NodeList(node.lineno, node.colno); + args.addChild(node.name.target); + const filter = new nunjucks.nodes.Filter( + node.lineno, + node.colno, + new nunjucks.nodes.Symbol(node.lineno, node.colno, EXECUTE_FILTER_NAME), + args + ); + replace(filter); + } + } } diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts index 5eaa6f84..226edbc7 100644 --- a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts @@ -1,19 +1,31 @@ -import { IDataQueryBuilder } from '@vulcan-sql/core/data-query'; import { + DataResult, FilterRunner, FilterRunnerTransformOptions, VulcanInternalExtension, } from '@vulcan-sql/core/models'; +import { streamToArray } from '@vulcan-sql/core/utils'; import { EXECUTE_FILTER_NAME } from './constants'; +const isDataResult = (response: any): response is DataResult => { + return response.getColumns && response.getData; +}; + @VulcanInternalExtension() export class ExecutorRunner extends FilterRunner { public filterName = EXECUTE_FILTER_NAME; public async transform({ - value, + value: builder, }: FilterRunnerTransformOptions): Promise { - const builder: IDataQueryBuilder = value; - return builder.value(); + const response = await builder.value(); + + // if input value is not a query builder, call the function .value and do nothing. + if (!isDataResult(response)) return response; + + const { getData } = response; + const dataStream = getData(); + const data = await streamToArray(dataStream); + return data; } } diff --git a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts index 70ca48f4..c61072ef 100644 --- a/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts +++ b/packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagBuilder.ts @@ -1,12 +1,5 @@ -import { ReplaceChildFunc, visitChildren } from '../../extension-utils'; import * as nunjucks from 'nunjucks'; -import { - EXECUTE_COMMAND_NAME, - EXECUTE_FILTER_NAME, - FINIAL_BUILDER_NAME, - METADATA_NAME, - REFERENCE_SEARCH_MAX_DEPTH, -} from './constants'; +import { FINIAL_BUILDER_NAME, METADATA_NAME } from './constants'; import { TagBuilder, VulcanInternalExtension } from '@vulcan-sql/core/models'; interface DeclarationLocation { @@ -100,8 +93,6 @@ export class ReqTagBuilder extends TagBuilder { ) { this.checkBuilder(node); this.checkMainBuilder(node); - } else { - visitChildren(node, this.replaceExecuteFunction.bind(this)); } } @@ -109,6 +100,7 @@ export class ReqTagBuilder extends TagBuilder { if (!this.hasMainBuilder) { this.wrapOutputWithBuilder(); } + this.reset(); } public override getMetadata() { @@ -157,53 +149,9 @@ export class ReqTagBuilder extends TagBuilder { this.root.children = [builder]; } - private replaceExecuteFunction( - node: nunjucks.nodes.Node, - replace: ReplaceChildFunc - ) { - if ( - node instanceof nunjucks.nodes.FunCall && - node.name instanceof nunjucks.nodes.LookupVal && - node.name.val.value === EXECUTE_COMMAND_NAME - ) { - let targetNode: typeof node.name.target | null = node.name.target; - let depth = 0; - while (targetNode) { - depth++; - if (depth > REFERENCE_SEARCH_MAX_DEPTH) { - throw new Error('Max depth reached'); - } - if (targetNode instanceof nunjucks.nodes.LookupVal) { - targetNode = targetNode.target; - } else if (targetNode instanceof nunjucks.nodes.FunCall) { - targetNode = targetNode.name; - } else if (targetNode instanceof nunjucks.nodes.Symbol) { - break; - } else { - throw new Error( - `Unexpected node type: ${ - (targetNode as nunjucks.nodes.Node)?.typename - }` - ); - } - } - - // If the target node is a variable from {% req xxx %}, replace it with execute filter - if (this.variableList.has((targetNode as nunjucks.nodes.Symbol).value)) { - const args = new nunjucks.nodes.NodeList(node.lineno, node.colno); - args.addChild(node.name.target); - const filter = new nunjucks.nodes.Filter( - node.lineno, - node.colno, - new nunjucks.nodes.Symbol( - node.lineno, - node.colno, - EXECUTE_FILTER_NAME - ), - args - ); - replace(filter); - } - } + private reset() { + this.variableList.clear(); + this.root = undefined; + this.hasMainBuilder = false; } } diff --git a/packages/core/src/lib/template-engine/nunjucksCompiler.ts b/packages/core/src/lib/template-engine/nunjucksCompiler.ts index f13742b1..a4cbe9b1 100644 --- a/packages/core/src/lib/template-engine/nunjucksCompiler.ts +++ b/packages/core/src/lib/template-engine/nunjucksCompiler.ts @@ -21,6 +21,7 @@ import { TagRunner, FilterBuilder, FilterRunner, + DataResult, } from '@vulcan-sql/core/models'; @injectable() @@ -78,7 +79,7 @@ export class NunjucksCompiler implements Compiler { templateName: string, data: T, pagination?: Pagination - ): Promise { + ): Promise { await this.initializeExtensions(); const builder = await this.renderAndGetMainBuilder(templateName, data); if (pagination) builder.paginate(pagination); diff --git a/packages/core/src/lib/utils/index.ts b/packages/core/src/lib/utils/index.ts index e6bc2a05..b26f014b 100644 --- a/packages/core/src/lib/utils/index.ts +++ b/packages/core/src/lib/utils/index.ts @@ -1,3 +1,4 @@ export * from './normalizedStringValue'; export * from './logger'; export * from './module'; +export * from './streams'; diff --git a/packages/core/src/lib/utils/streams.ts b/packages/core/src/lib/utils/streams.ts new file mode 100644 index 00000000..deddc9e8 --- /dev/null +++ b/packages/core/src/lib/utils/streams.ts @@ -0,0 +1,29 @@ +import { Readable } from 'stream'; + +export const arrayToStream = (array: any[]) => { + let index = 0; + const stream = new Readable({ + objectMode: true, + read() { + if (index >= array.length) { + this.push(null); + return; + } + this.push(array[index++]); + }, + }); + return stream; +}; + +export const streamToArray = (stream: Readable) => { + const rows: any[] = []; + return new Promise((resolve, reject) => { + stream.on('data', (data) => { + rows.push(data); + }); + stream.on('end', () => { + resolve(rows); + }); + stream.on('error', (err) => reject(err)); + }); +}; diff --git a/packages/core/src/models/extensions/dataSource.ts b/packages/core/src/models/extensions/dataSource.ts index 9db8daa7..00be97d7 100644 --- a/packages/core/src/models/extensions/dataSource.ts +++ b/packages/core/src/models/extensions/dataSource.ts @@ -1,7 +1,7 @@ import { SQLClauseOperation } from '@vulcan-sql/core/data-query'; import { Pagination } from '@vulcan-sql/core/models'; import { TYPES } from '@vulcan-sql/core/types'; -import { Stream } from 'stream'; +import { Readable } from 'stream'; import { ExtensionBase } from './base'; import { VulcanExtension } from './decorators'; @@ -24,7 +24,7 @@ export type DataColumn = { name: string; type: string }; export interface DataResult { getColumns: () => DataColumn[]; - getData: () => Stream; + getData: () => Readable; } export interface ExecuteOptions { diff --git a/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts index c2ef4c48..de1326e7 100644 --- a/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts @@ -1,3 +1,4 @@ +import { arrayToStream, streamToArray } from '@vulcan-sql/core'; import { createTestCompiler } from '../../testCompiler'; it('Extension should execute correct query and set/export the variable', async () => { @@ -8,18 +9,22 @@ it('Extension should execute correct query and set/export the variable', async ( select count(*) as count from user where user.id = {{ params.userId }}; {% endreq %} `); - builder.value.onFirstCall().resolves([{ count: 1 }]); + builder.value.onFirstCall().resolves({ + getColumns: () => [], + getData: () => arrayToStream([{ count: 1 }]), + }); // Action loader.setSource('test', compiledData); const result = await compiler.execute('test', { params: { userId: 'user-id' }, }); + const resultData = await streamToArray(result.getData()); // Assert expect(executor.createBuilder.firstCall.args[0]).toBe( `select count(*) as count from user where user.id = $1;` ); expect(executor.createBuilder.firstCall.args[1].get('$1')).toBe(`user-id`); - expect(result).toEqual([{ count: 1 }]); + expect(resultData).toEqual([{ count: 1 }]); }); it('If argument is not a symbol, extension should throw', async () => { @@ -122,3 +127,21 @@ it('Extension should throw an error if there are multiple builders using same na `We can't declare multiple builder with same name. Duplicated name: user (declared at 1:7 and 2:7)` ); }); + +it('Extension should reset after compiled each template', async () => { + // Arrange + const { compiler } = await createTestCompiler(); + compiler.compile( + ` + {% req user main %} select * from users; {% endreq %} + ` + ); + // Act, Arrange + await expect( + compiler.compile( + ` + {% req user main %} select * from users; {% endreq %} + ` + ) + ).resolves.not.toThrow(); +}); diff --git a/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts index 4b5e1376..850457ec 100644 --- a/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/execute.spec.ts @@ -1,6 +1,8 @@ +import { arrayToStream } from '@vulcan-sql/core'; +import { Readable } from 'stream'; import { createTestCompiler } from '../../testCompiler'; -it('Extension should call the .value() function of builder', async () => { +it('Extension should call the .value() function of builder and consume the stream', async () => { // Arrange const { compiler, loader, executor, builder } = await createTestCompiler(); const { compiledData } = await compiler.compile( @@ -12,7 +14,11 @@ select * from users; select * from group where userId = {{ user.count(3).value()[0].id }}; ` ); - builder.value.onFirstCall().resolves([{ id: 'user-id' }]); + builder.count.returns(builder); + builder.value.onFirstCall().resolves({ + getColumns: () => [], + getData: () => arrayToStream([{ id: 'user-id' }]), + }); // Action loader.setSource('test', compiledData); await compiler.execute('test', {}); @@ -59,3 +65,46 @@ select * from group where userId = '{{ user.value()[0].id }}'; 'something went wrong' ); }); + +it('Extension should return the raw value of .value() when input is not a builder', async () => { + // Arrange + const { compiler, loader, executor } = await createTestCompiler(); + const { compiledData } = await compiler.compile( + `{{ context.params.someProp.value() }}` + ); + // Action + loader.setSource('test', compiledData); + await compiler.execute('test', { + context: { params: { someProp: { value: () => `someRawVal` } } }, + }); + // Assert + expect(executor.createBuilder.firstCall.args[0]).toBe(`$1`); + expect(executor.createBuilder.firstCall.args[1].get('$1')).toBe(`someRawVal`); +}); + +it('Extension should throw an error if the data stream emit an error', async () => { + // Arrange + const { compiler, loader, builder } = await createTestCompiler(); + const { compiledData } = await compiler.compile( + ` +{% req user %} select * from users; {% endreq %} +select * from group where userId = '{{ user.value()[0].id }}'; +` + ); + const stream = new Readable({ + objectMode: true, + read() { + this.emit('error', new Error('something went wrong')); + }, + }); + builder.value.onFirstCall().resolves({ + getColumns: () => [], + getData: () => stream, + }); + + // Action, Assert + loader.setSource('test', compiledData); + await expect(compiler.execute('test', {})).rejects.toThrow( + 'something went wrong' + ); +}); diff --git a/packages/core/test/template-engine/built-in-extensions/query-builder/parameterize.spec.ts b/packages/core/test/template-engine/built-in-extensions/query-builder/parameterize.spec.ts index b899b38c..6a91f96c 100644 --- a/packages/core/test/template-engine/built-in-extensions/query-builder/parameterize.spec.ts +++ b/packages/core/test/template-engine/built-in-extensions/query-builder/parameterize.spec.ts @@ -1,3 +1,4 @@ +import { arrayToStream } from '@vulcan-sql/core'; import { createTestCompiler } from '../../testCompiler'; const queryTest = async ( @@ -9,7 +10,12 @@ const queryTest = async ( // Arrange const { compiler, loader, builder, executor } = await createTestCompiler(); const { compiledData } = await compiler.compile(template); - builder.value.onFirstCall().resolves([{ id: 1, name: 'freda' }]); + builder.value + .onFirstCall() + .resolves({ + getColumns: () => [], + getData: () => arrayToStream([{ id: 1, name: 'freda' }]), + }); // Action loader.setSource('test', compiledData); await compiler.execute('test', {