diff --git a/package.json b/package.json index 0d8f1ac8..50063c52 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,11 @@ }, "private": true, "dependencies": { - "class-validator": "^0.13.2", "@koa/cors": "^3.3.0", + "class-validator": "^0.13.2", + "dayjs": "^1.11.2", "glob": "^8.0.1", "inversify": "^6.0.1", - "dayjs": "^1.11.2", "joi": "^17.6.0", "js-yaml": "^4.1.0", "koa": "^2.13.4", @@ -35,6 +35,7 @@ "@nrwl/js": "14.0.3", "@nrwl/linter": "14.0.3", "@nrwl/workspace": "14.0.3", + "@types/from2": "^2.3.1", "@types/glob": "^7.2.0", "@types/jest": "27.4.1", "@types/js-yaml": "^4.0.5", @@ -52,6 +53,7 @@ "cz-conventional-changelog": "3.3.0", "eslint": "~8.12.0", "eslint-config-prettier": "8.1.0", + "from2": "^2.3.0", "jest": "27.5.1", "nx": "14.0.3", "prettier": "^2.5.1", diff --git a/packages/core/src/containers/modules/executor.ts b/packages/core/src/containers/modules/executor.ts index e3cea6c8..74091f2b 100644 --- a/packages/core/src/containers/modules/executor.ts +++ b/packages/core/src/containers/modules/executor.ts @@ -4,9 +4,10 @@ import { SQLClauseOperation, } from '@vulcan-sql/core/data-query'; import { Pagination } from '../../models/pagination'; -import { IDataSource } from '@vulcan-sql/core/data-source'; +import { DataResult, IDataSource } from '@vulcan-sql/core/data-source'; import { AsyncContainerModule } from 'inversify'; import { TYPES } from '../types'; +import { Stream } from 'stream'; /** * TODO: Mock data source to make data query builder could create by IoC @@ -24,10 +25,13 @@ class MockDataSource implements IDataSource { pagination?: Pagination | undefined; }) { return { - statement, - operations, - pagination, - }; + getColumns: () => { + return []; + }, + getData: () => { + return new Stream.Readable(); + }, + } as DataResult; } } diff --git a/packages/core/src/lib/data-source/dataSource.ts b/packages/core/src/lib/data-source/dataSource.ts index eddf679c..f2c0e43a 100644 --- a/packages/core/src/lib/data-source/dataSource.ts +++ b/packages/core/src/lib/data-source/dataSource.ts @@ -1,6 +1,13 @@ import { SQLClauseOperation } from '@vulcan-sql/core/data-query'; import { Pagination } from '@vulcan-sql/core/models'; +import { Stream } from 'stream'; +export type DataColumn = { name: string; type: string }; + +export type DataResult = { + getColumns: () => DataColumn[]; + getData: () => Stream; +}; export interface IDataSource { execute({ statement, @@ -10,5 +17,5 @@ export interface IDataSource { statement: string; operations: SQLClauseOperation; pagination?: Pagination; - }): Promise; + }): Promise; } diff --git a/packages/serve/src/lib/app.ts b/packages/serve/src/lib/app.ts index e7c81277..3243cd2d 100644 --- a/packages/serve/src/lib/app.ts +++ b/packages/serve/src/lib/app.ts @@ -1,15 +1,8 @@ -import { APISchema, ClassType } from '@vulcan-sql/core'; +import { APISchema } from '@vulcan-sql/core'; import * as Koa from 'koa'; import * as KoaRouter from 'koa-router'; import { isEmpty, uniq } from 'lodash'; -import { - AuditLoggingMiddleware, - BaseRouteMiddleware, - CorsMiddleware, - RequestIdMiddleware, - loadExtensions, - RateLimitMiddleware, -} from './middleware'; +import { BaseRouteMiddleware, BuiltInRouteMiddlewares } from './middleware'; import { RestfulRoute, BaseRoute, @@ -18,6 +11,7 @@ import { RouteGenerator, } from './route'; import { AppConfig } from '../models'; +import { importExtensions, loadComponents } from './loader'; export class VulcanApplication { private app: Koa; @@ -81,29 +75,17 @@ export class VulcanApplication { this.app.use(this.restfulRouter.allowedMethods()); } - public async buildMiddleware() { - // load built-in middleware - await this.use(CorsMiddleware); - await this.use(RateLimitMiddleware); - await this.use(RequestIdMiddleware); - await this.use(AuditLoggingMiddleware); - - // load extension middleware - const extensions = await loadExtensions(this.config.extensions); - await this.use(...extensions); - } - /** add middleware classes for app used */ - private async use(...classes: ClassType[]) { - const map: { [name: string]: BaseRouteMiddleware } = {}; - for (const cls of classes) { - const middleware = new cls(this.config.middlewares); - if (middleware.name in map) { - throw new Error( - `The identifier name "${middleware.name}" of middleware class ${cls.name} has been defined in other extensions` - ); - } - map[middleware.name] = middleware; - } + /** load built-in and extensions middleware classes for app used */ + public async useMiddleware() { + // import extension middleware classes + const classesOfExtension = await importExtensions( + 'middlewares', + this.config.extensions + ); + const map = await loadComponents( + [...BuiltInRouteMiddlewares, ...classesOfExtension], + this.config + ); for (const name of Object.keys(map)) { const middleware = map[name]; this.app.use(middleware.handle.bind(middleware)); diff --git a/packages/serve/src/lib/loader.ts b/packages/serve/src/lib/loader.ts new file mode 100644 index 00000000..b244f395 --- /dev/null +++ b/packages/serve/src/lib/loader.ts @@ -0,0 +1,55 @@ +import { BaseResponseFormatter } from './response-formatter'; +import { + defaultImport, + ClassType, + ModuleProperties, + mergedModules, + SourceOfExtensions, +} from '@vulcan-sql/core'; +import { BaseRouteMiddleware } from './middleware'; +import { AppConfig } from '../models'; +// The extension module interface +export interface ExtensionModule extends ModuleProperties { + ['middlewares']: ClassType[]; + ['response-formatter']: ClassType[]; +} + +type ExtensionName = 'middlewares' | 'response-formatter'; + +export const importExtensions = async ( + name: ExtensionName, + extensions?: SourceOfExtensions +) => { + // if extensions setup, load response formatter classes in the extensions + if (extensions) { + // import extension which user customized + const modules = await defaultImport(...extensions); + const module = await mergedModules(modules); + // return middleware classes in folder + return module[name] || []; + } + return []; +}; + +/** + * load components which inherit supper vulcan component class, may contains built-in or extensions + * @param classesOfComponent the classes of component which inherit supper vulcan component class + * @returns the created instance + */ +export const loadComponents = async ( + classesOfComponent: ClassType[], + config?: AppConfig +): Promise<{ [name: string]: T }> => { + const map: { [name: string]: T } = {}; + // create each extension + for (const cls of classesOfComponent) { + const component = new cls(config) as T; + if (component.name in map) { + throw new Error( + `The identifier name "${component.name}" of component class ${cls.name} has been defined in other extensions` + ); + } + map[component.name] = component; + } + return map; +}; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/auditLogMiddleware.ts b/packages/serve/src/lib/middleware/built-in-middleware/auditLogMiddleware.ts index 38814bec..7e6f555c 100644 --- a/packages/serve/src/lib/middleware/built-in-middleware/auditLogMiddleware.ts +++ b/packages/serve/src/lib/middleware/built-in-middleware/auditLogMiddleware.ts @@ -1,11 +1,11 @@ import { getLogger, ILogger, LoggerOptions } from '@vulcan-sql/core'; -import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; -import { MiddlewareConfig } from '@vulcan-sql/serve/models'; +import { BuiltInMiddleware } from '../middleware'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { AppConfig } from '@vulcan-sql/serve/models'; export class AuditLoggingMiddleware extends BuiltInMiddleware { private logger: ILogger; - constructor(config: MiddlewareConfig) { + constructor(config: AppConfig) { super('audit-log', config); // read logger options from config, if is undefined will set default value @@ -13,7 +13,7 @@ export class AuditLoggingMiddleware extends BuiltInMiddleware { this.logger = getLogger({ scopeName: 'AUDIT', options }); } - public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) { + public async handle(context: KoaRouterContext, next: KoaNext) { if (!this.enabled) return next(); const { path, request, params, response } = context; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/corsMiddleware.ts b/packages/serve/src/lib/middleware/built-in-middleware/corsMiddleware.ts index 0539470d..91f21b3e 100644 --- a/packages/serve/src/lib/middleware/built-in-middleware/corsMiddleware.ts +++ b/packages/serve/src/lib/middleware/built-in-middleware/corsMiddleware.ts @@ -1,20 +1,20 @@ import * as Koa from 'koa'; import * as cors from '@koa/cors'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; -import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware'; -import { MiddlewareConfig } from '@vulcan-sql/serve/models'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { BuiltInMiddleware } from '../middleware'; +import { AppConfig } from '@vulcan-sql/serve/models'; export type CorsOptions = cors.Options; export class CorsMiddleware extends BuiltInMiddleware { private koaCors: Koa.Middleware; - constructor(config: MiddlewareConfig) { + constructor(config: AppConfig) { super('cors', config); const options = this.getOptions() as CorsOptions; this.koaCors = cors(options); } - public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) { + public async handle(context: KoaRouterContext, next: KoaNext) { if (!this.enabled) return next(); return this.koaCors(context, next); } diff --git a/packages/serve/src/lib/middleware/built-in-middleware/index.ts b/packages/serve/src/lib/middleware/built-in-middleware/index.ts index 608f32ba..b0e5a17c 100644 --- a/packages/serve/src/lib/middleware/built-in-middleware/index.ts +++ b/packages/serve/src/lib/middleware/built-in-middleware/index.ts @@ -2,15 +2,19 @@ export * from './corsMiddleware'; export * from './requestIdMiddleware'; export * from './auditLogMiddleware'; export * from './rateLimitMiddleware'; +export * from './response-format'; import { CorsMiddleware } from './corsMiddleware'; import { RateLimitMiddleware } from './rateLimitMiddleware'; import { RequestIdMiddleware } from './requestIdMiddleware'; import { AuditLoggingMiddleware } from './auditLogMiddleware'; +import { ResponseFormatMiddleware } from './response-format'; -export default [ +// The order is the middleware running order +export const BuiltInRouteMiddlewares = [ CorsMiddleware, RateLimitMiddleware, RequestIdMiddleware, AuditLoggingMiddleware, + ResponseFormatMiddleware, ]; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/rateLimitMiddleware.ts b/packages/serve/src/lib/middleware/built-in-middleware/rateLimitMiddleware.ts index c75d60c0..aeeb75fa 100644 --- a/packages/serve/src/lib/middleware/built-in-middleware/rateLimitMiddleware.ts +++ b/packages/serve/src/lib/middleware/built-in-middleware/rateLimitMiddleware.ts @@ -1,21 +1,21 @@ import * as Koa from 'koa'; import { RateLimit, RateLimitOptions } from 'koa2-ratelimit'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; -import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware'; -import { MiddlewareConfig } from '@vulcan-sql/serve/models'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { BuiltInMiddleware } from '../middleware'; +import { AppConfig } from '@vulcan-sql/serve/models'; export { RateLimitOptions }; export class RateLimitMiddleware extends BuiltInMiddleware { private koaRateLimit: Koa.Middleware; - constructor(config: MiddlewareConfig) { + constructor(config: AppConfig) { super('rate-limit', config); const options = this.getOptions() as RateLimitOptions; this.koaRateLimit = RateLimit.middleware(options); } - public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) { + public async handle(context: KoaRouterContext, next: KoaNext) { if (!this.enabled) return next(); return this.koaRateLimit(context, next); } diff --git a/packages/serve/src/lib/middleware/built-in-middleware/requestIdMiddleware.ts b/packages/serve/src/lib/middleware/built-in-middleware/requestIdMiddleware.ts index 48eec710..f8c4d40d 100644 --- a/packages/serve/src/lib/middleware/built-in-middleware/requestIdMiddleware.ts +++ b/packages/serve/src/lib/middleware/built-in-middleware/requestIdMiddleware.ts @@ -1,8 +1,8 @@ import * as uuid from 'uuid'; import { FieldInType, asyncReqIdStorage } from '@vulcan-sql/core'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; -import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware'; -import { MiddlewareConfig } from '@vulcan-sql/serve/models'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { BuiltInMiddleware } from '../middleware'; +import { AppConfig } from '@vulcan-sql/serve/models'; export interface RequestIdOptions { name: string; @@ -12,7 +12,7 @@ export interface RequestIdOptions { export class RequestIdMiddleware extends BuiltInMiddleware { private options: RequestIdOptions; - constructor(config: MiddlewareConfig) { + constructor(config: AppConfig) { super('request-id', config); // read request-id options from config. this.options = (this.getOptions() as RequestIdOptions) || { @@ -23,7 +23,7 @@ export class RequestIdMiddleware extends BuiltInMiddleware { if (!this.options['name']) this.options['name'] = 'X-Request-ID'; if (!this.options['fieldIn']) this.options['fieldIn'] = FieldInType.HEADER; } - public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) { + public async handle(context: KoaRouterContext, next: KoaNext) { if (!this.enabled) return next(); const { request } = context; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/response-format/helpers.ts b/packages/serve/src/lib/middleware/built-in-middleware/response-format/helpers.ts new file mode 100644 index 00000000..e40e77f9 --- /dev/null +++ b/packages/serve/src/lib/middleware/built-in-middleware/response-format/helpers.ts @@ -0,0 +1,50 @@ +import { KoaRouterContext } from '@vulcan-sql/serve/route'; +import { BaseResponseFormatter } from '@vulcan-sql/serve/response-formatter'; + +export type ResponseFormatterMap = { + [name: string]: BaseResponseFormatter; +}; + +/** + * start to formatting if path is end with the format or "Accept" in the header contains the format + * @param context koa context + * @param format the formate name + * @returns boolean, is received + */ +export const isReceivedFormatRequest = ( + context: KoaRouterContext, + format: string +) => { + if (context.request.path.endsWith(`.${format}`)) return true; + if (context.request.accepts(format)) return true; + return false; +}; + +/** + * + * @param context koa context + * @param formatters the formatters which built-in and loaded extensions. + * @returns the format name used to format response + */ +export const checkUsableFormat = ({ + context, + formatters, + supportedFormats, + defaultFormat, +}: { + context: KoaRouterContext; + formatters: ResponseFormatterMap; + supportedFormats: string[]; + defaultFormat: string; +}) => { + for (const format of supportedFormats) { + if (!(format in formatters)) continue; + if (!isReceivedFormatRequest(context, format)) continue; + return format; + } + // if not found, use default format + if (!(defaultFormat in formatters)) + throw new Error(`Not find implemented formatters named ${defaultFormat}`); + + return defaultFormat; +}; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/response-format/index.ts b/packages/serve/src/lib/middleware/built-in-middleware/response-format/index.ts new file mode 100644 index 00000000..aadcb653 --- /dev/null +++ b/packages/serve/src/lib/middleware/built-in-middleware/response-format/index.ts @@ -0,0 +1,2 @@ +export * from './helpers'; +export * from './middleware'; diff --git a/packages/serve/src/lib/middleware/built-in-middleware/response-format/middleware.ts b/packages/serve/src/lib/middleware/built-in-middleware/response-format/middleware.ts new file mode 100644 index 00000000..4aeda792 --- /dev/null +++ b/packages/serve/src/lib/middleware/built-in-middleware/response-format/middleware.ts @@ -0,0 +1,57 @@ +import { AppConfig } from '@vulcan-sql/serve/models'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { BuiltInMiddleware } from '../../middleware'; +import { checkUsableFormat } from './helpers'; +import { importExtensions, loadComponents } from '@vulcan-sql/serve/loader'; +import { + BaseResponseFormatter, + BuiltInFormatters, +} from '@vulcan-sql/serve/response-formatter'; + +export type ResponseFormatOptions = { + formats: string[]; + default: string; +}; + +export class ResponseFormatMiddleware extends BuiltInMiddleware { + public readonly defaultFormat; + public readonly supportedFormats: string[]; + + constructor(config: AppConfig) { + super('response-format', config); + + const options = (this.getOptions() as ResponseFormatOptions) || {}; + const formats = options.formats || []; + + this.supportedFormats = formats.map((format) => format.toLowerCase()); + this.defaultFormat = !options.default ? 'json' : options.default; + } + public async handle(context: KoaRouterContext, next: KoaNext) { + // return to skip the middleware, if disabled + if (!this.enabled) return next(); + + const classesOfExtension = await importExtensions( + 'response-formatter', + this.config.extensions + ); + const formatters = await loadComponents([ + ...BuiltInFormatters, + ...classesOfExtension, + ]); + + // get supported and request format to use. + const format = checkUsableFormat({ + context, + formatters, + supportedFormats: this.supportedFormats, + defaultFormat: this.defaultFormat, + }); + + context.request.path = context.request.path.split('.')[0]; + // go to next to run middleware and route + await next(); + // format the response and route handler ran. + formatters[format].formatToResponse(context); + return; + } +} diff --git a/packages/serve/src/lib/middleware/index.ts b/packages/serve/src/lib/middleware/index.ts index 7d1d75c5..3cdea892 100644 --- a/packages/serve/src/lib/middleware/index.ts +++ b/packages/serve/src/lib/middleware/index.ts @@ -1,4 +1,3 @@ // export non-default -export * from './middleware'; -export * from './loader'; +export { BaseRouteMiddleware } from './middleware'; export * from './built-in-middleware'; diff --git a/packages/serve/src/lib/middleware/loader.ts b/packages/serve/src/lib/middleware/loader.ts deleted file mode 100644 index 0924c0fb..00000000 --- a/packages/serve/src/lib/middleware/loader.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { BaseRouteMiddleware } from './middleware'; -import { - defaultImport, - ClassType, - ModuleProperties, - mergedModules, - SourceOfExtensions, -} from '@vulcan-sql/core'; -// The extension module interface -export interface ExtensionModule extends ModuleProperties { - ['middlewares']: ClassType[]; -} - -export const loadExtensions = async (extensions?: SourceOfExtensions) => { - // if extensions setup, load middlewares classes in the extensions - if (extensions) { - // import extension which user customized - const modules = await defaultImport(...extensions); - const module = await mergedModules(modules); - // return middleware classes in folder - return module['middlewares'] || []; - } - return []; -}; diff --git a/packages/serve/src/lib/middleware/middleware.ts b/packages/serve/src/lib/middleware/middleware.ts index 4af5e14a..885f2457 100644 --- a/packages/serve/src/lib/middleware/middleware.ts +++ b/packages/serve/src/lib/middleware/middleware.ts @@ -1,36 +1,36 @@ -import { - BuiltInOptions, - AppConfig, - MiddlewareConfig, -} from '@vulcan-sql/serve/models'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; -import { Next } from 'koa'; - -export type RouteMiddlewareNext = Next; +import { AppConfig, BuiltInOptions } from '@vulcan-sql/serve/models'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; +import { isUndefined } from 'lodash'; export abstract class BaseRouteMiddleware { - protected config: MiddlewareConfig; - // middleware is enabled or not, default is enabled beside you give "enabled: false" in config. - protected enabled: boolean; + protected config: AppConfig; // An identifier to check the options set or not in the middlewares section of serve config public readonly name: string; - constructor(name: string, config: MiddlewareConfig) { + constructor(name: string, config: AppConfig) { this.name = name; this.config = config; - this.enabled = (this.getConfig()?.['enabled'] as boolean) || true; } public abstract handle( context: KoaRouterContext, - next: RouteMiddlewareNext + next: KoaNext ): Promise; protected getConfig() { - if (this.config && this.config[this.name]) return this.config[this.name]; + if (this.config && this.config.middlewares) + return this.config.middlewares[this.name]; return undefined; } } export abstract class BuiltInMiddleware extends BaseRouteMiddleware { + // middleware is enabled or not, default is enabled beside you give "enabled: false" in config. + protected enabled: boolean; + constructor(name: string, config: AppConfig) { + super(name, config); + + const value = this.getConfig()?.['enabled'] as boolean; + this.enabled = isUndefined(value) ? true : value; + } protected getOptions() { if (this.getConfig()) return this.getConfig()?.['options'] as BuiltInOptions; diff --git a/packages/serve/src/lib/response-formatter/csvFormatter.ts b/packages/serve/src/lib/response-formatter/csvFormatter.ts new file mode 100644 index 00000000..5e92afda --- /dev/null +++ b/packages/serve/src/lib/response-formatter/csvFormatter.ts @@ -0,0 +1,119 @@ +import * as Stream from 'stream'; +import { DataColumn, getLogger } from '@vulcan-sql/core'; +import { isArray, isObject, isUndefined } from 'lodash'; +import { KoaRouterContext } from '../route'; +import { BaseResponseFormatter, toBuffer } from './responseFormatter'; + +const logger = getLogger({ scopeName: 'SERVE' }); + +/** + * convert the array string to one line string for csv format + * @param arrString + */ +export const arrStringToCsvString = (arrString: string) => { + return arrString.replace(/^\[/, '').replace(/\\"/g, '""').replace(/\]$/, ''); +}; + +class CsvTransformer extends Stream.Transform { + private columns: string[]; + private readonly PREPEND_UTF8_BOM = '\ufeff'; + + constructor({ + columns, + options, + }: { + columns: string[]; + options?: Stream.TransformOptions; + }) { + /** + * make the csv stream source (writable stream) is object mode to get data row directly from data readable stream. + * make the csv stream transformed destination (readable stream) is not object mode + */ + options = options || { + writableObjectMode: true, + readableObjectMode: false, + }; + if (isUndefined(options.readableObjectMode)) + options.readableObjectMode = false; + if (isUndefined(options.writableObjectMode)) + options.writableObjectMode = true; + + super(options); + this.columns = columns; + + /** + * add columns name by comma through join for csv title. + * in order to avoid the non-alphabet characters transform wrong, add PREPEND_UTF8_BOM prefix + */ + this.push(toBuffer(this.PREPEND_UTF8_BOM)); + this.push(toBuffer(columns.join())); + this.push(toBuffer('\n')); + } + + public override _transform( + chunk: any, + _encoding: BufferEncoding, + callback: Stream.TransformCallback + ) { + // chuck => { name: 'jack', age: 18, hobby:['book', 'travel'] } + // pick value and join it by semicolon, e.g: "\"jack\",18,\"['book', 'travel']\"" + const valuesRow = this.columns.map((column) => + // if value is array or object, stringify to fix in one column, e.g: ['book', 'travel'] => "['book', 'travel']" + isObject(chunk[column]) || isArray(chunk[column]) + ? JSON.stringify(chunk[column]) + : chunk[column] + ); + // transform format data to buffer + const dataBuffer = toBuffer( + arrStringToCsvString(JSON.stringify(valuesRow)) + ); + // run callback and pass the transformed data buffer to transform.push() + this.push(dataBuffer); + this.push(toBuffer('\n')); + callback(null); + } +} + +export class CsvFormatter extends BaseResponseFormatter { + constructor() { + super('csv'); + } + + public format(data: Stream.Readable, columns?: DataColumn[]) { + if (!columns) throw new Error('must provide columns'); + // create csv transform stream and define transform to csv way. + const csvStream = new CsvTransformer({ + columns: columns.map((column) => column.name), + }); + // start to transform data to csv stream + data + .pipe(csvStream) + .on('error', (err) => { + logger.warn(`read stream failed, detail error ${err}`); + throw new Error( + `read data in the stream for formatting to csv failed.` + ); + }) + .on('end', () => { + logger.debug('convert to csv format stream > done.'); + }); + + return csvStream; + } + + public toResponse( + stream: Stream.Readable | Stream.Transform, + ctx: KoaRouterContext + ) { + // get file name by url path. e.g: url = '/urls/orders', result = orders + const size = ctx.url.split('/').length; + const filename = ctx.url.split('/')[size - 1]; + // set csv stream to response and header to note the stream will download + ctx.response.body = stream; + ctx.response.set( + 'Content-disposition', + `attachment; filename=${filename}.csv` + ); + ctx.response.set('Content-type', 'text/csv'); + } +} diff --git a/packages/serve/src/lib/response-formatter/index.ts b/packages/serve/src/lib/response-formatter/index.ts new file mode 100644 index 00000000..6cd18d88 --- /dev/null +++ b/packages/serve/src/lib/response-formatter/index.ts @@ -0,0 +1,9 @@ +export * from './responseFormatter'; +export * from './csvFormatter'; +export * from './jsonFormatter'; +export * from '../loader'; + +import { CsvFormatter } from './csvFormatter'; +import { JsonFormatter } from './jsonFormatter'; + +export const BuiltInFormatters = [CsvFormatter, JsonFormatter]; diff --git a/packages/serve/src/lib/response-formatter/jsonFormatter.ts b/packages/serve/src/lib/response-formatter/jsonFormatter.ts new file mode 100644 index 00000000..4ddc20e4 --- /dev/null +++ b/packages/serve/src/lib/response-formatter/jsonFormatter.ts @@ -0,0 +1,80 @@ +import * as Stream from 'stream'; +import { getLogger } from '@vulcan-sql/core'; +import { BaseResponseFormatter, toBuffer } from './responseFormatter'; +import { isUndefined } from 'lodash'; +import { KoaRouterContext } from '../route'; + +const logger = getLogger({ scopeName: 'SERVE' }); + +class JsonStringTransformer extends Stream.Transform { + private first: boolean; + constructor(options?: Stream.TransformOptions) { + /** + * make the json stream source (writable stream) is object mode to get data row directly from data readable stream. + * make the json stream transformed destination (readable stream) is not object mode + */ + options = options || { + writableObjectMode: true, + readableObjectMode: false, + }; + if (isUndefined(options.readableObjectMode)) + options.readableObjectMode = false; + if (isUndefined(options.writableObjectMode)) + options.writableObjectMode = true; + + super(options); + this.first = true; + } + override _transform( + chunk: any, + _encoding: BufferEncoding, + callback: Stream.TransformCallback + ) { + if (this.first) { + this.push(toBuffer('[')); + this.first = false; + } else { + this.push(toBuffer(',')); + } + + this.push(toBuffer(JSON.stringify(chunk))); + callback(null); + } + override _final(callback: (error?: Error | null) => void) { + this.push(toBuffer(']')); + callback(null); + } +} + +export class JsonFormatter extends BaseResponseFormatter { + constructor() { + super('json'); + } + + public format(data: Stream.Readable) { + const jsonStream = new JsonStringTransformer(); + // Read data stream and convert the format to json format stream. + data + .pipe(jsonStream) + .on('error', (err: Error) => { + logger.warn(`read stream failed, detail error ${err}`); + throw new Error( + `read data in the stream for formatting to json failed.` + ); + }) + .on('end', () => { + logger.debug('convert to json format stream > done.'); + }); + + return jsonStream; + } + + public toResponse( + stream: Stream.Readable | Stream.Transform, + ctx: KoaRouterContext + ) { + // set json stream to response in context ( data is json stream, no need to convert. ) + ctx.response.body = stream; + ctx.response.set('Content-type', 'application/json'); + } +} diff --git a/packages/serve/src/lib/response-formatter/responseFormatter.ts b/packages/serve/src/lib/response-formatter/responseFormatter.ts new file mode 100644 index 00000000..027555a1 --- /dev/null +++ b/packages/serve/src/lib/response-formatter/responseFormatter.ts @@ -0,0 +1,73 @@ +import { DataColumn } from '@vulcan-sql/core'; +import { has } from 'lodash'; +import * as Stream from 'stream'; +import { KoaRouterContext } from '../route'; + +export type BodyResponse = { + data: Stream.Readable; + columns: DataColumn[]; + [key: string]: any; +}; + +export const toBuffer = (str: string) => { + return Buffer.from(str, 'utf8'); +}; + +export interface IFormatter { + // format name, e.g: json, csv + readonly name: string; + + format( + data: Stream.Readable, + columns?: DataColumn[] + ): Stream.Readable | Stream.Transform; + + toResponse( + stream: Stream.Readable | Stream.Transform, + ctx: KoaRouterContext + ): void; + formatToResponse(ctx: KoaRouterContext): void; +} + +export abstract class BaseResponseFormatter implements IFormatter { + public readonly name: string; + constructor(name: string) { + this.name = name; + } + + public formatToResponse(ctx: KoaRouterContext) { + // return empty csv stream data or column is not exist + if (!has(ctx.response.body, 'data') || !has(ctx.response.body, 'columns')) { + const stream = new Stream.Readable(); + stream.push(null); + this.toResponse(stream, ctx); + return; + } + // if response has data and columns. + const { data, columns } = ctx.response.body as BodyResponse; + const formatted = this.format(data, columns); + // set formatted stream to response in context + this.toResponse(formatted, ctx); + return; + } + + /** + * Define how to format original data stream with option columns to formatted stream. + * @param data data stream + * @param columns data columns + */ + public abstract format( + data: Stream.Readable, + columns?: DataColumn[] + ): Stream.Readable | Stream.Transform; + + /** + * Define how to set the formatted stream to context in response + * @param stream formatted stream + * @param ctx koa context + */ + public abstract toResponse( + stream: Stream.Readable | Stream.Transform, + ctx: KoaRouterContext + ): void; +} diff --git a/packages/serve/src/lib/route/route-component/baseRoute.ts b/packages/serve/src/lib/route/route-component/baseRoute.ts index 2abce942..612d28b1 100644 --- a/packages/serve/src/lib/route/route-component/baseRoute.ts +++ b/packages/serve/src/lib/route/route-component/baseRoute.ts @@ -52,10 +52,7 @@ export abstract class BaseRoute implements IRoute { const { reqParams } = transformed; // could template name or template path, use for template engine const { templateSource } = this.apiSchema; - const statement = await this.templateEngine.execute( - templateSource, - reqParams - ); - return statement; + const result = await this.templateEngine.execute(templateSource, reqParams); + return result; } } diff --git a/packages/serve/src/lib/server.ts b/packages/serve/src/lib/server.ts index 5a482069..4243c0af 100644 --- a/packages/serve/src/lib/server.ts +++ b/packages/serve/src/lib/server.ts @@ -26,7 +26,7 @@ export class VulcanServer { // Create application const app = new VulcanApplication(omit(this.config, 'template'), generator); - await app.buildMiddleware(); + await app.useMiddleware(); await app.buildRoutes(this.schemas, this.config.types); // Run server this.server = http.createServer(app.getHandler()).listen(port); diff --git a/packages/serve/src/models/middlewareConfig.ts b/packages/serve/src/models/middlewareConfig.ts index 002b3ce1..874cc210 100644 --- a/packages/serve/src/models/middlewareConfig.ts +++ b/packages/serve/src/models/middlewareConfig.ts @@ -3,6 +3,7 @@ import { CorsOptions, RateLimitOptions, RequestIdOptions, + ResponseFormatOptions, } from '../lib/middleware'; // built-in options for middleware @@ -10,9 +11,10 @@ export type BuiltInOptions = | RequestIdOptions | LoggerOptions | RateLimitOptions - | CorsOptions; + | CorsOptions + | ResponseFormatOptions; -export type CustomOptions = string | number | boolean | object; +export type CustomOptions = any; /** * The identifier name represent to load middleware if it is custom, diff --git a/packages/serve/test/app.spec.ts b/packages/serve/test/app.spec.ts index fdc38744..c7229386 100644 --- a/packages/serve/test/app.spec.ts +++ b/packages/serve/test/app.spec.ts @@ -69,9 +69,8 @@ describe('Test vulcan server for practicing middleware', () => { }, container.get(TYPES.RouteGenerator) ); - await app.buildMiddleware(); + await app.useMiddleware(); await app.buildRoutes([fakeSchema], [APIProviderType.RESTFUL]); - const server = http .createServer(app.getHandler()) .listen(faker.internet.port()); @@ -83,7 +82,6 @@ describe('Test vulcan server for practicing middleware', () => { // Act const reqOperation = supertest(server).get(fakeSchema.urlPath); - const response = await reqOperation; // Assert expect(response.headers).toEqual(expect.objectContaining(expected)); @@ -269,12 +267,18 @@ describe('Test vulcan server for calling restful APIs', () => { ])( 'Should be correct when given validated koa context request from %p', async (_: string, schema: APISchema, ctx: KoaRouterContext) => { - // Arrange + // Arrange, close response format middlewares to make expected work. const app = new VulcanApplication( - {}, + { + middlewares: { + 'response-format': { + enabled: false, + }, + }, + }, container.get(TYPES.RouteGenerator) ); - await app.buildMiddleware(); + await app.useMiddleware(); await app.buildRoutes([schema], [APIProviderType.RESTFUL]); const server = http .createServer(app.getHandler()) diff --git a/packages/serve/test/middlewares/built-in-middlewares/auditLogMiddleware.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/auditLogMiddleware.spec.ts index 51ec6cd6..509c2312 100644 --- a/packages/serve/test/middlewares/built-in-middlewares/auditLogMiddleware.spec.ts +++ b/packages/serve/test/middlewares/built-in-middlewares/auditLogMiddleware.spec.ts @@ -11,6 +11,7 @@ import { import * as core from '@vulcan-sql/core'; import * as uuid from 'uuid'; import { LoggerOptions } from '@vulcan-sql/core'; +import { MiddlewareConfig } from '@vulcan-sql/serve'; describe('Test audit logging middlewares', () => { afterEach(() => { @@ -104,11 +105,13 @@ describe('Test audit logging middlewares', () => { // setup request-id middleware run first. const stubReqIdMiddleware = new RequestIdMiddleware({}); const middleware = new AuditLoggingMiddleware({ - 'audit-log': { - options: { - displayRequestId: true, - } as LoggerOptions, - }, + middlewares: { + 'audit-log': { + options: { + displayRequestId: true, + } as LoggerOptions, + }, + } as MiddlewareConfig, }); // Use spy to trace the logger from getLogger( scopeName: 'AUDIT' }) to know in logger.info(...) // it will get the setting of logger from above new audit logging middleware diff --git a/packages/serve/test/middlewares/built-in-middlewares/corsMiddleware.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/corsMiddleware.spec.ts index cd5f11b1..4b4743ea 100644 --- a/packages/serve/test/middlewares/built-in-middlewares/corsMiddleware.spec.ts +++ b/packages/serve/test/middlewares/built-in-middlewares/corsMiddleware.spec.ts @@ -3,6 +3,7 @@ import * as Koa from 'koa'; import * as supertest from 'supertest'; import { CorsOptions, CorsMiddleware } from '@vulcan-sql/serve/middleware'; import { Server } from 'http'; +import { MiddlewareConfig } from '@vulcan-sql/serve'; describe('Test cors middlewares', () => { let server: Server; @@ -12,10 +13,12 @@ describe('Test cors middlewares', () => { const app = new Koa(); const middleware = new CorsMiddleware({ - cors: { - options: { - origin: domain, - } as CorsOptions, + middlewares: { + cors: { + options: { + origin: domain, + } as CorsOptions, + } as MiddlewareConfig, }, }); // use middleware in koa app diff --git a/packages/serve/test/middlewares/built-in-middlewares/rateLimitMiddleware.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/rateLimitMiddleware.spec.ts index 469e9c0e..73891498 100644 --- a/packages/serve/test/middlewares/built-in-middlewares/rateLimitMiddleware.spec.ts +++ b/packages/serve/test/middlewares/built-in-middlewares/rateLimitMiddleware.spec.ts @@ -7,6 +7,7 @@ import { RateLimitMiddleware, RateLimitOptions, } from '@vulcan-sql/serve/middleware'; +import { MiddlewareConfig } from '@vulcan-sql/serve'; // Should use koa app and supertest for testing, because it will call koa context method in ratelimit middleware. describe('Test rate limit middlewares', () => { @@ -15,12 +16,14 @@ describe('Test rate limit middlewares', () => { const app = new Koa(); const router = new KoaRouter(); const middleware = new RateLimitMiddleware({ - 'rate-limit': { - options: { - max: 2, - interval: 2000, - } as RateLimitOptions, - }, + middlewares: { + 'rate-limit': { + options: { + max: 2, + interval: 2000, + } as RateLimitOptions, + }, + } as MiddlewareConfig, }); // use middleware in koa app app.use(middleware.handle.bind(middleware)); diff --git a/packages/serve/test/middlewares/built-in-middlewares/requestIdMiddleware.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/requestIdMiddleware.spec.ts index f0050bcf..33cf061f 100644 --- a/packages/serve/test/middlewares/built-in-middlewares/requestIdMiddleware.spec.ts +++ b/packages/serve/test/middlewares/built-in-middlewares/requestIdMiddleware.spec.ts @@ -10,6 +10,7 @@ import { RequestIdOptions, } from '@vulcan-sql/serve/middleware'; import * as uuid from 'uuid'; +import { MiddlewareConfig } from '@vulcan-sql/serve'; describe('Test request-id middlewares', () => { afterEach(() => { @@ -58,12 +59,14 @@ describe('Test request-id middlewares', () => { }; // Act const middleware = new RequestIdMiddleware({ - 'request-id': { - options: { - name: 'Test-Request-ID', - fieldIn: FieldInType.QUERY, - } as RequestIdOptions, - }, + middlewares: { + 'request-id': { + options: { + name: 'Test-Request-ID', + fieldIn: FieldInType.QUERY, + } as RequestIdOptions, + }, + } as MiddlewareConfig, }); // spy the asyncReqIdStorage behavior @@ -88,11 +91,13 @@ describe('Test request-id middlewares', () => { }; // Act const middleware = new RequestIdMiddleware({ - 'request-id': { - options: { - fieldIn: FieldInType.QUERY, - } as RequestIdOptions, - }, + middlewares: { + 'request-id': { + options: { + fieldIn: FieldInType.QUERY, + } as RequestIdOptions, + }, + } as MiddlewareConfig, }); // spy the asyncReqIdStorage behavior diff --git a/packages/serve/test/middlewares/built-in-middlewares/response-format/formatResponseMiddleware.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/response-format/formatResponseMiddleware.spec.ts new file mode 100644 index 00000000..9313c015 --- /dev/null +++ b/packages/serve/test/middlewares/built-in-middlewares/response-format/formatResponseMiddleware.spec.ts @@ -0,0 +1,99 @@ +import * as sinon from 'ts-sinon'; +import { ResponseFormatMiddleware } from '@vulcan-sql/serve/middleware'; +import * as loader from '@vulcan-sql/serve/loader'; +import { KoaRouterContext, MiddlewareConfig } from '@vulcan-sql/serve'; + +describe('Test format response middleware', () => { + afterEach(() => { + sinon.default.restore(); + }); + + it('Test to skip response format when enabled = false', async () => { + // Arrange + const ctx: KoaRouterContext = { + ...sinon.stubInterface(), + }; + // Act + const middleware = new ResponseFormatMiddleware({ + middlewares: { + 'response-format': { + enabled: false, + }, + } as MiddlewareConfig, + }); + // spy the async function to do test + const spy = jest.spyOn(loader, 'importExtensions'); + + await middleware.handle(ctx, async () => Promise.resolve()); + + expect(spy).not.toHaveBeenCalled(); + }); + + it('Test to get default json format and empty supported format when not set any config for response formatter', async () => { + // Arrange + const ctx: KoaRouterContext = { + ...sinon.stubInterface(), + }; + // Act + const middleware = new ResponseFormatMiddleware({ + middlewares: { + 'response-format': { + enabled: false, + }, + } as MiddlewareConfig, + }); + + await middleware.handle(ctx, async () => Promise.resolve()); + + expect(middleware.supportedFormats).toEqual([]); + expect(middleware.defaultFormat).toEqual('json'); + }); + + it('Test to get default "csv" format and empty supported format when set "default" is "csv', async () => { + // Arrange + const ctx: KoaRouterContext = { + ...sinon.stubInterface(), + }; + // Act + const middleware = new ResponseFormatMiddleware({ + middlewares: { + 'response-format': { + enabled: false, + options: { + default: 'csv', + }, + }, + } as MiddlewareConfig, + }); + + await middleware.handle(ctx, async () => Promise.resolve()); + + expect(middleware.supportedFormats).toEqual([]); + expect(middleware.defaultFormat).toEqual('csv'); + }); + + it('Test to get ["hyper", "csv"] formats when set "formats" to ["hyper", "csv"]', async () => { + // Arrange + const ctx: KoaRouterContext = { + ...sinon.stubInterface(), + }; + // Act + const middleware = new ResponseFormatMiddleware({ + middlewares: { + 'response-format': { + enabled: false, + options: { + formats: ['hyper', 'csv'], + }, + }, + } as MiddlewareConfig, + }); + + await middleware.handle(ctx, async () => Promise.resolve()); + + expect(middleware.supportedFormats).toEqual(['hyper', 'csv']); + expect(middleware.defaultFormat).toEqual('json'); + }); + + // TODO: test handle to get context response +}); diff --git a/packages/serve/test/middlewares/built-in-middlewares/response-format/helpers.spec.ts b/packages/serve/test/middlewares/built-in-middlewares/response-format/helpers.spec.ts new file mode 100644 index 00000000..3f4ec644 --- /dev/null +++ b/packages/serve/test/middlewares/built-in-middlewares/response-format/helpers.spec.ts @@ -0,0 +1,281 @@ +import { Request } from 'koa'; +import * as sinon from 'ts-sinon'; +import faker from '@faker-js/faker'; +import { + checkUsableFormat, + isReceivedFormatRequest, + ResponseFormatterMap, +} from '@vulcan-sql/serve/middleware'; +import * as responseHelpers from '@vulcan-sql/serve/middleware/built-in-middleware/response-format/helpers'; +import { + BaseResponseFormatter, + BuiltInFormatters, + CsvFormatter, + JsonFormatter, + loadComponents, +} from '@vulcan-sql/serve/response-formatter'; +import { KoaRouterContext } from '@vulcan-sql/serve'; +import { importExtensions } from '@vulcan-sql/serve/loader'; + +it('Test to get built-in formatters when call load usable formatters with no extensions', async () => { + // Act + const classesOfExtension = await importExtensions('response-formatter'); + const result = await loadComponents([ + ...BuiltInFormatters, + ...classesOfExtension, + ]); + // Assert + expect(result).toEqual({ + csv: new CsvFormatter(), + json: new JsonFormatter(), + }); +}); + +it.each([ + { + request: { + path: `${faker.internet.url()}.json`, + accepts: jest.fn().mockReturnValue(false), + }, + format: 'json', + expected: true, + }, + { + request: { + path: faker.internet.url(), + accepts: jest.fn().mockReturnValue('application/json'), + }, + format: 'json', + expected: true, + }, + { + request: { + path: `${faker.internet.url()}.json`, + accepts: jest.fn().mockReturnValue(false), + }, + format: 'csv', + expected: false, + }, +])( + 'Test to get $expected when call received format request with $request, format "$format"', + ({ request, format, expected }) => { + // Arrange + const context = { + ...sinon.stubInterface(), + request: { + ...sinon.stubInterface(), + path: request.path, + accepts: request.accepts, + } as Request, + }; + // Act + const result = isReceivedFormatRequest(context, format); + // Assert + expect(result).toEqual(expected); + } +); + +describe('Test to call check usable format function', () => { + afterEach(() => { + sinon.default.restore(); + }); + it.each([ + { + defaultFormat: 'json', + expected: 'json', + }, + { + defaultFormat: 'csv', + expected: 'csv', + }, + { + defaultFormat: 'hyper', + expected: 'hyper', + }, + ])( + 'Test to get default format "$expected" when check usable format with empty support formats', + ({ defaultFormat, expected }) => { + // Arrange + const input = { + formatters: { + csv: new CsvFormatter(), + json: new JsonFormatter(), + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: [], + }; + + // Act + const result = checkUsableFormat({ + context: sinon.stubInterface(), + formatters: input.formatters, + supportedFormats: input.supportedFormats, + defaultFormat, + }); + + // Assert + expect(result).toEqual(expected); + } + ); + + it.each([ + { + defaultFormat: 'hyper', + }, + { + defaultFormat: 'json', + }, + ])( + 'Test to throw error when check usable format with empty support formats, but default format "$defaultFormat" not existed in formatters', + ({ defaultFormat }) => { + // Arrange + const expected = new Error( + `Not find implemented formatters named ${defaultFormat}` + ); + + // Act + const checkUsableFormatAction = () => + checkUsableFormat({ + context: sinon.stubInterface(), + formatters: {}, + supportedFormats: [], + defaultFormat, + }); + + // Assert + expect(checkUsableFormatAction).toThrowError(expected); + } + ); + + it.each([ + { + formatters: { + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: ['csv', 'json'], + defaultFormat: 'hyper', + expected: 'hyper', + }, + { + formatters: { + json: new JsonFormatter(), + } as ResponseFormatterMap, + supportedFormats: ['csv', 'hyper'], + defaultFormat: 'json', + expected: 'json', + }, + ])( + 'Test to get default format "$expected" when check usable format with supported formats "$supportedFormats" but formatters not matched', + ({ formatters, supportedFormats, defaultFormat, expected }) => { + // Arrange + + sinon.default + .stub(responseHelpers, 'isReceivedFormatRequest') + .returns(true); + + // Act + const result = checkUsableFormat({ + context: sinon.stubInterface(), + formatters, + supportedFormats, + defaultFormat, + }); + + // Assert + expect(result).toEqual(expected); + } + ); + + it.each([ + { + formatters: { + json: new JsonFormatter(), + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: ['csv', 'hyper'], + defaultFormat: 'json', + expected: 'json', + }, + { + formatters: { + json: new JsonFormatter(), + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: ['csv', 'json'], + defaultFormat: 'hyper', + expected: 'hyper', + }, + ])( + 'Test to get default format "$expected" when check usable format with matched formatter in supported formats "$supportedFormats" but not received format request', + ({ formatters, supportedFormats, defaultFormat, expected }) => { + // Arrange + + sinon.default + .stub(responseHelpers, 'isReceivedFormatRequest') + .returns(false); + + // Act + const result = checkUsableFormat({ + context: sinon.stubInterface(), + formatters, + supportedFormats, + defaultFormat, + }); + + // Assert + expect(result).toEqual(expected); + } + ); + + it.each([ + { + formatters: { + json: new JsonFormatter(), + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: ['csv', 'json', 'hyper'], + defaultFormat: 'hyper', + expected: 'json', + }, + { + formatters: { + json: new JsonFormatter(), + hyper: { + name: 'hyper', + } as BaseResponseFormatter, + } as ResponseFormatterMap, + supportedFormats: ['csv', 'hyper', 'json'], + defaultFormat: 'json', + expected: 'hyper', + }, + ])( + 'Test to get format "$expected" when check usable format with matched formatter in supported formats "$supportedFormats" and received format request', + ({ formatters, supportedFormats, defaultFormat, expected }) => { + // Arrange + sinon.default + .stub(responseHelpers, 'isReceivedFormatRequest') + .returns(true); + + // Act + const result = checkUsableFormat({ + context: sinon.stubInterface(), + formatters, + supportedFormats, + defaultFormat, + }); + + // Assert + expect(result).toEqual(expected); + } + ); +}); diff --git a/packages/serve/test/middlewares/loader.spec.ts b/packages/serve/test/middlewares/loader.spec.ts index 7a95448d..792d1893 100644 --- a/packages/serve/test/middlewares/loader.spec.ts +++ b/packages/serve/test/middlewares/loader.spec.ts @@ -1,41 +1,12 @@ import * as path from 'path'; import * as sinon from 'ts-sinon'; -import { - BaseRouteMiddleware, - loadExtensions, -} from '@vulcan-sql/serve/middleware'; -import middlewares from '@vulcan-sql/serve/middleware/built-in-middleware'; +import { importExtensions } from '@vulcan-sql/serve/loader'; +import { BaseRouteMiddleware } from '@vulcan-sql/serve/middleware'; import { TestModeMiddleware } from './test-custom-middlewares'; -import { ClassType, defaultImport } from '@vulcan-sql/core'; +import { ClassType } from '@vulcan-sql/core'; import { AppConfig } from '@vulcan-sql/serve/models'; -import { flatten } from 'lodash'; - -// the load Built-in used for tests -const loadBuiltIn = async () => { - // built-in middleware folder - const builtInFolder = path.resolve( - __dirname, - '../../src/lib/middleware', - 'built-in-middleware' - ); - // read built-in middlewares in index.ts, the content is an array middleware class - const modules = - flatten( - await defaultImport[]>(builtInFolder) - ) || []; - return modules || []; -}; describe('Test middleware loader', () => { - it('Should load successfully when loading built-in middlewares', async () => { - // Arrange - - const expected = [...middlewares] as ClassType[]; - // Act - const actual = await loadBuiltIn(); - // Assert - expect(actual).toEqual(expect.arrayContaining(expected)); - }); it('Should load successfully when loading extension middlewares', async () => { // Arrange const expected = [TestModeMiddleware] as ClassType[]; @@ -45,7 +16,7 @@ describe('Test middleware loader', () => { } as AppConfig; // Act - const actual = await loadExtensions(config.extensions); + const actual = await importExtensions('middlewares', config.extensions); // Assert expect(actual).toEqual(expect.arrayContaining(expected)); }); @@ -61,7 +32,7 @@ describe('Test middleware loader', () => { } as AppConfig; // Act - const actual = await loadExtensions(config.extensions); + const actual = await importExtensions('middlewares', config.extensions); // Assert expect(actual).not.toEqual(expect.arrayContaining(expected)); }); diff --git a/packages/serve/test/middlewares/test-custom-middlewares/testModeMiddleware.ts b/packages/serve/test/middlewares/test-custom-middlewares/testModeMiddleware.ts index 877370d3..ba253c5c 100644 --- a/packages/serve/test/middlewares/test-custom-middlewares/testModeMiddleware.ts +++ b/packages/serve/test/middlewares/test-custom-middlewares/testModeMiddleware.ts @@ -1,9 +1,6 @@ import { MiddlewareConfig } from '@vulcan-sql/serve/models'; -import { - BaseRouteMiddleware, - RouteMiddlewareNext, -} from '@vulcan-sql/serve/middleware'; -import { KoaRouterContext } from '@vulcan-sql/serve/route'; +import { BaseRouteMiddleware } from '@vulcan-sql/serve/middleware'; +import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route'; export interface TestModeOptions { mode: boolean; @@ -15,7 +12,7 @@ export class TestModeMiddleware extends BaseRouteMiddleware { super('test-mode', config); this.mode = (this.getConfig()?.['mode'] as boolean) || false; } - public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) { + public async handle(context: KoaRouterContext, next: KoaNext) { context.response.set('test-mode', String(this.mode)); await next(); } diff --git a/packages/serve/test/response-formatter/csv.spec.ts b/packages/serve/test/response-formatter/csv.spec.ts new file mode 100644 index 00000000..f59ad6cb --- /dev/null +++ b/packages/serve/test/response-formatter/csv.spec.ts @@ -0,0 +1,121 @@ +import faker from '@faker-js/faker'; +import * as sinon from 'ts-sinon'; +import * as Stream from 'stream'; +import { Response } from 'koa'; +import { + arrStringToCsvString, + CsvFormatter, +} from '@vulcan-sql/serve/response-formatter'; +import { KoaRouterContext } from '@vulcan-sql/serve'; +import { arrayToStream, streamToString } from '../test-utils'; + +describe('Test array string to csv string', () => { + it.each([ + { + input: ['val', 0, true, JSON.stringify({ key: 1, value: 'subVal' })], + expected: '"val",0,true,"{""key"":1,""value"":""subVal""}"', + }, + { + input: ['val', 0, true, JSON.stringify([1, 'value'])], + expected: '"val",0,true,"[1,""value""]"', + }, + ])('Test array to string to csv', ({ input, expected }) => { + // Act + const result = arrStringToCsvString(JSON.stringify(input)); + // Assert + expect(result).toBe(expected); + }); +}); + +describe('Test to respond to csv', () => { + it('Test to get empty stream when not found "data" or "columns" in ctx.response.body', () => { + // Arrange + const stubResponse = sinon.stubInterface(); + stubResponse.set.callsFake(() => null); + const ctx = { + ...sinon.stubInterface(), + url: faker.internet.url(), + response: stubResponse, + }; + const expected = new Stream.Readable(); + expected.push(null); + + // Act + const formatter = new CsvFormatter(); + formatter.formatToResponse(ctx); + // Assert + expect(ctx.response.body).toEqual(expected); + }); + + it.each([ + { + input: { + data: arrayToStream([ + { + column1: '5ccbe099-3647-47f6-b16a-847184dc8349', + column2: 'abc', + column3: 1, + }, + { + column1: 'b77f2033-015b-4c0c-bfa1-b354bcb18a6e', + column2: 'deg', + column3: 2, + }, + ]), + columns: [ + { name: 'column1', type: 'uuid' }, + { name: 'column2', type: 'varchar' }, + { name: 'column3', type: 'integer' }, + ], + }, + expected: `\ufeffcolumn1,column2,column3\n"5ccbe099-3647-47f6-b16a-847184dc8349","abc",1\n"b77f2033-015b-4c0c-bfa1-b354bcb18a6e","deg",2\n`, + }, + { + input: { + data: arrayToStream([ + { + name: 'jack', + age: 18, + hobby: ['novels', 'basketball'], + }, + { + name: 'mercy', + age: 20, + hobby: ['shopping', 'jogging'], + }, + ]), + columns: [ + { name: 'name', type: 'varchar' }, + { name: 'age', type: 'integer' }, + { name: 'hobby', type: 'array' }, + ], + }, + expected: `\ufeffname,age,hobby\n"jack",18,"[""novels"",""basketball""]"\n"mercy",20,"[""shopping"",""jogging""]"\n`, + }, + ])( + 'Test success when formatting to csv stream', + async ({ input, expected }) => { + // Arrange + const stubResponse = sinon.stubInterface(); + stubResponse.set.callsFake(() => null); + // source data & columns + stubResponse.body = { + data: input.data, + columns: input.columns, + }; + const ctx = { + ...sinon.stubInterface(), + url: faker.internet.url(), + response: stubResponse, + }; + + // Act + const formatter = new CsvFormatter(); + formatter.formatToResponse(ctx); + // Assert + + const result = await streamToString(ctx.response.body as Stream); + expect(result).toEqual(expected); + } + ); +}); diff --git a/packages/serve/test/response-formatter/json.spec.ts b/packages/serve/test/response-formatter/json.spec.ts new file mode 100644 index 00000000..d6acc371 --- /dev/null +++ b/packages/serve/test/response-formatter/json.spec.ts @@ -0,0 +1,88 @@ +import { Response } from 'koa'; +import * as sinon from 'ts-sinon'; +import * as Stream from 'stream'; +import { JsonFormatter } from '@vulcan-sql/serve/response-formatter'; +import { KoaRouterContext } from '@vulcan-sql/serve'; +import faker from '@faker-js/faker'; +import { arrayToStream, streamToString } from '../test-utils'; + +describe('Test to respond to json', () => { + it('Test to get empty stream when not found "data" or "columns" in ctx.response.body', () => { + // Arrange + const stubResponse = sinon.stubInterface(); + stubResponse.set.callsFake(() => null); + const ctx = { + ...sinon.stubInterface(), + url: faker.internet.url(), + response: stubResponse, + }; + const expected = new Stream.Readable(); + expected.push(null); + // Act + const formatter = new JsonFormatter(); + formatter.formatToResponse(ctx); + // Assert + expect(ctx.response.body).toEqual(expected); + }); + + it.each([ + { + input: { + data: arrayToStream([ + { + column1: '5ccbe099-3647-47f6-b16a-847184dc8349', + column2: 'abc', + column3: 1, + }, + { + column1: 'b77f2033-015b-4c0c-bfa1-b354bcb18a6e', + column2: 'deg', + column3: 2, + }, + ]), + columns: [ + { name: 'column1', type: 'uuid' }, + { name: 'column2', type: 'varchar' }, + { name: 'column3', type: 'integer' }, + ], + }, + expected: [ + { + column1: '5ccbe099-3647-47f6-b16a-847184dc8349', + column2: 'abc', + column3: 1, + }, + { + column1: 'b77f2033-015b-4c0c-bfa1-b354bcb18a6e', + column2: 'deg', + column3: 2, + }, + ], + }, + ])( + 'Test success when formatting to json stream', + async ({ input, expected }) => { + // Arrange + const stubResponse = sinon.stubInterface(); + stubResponse.set.callsFake(() => null); + // source data & columns + stubResponse.body = { + data: input.data, + columns: input.columns, + }; + const ctx = { + ...sinon.stubInterface(), + url: faker.internet.url(), + response: stubResponse, + }; + + // Act + const formatter = new JsonFormatter(); + formatter.formatToResponse(ctx); + // Assert + + const result = await streamToString(ctx.response.body as Stream); + expect(result).toEqual(JSON.stringify(expected)); + } + ); +}); diff --git a/packages/serve/test/test-utils.ts b/packages/serve/test/test-utils.ts new file mode 100644 index 00000000..72452118 --- /dev/null +++ b/packages/serve/test/test-utils.ts @@ -0,0 +1,21 @@ +import * as Stream from 'stream'; + +/* istanbul ignore file */ +export const arrayToStream = (data: Array): Stream => { + return new Stream.Readable({ + objectMode: true, + read() { + // make the data push by array order. + this.push(data.shift() || null); + }, + }); +}; + +export const streamToString = (stream: Stream) => { + const chunks: any = []; + return new Promise((resolve, reject) => { + stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + stream.on('error', (err) => reject(err)); + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + }); +}; diff --git a/tsconfig.base.json b/tsconfig.base.json index c326e5f7..7bdaef27 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -79,6 +79,16 @@ "@vulcan-sql/serve/pagination/*": ["packages/serve/src/lib/pagination/*"], "@vulcan-sql/serve/route": ["packages/serve/src/lib/route/index"], "@vulcan-sql/serve/route/*": ["packages/serve/src/lib/route/*"], + "@vulcan-sql/serve/utils": ["packages/serve/src/lib/utils/index"], + "@vulcan-sql/serve/utils/*": ["packages/serve/src/lib/utils/*"], + "@vulcan-sql/serve/response-formatter": [ + "packages/serve/src/lib/response-formatter/index" + ], + "@vulcan-sql/serve/response-formatter/*": [ + "packages/serve/src/lib/response-formatter/*" + ], + + "@vulcan-sql/serve/loader": ["packages/serve/src/lib/loader"], "@vulcan-sql/test-utility": ["packages/test-utility/src/index.ts"] } }, diff --git a/yarn.lock b/yarn.lock index 9236c94f..360ce29d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1066,6 +1066,13 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/from2@^2.3.1": + version "2.3.1" + resolved "https://registry.yarnpkg.com/@types/from2/-/from2-2.3.1.tgz#5620cc93c1a08c76691c1ba3d78ce4a37bf58277" + integrity sha512-l7kKtohAc5h0CKh6vFMv5WcWvQx40KE6dQneUg22i8c1mwxhVPbN781bYts/mYXxSBrQMhNxkhwg18QY0MfeOg== + dependencies: + "@types/node" "*" + "@types/glob@^7.2.0": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -2033,6 +2040,11 @@ copy-to@^2.0.1: resolved "https://registry.yarnpkg.com/copy-to/-/copy-to-2.0.1.tgz#2680fbb8068a48d08656b6098092bdafc906f4a5" integrity sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU= +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + cosmiconfig-typescript-loader@^1.0.0: version "1.0.9" resolved "https://registry.yarnpkg.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-1.0.9.tgz#69c523f7e8c3d9f27f563d02bbeadaf2f27212d3" @@ -2682,6 +2694,14 @@ fresh@~0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= +from2@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" + integrity sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g== + dependencies: + inherits "^2.0.1" + readable-stream "^2.0.0" + fs-constants@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" @@ -3029,7 +3049,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -3168,6 +3188,11 @@ isarray@0.0.1: resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -4408,6 +4433,11 @@ pretty-format@^27.0.0, pretty-format@^27.5.1: ansi-styles "^5.0.0" react-is "^17.0.1" +process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -4458,6 +4488,19 @@ react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== +readable-stream@^2.0.0: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" @@ -4592,7 +4635,7 @@ safe-buffer@5.2.1, safe-buffer@~5.2.0: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-buffer@~5.1.1: +safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== @@ -4798,6 +4841,13 @@ string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + strip-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" @@ -5185,7 +5235,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -util-deprecate@^1.0.1: +util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= @@ -5218,6 +5268,7 @@ validator@^13.7.0: version "13.7.0" resolved "https://registry.yarnpkg.com/validator/-/validator-13.7.0.tgz#4f9658ba13ba8f3d82ee881d3516489ea85c0857" integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw== + vary@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"