Feature: Implementing built-in middleware and middleware loader

package.json

@@ -7,6 +7,7 @@
   },
   "private": true,
   "dependencies": {
+    "@koa/cors": "^3.3.0",
     "dayjs": "^1.11.2",
     "glob": "^8.0.1",
     "joi": "^17.6.0",
@@ -15,9 +16,11 @@
     "koa-bodyparser": "^4.3.0",
     "koa-compose": "^4.1.0",
     "koa-router": "^10.1.1",
+    "koa2-ratelimit": "^1.1.1",
     "lodash": "^4.17.21",
     "nunjucks": "^3.2.3",
     "tslib": "^2.3.0",
+    "tslog": "^3.3.3",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
@@ -34,6 +37,8 @@
     "@types/koa": "^2.13.4",
     "@types/koa-compose": "^3.2.5",
     "@types/koa-router": "^7.4.4",
+    "@types/koa2-ratelimit": "^0.9.3",
+    "@types/koa__cors": "^3.3.0",
     "@types/lodash": "^4.14.182",
     "@types/node": "16.11.7",
     "@types/supertest": "^2.0.12",

packages/core/src/index.ts

@@ -1,3 +1,4 @@
+export * from './lib/utils';
 export * from './lib/validators';
 // Export all other modules
 export * from './models';

packages/core/src/lib/utils/index.ts

@@ -1 +1,3 @@
 export * from './normalizedStringValue';
+export * from './logger';
+export * from './module';

packages/core/src/lib/utils/logger.ts

@@ -0,0 +1,115 @@
+import { Logger } from 'tslog';
+import { AsyncLocalStorage } from 'async_hooks';
+export { Logger as ILogger };
+// The category according to package name
+export enum LoggingScope {
+  CORE = 'CORE',
+  BUILD = 'BUILD',
+  SERVE = 'SERVE',
+  AUDIT = 'AUDIT',
+}
+
+type LoggingScopeTypes = keyof typeof LoggingScope;
+
+export enum LoggingLevel {
+  SILLY = 'silly',
+  TRACE = 'trace',
+  DEBUG = 'debug',
+  INFO = 'info',
+  WARN = 'warn',
+  ERROR = 'error',
+  FATAL = 'fatal',
+}
+
+export interface LoggerOptions {
+  level?: LoggingLevel;
+  displayRequestId?: boolean;
+}
+
+type LoggerMapConfig = {
+  [scope in LoggingScope]: LoggerOptions;
+};
+
+const defaultMapConfig: LoggerMapConfig = {
+  [LoggingScope.CORE]: {
+    level: LoggingLevel.DEBUG,
+    displayRequestId: false,
+  },
+  [LoggingScope.BUILD]: {
+    level: LoggingLevel.DEBUG,
+    displayRequestId: false,
+  },
+  [LoggingScope.SERVE]: {
+    level: LoggingLevel.DEBUG,
+    displayRequestId: false,
+  },
+  [LoggingScope.AUDIT]: {
+    level: LoggingLevel.DEBUG,
+    displayRequestId: false,
+  },
+};
+
+export type AsyncRequestIdStorage = AsyncLocalStorage<{ requestId: string }>;
+class LoggerFactory {
+  private loggerMap: { [scope: string]: Logger };
+  public readonly asyncReqIdStorage: AsyncRequestIdStorage;
+
+  constructor() {
+    this.asyncReqIdStorage = new AsyncLocalStorage();
+
+    this.loggerMap = {
+      [LoggingScope.CORE]: this.createLogger(LoggingScope.CORE),
+      [LoggingScope.BUILD]: this.createLogger(LoggingScope.BUILD),
+      [LoggingScope.SERVE]: this.createLogger(LoggingScope.SERVE),
+    };
+  }
+
+  public getLogger({
+    scopeName,
+    options,
+  }: {
+    scopeName: LoggingScopeTypes;
+    options?: LoggerOptions;
+  }) {
+    if (!(scopeName in LoggingScope))
+      throw new Error(
+        `The ${scopeName} does not belong to ${Object.keys(LoggingScope)}`
+      );
+    // if scope name exist in mapper and not update config
+    if (scopeName in this.loggerMap) {
+      if (!options) return this.loggerMap[scopeName];
+      // if options existed, update settings.
+      const logger = this.loggerMap[scopeName];
+      this.updateSettings(logger, options);
+      return logger;
+    }
+    // if scope name does not exist in map or exist but would like to update config
+    const newLogger = this.createLogger(scopeName as LoggingScope, options);
+    this.loggerMap[scopeName] = newLogger;
+    return newLogger;
+  }
+
+  private updateSettings(logger: Logger, options: LoggerOptions) {
+    const prevSettings = logger.settings;
+    logger.setSettings({
+      minLevel: options.level || prevSettings.minLevel,
+      displayRequestId:
+        options.displayRequestId || prevSettings.displayRequestId,
+    });
+  }
+
+  private createLogger(name: LoggingScope, options?: LoggerOptions) {
+    return new Logger({
+      name,
+      minLevel: options?.level || defaultMapConfig[name].level,
+      // use function call for requestId, then when logger get requestId, it will get newest store again
+      requestId: () => this.asyncReqIdStorage.getStore()?.requestId as string,
+      displayRequestId:
+        options?.displayRequestId || defaultMapConfig[name].displayRequestId,
+    });
+  }
+}
+
+const factory = new LoggerFactory();
+export const getLogger = factory.getLogger.bind(factory);
+export const asyncReqIdStorage = factory.asyncReqIdStorage;

packages/core/src/lib/utils/module.ts

@@ -0,0 +1,14 @@
+// The type for class T
+export interface ClassType<T> extends Function {
+  new (...args: any[]): T;
+}
+
+/**
+ * dynamic import default module.
+ * @param folderOrFile The folder / file path
+ * @returns default module
+ */
+export const defaultImport = async <T = any>(folderOrFile: string) => {
+  const module = await import(folderOrFile);
+  return module.default as T;
+};

packages/core/src/lib/validators/data-type-validators/dateTypeValidator.ts

@@ -2,7 +2,7 @@ import * as Joi from 'joi';
 import { isUndefined } from 'lodash';
 import * as dayjs from 'dayjs';
 import customParseFormat = require('dayjs/plugin/customParseFormat');
-import IValidator from '../validator';
+import { IValidator } from '../validator';
 
 // Support custom date format -> dayjs.format(...)
 dayjs.extend(customParseFormat);
@@ -13,7 +13,7 @@ export interface DateInputArgs {
   format?: string;
 }
 
-export default class DateTypeValidator implements IValidator {
+export class DateTypeValidator implements IValidator {
   public readonly name = 'date';
   // Validator for arguments schema in schema.yaml, should match DateInputArgs
   private argsValidator = Joi.object({

packages/core/src/lib/validators/data-type-validators/index.ts

@@ -0,0 +1,18 @@
+// export all other non-default objects of validators module
+export * from './dateTypeValidator';
+export * from './integerTypeValidator';
+export * from './stringTypeValidator';
+export * from './uuidTypeValidator';
+
+// import default objects and export
+import { DateTypeValidator } from './dateTypeValidator';
+import { IntegerTypeValidator } from './integerTypeValidator';
+import { StringTypeValidator } from './stringTypeValidator';
+import { UUIDTypeValidator } from './uuidTypeValidator';
+
+export default [
+  DateTypeValidator,
+  IntegerTypeValidator,
+  StringTypeValidator,
+  UUIDTypeValidator,
+];

packages/core/src/lib/validators/data-type-validators/integerTypeValidator.ts

@@ -1,6 +1,6 @@
 import * as Joi from 'joi';
 import { isUndefined } from 'lodash';
-import IValidator from '../validator';
+import { IValidator } from '../validator';
 
 export interface IntInputArgs {
   // The integer minimum value
@@ -13,7 +13,7 @@ export interface IntInputArgs {
   less?: number;
 }
 
-export default class IntegerTypeValidator implements IValidator {
+export class IntegerTypeValidator implements IValidator {
   public readonly name = 'integer';
   // Validator for arguments schema in schema.yaml, should match IntInputArgs
   private argsValidator = Joi.object({

packages/core/src/lib/validators/data-type-validators/stringTypeValidator.ts

@@ -1,6 +1,6 @@
 import * as Joi from 'joi';
 import { isUndefined } from 'lodash';
-import IValidator from '../validator';
+import { IValidator } from '../validator';
 
 export interface StringInputArgs {
   // The string regex format pattern
@@ -13,7 +13,7 @@ export interface StringInputArgs {
   max?: number;
 }
 
-export default class StringTypeValidator implements IValidator {
+export class StringTypeValidator implements IValidator {
   public readonly name = 'string';
   // Validator for arguments schema in schema.yaml, should match StringInputArgs
   private argsValidator = Joi.object({

packages/core/src/lib/validators/data-type-validators/uuidTypeValidator.ts

@@ -1,7 +1,7 @@
 import * as Joi from 'joi';
 import { GuidVersions } from 'joi';
 import { isUndefined } from 'lodash';
-import IValidator from '../validator';
+import { IValidator } from '../validator';
 
 type UUIDVersion = 'uuid_v1' | 'uuid_v4' | 'uuid_v5';
 
@@ -10,7 +10,7 @@ export interface UUIDInputArgs {
   version?: UUIDVersion;
 }
 
-export default class UUIDTypeValidator implements IValidator {
+export class UUIDTypeValidator implements IValidator {
   public readonly name = 'uuid';
   // Validator for arguments schema in schema.yaml, should match UUIDInputArgs
   private argsValidator = Joi.object({

packages/core/src/lib/validators/index.ts

@@ -1,21 +1,3 @@
-// export all other non-default objects of validators module
-export * from './data-type-validators/dateTypeValidator';
-export * from './data-type-validators/integerTypeValidator';
-export * from './data-type-validators/stringTypeValidator';
-export * from './data-type-validators/uuidTypeValidator';
+export * from './data-type-validators';
 export * from './validatorLoader';
-
-// import default objects and export
-import IValidator from './validator';
-import DateTypeValidator from './data-type-validators/dateTypeValidator';
-import IntegerTypeValidator from './data-type-validators/integerTypeValidator';
-import StringTypeValidator from './data-type-validators/stringTypeValidator';
-import UUIDTypeValidator from './data-type-validators/uuidTypeValidator';
-
-export {
-  IValidator,
-  DateTypeValidator,
-  IntegerTypeValidator,
-  StringTypeValidator,
-  UUIDTypeValidator,
-};
+export * from './validator';

packages/core/src/lib/validators/validator.ts

@@ -1,4 +1,4 @@
-export default interface IValidator<T = any> {
+export interface IValidator<T = any> {
   // validator name
   readonly name: string;
   // validate Schema format

packages/core/src/lib/validators/validatorLoader.ts

@@ -1,53 +1,48 @@
-import IValidator from './validator';
-import * as glob from 'glob';
+import { IValidator } from './validator';
+
 import * as path from 'path';
+import { defaultImport, ClassType } from '../utils';
+
+// The extension module interface
+export interface ExtensionModule {
+  validators?: ClassType<IValidator>[];
+}
 
 export interface IValidatorLoader {
   load(validatorName: string): Promise<IValidator>;
 }
 
 export class ValidatorLoader implements IValidatorLoader {
   // only found built-in validators in sub folders
-  private builtInFolderPath: string = path.resolve(__dirname, '*', '*.ts');
-  private userDefinedFolderPath?: string;
+  private builtInFolder: string = path.join(__dirname, 'data-type-validators');
+  private extensionPath?: string;
 
   constructor(folderPath?: string) {
-    this.userDefinedFolderPath = folderPath;
+    this.extensionPath = folderPath;
   }
   public async load(validatorName: string) {
-    let validatorFiles = [
-      ...(await this.getValidatorFilePaths(this.builtInFolderPath)),
-    ];
-    if (this.userDefinedFolderPath) {
-      // include sub-folder or non sub-folders
-      const userDefinedValidatorFiles = await this.getValidatorFilePaths(
-        path.resolve(this.userDefinedFolderPath, '**', '*.ts')
-      );
-      validatorFiles = validatorFiles.concat(userDefinedValidatorFiles);
+    // read built-in validators in index.ts, the content is an array middleware class
+    const builtInClass = await defaultImport<ClassType<IValidator>[]>(
+      this.builtInFolder
+    );
+
+    // if extension path setup, load extension middlewares classes
+    let extensionClass: ClassType<IValidator>[] = [];
+    if (this.extensionPath) {
+      // import extension which user customized
+      const module = await defaultImport<ExtensionModule>(this.extensionPath);
+      extensionClass = module.validators || [];
     }
 
-    for (const file of validatorFiles) {
-      // import validator files to module
-      const validatorModule = await import(file);
-      // get validator class by getting default.
-      if (validatorModule && validatorModule.default) {
-        const validatorClass = validatorModule.default;
-        const validator = new validatorClass() as IValidator;
-        if (validator.name === validatorName) return validator;
-      }
+    // create all middlewares by new it.
+    for (const validatorClass of [...builtInClass, ...extensionClass]) {
+      const validator = new validatorClass() as IValidator;
+      if (validator.name === validatorName) return validator;
     }
+
     // throw error if not found
     throw new Error(
       `The name "${validatorName}" of validator not defined in built-in validators and passed folder path, or the defined validator not export as default.`
     );
   }
-
-  private async getValidatorFilePaths(sourcePath: string): Promise<string[]> {
-    return new Promise((resolve, reject) => {
-      glob(sourcePath, { nodir: true }, (err, files) => {
-        if (err) return reject(err);
-        else return resolve(files);
-      });
-    });
-  }
 }

packages/core/test/validators/data-type-validators/dataTypeValidator.spec.ts

@@ -13,7 +13,6 @@ describe('Test "date" type validator', () => {
       const args = JSON.parse(inputArgs);
       // Act
       const validator = new DateTypeValidator();
-      const result = validator.validateSchema(args);
       // Assert
       expect(() => validator.validateSchema(args)).not.toThrow();
     }

packages/core/test/validators/test-custom-validators/index.ts

@@ -0,0 +1,7 @@
+import { IPTypeValidator } from './ip-type-validator';
+
+// Imitate extension for testing
+export default {
+  validators: [IPTypeValidator],
+  middlewares: [],
+};