feat: mwcc multi stage compilation support (#523)

packages/faas-cli-plugin-invoke/package.json

@@ -8,7 +8,7 @@
     "@midwayjs/faas-code-analysis": "^1.0.1",
     "@midwayjs/fcli-command-core": "^1.0.7",
     "@midwayjs/locate": "^1.0.3",
-    "@midwayjs/mwcc": "^0.2.2",
+    "@midwayjs/mwcc": "^0.3.0",
     "@midwayjs/runtime-engine": "^1.0.1",
     "@midwayjs/runtime-mock": "^1.0.1",
     "@midwayjs/serverless-fc-starter": "^1.0.8",

packages/faas-cli-plugin-invoke/src/index.ts

@@ -5,7 +5,7 @@ import {
   compareFileChange,
   copyFiles,
 } from '@midwayjs/faas-code-analysis';
-import { compileInProject } from '@midwayjs/mwcc';
+import { CompilerHost, Program, resolveTsConfigFile } from '@midwayjs/mwcc';
 import { writeWrapper } from '@midwayjs/serverless-spec-builder';
 import { createRuntime } from '@midwayjs/runtime-mock';
 import * as FCTrigger from '@midwayjs/serverless-fc-trigger';
@@ -20,9 +20,9 @@ import {
   remove,
   readFileSync,
   copy,
-  mkdirSync,
   ensureDirSync,
   symlinkSync,
+  mkdirSync,
 } from 'fs-extra';
 export * from './invoke';
 const commonLock: any = {};
@@ -43,7 +43,11 @@ export class FaaSInvokePlugin extends BasePlugin {
   analysisCodeInfoPath: string;
   entryInfo: any;
   fileChanges: any;
-  relativeTsCodeRoot: string;
+  analyzedTsCodeRoot: string;
+
+  private compilerHost: CompilerHost;
+  private program: Program;
+
   get defaultTmpFaaSOut() {
     return resolve(
       this.core.config.servicePath,
@@ -58,8 +62,8 @@ export class FaaSInvokePlugin extends BasePlugin {
         'locator', // 分析目录结构
         'copyFile', // 拷贝文件
         'checkFileChange', // 检查文件是否更新
-        'analysisCode', // 代码分析
-        'compile', // ts 编译
+        'compile', // ts 代码编译
+        'emit', // ts 代码输出
         'entry', // 生成执行入口
         'getInvoke', // 获取runtime
         'callInvoke', // 进行调用
@@ -94,8 +98,8 @@ export class FaaSInvokePlugin extends BasePlugin {
     'invoke:locator': this.locator.bind(this),
     'invoke:copyFile': this.copyFile.bind(this),
     'invoke:checkFileChange': this.checkFileChange.bind(this),
-    'invoke:analysisCode': this.analysisCode.bind(this),
     'invoke:compile': this.compile.bind(this),
+    'invoke:emit': this.emit.bind(this),
     'invoke:entry': this.entry.bind(this),
     'invoke:getInvoke': this.getInvoke.bind(this),
     'invoke:callInvoke': this.callInvoke.bind(this),
@@ -206,8 +210,17 @@ export class FaaSInvokePlugin extends BasePlugin {
       '.faasTSBuildInfo.log'
     ));
     this.analysisCodeInfoPath = resolve(this.buildLogDir, '.faasFuncList.log');
-    this.relativeTsCodeRoot =
-      relative(this.baseDir, this.codeAnalyzeResult.tsCodeRoot) || '.';
+
+    let directoryToScan: string;
+    const tmpOutDir = resolve(this.defaultTmpFaaSOut, 'src');
+    if (existsSync(tmpOutDir)) {
+      this.analyzedTsCodeRoot = tmpOutDir;
+      directoryToScan = tmpOutDir;
+    } else {
+      this.analyzedTsCodeRoot = this.codeAnalyzeResult.tsCodeRoot;
+      directoryToScan = relative(this.baseDir, this.analyzedTsCodeRoot);
+    }
+
     const isTsMode = this.checkIsTsMode();
     if (isTsMode) {
       process.env.MIDWAY_TS_MODE = 'true';
@@ -227,11 +240,7 @@ export class FaaSInvokePlugin extends BasePlugin {
     if (existsSync(buildLockPath)) {
       this.core.debug('buildLockPath', buildLockPath);
       this.fileChanges = await compareFileChange(
-        [
-          specFile,
-          `${this.relativeTsCodeRoot}/**/*`,
-          `${this.defaultTmpFaaSOut}/src/**/*`, // 允许用户将ts代码生成到此文件夹
-        ],
+        [specFile, `${directoryToScan}/**/*`],
         [buildLockPath],
         { cwd: this.baseDir }
       );
@@ -244,10 +253,7 @@ export class FaaSInvokePlugin extends BasePlugin {
         return;
       }
     } else {
-      this.fileChanges = [
-        `${this.relativeTsCodeRoot}/**/*`,
-        `${this.defaultTmpFaaSOut}/src/**/*`,
-      ];
+      this.fileChanges = [`${directoryToScan}/**/*`];
       // 如果没有构建锁，但是存在代码分析，这时候认为上一次是获取了函数列表
       // if (existsSync(this.analysisCodeInfoPath)) {
       //   this.getAnaLysisCodeInfo();
@@ -257,17 +263,36 @@ export class FaaSInvokePlugin extends BasePlugin {
     this.setLock(this.buildLockPath, LOCK_TYPE.WAITING);
   }
 
-  async analysisCode() {
+  async compile() {
     // 如果在代码分析中止，那么可以从store中获取function信息
     this.setStore('functions', this.core.service.functions);
+
     // 如果跳过了ts编译，那么也就是说曾经编译过，那么也跳过代码分析
     if (this.skipTsBuild) {
       return;
     }
+
+    const dest = join(this.buildDir, 'dist');
+    const { config } = resolveTsConfigFile(
+      this.baseDir,
+      dest,
+      undefined,
+      this.getStore('mwccHintConfig', 'global'),
+      {
+        include: this.fileChanges,
+        compilerOptions: {
+          incremental: this.options.incremental,
+          rootDir: this.analyzedTsCodeRoot,
+        },
+      }
+    );
+    this.compilerHost = new CompilerHost(this.baseDir, config);
+    this.program = new Program(this.compilerHost);
+
     // 当spec上面没有functions的时候，启动代码分析
     if (!this.core.service.functions) {
       const newSpec = await analysis([
-        resolve(this.baseDir, this.relativeTsCodeRoot),
+        resolve(this.baseDir, this.analyzedTsCodeRoot),
         resolve(this.defaultTmpFaaSOut, 'src'),
       ]);
       this.core.debug('Code Analysis Result', newSpec);
@@ -278,9 +303,7 @@ export class FaaSInvokePlugin extends BasePlugin {
         JSON.stringify(newSpec.functions)
       );
     }
-    if (
-      this.core.pluginManager.options.stopLifecycle === 'invoke:analysisCode'
-    ) {
+    if (this.core.pluginManager.options.stopLifecycle === 'invoke:compile') {
       // LOCK_TYPE.INITIAL 是因为跳过了ts编译，下一次来的时候还是得进行ts编译
       this.setLock(this.buildLockPath, LOCK_TYPE.INITIAL);
     }
@@ -299,44 +322,21 @@ export class FaaSInvokePlugin extends BasePlugin {
       }
     }
   }
-  async compile() {
+
+  async emit() {
     const isTsMode = this.checkIsTsMode();
     if (isTsMode || this.skipTsBuild) {
       return;
     }
 
-    this.core.debug('Compile', this.codeAnalyzeResult);
+    this.core.debug('emit', this.codeAnalyzeResult);
     try {
+      this.program.emit();
+
       const dest = join(this.buildDir, 'dist');
       if (!existsSync(dest)) {
         mkdirSync(dest);
       }
-      const source = [];
-      const tmp = [];
-      this.fileChanges.forEach((file: string) => {
-        if (file.indexOf(this.defaultTmpFaaSOut) !== -1) {
-          tmp.push(file);
-        } else {
-          source.push(file);
-        }
-      });
-      if (source.length) {
-        await compileInProject(this.baseDir, dest, undefined, {
-          include: source,
-          compilerOptions: {
-            incremental: this.options.incremental,
-            rootDir: this.codeAnalyzeResult.tsCodeRoot,
-          },
-        });
-      }
-      if (tmp.length) {
-        await compileInProject(this.baseDir, dest, undefined, {
-          include: tmp,
-          compilerOptions: {
-            rootDir: resolve(this.defaultTmpFaaSOut, 'src'),
-          },
-        });
-      }
     } catch (e) {
       await remove(this.buildLockPath);
       this.setLock(this.buildLockPath, LOCK_TYPE.COMPLETE);

packages/faas-cli-plugin-invoke/src/invoke.ts

@@ -87,7 +87,7 @@ export async function getFuncList(options: IGetFuncList) {
     return spec.functions;
   }
   const invokeFun = getFunction({
-    stopLifecycle: 'invoke:analysisCode',
+    stopLifecycle: 'invoke:compile',
     key: 'functions',
     specFile,
     spec,

packages/faas-cli-plugin-package/package.json

@@ -7,7 +7,7 @@
     "@midwayjs/faas-code-analysis": "^1.0.1",
     "@midwayjs/fcli-command-core": "^1.0.7",
     "@midwayjs/locate": "^1.0.3",
-    "@midwayjs/mwcc": "^0.2.2",
+    "@midwayjs/mwcc": "^0.3.0",
     "archiver": "^3.1.1",
     "fs-extra": "^8.1.0",
     "globby": "^10.0.1",

packages/faas-cli-plugin-package/src/index.ts

@@ -20,7 +20,7 @@ import {
 import * as micromatch from 'micromatch';
 import { commonPrefix, formatLayers } from './utils';
 import { analysis, copyFiles } from '@midwayjs/faas-code-analysis';
-import { compileInProject, MwccConfig } from '@midwayjs/mwcc';
+import { CompilerHost, Program, resolveTsConfigFile } from '@midwayjs/mwcc';
 import { exec } from 'child_process';
 import * as archiver from 'archiver';
 import { AnalyzeResult, Locator } from '@midwayjs/locate';
@@ -38,7 +38,9 @@ export class PackagePlugin extends BasePlugin {
   codeAnalyzeResult: AnalyzeResult;
   integrationDistTempDirectory = 'integration_dist'; // 一体化构建的临时目录
   zipCodeDefaultName = 'serverless.zip';
-  mwccHintConfig: MwccConfig = {};
+
+  private compilerHost: CompilerHost;
+  private program: Program;
 
   commands = {
     package: {
@@ -47,8 +49,8 @@ export class PackagePlugin extends BasePlugin {
         'cleanup', // 清理构建目录
         'installDevDep', // 安装开发期依赖
         'copyFile', // 拷贝文件: package.include 和 shared content
-        'codeAnalysis', // 代码分析
-        'tscompile', // 编译函数  'package:after:tscompile'
+        'compile', // 代码分析
+        'emit', // 编译函数  'package:after:tscompile'
         'checkAggregation', // 检测高密度部署
         'generateSpec', // 生成对应平台的描述文件，例如 serverless.yml 等
         'generateEntry', // 生成对应平台的入口文件
@@ -95,14 +97,14 @@ export class PackagePlugin extends BasePlugin {
     'package:cleanup': this.cleanup.bind(this),
     'package:installDevDep': this.installDevDep.bind(this),
     'package:copyFile': this.copyFile.bind(this),
-    'package:codeAnalysis': this.codeAnalysis.bind(this),
+    'package:compile': this.compile.bind(this),
     'package:installLayer': this.installLayer.bind(this),
     'package:installDep': this.installDep.bind(this),
     'package:checkAggregation': this.checkAggregation.bind(this),
     'package:package': this.package.bind(this),
     'after:package:generateEntry': this.defaultGenerateEntry.bind(this),
     'before:package:finalize': this.finalize.bind(this),
-    'package:tscompile': this.tsCompile.bind(this),
+    'package:emit': this.emit.bind(this),
   };
 
   async cleanup() {
@@ -308,7 +310,31 @@ export class PackagePlugin extends BasePlugin {
     this.core.cli.log(' - Dependencies install complete');
   }
 
-  async codeAnalysis() {
+  async compile() {
+    let tsCodeRoot: string;
+    const tmpOutDir = resolve(this.defaultTmpFaaSOut, 'src');
+    if (existsSync(tmpOutDir)) {
+      tsCodeRoot = tmpOutDir;
+    } else {
+      tsCodeRoot = this.codeAnalyzeResult.tsCodeRoot;
+    }
+
+    const { config } = resolveTsConfigFile(
+      this.servicePath,
+      join(this.midwayBuildPath, 'dist'),
+      undefined,
+      this.getStore('mwccHintConfig', 'global'),
+      {
+        compilerOptions: {
+          sourceRoot: '../src',
+          rootDir: tsCodeRoot,
+        },
+        include: [tsCodeRoot],
+      }
+    );
+    this.compilerHost = new CompilerHost(this.servicePath, config);
+    this.program = new Program(this.compilerHost);
+
     if (this.core.service.functions) {
       return this.core.service.functions;
     }
@@ -320,38 +346,15 @@ export class PackagePlugin extends BasePlugin {
     this.core.service.functions = newSpec.functions;
   }
 
-  async tsCompile() {
+  async emit() {
     const isTsDir = existsSync(join(this.servicePath, 'tsconfig.json'));
     this.core.cli.log('Building Midway FaaS directory files...');
     if (!isTsDir) {
       this.core.cli.log(' - Not found tsconfig.json and skip build');
       return;
     }
     this.core.cli.log(' - Using tradition build mode');
-    await compileInProject(
-      this.servicePath,
-      join(this.midwayBuildPath, 'dist'),
-      this.mwccHintConfig,
-      {
-        compilerOptions: {
-          sourceRoot: '../src',
-          rootDir: this.codeAnalyzeResult.tsCodeRoot,
-        },
-        include: [this.codeAnalyzeResult.tsCodeRoot],
-      }
-    );
-    const tmpOutDir = resolve(this.defaultTmpFaaSOut, 'src');
-    if (existsSync(tmpOutDir)) {
-      await compileInProject(
-        this.servicePath,
-        join(this.midwayBuildPath, 'dist'),
-        this.mwccHintConfig,
-        {
-          compilerOptions: { rootDir: tmpOutDir },
-          include: [tmpOutDir],
-        }
-      );
-    }
+    this.program.emit();
     this.core.cli.log(' - Build Midway FaaS complete');
   }
 

packages/faas-cli-plugin-package/test/noyaml.test.ts

@@ -4,7 +4,6 @@ import { AliyunFCPlugin } from '../../faas-cli-plugin-fc/src/index';
 import { FaaSTmpOutPlugin } from './fixtures/plugins/faas_tmp_out';
 import { resolve } from 'path';
 import { remove, existsSync, readFileSync } from 'fs-extra';
-import { transform } from '@midwayjs/serverless-spec-builder';
 import * as assert from 'assert';
 
 describe('/test/noyaml.test.ts', () => {
@@ -28,15 +27,6 @@ describe('/test/noyaml.test.ts', () => {
       core.addPlugin(FaaSTmpOutPlugin);
       await core.ready();
       await core.invoke(['package']);
-      const yaml = transform(resolve(buildDir, 'template.yml'));
-      assert(
-        yaml.Resources['serverless-midway-test']['service'].Properties
-          .Handler === 'service.handler'
-      );
-      assert(
-        yaml.Resources['serverless-midway-test']['test-hand'].Properties
-          .Handler === 'test.hand'
-      );
       assert(!existsSync(resolve(buildDir, 'faas_tmp_out')));
       assert(
         /console.log\('test\.js'\)/.test(

tsconfig.json

@@ -13,6 +13,7 @@
     "strict": false,
     // TODO: Emit test files to outDir
     "noEmitOnError": false,
+    "skipLibCheck": true,
   },
   "exclude": [
     "dist",