feat(eval): support esbuild & typescript loader

build/compile.ts

@@ -53,8 +53,10 @@ const KOISHI_VERSION = JSON.stringify(version)
     if (name === 'koishi') {
       entryPoints.push(base + '/src/worker.ts')
     } else if (name === 'plugin-eval') {
+      const loaders = await readdir(base + '/src/loaders')
       entryPoints.push(base + '/src/worker/index.ts')
       entryPoints.push(base + '/src/transfer.ts')
+      entryPoints.push(...loaders.map(name => `${base}/src/loaders/${name}`))
     } else if (name === 'plugin-eval-addons') {
       entryPoints.push(base + '/src/worker.ts')
     } else if (name === 'koishi-test-utils') {
@@ -77,7 +79,10 @@ const KOISHI_VERSION = JSON.stringify(version)
       plugins: [{
         name: 'external library',
         setup(build) {
-          build.onResolve({ filter }, () => ({ external: true }))
+          build.onResolve({ filter }, (args) => {
+            if (args.path.match(/\./)) console.log(args.importer, '->', args.path)
+            return { external: true }
+          })
         },
       }],
     }
@@ -86,7 +91,7 @@ const KOISHI_VERSION = JSON.stringify(version)
       return tasks[name] = bundle(options)
     }
 
-    filter = /^([/\w-]+|\.\/transfer)$/
+    filter = /^([@/\w-]+|.+\/transfer)$/
     tasks[name] = Promise.all([options, {
       ...options,
       outdir: `${root}/${name}/lib/worker`,

packages/plugin-eval/package.json

@@ -35,12 +35,15 @@
   "peerDependencies": {
     "koishi-core": "^3.7.2"
   },
-  "dependencies": {
-    "js-yaml": "^4.0.0",
+  "optionalDependencies": {
+    "esbuild": "^0.11.6",
     "json5": "^2.2.0",
-    "simple-git": "^2.37.0",
     "typescript": "^4.2.3"
   },
+  "dependencies": {
+    "js-yaml": "^4.0.0",
+    "simple-git": "^2.37.0"
+  },
   "devDependencies": {
     "@types/js-yaml": "^4.0.0",
     "koishi-test-utils": "^6.0.0-beta.11"

packages/plugin-eval/src/index.ts

@@ -5,8 +5,7 @@ import { resolve } from 'path'
 import { load } from 'js-yaml'
 import { promises as fs } from 'fs'
 import Git, { CheckRepoActions } from 'simple-git'
-import { WorkerResponse } from './worker'
-import { Loader } from './transfer'
+import { WorkerResponse, Loader } from './worker'
 
 export * from './main'
 
@@ -54,17 +53,17 @@ interface Manifest {
   commands?: CommandManifest[]
 }
 
-const builtinLoaders = ['native']
+const builtinLoaders = ['default', 'esbuild', 'typescript']
 
 const defaultConfig: EvalConfig = {
   prefix: '>',
   authority: 2,
   timeout: 1000,
   setupFiles: {},
-  loader: 'native',
+  loader: 'default',
   channelFields: ['id'],
   userFields: ['id', 'authority'],
-  dataKeys: ['inspect', 'setupFiles'],
+  dataKeys: ['inspect', 'loader', 'setupFiles'],
 }
 
 const logger = new Logger('eval')
@@ -164,12 +163,18 @@ export function apply(ctx: Context, config: Config = {}) {
   }
 
   Argv.interpolate('${', '}', (source) => {
-    const expr = segment.unescape(source)
-    const result = loader.extract(expr)
-    if (result) return { source, command, args: [result], rest: segment.escape(source.slice(result.length + 1)) }
-    const index = source.indexOf('}')
-    if (index >= 0) return { source, rest: source.slice(index + 1), tokens: [] }
-    return { source, rest: '', tokens: [] }
+    const result = loader.extract(segment.unescape(source))
+    if (!result) {
+      const index = source.indexOf('}')
+      if (index >= 0) return { source, rest: source.slice(index + 1), tokens: [] }
+      return { source, rest: '', tokens: [] }
+    }
+    return {
+      source,
+      command,
+      args: [result],
+      rest: segment.escape(source.slice(result.length + 1)),
+    }
   })
 }
 

packages/plugin-eval/src/loaders/esbuild.ts

@@ -0,0 +1,32 @@
+import esbuild from 'esbuild'
+
+export async function transform(expr: string) {
+  try {
+    const { code } = await esbuild.transform(expr, {
+      sourcemap: 'inline',
+      loader: 'ts',
+      charset: 'utf8',
+      target: 'es2020',
+    })
+    return code
+  } catch (e) {
+    const [{ location, text }] = e.errors as esbuild.Message[]
+    throw new Error(`${text}\n    at stdin:${location.line}:${location.column}`)
+  }
+}
+
+export function extract(expr: string) {
+  try {
+    esbuild.transformSync(expr, {
+      loader: 'ts',
+      charset: 'utf8',
+      target: 'es2020',
+    })
+  } catch (e) {
+    const [{ location, text }] = e.errors as esbuild.Message[]
+    if (text === 'Unexpected "}"') {
+      const sLines = expr.split('\n')
+      return [...sLines.slice(0, location.line - 1), location.lineText.slice(0, location.column)].join('\n')
+    }
+  }
+}

packages/plugin-eval/src/loaders/typescript.ts

@@ -0,0 +1,29 @@
+import ts from 'typescript'
+import json5 from 'json5'
+import { promises as fs } from 'fs'
+import { WorkerData } from '../worker'
+import { resolve } from 'path'
+import { Logger } from 'koishi-utils'
+
+export { extract } from './default'
+
+const compilerOptions: ts.CompilerOptions = {
+  inlineSourceMap: true,
+  module: ts.ModuleKind.ES2020,
+  target: ts.ScriptTarget.ES2020,
+}
+
+export async function prepare(config: WorkerData) {
+  const logger = new Logger('eval:loader')
+  const tsconfigPath = resolve(config.root, 'tsconfig.json')
+  return fs.readFile(tsconfigPath, 'utf8').then((tsconfig) => {
+    Object.assign(compilerOptions, json5.parse(tsconfig))
+  }, () => {
+    logger.info('auto generating tsconfig.json...')
+    return fs.writeFile(tsconfigPath, json5.stringify({ compilerOptions }, null, 2))
+  })
+}
+
+export async function transform(expr: string) {
+  return ts.transpile(expr, compilerOptions)
+}

packages/plugin-eval/src/main.ts

@@ -11,7 +11,6 @@ export interface MainConfig extends Trap.Config {
   prefix?: string
   authority?: number
   timeout?: number
-  loader?: string
   resourceLimits?: ResourceLimits
   dataKeys?: (keyof WorkerData)[]
   gitRemote?: string

packages/plugin-eval/src/transfer.ts

@@ -59,10 +59,3 @@ export function expose(ep: Endpoint, object: {}) {
     ep.postMessage(JSON.stringify({ uuid, value }))
   })
 }
-
-// createLoader() is not relavant to transfer
-// we put it here since it's a cross-thread feature
-export interface Loader {
-  extract(expr: string): string
-  transform(expr: string): string | Promise<string>
-}

packages/plugin-eval/src/worker/index.ts

@@ -29,6 +29,7 @@ export * from './loader'
 
 export interface WorkerConfig {
   root?: string
+  loader?: string
   inspect?: InspectOptions
   cacheFile?: string
   storageFile?: string
@@ -39,6 +40,14 @@ export interface WorkerData extends WorkerConfig {
   addonNames?: string[]
 }
 
+// createLoader() is not relavant to transfer
+// we put it here since it's a cross-thread feature
+export interface Loader {
+  extract(expr: string): string
+  prepare(config: WorkerData): void | Promise<void>
+  transform(expr: string): string | Promise<string>
+}
+
 interface EvalOptions {
   silent: boolean
   source: string

packages/plugin-eval/src/worker/loader.ts

@@ -1,11 +1,10 @@
-import { config, context, internal } from '.'
+import { config, context, internal, Loader } from '.'
 import { resolve, posix, dirname } from 'path'
 import { promises as fs } from 'fs'
 import { deserialize, serialize, cachedDataVersionTag } from 'v8'
 import { Logger } from 'koishi-utils'
-import json5 from 'json5'
-import ts from 'typescript'
 
+let loader: Loader
 const logger = new Logger('eval:loader')
 
 // TODO pending @types/node
@@ -64,12 +63,6 @@ async function linker(specifier: string, { identifier }: Module) {
   throw new Error(`Unable to resolve dependency "${specifier}" in "${identifier}"`)
 }
 
-const compilerOptions: ts.CompilerOptions = {
-  inlineSourceMap: true,
-  module: ts.ModuleKind.ES2020,
-  target: ts.ScriptTarget.ES2020,
-}
-
 interface FileCache {
   outputText: string
   cachedData: Buffer
@@ -100,15 +93,10 @@ export async function safeWriteFile(filename: string, data: any) {
 export default async function prepare() {
   if (!config.root) return
 
-  const tsconfigPath = resolve(config.root, 'tsconfig.json')
+  loader = require(config.loader) as Loader
   const cachePath = resolve(config.root, config.cacheFile || '.koishi/cache')
   await Promise.all([
-    fs.readFile(tsconfigPath, 'utf8').then((tsconfig) => {
-      Object.assign(compilerOptions, json5.parse(tsconfig))
-    }, () => {
-      logger.info('auto generating tsconfig.json...')
-      return fs.writeFile(tsconfigPath, json5.stringify({ compilerOptions }, null, 2))
-    }),
+    loader.prepare?.(config),
     readSerialized(cachePath).then((data) => {
       if (data && data.tag === CACHE_TAG && data.v8tag === V8_TAG) {
         Object.assign(cachedFiles, data.files)
@@ -154,9 +142,7 @@ async function createModule(path: string) {
       module = new SourceTextModule(outputText, { context, identifier, cachedData })
     } else {
       type = 'source text'
-      const { outputText } = ts.transpileModule(source, {
-        compilerOptions,
-      })
+      const outputText = await loader.transform(source)
       module = new SourceTextModule(outputText, { context, identifier })
       const cachedData = module.createCachedData()
       files[source] = { outputText, cachedData }

packages/plugin-eval/tests/sandbox.spec.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-new-wrappers, max-len */
 
 import 'koishi-test-utils'
-import { Sandbox } from 'koishi-plugin-eval/src/sandbox'
+import { Sandbox } from 'koishi-plugin-eval/src/worker/sandbox'
 import { inspect } from 'util'
 import { expect } from 'chai'