Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support node workers out of the box #3932

Open
wants to merge 19 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions docs/guide/features.md
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,22 @@ By default, the worker script will be emitted as a separate chunk in the product
import MyWorker from './worker?worker&inline'
```

## Node Worker Threads

Just like Web Workers Node.js worker threads are supported out of the box. You will need to set build target to nodeN and add native modules as external dependencies in rollup options in config. A Node.js worker script can imported by appending `?worker` to the import request. The default export will be the Worker class from the native 'worker_threads' module. Typescript worker file is also supported.

```js
import MyWorker from './worker?worker'

const worker = new MyWorker()
```

By default, the worker script will be emitted as a separate chunk in the production build. If you wish to inline the worker, add the `inline` query. Inline code will be added with `{eval: true}`

```js
import MyWorker from './worker?worker&inline'
```

## Build Optimizations

> Features listed below are automatically applied as part of the build process and there is no need for explicit configuration unless you want to disable them.
Expand Down
34 changes: 34 additions & 0 deletions packages/playground/worker-thread-node/__tests__/serve.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
const { join } = require('path')
/**
* @param {string} root
* @param {boolean} isProd
*/
exports.serve = async function serve(root, isProd) {
// make a build in either cases
// make minified build if prod
if (isProd) {
const { build } = require('vite')
await build({
configFile: join(root, 'vite.config.ts'),
root,
logLevel: 'silent'
})
} else {
const { build } = require('vite')

const config = require(join(root, 'vite.config.ts')).serveConfig
await build({
...config,
root,
logLevel: 'silent'
})
}

return new Promise((resolve, _) => {
resolve({
close: async () => {
// no need to close anything
}
})
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import { isBuild, testDir } from '../../testUtils'
import fs from 'fs-extra'
import path from 'path'

test('response from worker', async () => {
const distDir = path.resolve(testDir, 'dist')
const { run } = require(path.join(distDir, 'main.cjs'))
expect(await run('ping')).toBe('pong')
})

test('response from inline worker', async () => {
const distDir = path.resolve(testDir, 'dist')
const { inlineWorker } = require(path.join(distDir, 'main.cjs'))
expect(await inlineWorker('ping')).toBe('this is inline node worker')
})

if (isBuild) {
test('worker code generation', async () => {
const assetsDir = path.resolve(testDir, 'dist/assets')
const distDir = path.resolve(testDir, 'dist')
const files = fs.readdirSync(assetsDir)
const mainContent = fs.readFileSync(
path.resolve(distDir, 'main.cjs.js'),
'utf-8'
)

const workerFile = files.find((f) => f.includes('worker'))
const workerContent = fs.readFileSync(
path.resolve(assetsDir, workerFile),
'utf-8'
)

expect(files.length).toBe(1)

// main file worker chunk content
expect(mainContent).toMatch(`require("worker_threads")`)
expect(mainContent).toMatch(`Worker`)

// main content should contain __dirname to resolve module as relation path from main module
expect(mainContent).toMatch(`__dirname`)

// should resolve worker_treads from external dependency
expect(workerContent).toMatch(`require("worker_threads")`)

// inline nodejs worker
expect(mainContent).toMatch(`{eval:!0}`)
})
}
35 changes: 35 additions & 0 deletions packages/playground/worker-thread-node/main.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import MyWorker from './worker?worker'
import InlineWorker from './worker-inline?worker&inline'
import { Worker } from 'worker_threads'

export const run = async (message: string): Promise<string> => {
return new Promise((resolve, _) => {
const worker = new MyWorker() as unknown as Worker
Shinigami92 marked this conversation as resolved.
Show resolved Hide resolved
worker.postMessage(message)
worker.on('message', (msg) => {
worker.terminate()
resolve(msg)
})
})
}

export const inlineWorker = async (message: string): Promise<string> => {
return new Promise((resolve, _) => {
const worker = new InlineWorker() as unknown as Worker
worker.postMessage(message)
worker.on('message', (msg) => {
worker.terminate()
resolve(msg)
})
})
}

if (require.main === module) {
Promise.all([run('ping'), inlineWorker('ping')]).then(
([chunkResponse, inlineResponse]) => {
console.log('Response from chunk worker - ', chunkResponse)
console.log('Response from inline worker - ', inlineResponse)
process.exit()
}
)
}
10 changes: 10 additions & 0 deletions packages/playground/worker-thread-node/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "worker-thread-node",
"private": true,
"version": "0.0.0",
"scripts": {
"dev": "node scripts/build.js",
"build": "node scripts/build.js",
"start": "node dist/main.cjs.js"
}
}
3 changes: 3 additions & 0 deletions packages/playground/worker-thread-node/scripts/build.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
const vite = require('vite')
const { build } = vite
build({ configFile: 'vite.config.ts' })
35 changes: 35 additions & 0 deletions packages/playground/worker-thread-node/vite.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { defineConfig } from 'vite'
import { builtinModules } from 'module'

export default defineConfig({
build: {
target: 'node16',
outDir: 'dist',
lib: {
entry: 'main.ts',
formats: ['cjs'],
fileName: 'main'
},
rollupOptions: {
external: [...builtinModules]
},
emptyOutDir: true
}
})

export const serveConfig = defineConfig({
build: {
target: 'node16',
outDir: 'dist',
lib: {
entry: 'main.ts',
formats: ['cjs'],
fileName: 'main'
},
rollupOptions: {
external: [...builtinModules]
},
minify: false,
emptyOutDir: true
}
})
5 changes: 5 additions & 0 deletions packages/playground/worker-thread-node/worker-inline.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import { parentPort } from 'worker_threads'

parentPort.on('message', () => {
parentPort.postMessage('this is inline node worker')
})
5 changes: 5 additions & 0 deletions packages/playground/worker-thread-node/worker.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import { parentPort } from 'worker_threads'

parentPort.on('message', (message) => {
parentPort.postMessage('pong')
})
48 changes: 42 additions & 6 deletions packages/vite/src/node/plugins/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Plugin } from '../plugin'
import { resolvePlugins } from '../plugins'
import { parse as parseUrl, URLSearchParams } from 'url'
import { fileToUrl, getAssetHash } from './asset'
import { cleanUrl, injectQuery } from '../utils'
import { cleanUrl, injectQuery, isTargetNode } from '../utils'
import Rollup from 'rollup'
import { ENV_PUBLIC_PATH } from '../constants'
import path from 'path'
Expand Down Expand Up @@ -52,28 +52,52 @@ export function webWorkerPlugin(config: ResolvedConfig): Plugin {
}

let url: string
const isNode = isTargetNode(config?.build?.target)

if (isBuild) {
// bundle the file as entry to support imports
const rollup = require('rollup') as typeof Rollup
const bundle = await rollup.rollup({
input: cleanUrl(id),
...config?.build?.rollupOptions,
plugins: await resolvePlugins({ ...config }, [], [], []),
onwarn(warning, warn) {
onRollupWarning(warning, warn, config)
}
})

let code: string
try {
const { output } = await bundle.generate({
format: 'iife',
sourcemap: config.build.sourcemap
})
code = output[0].code
if (isNode) {
const { output } = await bundle.generate({
format: 'cjs',
sourcemap: config.build.sourcemap
})
code = output[0].code
} else {
const { output } = await bundle.generate({
format: 'iife',
sourcemap: config.build.sourcemap
})
code = output[0].code
}
} finally {
await bundle.close()
}
const content = Buffer.from(code)
if (query.inline != null) {
if (isNode) {
let code = content.toString().trim()
code = code.replace(/\r?\n|\r/g, '')

return `
import { Worker } from "worker_threads" \n
import { join } from "path" \n
export default function WorkerWrapper() {
return new Worker(\'${code}', { eval: true })
}
`
}
// inline as blob data url
return `const encodedJs = "${content.toString('base64')}";
const blob = typeof window !== "undefined" && window.Blob && new Blob([atob(encodedJs)], { type: "text/javascript;charset=utf-8" });
Expand Down Expand Up @@ -107,6 +131,18 @@ export function webWorkerPlugin(config: ResolvedConfig): Plugin {
query.sharedworker != null ? 'SharedWorker' : 'Worker'
const workerOptions = { type: 'module' }

if (isNode) {
return `
import { Worker } from "worker_threads" \n
import { join } from "path" \n
export default function WorkerWrapper() {
return new Worker(join(__dirname, ${JSON.stringify(
url
)}), ${JSON.stringify(workerOptions, null, 2)})
}
`
}

return `export default function WorkerWrapper() {
return new ${workerConstructor}(${JSON.stringify(
url
Expand Down
17 changes: 17 additions & 0 deletions packages/vite/src/node/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,22 @@ export function resolveHostname(
return { host, name }
}

export function isTargetNode(target: string | false | string[]): boolean {
if (!target) {
return false
}

if (typeof target === 'string') {
return target.includes('node')
}

if (Array.isArray(target)) {
return target.some((f) => f.includes('node'))
}

return false
}

export function arraify<T>(target: T | T[]): T[] {
return Array.isArray(target) ? target : [target]
}
Expand All @@ -567,4 +583,5 @@ export function toUpperCaseDriveLetter(pathName: string): string {
}

export const multilineCommentsRE = /\/\*(.|[\r\n])*?\*\//gm

export const singlelineCommentsRE = /\/\/.*/g