fix(bench): isolate measured workloads

Author: chrisbbreuer

bench/memory/README.md

@@ -22,7 +22,10 @@ and Stacks dev fixtures before becoming executable profiles.
 
 The runner uses the same byte-for-byte response parity checks as
 `bench/routing`. It samples the entire server process tree, so launchers cannot
-hide worker memory. The separately launched load generator is not counted.
+hide worker memory. The separately launched load generator is not counted. Each
+server registers only the selected scenario, so an unrelated validator or
+database route cannot inflate one framework's static JSON result. This applies
+identically to every target.
 
 ## Reported value
 

bench/memory/run.ts

@@ -196,7 +196,7 @@ async function measure(
   | { skipped: string }
   | { measurement: Omit<MemoryMeasurement, 'targetId' | 'run'>, samples: MemorySample[], load: LoadResult }
 > {
-  const booted = await boot(target, Boolean(scenario.requiresDb))
+  const booted = await boot(target, Boolean(scenario.requiresDb), scenario)
   if ('skipped' in booted) return booted
 
   try {

bench/routing/bunfig.toml

@@ -0,0 +1,3 @@
+# Benchmark servers must not inherit the application's environment and
+# auto-import preloads. Each target imports only the framework it measures.
+preload = []

bench/routing/runtime.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'bun:test'
+import { BENCH_ROOT, serverCommand, serverEnvironment } from './runtime'
+
+describe('benchmark server isolation', () => {
+  it('boots targets without the application bunfig preloads', () => {
+    expect(serverCommand('bun-raw.ts')).toEqual([
+      process.execPath,
+      `--config=${BENCH_ROOT}bunfig.toml`,
+      `${BENCH_ROOT}servers/bun-raw.ts`,
+    ])
+  })
+
+  it('boots every framework in production mode', () => {
+    const env = serverEnvironment({
+      id: 'test',
+      label: 'test',
+      server: 'bun-raw.ts',
+      env: { BENCH_MODE: 'minimal' },
+    }, false, 'static-json')
+
+    expect(env.APP_ENV).toBe('production')
+    expect(env.NODE_ENV).toBe('production')
+    expect(env.BENCH_MODE).toBe('minimal')
+    expect(env.BENCH_SCENARIO).toBe('static-json')
+  })
+})

bench/routing/runtime.ts

@@ -18,19 +18,36 @@ export interface BootedServer {
   pid: number
 }
 
-export async function boot(target: Target, withDb: boolean): Promise<BootedServer | { skipped: string }> {
-  const proc = Bun.spawn([process.execPath, join(BENCH_ROOT, 'servers', target.server)], {
+/** Keep benchmark targets out of the application's preload graph. */
+export function serverCommand(server: string): string[] {
+  return [
+    process.execPath,
+    `--config=${join(BENCH_ROOT, 'bunfig.toml')}`,
+    join(BENCH_ROOT, 'servers', server),
+  ]
+}
+
+/** Give every framework the same production environment. */
+export function serverEnvironment(target: Target, withDb: boolean, scenarioId?: string): Record<string, string> {
+  return {
+    ...process.env,
+    APP_ENV: 'production',
+    NODE_ENV: 'production',
+    BENCH_PORT: String(PORT),
+    BENCH_DB: withDb ? '1' : '0',
+    BENCH_DB_FILE: FIXTURE,
+    DB_DATABASE_PATH: FIXTURE,
+    ...(scenarioId ? { BENCH_SCENARIO: scenarioId } : {}),
+    ...target.env,
+  } as Record<string, string>
+}
+
+export async function boot(target: Target, withDb: boolean, scenario?: Scenario): Promise<BootedServer | { skipped: string }> {
+  const proc = Bun.spawn(serverCommand(target.server), {
     cwd: REPO_ROOT,
     stdout: 'pipe',
     stderr: 'pipe',
-    env: {
-      ...process.env,
-      BENCH_PORT: String(PORT),
-      BENCH_DB: withDb ? '1' : '0',
-      BENCH_DB_FILE: FIXTURE,
-      DB_DATABASE_PATH: FIXTURE,
-      ...target.env,
-    } as Record<string, string>,
+    env: serverEnvironment(target, withDb, scenario?.id),
   })
 
   const deadline = Date.now() + 60_000
@@ -43,7 +60,7 @@ export async function boot(target: Target, withDb: boolean): Promise<BootedServe
       throw new Error(`${target.id} server exited ${proc.exitCode}:\n${err}`)
     }
     try {
-      const res = await fetch(`http://127.0.0.1:${PORT}/bench/json`)
+      const res = await fetch(`http://127.0.0.1:${PORT}${scenario?.path ?? '/bench/json'}`)
       if (res.ok) {
         await res.arrayBuffer()
         break

bench/routing/servers/bun-raw.ts

@@ -7,16 +7,18 @@
  * already believed.
  */
 
-import { Database } from 'bun:sqlite'
 import process from 'node:process'
 
 const port = Number(process.env.BENCH_PORT ?? 3999)
 const withDb = process.env.BENCH_DB === '1'
+const scenario = process.env.BENCH_SCENARIO
+const serves = (id: string) => !scenario || scenario === id
 
 const JSON_HEADERS = { 'content-type': 'application/json' } as const
 
 let selectItem: import('bun:sqlite').Statement | undefined
-if (withDb) {
+if (withDb && serves('db-roundtrip')) {
+  const { Database } = await import('bun:sqlite')
   const db = new Database(process.env.BENCH_DB_FILE!, { readonly: true })
   selectItem = db.prepare('SELECT id, name FROM bench_items WHERE id = 1')
 }
@@ -28,13 +30,13 @@ Bun.serve({
     const pathStart = url.indexOf('/', url.indexOf('://') + 3)
     const path = pathStart === -1 ? '/' : url.slice(pathStart)
 
-    if (path === '/bench/json')
+    if (serves('static-json') && path === '/bench/json')
       return new Response('{"hello":"world"}', { headers: JSON_HEADERS })
 
-    if (path.startsWith('/bench/users/'))
+    if (serves('path-param') && path.startsWith('/bench/users/'))
       return new Response(JSON.stringify({ id: path.slice('/bench/users/'.length) }), { headers: JSON_HEADERS })
 
-    if (path === '/bench/echo' && req.method === 'POST') {
+    if (serves('post-validate') && path === '/bench/echo' && req.method === 'POST') {
       const body = await req.json() as { name?: unknown, count?: unknown }
       if (typeof body.name !== 'string' || typeof body.count !== 'number')
         return new Response('{"errors":{}}', { status: 422, headers: JSON_HEADERS })

bench/routing/servers/elysia.ts

@@ -11,6 +11,8 @@ import process from 'node:process'
 
 const port = Number(process.env.BENCH_PORT ?? 3999)
 const withDb = process.env.BENCH_DB === '1'
+const scenario = process.env.BENCH_SCENARIO
+const serves = (id: string) => !scenario || scenario === id
 
 let Elysia: any
 let t: any
@@ -23,15 +25,19 @@ catch {
 }
 
 const app = new Elysia()
-  .get('/bench/json', () => ({ hello: 'world' }))
-  .get('/bench/users/:id', ({ params }: any) => ({ id: params.id }))
-  .post(
+if (serves('static-json'))
+  app.get('/bench/json', () => ({ hello: 'world' }))
+if (serves('path-param'))
+  app.get('/bench/users/:id', ({ params }: any) => ({ id: params.id }))
+if (serves('post-validate')) {
+  app.post(
     '/bench/echo',
     ({ body }: any) => ({ name: body.name, count: body.count }),
     { body: t.Object({ name: t.String(), count: t.Number() }) },
   )
+}
 
-if (withDb) {
+if (withDb && serves('db-roundtrip')) {
   const { Database } = await import('bun:sqlite')
   const db = new Database(process.env.BENCH_DB_FILE!, { readonly: true })
   const selectItem = db.prepare('SELECT id, name FROM bench_items WHERE id = 1')

bench/routing/servers/express.ts

@@ -2,6 +2,8 @@ import process from 'node:process'
 
 const port = Number(process.env.BENCH_PORT ?? 3999)
 const withDb = process.env.BENCH_DB === '1'
+const scenario = process.env.BENCH_SCENARIO
+const serves = (id: string) => !scenario || scenario === id
 
 let express: any
 try {
@@ -13,16 +15,20 @@ catch {
 }
 
 const app = express()
-app.use(express.json())
-app.get('/bench/json', (_req: any, res: any) => res.json({ hello: 'world' }))
-app.get('/bench/users/:id', (req: any, res: any) => res.json({ id: req.params.id }))
-app.post('/bench/echo', (req: any, res: any) => {
-  const { name, count } = req.body ?? {}
-  if (typeof name !== 'string' || typeof count !== 'number') return res.status(422).json({ errors: {} })
-  return res.json({ name, count })
-})
+if (serves('static-json'))
+  app.get('/bench/json', (_req: any, res: any) => res.json({ hello: 'world' }))
+if (serves('path-param'))
+  app.get('/bench/users/:id', (req: any, res: any) => res.json({ id: req.params.id }))
+if (serves('post-validate')) {
+  app.use(express.json())
+  app.post('/bench/echo', (req: any, res: any) => {
+    const { name, count } = req.body ?? {}
+    if (typeof name !== 'string' || typeof count !== 'number') return res.status(422).json({ errors: {} })
+    return res.json({ name, count })
+  })
+}
 
-if (withDb) {
+if (withDb && serves('db-roundtrip')) {
   const { Database } = await import('bun:sqlite')
   const db = new Database(process.env.BENCH_DB_FILE!, { readonly: true })
   const selectItem = db.prepare('SELECT id, name FROM bench_items WHERE id = 1')

bench/routing/servers/fastify.ts

@@ -2,6 +2,8 @@ import process from 'node:process'
 
 const port = Number(process.env.BENCH_PORT ?? 3999)
 const withDb = process.env.BENCH_DB === '1'
+const scenario = process.env.BENCH_SCENARIO
+const serves = (id: string) => !scenario || scenario === id
 
 let fastify: any
 try {
@@ -13,19 +15,23 @@ catch {
 }
 
 const app = fastify({ logger: false })
-app.get('/bench/json', () => ({ hello: 'world' }))
-app.get('/bench/users/:id', (request: any) => ({ id: request.params.id }))
-app.post('/bench/echo', {
-  schema: {
-    body: {
-      type: 'object',
-      required: ['name', 'count'],
-      properties: { name: { type: 'string' }, count: { type: 'number' } },
+if (serves('static-json'))
+  app.get('/bench/json', () => ({ hello: 'world' }))
+if (serves('path-param'))
+  app.get('/bench/users/:id', (request: any) => ({ id: request.params.id }))
+if (serves('post-validate')) {
+  app.post('/bench/echo', {
+    schema: {
+      body: {
+        type: 'object',
+        required: ['name', 'count'],
+        properties: { name: { type: 'string' }, count: { type: 'number' } },
+      },
     },
-  },
-}, (request: any) => ({ name: request.body.name, count: request.body.count }))
+  }, (request: any) => ({ name: request.body.name, count: request.body.count }))
+}
 
-if (withDb) {
+if (withDb && serves('db-roundtrip')) {
   const { Database } = await import('bun:sqlite')
   const db = new Database(process.env.BENCH_DB_FILE!, { readonly: true })
   const selectItem = db.prepare('SELECT id, name FROM bench_items WHERE id = 1')

bench/routing/servers/hono.ts

@@ -12,12 +12,12 @@ import process from 'node:process'
 
 const port = Number(process.env.BENCH_PORT ?? 3999)
 const withDb = process.env.BENCH_DB === '1'
+const scenario = process.env.BENCH_SCENARIO
+const serves = (id: string) => !scenario || scenario === id
 
 let Hono: any
-let validator: any
 try {
   ;({ Hono } = await import('hono') as any)
-  ;({ validator } = await import('hono/validator') as any)
 }
 catch {
   console.error('[bench] hono is not installed — run `bun add -d hono` to include it')
@@ -26,22 +26,27 @@ catch {
 
 const app = new Hono()
 
-app.get('/bench/json', (c: any) => c.json({ hello: 'world' }))
-app.get('/bench/users/:id', (c: any) => c.json({ id: c.req.param('id') }))
-app.post(
-  '/bench/echo',
-  validator('json', (value: any, c: any) => {
-    if (typeof value.name !== 'string' || typeof value.count !== 'number')
-      return c.json({ errors: {} }, 422)
-    return value
-  }),
-  (c: any) => {
-    const body = c.req.valid('json')
-    return c.json({ name: body.name, count: body.count })
-  },
-)
+if (serves('static-json'))
+  app.get('/bench/json', (c: any) => c.json({ hello: 'world' }))
+if (serves('path-param'))
+  app.get('/bench/users/:id', (c: any) => c.json({ id: c.req.param('id') }))
+if (serves('post-validate')) {
+  const { validator } = await import('hono/validator') as any
+  app.post(
+    '/bench/echo',
+    validator('json', (value: any, c: any) => {
+      if (typeof value.name !== 'string' || typeof value.count !== 'number')
+        return c.json({ errors: {} }, 422)
+      return value
+    }),
+    (c: any) => {
+      const body = c.req.valid('json')
+      return c.json({ name: body.name, count: body.count })
+    },
+  )
+}
 
-if (withDb) {
+if (withDb && serves('db-roundtrip')) {
   const { Database } = await import('bun:sqlite')
   const db = new Database(process.env.BENCH_DB_FILE!, { readonly: true })
   const selectItem = db.prepare('SELECT id, name FROM bench_items WHERE id = 1')