fix(bench): discard autocannon warmup before measuring

Author: chrisbbreuer

bench/routing/drivers.test.ts

@@ -1,5 +1,5 @@
-import { describe, expect, it } from 'bun:test'
-import { ohaArgs } from './drivers'
+import { describe, expect, it, spyOn } from 'bun:test'
+import { DRIVERS, ohaArgs } from './drivers'
 
 describe('oha command', () => {
   it('applies one global fixed request rate with latency correction', () => {
@@ -18,3 +18,42 @@ describe('oha command', () => {
     ])
   })
 })
+
+describe('autocannon warmup', () => {
+  it.each([0, 3])('discards %s seconds of warmup without changing workers', async (warmupSeconds) => {
+    const output = (requests: number) => ({
+      stdout: new Response(JSON.stringify({ requests: { total: requests, average: requests } })).body,
+      stderr: new Response('').body,
+      exited: Promise.resolve(0),
+    }) as unknown as ReturnType<typeof Bun.spawn>
+    const spawn = spyOn(Bun, 'spawn')
+    if (warmupSeconds > 0) spawn.mockReturnValueOnce(output(999))
+    spawn.mockReturnValueOnce(output(42))
+    try {
+      const result = await DRIVERS.find(driver => driver.name === 'autocannon')!.run({
+        url: 'http://127.0.0.1:39400/bench/echo',
+        method: 'POST',
+        body: '{"name":"bench","count":7}',
+        headers: { 'content-type': 'application/json', 'x-bench': 'preserved' },
+        connections: 7,
+        warmupSeconds,
+        durationSeconds: 11,
+      })
+      expect(result.requests).toBe(42)
+      expect(JSON.parse(result.raw).requests.total).toBe(42)
+      const commands = spawn.mock.calls.map(call => call[0] as string[])
+      expect(commands.map(args => args[args.indexOf('-d') + 1])).toEqual(warmupSeconds > 0 ? ['3', '11'] : ['11'])
+      for (const args of commands) {
+        expect(args).not.toContain('-w')
+        expect(args[args.indexOf('-c') + 1]).toBe('7')
+        expect(args[args.indexOf('-m') + 1]).toBe('POST')
+        expect(args[args.indexOf('-b') + 1]).toBe('{"name":"bench","count":7}')
+        expect(args).toContain('content-type: application/json')
+        expect(args).toContain('x-bench: preserved')
+      }
+    }
+    finally {
+      spawn.mockRestore()
+    }
+  })
+})

bench/routing/drivers.ts

@@ -172,15 +172,12 @@ const autocannon: Driver = {
   supportsFixedRate: false,
   isAvailable: () => which('autocannon'),
   async run(req) {
-    const args = ['autocannon', '-d', String(req.durationSeconds), '-w', String(req.warmupSeconds), '-c', String(req.connections), '-j']
-    if (req.method !== 'GET') {
-      args.push('-m', req.method)
-      if (req.body != null) args.push('-b', req.body)
-    }
-    for (const [name, value] of Object.entries(req.headers))
-      args.push('-H', `${name}: ${value}`)
-    args.push(req.url)
-    const raw = await capture(args)
+    const args = ['autocannon', '-c', String(req.connections), '-j', ...methodArgs(req, '-m', '-b', '-H')]
+    // Keep warmup separate, as in the native adapters. Autocannon's -w
+    // selects worker threads, not a discarded warmup duration.
+    if (req.warmupSeconds > 0)
+      await capture([...args, '-d', String(req.warmupSeconds), req.url])
+    const raw = await capture([...args, '-d', String(req.durationSeconds), req.url])
     const json = JSON.parse(raw)
     return {
       rpsMean: json.requests?.average ?? 0,