perf: compile default bot matcher at build time (#318)

Author: harlan-zw

build.config.ts

@@ -1,7 +1,19 @@
+import { writeFile } from 'node:fs/promises'
+import { resolve } from 'pathe'
 import { defineBuildConfig } from 'unbuild'
+import { generateBotMatcherModule } from './scripts/generate-bot-matcher'
+import { BOT_MAP } from './src/const-bots'
 
 export default defineBuildConfig({
   declaration: true,
+  hooks: {
+    'build:before': async function (ctx) {
+      await writeFile(
+        resolve(ctx.options.rootDir, 'src/generated-bot-matcher.ts'),
+        generateBotMatcherModule(BOT_MAP),
+      )
+    },
+  },
   rollup: {
     emitCJS: true,
   },

scripts/generate-bot-matcher.ts

@@ -0,0 +1,84 @@
+interface BotPatternDefinition {
+  name: string
+  pattern: string
+  secondaryPatterns?: string[]
+}
+
+interface BotCategoryDefinition {
+  bots: BotPatternDefinition[]
+  trusted: boolean
+  type: string
+}
+
+interface PatternEntry {
+  botIndex: number
+  definitionIndex: number
+  pattern: string
+}
+
+function collectPatterns(definitions: readonly BotCategoryDefinition[]): PatternEntry[] {
+  const patterns = new Map<string, PatternEntry>()
+  for (const [definitionIndex, definition] of definitions.entries()) {
+    for (const [botIndex, bot] of definition.bots.entries()) {
+      for (const sourcePattern of [bot.pattern, ...(bot.secondaryPatterns || [])]) {
+        const pattern = sourcePattern.toLowerCase()
+        patterns.set(pattern, {
+          botIndex,
+          definitionIndex,
+          pattern,
+        })
+      }
+    }
+  }
+  return [...patterns.values()]
+}
+
+function escapeRegExpPattern(value: string): string {
+  return value
+    .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+    .replaceAll('/', '\\/')
+    .replaceAll('\n', '\\n')
+    .replaceAll('\r', '\\r')
+    .replaceAll('\u2028', '\\u2028')
+    .replaceAll('\u2029', '\\u2029')
+}
+
+function removeShadowedPatterns(patterns: PatternEntry[]): PatternEntry[] {
+  return patterns.filter((entry, index) => !patterns
+    .slice(0, index)
+    .some(candidate => entry.pattern.includes(candidate.pattern)))
+}
+
+export function generateBotMatcherModule(definitions: readonly BotCategoryDefinition[]): string {
+  const patterns = removeShadowedPatterns(collectPatterns(definitions))
+  const botTargets = patterns.map(({ definitionIndex, botIndex }) => {
+    if (definitionIndex > 255 || botIndex > 255)
+      throw new Error('Bot matcher target exceeds its 8-bit index limit.')
+    return (definitionIndex << 8) | botIndex
+  })
+  const alternatives = patterns.map(({ pattern }) => `(${escapeRegExpPattern(pattern)})`).join('|')
+
+  return `// Generated by build.config.ts. Do not edit.
+
+const defaultBotPattern = /(?=(?:${alternatives}))/g
+const botTargets: number[] = [${botTargets.join(', ')}]
+
+export function matchDefaultBotTarget(value: string): number | undefined {
+  let bestPriority = -1
+  defaultBotPattern.lastIndex = 0
+  let match = defaultBotPattern.exec(value)
+  while (match) {
+    let priority = 0
+    while (typeof match[priority + 1] === 'undefined')
+      priority++
+    if (bestPriority === -1 || priority < bestPriority)
+      bestPriority = priority
+    if (bestPriority === 0)
+      break
+    defaultBotPattern.lastIndex = match.index + 1
+    match = defaultBotPattern.exec(value)
+  }
+  return bestPriority === -1 ? undefined : botTargets[bestPriority]
+}
+`
+}

src/generated-bot-matcher.ts

@@ -0,0 +1,22 @@
+// Generated by build.config.ts. Do not edit.
+
+const defaultBotPattern = /(?=(?:(googlebot)|(google\.com\/bot\.html)|(bingbot)|(msnbot)|(yandexbot)|(baiduspider)|(baidu\.com)|(duckduckbot)|(duckduckgo\.com)|(slurp)|(applebot)|(twitterbot)|(twitter)|(facebookexternalhit)|(facebook\.com)|(linkedinbot)|(linkedin)|(pinterestbot)|(pinterest)|(discordbot)|(discordapp)|(mj12bot)|(majestic12\.co\.uk\/bot)|(ahrefsbot)|(ahrefs\.com)|(semrushbot)|(semrush\.com\/bot)|(screaming frog)|(screamingfrog\.co\.uk)|(rogerbot)|(anthropic)|(claude)|(gptbot)|(openai\.com)|(google-extended)|(bytespider)|(diffbot)|(cohere)|(ccbot)|(commoncrawl\.org)|(perplexitybot)|(perplexity\.ai)|(bot)|(spider)|(crawler)|(scraper)|(phantomjs)|(headless)|(playwright)|(selenium)|(webdriver)|(puppeteer)|(python-requests)|(python)|(wget)|(curl)|(zgrab)|(masscan)|(nmap)|(insecure\.org)|(nikto)|(wpscan)|(scrapy)))/g
+const botTargets: number[] = [0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 256, 256, 257, 257, 258, 258, 259, 259, 260, 260, 512, 512, 513, 513, 514, 514, 515, 515, 516, 768, 769, 770, 770, 771, 773, 774, 776, 777, 777, 778, 778, 1024, 1025, 1026, 1027, 1280, 1284, 1282, 1283, 1283, 1284, 1536, 1536, 1537, 1538, 1792, 1793, 1794, 1794, 1795, 1796, 2048]
+
+export function matchDefaultBotTarget(value: string): number | undefined {
+  let bestPriority = -1
+  defaultBotPattern.lastIndex = 0
+  let match = defaultBotPattern.exec(value)
+  while (match) {
+    let priority = 0
+    while (typeof match[priority + 1] === 'undefined')
+      priority++
+    if (bestPriority === -1 || priority < bestPriority)
+      bestPriority = priority
+    if (bestPriority === 0)
+      break
+    defaultBotPattern.lastIndex = match.index + 1
+    match = defaultBotPattern.exec(value)
+  }
+  return bestPriority === -1 ? undefined : botTargets[bestPriority]
+}

src/util.ts

@@ -15,6 +15,7 @@ import {
   SEO_BOTS,
   SOCIAL_BOTS,
 } from './const-bots'
+import { matchDefaultBotTarget } from './generated-bot-matcher'
 
 export * from './const-bots'
 export { AiBots, NonHelpfulBots }
@@ -538,22 +539,39 @@ export function isBotFromHeaders(
 
   const userAgentLower = userAgent.toLowerCase()
 
-  // Use provided pattern map or create on demand for standalone usage
-  const resolvedPatternMap = patternMap || createPatternMap()
-
-  // Use optimized pattern lookup - O(n) instead of O(n³)
-  for (const [pattern, botData] of resolvedPatternMap) {
-    if (userAgentLower.includes(pattern)) {
-      return {
-        isBot: true,
-        data: {
-          botName: botData.botName,
-          botCategory: botData.botCategory,
-          trusted: botData.trusted,
-        },
+  const defaultBotTarget = patternMap ? undefined : matchDefaultBotTarget(userAgentLower)
+  if (typeof defaultBotTarget !== 'undefined') {
+    const definition = BOT_MAP[defaultBotTarget >> 8]!
+    const bot = definition.bots[defaultBotTarget & 255]!
+    return {
+      isBot: true,
+      data: {
+        botName: bot.name as BotName,
+        botCategory: definition.type,
+        trusted: definition.trusted,
+      },
+    }
+  }
+
+  let botData: PatternMapValue | undefined
+  if (patternMap) {
+    for (const [pattern, candidate] of patternMap) {
+      if (userAgentLower.includes(pattern)) {
+        botData = candidate
+        break
       }
     }
   }
+  if (botData) {
+    return {
+      isBot: true,
+      data: {
+        botName: botData.botName,
+        botCategory: botData.botCategory,
+        trusted: botData.trusted,
+      },
+    }
+  }
 
   return { isBot: false }
 }

test/unit/botDetection.test.ts

@@ -1,5 +1,6 @@
+import type { PatternMapValue } from '../../src/runtime/types'
 import { describe, expect, it } from 'vitest'
-import { isBotFromHeaders } from '../../src/util'
+import { createPatternMap, getBotDetection, isBotFromHeaders } from '../../src/util'
 
 const ValidHeaders = {
   'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
@@ -11,6 +12,12 @@ const ValidHeaders = {
 }
 
 describe('bot Detection from Headers', () => {
+  it.each([...createPatternMap().keys()])('matches the default %s pattern with reusable-map semantics', (pattern) => {
+    const headers = { 'user-agent': `Mozilla/5.0 compatible ${pattern}/1.0` }
+
+    expect(isBotFromHeaders(headers)).toEqual(isBotFromHeaders(headers, createPatternMap()))
+  })
+
   // Test empty user agent handling
   it('handles empty user agent properly', () => {
     const result = isBotFromHeaders({})
@@ -103,6 +110,24 @@ describe('bot Detection from Headers', () => {
     expect(result.data?.trusted).toBe(false)
   })
 
+  it('preserves definition priority over match position', () => {
+    const result = isBotFromHeaders({ 'user-agent': 'SomeRandomBot/1.0 Googlebot/2.1' })
+
+    expect(result.data?.botName).toBe('googlebot')
+  })
+
+  it('preserves reusable-map priority for every pair of default patterns', () => {
+    const patternMap = createPatternMap()
+    const patterns = [...patternMap.keys()]
+
+    for (const firstPattern of patterns) {
+      for (const secondPattern of patterns) {
+        const headers = { 'user-agent': `${firstPattern}/${secondPattern}` }
+        expect(isBotFromHeaders(headers)).toEqual(isBotFromHeaders(headers, patternMap))
+      }
+    }
+  })
+
   // Test human traffic
   it('identifies likely human traffic', () => {
     const result = isBotFromHeaders(ValidHeaders)
@@ -129,4 +154,49 @@ describe('bot Detection from Headers', () => {
 
     expect(result.isBot).toBe(false)
   })
+
+  it('uses a caller-provided bot map', () => {
+    const customMap = new Map<string, PatternMapValue>([
+      ['private-crawler', { botName: 'private', botCategory: 'scraping', trusted: false }],
+    ])
+
+    const result = getBotDetection({ 'user-agent': 'Private-Crawler/1.0' }, customMap)
+
+    expect(result).toEqual({
+      isBot: true,
+      userAgent: 'Private-Crawler/1.0',
+      detectionMethod: 'headers',
+      botName: 'private',
+      botCategory: 'scraping',
+      trusted: false,
+    })
+  })
+
+  it('keeps caller-provided bot maps isolated', async () => {
+    const firstMap = new Map<string, PatternMapValue>([
+      ['first-crawler', { botName: 'first', botCategory: 'scraping', trusted: false }],
+    ])
+    const secondMap = new Map<string, PatternMapValue>([
+      ['second-crawler', { botName: 'second', botCategory: 'search-engine', trusted: true }],
+    ])
+
+    const [firstResult, secondResult] = await Promise.all([
+      Promise.resolve(isBotFromHeaders({ 'user-agent': 'First-Crawler/1.0' }, firstMap)),
+      Promise.resolve(isBotFromHeaders({ 'user-agent': 'Second-Crawler/1.0' }, secondMap)),
+    ])
+
+    expect(firstResult.data?.botName).toBe('first')
+    expect(secondResult.data?.botName).toBe('second')
+    expect(isBotFromHeaders({ 'user-agent': 'First-Crawler/1.0' }, secondMap)).toEqual({ isBot: false })
+  })
+
+  it('returns independently mutable pattern maps', () => {
+    const firstMap = createPatternMap()
+    const secondMap = createPatternMap()
+
+    firstMap.clear()
+
+    expect(isBotFromHeaders({ 'user-agent': 'Googlebot/2.1' }, firstMap)).toEqual({ isBot: false })
+    expect(isBotFromHeaders({ 'user-agent': 'Googlebot/2.1' }, secondMap).data?.botName).toBe('googlebot')
+  })
 })