Update bot detection list and add support for oxygen-do-not-stream-re…

…sponse header
Shopify · May 19, 2023 · 37d9343 · 37d9343
1 parent 25c4eec
commit 37d9343
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 33 deletions.
diff --git a/.changeset/happy-snails-pay.md b/.changeset/happy-snails-pay.md
@@ -0,0 +1,5 @@
+---
+'@shopify/hydrogen': patch
+---
+
+Update bot detection list and add support for oxygen-do-not-stream-response header
diff --git a/packages/hydrogen/src/entry-server.tsx b/packages/hydrogen/src/entry-server.tsx
@@ -323,7 +323,10 @@ async function processRequest(
     });
   }
 
-  if (isBotUA(url, request.headers.get('user-agent'))) {
+  if (
+    request.headers.get('oxygen-do-not-stream-response') === 'true' ||
+    isBotUA(url, request.headers.get('user-agent'))
+  ) {
     response.doNotStream();
   }
 

diff --git a/packages/hydrogen/src/utilities/bot-ua.ts b/packages/hydrogen/src/utilities/bot-ua.ts
@@ -4,55 +4,56 @@
  * https://github.com/GoogleChrome/rendertron/blob/6f681688737846b28754fbfdf5db173846a826df/middleware/src/middleware.ts#L24-L41
  */
 const botUserAgents = [
-  'AdsBot-Google',
-  'AdsBot-Google-Mobile',
+  '360spider',
+  'adsbot-google',
+  'adsbot-google-mobile',
   'applebot',
-  'Baiduspider',
   'baiduspider',
-  'Bytespider',
-  '360Spider',
-  'PetalBot',
-  'Yisouspider',
+  'baiduspider',
+  'bingbot',
   'bingbot',
-  'Bingbot',
-  'BingPreview',
+  'bingpreview',
   'bitlybot',
-  'Discordbot',
-  'DuckDuckBot',
-  'Embedly',
+  'bytespider',
+  'discordbot',
+  'duckduckbot',
+  'embedly',
   'facebookcatalog',
   'facebookexternalhit',
-  'Google-PageRenderer',
-  'Googlebot',
-  'Googlebot-Image',
-  'Googlebot-News',
-  'Googlebot-Video',
+  'google-inspectiontool',
+  'google-pagerenderer',
+  'googlebot',
+  'googlebot-image',
+  'googlebot-news',
+  'googlebot-video',
+  'googleother',
   'googleweblight',
   'ia_archive',
-  'LinkedInBot',
-  'Mediapartners-Google',
+  'ia_archiver',
+  'linkedinbot',
+  'mediapartners-google',
   'outbrain',
+  'petalbot',
   'pinterest',
   'quora link preview',
   'redditbot',
   'rogerbot',
+  'seoradar',
   'showyoubot',
-  'SkypeUriPreview',
-  'Slackbot',
-  'Slurp',
+  'skypeuripreview',
+  'slackbot',
+  'slurp',
   'sogou',
-  'Storebot-Google',
-  'TelegramBot',
+  'storebot-google',
+  'telegrambot',
   'tumblr',
-  'Twitterbot',
-  'vkShare',
-  'W3C_Validator',
-  'WhatsApp',
+  'twitterbot',
+  'vkshare',
+  'w3c html2txt',
+  'w3c_validator',
+  'whatsapp',
   'yandex',
-
-  // SEO Tools
-  'Seoradar',
-  'W3C html2txt',
+  'yisouspider',
 ];
 
 /**