fix(instagram-embed): use facebookexternalhit UA & don't cache empty shells (#806)

Author: harlan-zw

packages/script/src/runtime/server/instagram-embed.ts

@@ -1,8 +1,9 @@
 import { createError, defineEventHandler, getQuery, setHeader } from 'h3'
-import { useRuntimeConfig } from 'nitropack/runtime'
+import { defineCachedFunction, useRuntimeConfig } from 'nitropack/runtime'
+import { $fetch } from 'ofetch'
 import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml'
 import { createCachedJsonFetch } from './utils/cached-upstream'
-import { proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed'
+import { isEmbedShell, proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed'
 import { withSigning } from './utils/withSigning'
 
 export { proxyAssetUrl, proxyImageUrl, rewriteUrl, rewriteUrlsInText, scopeCss } from './utils/instagram-embed'
@@ -12,10 +13,35 @@ const SRCSET_SPLIT_RE = /\s+/
 
 // Instagram embed HTML is semi-fresh (likes, captions may update); 10min
 // matches the outbound Cache-Control header and dedupes per post+captions.
-const cachedEmbedFetch = createCachedJsonFetch<string>(
-  'nuxt-scripts-instagram-embed',
-  600,
-  url => url,
+// Throws on shell responses so nitro doesn't cache them.
+const cachedEmbedFetch = defineCachedFunction(
+  async (url: string, headers: Record<string, string>): Promise<string> => {
+    const html = await $fetch<string>(url, { timeout: 10000, headers })
+    if (isEmbedShell(html)) {
+      throw createError({
+        statusCode: 502,
+        statusMessage: 'Instagram returned an empty embed shell (post unavailable or upstream rate-limiting)',
+      })
+    }
+    return html
+  },
+  {
+    // v2 — bump to evict any v1 entries that cached the empty JS shell
+    // before the shell-detection / UA fix landed.
+    name: 'nuxt-scripts-instagram-embed-v2',
+    maxAge: 600,
+    swr: true,
+    staleMaxAge: 600,
+    // Vary on headers too — Instagram's response is UA-dependent, so
+    // different callers (e.g. unit tests, future UA changes) must not
+    // collide on the same key.
+    getKey: (url: string, headers: Record<string, string>) => {
+      const parts = [url]
+      for (const [k, v] of Object.entries(headers).sort(([a], [b]) => a.localeCompare(b)))
+        parts.push(`${k}=${v}`)
+      return parts.join('|')
+    },
+  },
 )
 
 // Static CSS from Instagram's CDN is versioned; 24h cache is safe because the
@@ -79,10 +105,12 @@ export default withSigning(defineEventHandler(async (event) => {
   const embedUrl = `${cleanUrl}embed/${captions ? 'captioned/' : ''}`
 
   const html = await cachedEmbedFetch(embedUrl, {
-    headers: {
-      'Accept': 'text/html',
-      'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
-    },
+    'Accept': 'text/html',
+    // Meta's own crawler UA. Googlebot's UA is also accepted by Instagram
+    // but is IP-verified, so it fails from hosts outside Google's ranges
+    // (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead
+    // of the SSR'd post.
+    'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
   }).catch((error: any) => {
     throw createError({
       statusCode: error.statusCode || 500,

packages/script/src/runtime/server/utils/instagram-embed.ts

@@ -1,6 +1,19 @@
 import { buildProxyUrl } from './proxy-url'
 
 export const RSRC_RE = /url\(\/rsrc\.php([^)]+)\)/g
+
+// Instagram serves a JS-only shell (splash-screen + comet sentinel, no SSR'd
+// post markup) when it can't or won't render server-side — e.g. for bot UAs
+// it can't verify, or for removed/private posts.
+const SHELL_BODY_RE = /id=["'](?:splash-screen|has-finished-comet-page)["']/
+// Match Embed / EmbeddedMedia / EmbeddedMediaImage as tokens inside any
+// class attribute (single- or double-quoted, multi-class lists).
+const HAS_POST_CONTENT_RE = /\bclass=(["'])[^"']*\b(?:Embed|EmbeddedMedia|EmbeddedMediaImage)\b[^"']*\1/i
+
+export function isEmbedShell(html: string): boolean {
+  return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html)
+}
+
 export const AMP_RE = /&/g
 export const SCONTENT_RE = /https:\/\/scontent[^"'\s),]+\.cdninstagram\.com[^"'\s),]+/g
 export const STATIC_CDN_RE = /https:\/\/static\.cdninstagram\.com[^"'\s),]+/g

test/unit/instagram-embed.test.ts

@@ -1,6 +1,7 @@
 import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml'
 import { describe, expect, it } from 'vitest'
 import {
+  isEmbedShell,
   proxyImageUrl,
   rewriteUrl,
   rewriteUrlsInText,
@@ -302,3 +303,43 @@ html, body { margin: 0; padding: 0; }
     expect(result).toContain(`${scope} [data-x="a,b"] .Embed`)
   })
 })
+
+describe('instagram-embed: isEmbedShell', () => {
+  it('detects JS-only shell with splash-screen and no post markup', () => {
+    const html = '<body><div id="splash-screen"></div><div id="has-finished-comet-page"></div></body>'
+    expect(isEmbedShell(html)).toBe(true)
+  })
+
+  it('accepts real SSR\'d post even when shell sentinels appear elsewhere', () => {
+    // Some Instagram responses include the comet sentinel alongside real content.
+    const html = '<body><a class="EmbeddedMedia"><img class="EmbeddedMediaImage" /></a><div id="has-finished-comet-page"></div></body>'
+    expect(isEmbedShell(html)).toBe(false)
+  })
+
+  it('accepts real post with Embed wrapper', () => {
+    const html = '<div class="Embed" data-media-id="123"><div>content</div></div>'
+    expect(isEmbedShell(html)).toBe(false)
+  })
+
+  it('returns false on unrelated HTML (no shell sentinels)', () => {
+    expect(isEmbedShell('<html><body>nothing here</body></html>')).toBe(false)
+  })
+
+  it('detects post content inside multi-class lists', () => {
+    // Real Instagram responses include splash-screen alongside the SSR'd post;
+    // multi-class lists like `class="post EmbeddedMedia foo"` must count.
+    const html = '<div id="splash-screen"></div><div class="post EmbeddedMedia foo"></div>'
+    expect(isEmbedShell(html)).toBe(false)
+  })
+
+  it('detects post content with single-quoted class attribute', () => {
+    const html = `<div id='splash-screen'></div><a class='EmbeddedMedia'></a>`
+    expect(isEmbedShell(html)).toBe(false)
+  })
+
+  it('does not match Embed inside an unrelated class token (word boundary)', () => {
+    // `EmbedSomething` should not count as post content.
+    const html = '<div id="splash-screen"></div><div class="NotAnEmbedThing"></div>'
+    expect(isEmbedShell(html)).toBe(true)
+  })
+})