fix: decouple page rendering from external blog feeds

src/utils/blog.functions.ts

@@ -84,13 +84,14 @@ function setExistingBlogListResponseHeaders() {
   )
 }
 
-async function getBlogCardPosts() {
-  const externalPosts = await getExternalBlogPosts()
+function getInternalBlogCardPosts() {
+  return sortBlogCardPosts(getVisiblePosts().map(postToBlogCardPost))
+}
 
-  return sortBlogCardPosts([
-    ...getVisiblePosts().map(postToBlogCardPost),
-    ...externalPosts,
-  ])
+async function getBlogCardPosts(options?: { libraryId?: LibraryId }) {
+  const externalPosts = await getExternalBlogPosts(options)
+
+  return sortBlogCardPosts([...getInternalBlogCardPosts(), ...externalPosts])
 }
 
 export const fetchBlogPost = createServerFn({ method: 'GET' })
@@ -153,7 +154,7 @@ export const fetchBlogPostsForLibrary = createServerFn({ method: 'GET' })
       return []
     }
 
-    return (await getBlogCardPosts()).filter((post) =>
+    return (await getBlogCardPosts({ libraryId: library.id })).filter((post) =>
       getBlogLibraries(post.library).some(
         (postLibrary) => postLibrary.id === library.id,
       ),
@@ -164,16 +165,18 @@ export const fetchRecentPosts = createServerFn({ method: 'GET' }).handler(
   async (): Promise<Array<RecentPost>> => {
     setExistingBlogListResponseHeaders()
 
-    return (await getBlogCardPosts()).slice(0, 3).map((post) => ({
-      slug: post.slug,
-      title: post.title,
-      published: post.published,
-      excerpt: post.excerpt,
-      headerImage: post.headerImage,
-      authors: post.authors,
-      externalUrl: post.externalUrl,
-      source: post.source,
-    }))
+    return getInternalBlogCardPosts()
+      .slice(0, 3)
+      .map((post) => ({
+        slug: post.slug,
+        title: post.title,
+        published: post.published,
+        excerpt: post.excerpt,
+        headerImage: post.headerImage,
+        authors: post.authors,
+        externalUrl: post.externalUrl,
+        source: post.source,
+      }))
   },
 )
 

src/utils/external-blog-posts.server.ts

@@ -4,9 +4,18 @@ import { fetchCached } from '~/utils/cache.server'
 
 const DEFAULT_STANDARD_SITE_TIMEOUT_MS = 5000 // 5 seconds
 const DEFAULT_STANDARD_SITE_CACHE_TTL_MS = 60 * 60 * 1000 // 1 hour
+const DEFAULT_EXTERNAL_BLOG_FAILURE_CACHE_TTL_MS = 60 * 1000 // 1 minute
 const STANDARD_SITE_DOCUMENT_COLLECTION = 'site.standard.document'
 const STANDARD_SITE_PAGE_LIMIT = 100
 
+declare global {
+  var externalBlogFailureCache: Map<string, number> | undefined
+}
+
+const externalBlogFailureCache =
+  globalThis.externalBlogFailureCache ??
+  (globalThis.externalBlogFailureCache = new Map())
+
 type ExternalLibraryId = Extract<LibraryId, 'query' | 'router'>
 
 type ExternalBlogItem = {
@@ -55,16 +64,18 @@ type StandardSiteExternalBlogSource = {
   collection?: string
   slugPrefix: string
   authors: Array<string>
+  libraries: ReadonlyArray<LibraryId>
   externalUrlSearchParams?: Record<string, string>
   cacheTtlMs?: number
+  failureCacheTtlMs?: number
   timeoutMs?: number
   maxPages?: number
   inferLibraries?: (item: ExternalBlogItem) => Array<LibraryId>
 }
 
 type ExternalBlogSource = StandardSiteExternalBlogSource
 
-const externalBlogSources = [
+const externalBlogSources: ReadonlyArray<ExternalBlogSource> = [
   {
     type: 'standard-site',
     id: 'tkdodo',
@@ -74,14 +85,15 @@ const externalBlogSources = [
     repo: 'did:plc:3nqrhu5mthmias3zc4a2ovzj',
     slugPrefix: 'tkdodo',
     authors: ['Dominik Dorfmeister'],
+    libraries: ['query', 'router'],
     externalUrlSearchParams: {
       utm_source: 'tanstack.com',
       utm_medium: 'referral',
       utm_campaign: 'tanstack_blog',
     },
     inferLibraries: inferTanStackQueryAndRouterLibraries,
   },
-] satisfies Array<ExternalBlogSource>
+]
 
 function normalizeSearchValue(value: string) {
   return value
@@ -351,24 +363,50 @@ async function fetchStandardSiteBlogPosts(
 }
 
 async function fetchExternalBlogPostsForSource(source: ExternalBlogSource) {
-  return fetchCached({
-    key: `external-blog-posts:${source.id}`,
-    ttl: source.cacheTtlMs ?? DEFAULT_STANDARD_SITE_CACHE_TTL_MS,
-    fn: async () => fetchStandardSiteBlogPosts(source),
-  }).catch((error) => {
+  const cacheKey = `external-blog-posts:${source.id}`
+  const failureExpiresAt = externalBlogFailureCache.get(cacheKey)
+
+  if (failureExpiresAt && failureExpiresAt > Date.now()) {
+    return []
+  }
+
+  externalBlogFailureCache.delete(cacheKey)
+
+  try {
+    const posts = await fetchCached({
+      key: cacheKey,
+      ttl: source.cacheTtlMs ?? DEFAULT_STANDARD_SITE_CACHE_TTL_MS,
+      fn: async () => fetchStandardSiteBlogPosts(source),
+    })
+
+    externalBlogFailureCache.delete(cacheKey)
+    return posts
+  } catch (error) {
+    externalBlogFailureCache.set(
+      cacheKey,
+      Date.now() +
+        (source.failureCacheTtlMs ??
+          DEFAULT_EXTERNAL_BLOG_FAILURE_CACHE_TTL_MS),
+    )
     console.warn(
       `Unable to load external blog posts from ${source.name}`,
       error,
     )
     return []
-  })
+  }
 }
 
-export async function getExternalBlogPosts() {
+export async function getExternalBlogPosts(options?: {
+  libraryId?: LibraryId
+}) {
+  const libraryId = options?.libraryId
+  const sources = libraryId
+    ? externalBlogSources.filter((source) =>
+        source.libraries.includes(libraryId),
+      )
+    : externalBlogSources
   const postsBySource = await Promise.all(
-    externalBlogSources.map((source) =>
-      fetchExternalBlogPostsForSource(source),
-    ),
+    sources.map((source) => fetchExternalBlogPostsForSource(source)),
   )
 
   return postsBySource.flat()

tests/external-blog-posts.test.ts

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict'
+import {
+  getExternalBlogPosts,
+  inferExternalPostLibraries,
+} from '../src/utils/external-blog-posts.server'
+
+assert.deepEqual(
+  inferExternalPostLibraries(
+    'Concurrent Optimistic Updates in React Query',
+    'https://tkdodo.eu/blog/concurrent-optimistic-updates-in-react-query',
+  ),
+  ['query'],
+)
+assert.deepEqual(
+  inferExternalPostLibraries(
+    'TanStack Router and Query',
+    'https://tkdodo.eu/blog/tanstack-router-and-query',
+  ),
+  ['query', 'router'],
+)
+assert.deepEqual(
+  inferExternalPostLibraries(
+    'Working with TypeScript',
+    'https://tkdodo.eu/blog/working-with-typescript',
+  ),
+  [],
+)
+
+const originalFetch = globalThis.fetch
+const originalWarn = console.warn
+let fetchCount = 0
+
+globalThis.fetch = async () => {
+  fetchCount++
+  throw new Error('External source unavailable')
+}
+console.warn = () => {}
+
+try {
+  assert.deepEqual(await getExternalBlogPosts({ libraryId: 'table' }), [])
+  assert.equal(fetchCount, 0, 'unrelated libraries skip external sources')
+
+  assert.deepEqual(await getExternalBlogPosts({ libraryId: 'query' }), [])
+  assert.equal(fetchCount, 1, 'supported libraries fetch their external source')
+
+  assert.deepEqual(await getExternalBlogPosts({ libraryId: 'query' }), [])
+  assert.equal(fetchCount, 1, 'failed external fetches use the short backoff')
+} finally {
+  globalThis.fetch = originalFetch
+  console.warn = originalWarn
+}
+
+console.log('external blog post tests passed')