fix(notion): bypass Cloudflare IP block via loadCachedPageChunkV2 + throttle/cache

apps/web/.env.example

@@ -10,4 +10,14 @@ QDRANT_API_KEY=
 QDRANT_URL=
 VECTOR_DIMENSION=768  # Upto 768 dimensions are supported
 
-CACHE_EXPIRE=1800 # expiration time of the cache memory
+CACHE_EXPIRE=1800 # expiration time of the cache memory
+
+# Notion session cookies from a logged-in notion.so session. Required in production so
+# that Notion's Cloudflare protection doesn't 403-block server-side content fetches from
+# datacenter IPs. Leave blank for local dev.
+NOTION_TOKEN_V2=
+NOTION_ACTIVE_USER=
+
+# Optional: route all Notion API calls through an egress proxy on a clean (non-datacenter)
+# IP. Needed if Notion's Cloudflare keeps blocking the server IP. e.g. http://user:pass@host:port
+NOTION_PROXY_URL=

apps/web/app/api/AddTracks/route.ts

@@ -1,6 +1,6 @@
 import { getServerSession } from "next-auth";
 import { NextRequest, NextResponse } from "next/server";
-import { NotionAPI } from "notion-client";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
 import { authOptions } from "../../../lib/auth";
 
 export async function POST(req: NextRequest) {
@@ -11,9 +11,12 @@ export async function POST(req: NextRequest) {
   }
   const body = await req.json();
   const notionId = body.notionId;
-  const notion = new NotionAPI();
+  const notion = getNotionClient();
   try {
-    const recordMap = await notion.getPage(notionId);
+    const recordMap = await fetchNotionPage(notion, notionId);
+    if (!recordMap?.block) {
+      return NextResponse.json({ message: "Failed to load Notion page" }, { status: 502 });
+    }
     const data = Object.keys(recordMap.block).filter((key) => {
       const block = recordMap.block[key];
       return block?.role !== "none"

apps/web/app/pdf/[...pdfId]/page.tsx

@@ -1,14 +1,13 @@
 import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved";
-import { NotionAPI } from "notion-client";
 import { redirect } from "next/navigation";
 import { Print } from "../../../components/Print";
 import { getProblem, getTrack } from "../../../components/utils";
 import { LessonView } from "../../../components/LessonView";
 import { getServerSession } from "next-auth";
 import { authOptions } from "../../../lib/auth";
-import { fetchNotionPage } from "../../../lib/notion";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
 
-const notion = new NotionAPI();
+const notion = getNotionClient();
 
 export default async function TrackComponent({ params }: { params: { pdfId: string[] } }) {
   const trackId: string = params.pdfId[0] || "";

apps/web/app/tracks/[...trackIds]/page.tsx

@@ -1,12 +1,11 @@
 import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved";
-import { NotionAPI } from "notion-client";
 import { redirect, notFound } from "next/navigation";
 import { getAllTracks, getProblem, getTrack } from "../../../components/utils";
 import { cache } from "react";
 import { LessonView } from "../../../components/LessonView";
-import { fetchNotionPage } from "../../../lib/notion";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
 
-const notion = new NotionAPI();
+const notion = getNotionClient();
 
 export const dynamic = "auto";
 // Dynamic Metadata

apps/web/components/NotionRenderer.tsx

@@ -23,6 +23,17 @@ export const NotionRenderer = ({ recordMap }: { recordMap: any }) => {
     []
   );
 
+  if (!recordMap?.block || Object.keys(recordMap.block).length === 0) {
+    return (
+      <div className="flex flex-col items-center justify-center gap-2 py-24 text-center">
+        <p className="text-lg font-medium">This lesson is temporarily unavailable</p>
+        <p className="text-muted-foreground text-sm">
+          We couldn&apos;t load the content right now. Please refresh in a little while.
+        </p>
+      </div>
+    );
+  }
+
   return (
     <NotionRendererLib
       bodyClassName="text-base sm:text-lg"

apps/web/lib/notion.ts

@@ -1,18 +1,19 @@
 import { NotionAPI } from "notion-client";
+import { HttpsProxyAgent } from "https-proxy-agent";
 
-function normalizeRecordMap(recordMap: any) {
-  if (!recordMap?.block) return recordMap;
+function normalizeBlocks(block: any) {
+  if (!block) return {};
   const normalizedBlock: any = {};
-  for (const [key, block] of Object.entries(recordMap.block) as any) {
-    if (!block?.value) continue;
-    const value = block.value;
+  for (const [key, b] of Object.entries(block) as any) {
+    if (!b?.value) continue;
+    const value = b.value;
     if (!value.type && value.value?.type) {
-      normalizedBlock[key] = { ...block, value: value.value };
+      normalizedBlock[key] = { ...b, value: value.value };
     } else {
-      normalizedBlock[key] = block;
+      normalizedBlock[key] = b;
     }
   }
-  return { ...recordMap, block: normalizedBlock };
+  return normalizedBlock;
 }
 
 function collectContentBlockIds(recordMap: any): string[] {
@@ -38,21 +39,162 @@ function collectContentBlockIds(recordMap: any): string[] {
   return Array.from(seen);
 }
 
-// Notion's API now returns blocks in a nested `value.value` shape. notion-client's
-// built-in missing-block traversal walks the raw map and can't see past that nesting,
-// so toggle children (and other nested descendants) never get fetched. We disable its
-// traversal, normalize the shape, then manually fetch descendants until the tree is
-// complete.
-export async function fetchNotionPage(notion: NotionAPI, pageId: string): Promise<any> {
-  let recordMap: any = await notion.getPage(pageId, { fetchMissingBlocks: false });
-  recordMap = normalizeRecordMap(recordMap);
-
-  for (let i = 0; i < 10; i++) {
-    const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]);
-    if (!missing.length) break;
-    const fetched = await notion.getBlocks(missing).then((r: any) => r.recordMap.block);
-    recordMap = normalizeRecordMap({ ...recordMap, block: { ...recordMap.block, ...fetched } });
+// --- Why this file looks the way it does -------------------------------------
+// Notion serves its unofficial `/api/v3` endpoints behind Cloudflare. Requests from our
+// k8s datacenter egress IP get hard-blocked with a 403 "Attention Required" Cloudflare
+// page on the endpoints notion-client normally uses (`loadPageChunk`, `syncRecordValues`,
+// `queryCollection`), which took down every track/problem page with a 500.
+//
+// Empirically, `loadCachedPageChunkV2` is NOT blocked from the cluster and returns the
+// full page recordMap in a single request, so we fetch through that endpoint directly
+// (via the client's public `fetch`) instead of `getPage`. We also authenticate with
+// NOTION_TOKEN_V2 (private-page access) and aggressively throttle + cache, because the
+// block is IP-reputation based: bursts of requests re-trigger a broader Cloudflare block,
+// so keeping request volume low is what keeps this endpoint working.
+let notionSingleton: NotionAPI | null = null;
+
+export function getNotionClient(): NotionAPI {
+  if (notionSingleton) return notionSingleton;
+  notionSingleton = new NotionAPI({
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    authToken: process.env.NOTION_TOKEN_V2 || undefined,
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    activeUser: process.env.NOTION_ACTIVE_USER || undefined,
+  });
+  return notionSingleton;
+}
+
+// Optional egress proxy. Because Notion's Cloudflare block is IP-reputation based, the
+// fully-robust fix is to route requests through a clean/non-datacenter IP. When
+// NOTION_PROXY_URL is set (e.g. http://user:pass@host:port), all Notion calls tunnel
+// through it — which also unblocks the endpoints loadCachedPageChunkV2 can't cover
+// (images via getSignedFileUrls, embedded DBs via queryCollection). Inert when unset.
+let gotOptionsCache: { agent: { https: HttpsProxyAgent<string> } } | undefined | null = null;
+
+function getGotOptions() {
+  if (gotOptionsCache !== null) return gotOptionsCache;
+  // eslint-disable-next-line turbo/no-undeclared-env-vars
+  const proxy = process.env.NOTION_PROXY_URL;
+  gotOptionsCache = proxy ? { agent: { https: new HttpsProxyAgent(proxy) } } : undefined;
+  return gotOptionsCache;
+}
+
+// Two-tier cache: `fresh` entries are served within TTL; `stale` entries never expire and
+// are the fallback when Notion is unreachable/blocked, so a transient block degrades to
+// slightly-stale content instead of a 500. Long TTL keeps refetch volume (and IP-flag
+// risk) low; content changes rarely.
+const CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes
+type CacheEntry = { recordMap: any; ts: number };
+const cache = new Map<string, CacheEntry>();
+const staleCache = new Map<string, any>();
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+// Global throttle across all concurrent renders sharing this process. Notion's Cloudflare
+// block is triggered by request bursts, and the PDF route in particular fetches every
+// problem of a track in parallel, so we cap concurrency and space requests out.
+const MAX_CONCURRENCY = 2;
+const MIN_GAP_MS = 250;
+let active = 0;
+let lastStart = 0;
+const waiters: Array<() => void> = [];
+
+async function acquireSlot() {
+  if (active >= MAX_CONCURRENCY) {
+    await new Promise<void>((resolve) => waiters.push(resolve));
+  }
+  active++;
+  const wait = lastStart + MIN_GAP_MS - Date.now();
+  if (wait > 0) await sleep(wait);
+  lastStart = Date.now();
+}
+
+function releaseSlot() {
+  active--;
+  const next = waiters.shift();
+  if (next) next();
+}
+
+function isForbidden(err: any): boolean {
+  const status = err?.response?.statusCode ?? err?.response?.status;
+  return status === 403 || /403/.test(err?.message || "");
+}
+
+async function loadPageViaCachedChunk(notion: NotionAPI, pageId: string): Promise<any> {
+  // notion-client's typed `fetch` reuses auth (token_v2 cookie) and error handling.
+  const res: any = await notion.fetch({
+    endpoint: "loadCachedPageChunkV2",
+    body: { pageId, limit: 100, cursor: { stack: [] }, chunkNumber: 0, verticalColumns: false },
+    gotOptions: getGotOptions(),
+  });
+
+  const recordMap = res?.recordMap ?? {};
+  recordMap.block = normalizeBlocks(recordMap.block);
+  // react-notion-x expects these maps to exist even when empty.
+  recordMap.collection = recordMap.collection ?? {};
+  recordMap.collection_view = recordMap.collection_view ?? {};
+  recordMap.notion_user = recordMap.notion_user ?? {};
+  recordMap.collection_query = recordMap.collection_query ?? {};
+  recordMap.signed_urls = recordMap.signed_urls ?? {};
+
+  if (!recordMap.block || Object.keys(recordMap.block).length === 0) {
+    throw new Error(`Notion page not found "${pageId}"`);
+  }
+
+  const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]);
+  if (missing.length) {
+    // loadCachedPageChunkV2 returns the full page tree in practice; if a handful of nested
+    // blocks are missing we render what we have rather than hitting the (blocked)
+    // syncRecordValues endpoint.
+    console.warn(`[notion] ${pageId}: ${missing.length} nested block(s) missing from cached chunk`);
   }
 
   return recordMap;
 }
+
+async function fetchWithRetry(notion: NotionAPI, pageId: string): Promise<any> {
+  const attempts = 4;
+  let lastErr: unknown;
+  for (let i = 0; i < attempts; i++) {
+    await acquireSlot();
+    try {
+      return await loadPageViaCachedChunk(notion, pageId);
+    } catch (err) {
+      lastErr = err;
+      // Back off harder on Cloudflare 403s to let the IP-reputation block cool down.
+      if (i < attempts - 1) {
+        const base = isForbidden(err) ? 1500 : 300;
+        await sleep(base * 2 ** i);
+      }
+    } finally {
+      releaseSlot();
+    }
+  }
+  throw lastErr;
+}
+
+export async function fetchNotionPage(notion: NotionAPI, pageId: string): Promise<any> {
+  if (!pageId) return null;
+
+  const cached = cache.get(pageId);
+  if (cached && Date.now() - cached.ts < CACHE_TTL_MS) {
+    return cached.recordMap;
+  }
+
+  try {
+    const recordMap = await fetchWithRetry(notion, pageId);
+    cache.set(pageId, { recordMap, ts: Date.now() });
+    staleCache.set(pageId, recordMap);
+    return recordMap;
+  } catch (err) {
+    // Serve the last good version if we have one, so an upstream Notion/Cloudflare failure
+    // doesn't 500 the whole page.
+    const stale = staleCache.get(pageId);
+    if (stale) {
+      console.error(`[notion] fetch failed for ${pageId}, serving stale content:`, (err as Error)?.message);
+      return stale;
+    }
+    console.error(`[notion] fetch failed for ${pageId} with no cached fallback:`, (err as Error)?.message);
+    return null;
+  }
+}

apps/web/lib/search.ts

@@ -1,5 +1,5 @@
 "use server";
-import { NotionAPI } from "notion-client";
+import { fetchNotionPage, getNotionClient } from "./notion";
 import { getTrack } from "../components/utils";
 import { GoogleGenerativeAI } from "@google/generative-ai";
 import { QdrantClient } from "@qdrant/js-client-rest";
@@ -16,13 +16,13 @@ const model = genAI.getGenerativeModel({ model: "embedding-001"});
 const VECTOR_SIZE = parseInt(process.env.VECTOR_SIZE!) || 768;
 
 export async function scrapeData({ trackId }: { trackId: string }) {
-  const notion = new NotionAPI();
+  const notion = getNotionClient();
   const track = await getTrack(trackId);
   const data = await Promise.all(
     track?.problems.map(async (problem: any) => {
       const notionDocId = problem.notionDocId;
-      const notionPage = await notion.getPage(notionDocId);
-      const titles = Object.values(notionPage.block)
+      const notionPage = await fetchNotionPage(notion, notionDocId);
+      const titles = Object.values(notionPage?.block ?? {})
         .map((block) => {
           const title = block?.value?.properties?.title;
           if (title && title[0] && title[0][0]) {

apps/web/package.json

@@ -23,6 +23,7 @@
     "clsx": "^2.1.0",
     "date-fns": "^3.6.0",
     "framer-motion": "^11.3.30",
+    "https-proxy-agent": "^7.0.6",
     "next": "^14.0.4",
     "next-auth": "^4.24.7",
     "next-themes": "^0.2.1",

turbo.json

@@ -12,7 +12,10 @@
     "GOOGLEAI_URL",
     "VECTOR_SIZE",
     "NEXTAUTH_SECRET",
-    "CACHE_EXPIRE"
+    "CACHE_EXPIRE",
+    "NOTION_TOKEN_V2",
+    "NOTION_ACTIVE_USER",
+    "NOTION_PROXY_URL"
   ],
   "globalDependencies": ["**/.env.*local", "**/.env", ".env", ".env.local", "tsconfig.json"],
   "tasks": {