diff --git a/apps/web/.env.example b/apps/web/.env.example
index 703891dc..6b33d817 100644
--- a/apps/web/.env.example
+++ b/apps/web/.env.example
@@ -10,4 +10,14 @@ QDRANT_API_KEY=
QDRANT_URL=
VECTOR_DIMENSION=768 # Upto 768 dimensions are supported
-CACHE_EXPIRE=1800 # expiration time of the cache memory
\ No newline at end of file
+CACHE_EXPIRE=1800 # expiration time of the cache memory
+
+# Notion session cookies from a logged-in notion.so session. Required in production so
+# that Notion's Cloudflare protection doesn't 403-block server-side content fetches from
+# datacenter IPs. Leave blank for local dev.
+NOTION_TOKEN_V2=
+NOTION_ACTIVE_USER=
+
+# Optional: route all Notion API calls through an egress proxy on a clean (non-datacenter)
+# IP. Needed if Notion's Cloudflare keeps blocking the server IP. e.g. http://user:pass@host:port
+NOTION_PROXY_URL=
\ No newline at end of file
diff --git a/apps/web/app/api/AddTracks/route.ts b/apps/web/app/api/AddTracks/route.ts
index 26b0111d..eca12f71 100644
--- a/apps/web/app/api/AddTracks/route.ts
+++ b/apps/web/app/api/AddTracks/route.ts
@@ -1,6 +1,6 @@
import { getServerSession } from "next-auth";
import { NextRequest, NextResponse } from "next/server";
-import { NotionAPI } from "notion-client";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
import { authOptions } from "../../../lib/auth";
export async function POST(req: NextRequest) {
@@ -11,9 +11,12 @@ export async function POST(req: NextRequest) {
}
const body = await req.json();
const notionId = body.notionId;
- const notion = new NotionAPI();
+ const notion = getNotionClient();
try {
- const recordMap = await notion.getPage(notionId);
+ const recordMap = await fetchNotionPage(notion, notionId);
+ if (!recordMap?.block) {
+ return NextResponse.json({ message: "Failed to load Notion page" }, { status: 502 });
+ }
const data = Object.keys(recordMap.block).filter((key) => {
const block = recordMap.block[key];
return block?.role !== "none"
diff --git a/apps/web/app/pdf/[...pdfId]/page.tsx b/apps/web/app/pdf/[...pdfId]/page.tsx
index f723361e..aafc3245 100644
--- a/apps/web/app/pdf/[...pdfId]/page.tsx
+++ b/apps/web/app/pdf/[...pdfId]/page.tsx
@@ -1,14 +1,13 @@
import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved";
-import { NotionAPI } from "notion-client";
import { redirect } from "next/navigation";
import { Print } from "../../../components/Print";
import { getProblem, getTrack } from "../../../components/utils";
import { LessonView } from "../../../components/LessonView";
import { getServerSession } from "next-auth";
import { authOptions } from "../../../lib/auth";
-import { fetchNotionPage } from "../../../lib/notion";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
-const notion = new NotionAPI();
+const notion = getNotionClient();
export default async function TrackComponent({ params }: { params: { pdfId: string[] } }) {
const trackId: string = params.pdfId[0] || "";
diff --git a/apps/web/app/tracks/[...trackIds]/page.tsx b/apps/web/app/tracks/[...trackIds]/page.tsx
index 419720a9..d16bc1cd 100644
--- a/apps/web/app/tracks/[...trackIds]/page.tsx
+++ b/apps/web/app/tracks/[...trackIds]/page.tsx
@@ -1,12 +1,11 @@
import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved";
-import { NotionAPI } from "notion-client";
import { redirect, notFound } from "next/navigation";
import { getAllTracks, getProblem, getTrack } from "../../../components/utils";
import { cache } from "react";
import { LessonView } from "../../../components/LessonView";
-import { fetchNotionPage } from "../../../lib/notion";
+import { fetchNotionPage, getNotionClient } from "../../../lib/notion";
-const notion = new NotionAPI();
+const notion = getNotionClient();
export const dynamic = "auto";
// Dynamic Metadata
diff --git a/apps/web/components/NotionRenderer.tsx b/apps/web/components/NotionRenderer.tsx
index 5ec17197..c3015dba 100644
--- a/apps/web/components/NotionRenderer.tsx
+++ b/apps/web/components/NotionRenderer.tsx
@@ -23,6 +23,17 @@ export const NotionRenderer = ({ recordMap }: { recordMap: any }) => {
[]
);
+ if (!recordMap?.block || Object.keys(recordMap.block).length === 0) {
+ return (
+
+
This lesson is temporarily unavailable
+
+ We couldn't load the content right now. Please refresh in a little while.
+
+
+ );
+ }
+
return (
{
- let recordMap: any = await notion.getPage(pageId, { fetchMissingBlocks: false });
- recordMap = normalizeRecordMap(recordMap);
-
- for (let i = 0; i < 10; i++) {
- const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]);
- if (!missing.length) break;
- const fetched = await notion.getBlocks(missing).then((r: any) => r.recordMap.block);
- recordMap = normalizeRecordMap({ ...recordMap, block: { ...recordMap.block, ...fetched } });
+// --- Why this file looks the way it does -------------------------------------
+// Notion serves its unofficial `/api/v3` endpoints behind Cloudflare. Requests from our
+// k8s datacenter egress IP get hard-blocked with a 403 "Attention Required" Cloudflare
+// page on the endpoints notion-client normally uses (`loadPageChunk`, `syncRecordValues`,
+// `queryCollection`), which took down every track/problem page with a 500.
+//
+// Empirically, `loadCachedPageChunkV2` is NOT blocked from the cluster and returns the
+// full page recordMap in a single request, so we fetch through that endpoint directly
+// (via the client's public `fetch`) instead of `getPage`. We also authenticate with
+// NOTION_TOKEN_V2 (private-page access) and aggressively throttle + cache, because the
+// block is IP-reputation based: bursts of requests re-trigger a broader Cloudflare block,
+// so keeping request volume low is what keeps this endpoint working.
+let notionSingleton: NotionAPI | null = null;
+
+export function getNotionClient(): NotionAPI {
+ if (notionSingleton) return notionSingleton;
+ notionSingleton = new NotionAPI({
+ // eslint-disable-next-line turbo/no-undeclared-env-vars
+ authToken: process.env.NOTION_TOKEN_V2 || undefined,
+ // eslint-disable-next-line turbo/no-undeclared-env-vars
+ activeUser: process.env.NOTION_ACTIVE_USER || undefined,
+ });
+ return notionSingleton;
+}
+
+// Optional egress proxy. Because Notion's Cloudflare block is IP-reputation based, the
+// fully-robust fix is to route requests through a clean/non-datacenter IP. When
+// NOTION_PROXY_URL is set (e.g. http://user:pass@host:port), all Notion calls tunnel
+// through it — which also unblocks the endpoints loadCachedPageChunkV2 can't cover
+// (images via getSignedFileUrls, embedded DBs via queryCollection). Inert when unset.
+let gotOptionsCache: { agent: { https: HttpsProxyAgent } } | undefined | null = null;
+
+function getGotOptions() {
+ if (gotOptionsCache !== null) return gotOptionsCache;
+ // eslint-disable-next-line turbo/no-undeclared-env-vars
+ const proxy = process.env.NOTION_PROXY_URL;
+ gotOptionsCache = proxy ? { agent: { https: new HttpsProxyAgent(proxy) } } : undefined;
+ return gotOptionsCache;
+}
+
+// Two-tier cache: `fresh` entries are served within TTL; `stale` entries never expire and
+// are the fallback when Notion is unreachable/blocked, so a transient block degrades to
+// slightly-stale content instead of a 500. Long TTL keeps refetch volume (and IP-flag
+// risk) low; content changes rarely.
+const CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes
+type CacheEntry = { recordMap: any; ts: number };
+const cache = new Map();
+const staleCache = new Map();
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+// Global throttle across all concurrent renders sharing this process. Notion's Cloudflare
+// block is triggered by request bursts, and the PDF route in particular fetches every
+// problem of a track in parallel, so we cap concurrency and space requests out.
+const MAX_CONCURRENCY = 2;
+const MIN_GAP_MS = 250;
+let active = 0;
+let lastStart = 0;
+const waiters: Array<() => void> = [];
+
+async function acquireSlot() {
+ if (active >= MAX_CONCURRENCY) {
+ await new Promise((resolve) => waiters.push(resolve));
+ }
+ active++;
+ const wait = lastStart + MIN_GAP_MS - Date.now();
+ if (wait > 0) await sleep(wait);
+ lastStart = Date.now();
+}
+
+function releaseSlot() {
+ active--;
+ const next = waiters.shift();
+ if (next) next();
+}
+
+function isForbidden(err: any): boolean {
+ const status = err?.response?.statusCode ?? err?.response?.status;
+ return status === 403 || /403/.test(err?.message || "");
+}
+
+async function loadPageViaCachedChunk(notion: NotionAPI, pageId: string): Promise {
+ // notion-client's typed `fetch` reuses auth (token_v2 cookie) and error handling.
+ const res: any = await notion.fetch({
+ endpoint: "loadCachedPageChunkV2",
+ body: { pageId, limit: 100, cursor: { stack: [] }, chunkNumber: 0, verticalColumns: false },
+ gotOptions: getGotOptions(),
+ });
+
+ const recordMap = res?.recordMap ?? {};
+ recordMap.block = normalizeBlocks(recordMap.block);
+ // react-notion-x expects these maps to exist even when empty.
+ recordMap.collection = recordMap.collection ?? {};
+ recordMap.collection_view = recordMap.collection_view ?? {};
+ recordMap.notion_user = recordMap.notion_user ?? {};
+ recordMap.collection_query = recordMap.collection_query ?? {};
+ recordMap.signed_urls = recordMap.signed_urls ?? {};
+
+ if (!recordMap.block || Object.keys(recordMap.block).length === 0) {
+ throw new Error(`Notion page not found "${pageId}"`);
+ }
+
+ const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]);
+ if (missing.length) {
+ // loadCachedPageChunkV2 returns the full page tree in practice; if a handful of nested
+ // blocks are missing we render what we have rather than hitting the (blocked)
+ // syncRecordValues endpoint.
+ console.warn(`[notion] ${pageId}: ${missing.length} nested block(s) missing from cached chunk`);
}
return recordMap;
}
+
+async function fetchWithRetry(notion: NotionAPI, pageId: string): Promise {
+ const attempts = 4;
+ let lastErr: unknown;
+ for (let i = 0; i < attempts; i++) {
+ await acquireSlot();
+ try {
+ return await loadPageViaCachedChunk(notion, pageId);
+ } catch (err) {
+ lastErr = err;
+ // Back off harder on Cloudflare 403s to let the IP-reputation block cool down.
+ if (i < attempts - 1) {
+ const base = isForbidden(err) ? 1500 : 300;
+ await sleep(base * 2 ** i);
+ }
+ } finally {
+ releaseSlot();
+ }
+ }
+ throw lastErr;
+}
+
+export async function fetchNotionPage(notion: NotionAPI, pageId: string): Promise {
+ if (!pageId) return null;
+
+ const cached = cache.get(pageId);
+ if (cached && Date.now() - cached.ts < CACHE_TTL_MS) {
+ return cached.recordMap;
+ }
+
+ try {
+ const recordMap = await fetchWithRetry(notion, pageId);
+ cache.set(pageId, { recordMap, ts: Date.now() });
+ staleCache.set(pageId, recordMap);
+ return recordMap;
+ } catch (err) {
+ // Serve the last good version if we have one, so an upstream Notion/Cloudflare failure
+ // doesn't 500 the whole page.
+ const stale = staleCache.get(pageId);
+ if (stale) {
+ console.error(`[notion] fetch failed for ${pageId}, serving stale content:`, (err as Error)?.message);
+ return stale;
+ }
+ console.error(`[notion] fetch failed for ${pageId} with no cached fallback:`, (err as Error)?.message);
+ return null;
+ }
+}
diff --git a/apps/web/lib/search.ts b/apps/web/lib/search.ts
index c100c230..0a0bacc9 100644
--- a/apps/web/lib/search.ts
+++ b/apps/web/lib/search.ts
@@ -1,5 +1,5 @@
"use server";
-import { NotionAPI } from "notion-client";
+import { fetchNotionPage, getNotionClient } from "./notion";
import { getTrack } from "../components/utils";
import { GoogleGenerativeAI } from "@google/generative-ai";
import { QdrantClient } from "@qdrant/js-client-rest";
@@ -16,13 +16,13 @@ const model = genAI.getGenerativeModel({ model: "embedding-001"});
const VECTOR_SIZE = parseInt(process.env.VECTOR_SIZE!) || 768;
export async function scrapeData({ trackId }: { trackId: string }) {
- const notion = new NotionAPI();
+ const notion = getNotionClient();
const track = await getTrack(trackId);
const data = await Promise.all(
track?.problems.map(async (problem: any) => {
const notionDocId = problem.notionDocId;
- const notionPage = await notion.getPage(notionDocId);
- const titles = Object.values(notionPage.block)
+ const notionPage = await fetchNotionPage(notion, notionDocId);
+ const titles = Object.values(notionPage?.block ?? {})
.map((block) => {
const title = block?.value?.properties?.title;
if (title && title[0] && title[0][0]) {
diff --git a/apps/web/package.json b/apps/web/package.json
index e4ec22a3..fc7e6d96 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -23,6 +23,7 @@
"clsx": "^2.1.0",
"date-fns": "^3.6.0",
"framer-motion": "^11.3.30",
+ "https-proxy-agent": "^7.0.6",
"next": "^14.0.4",
"next-auth": "^4.24.7",
"next-themes": "^0.2.1",
diff --git a/turbo.json b/turbo.json
index 7dabf435..57404986 100644
--- a/turbo.json
+++ b/turbo.json
@@ -12,7 +12,10 @@
"GOOGLEAI_URL",
"VECTOR_SIZE",
"NEXTAUTH_SECRET",
- "CACHE_EXPIRE"
+ "CACHE_EXPIRE",
+ "NOTION_TOKEN_V2",
+ "NOTION_ACTIVE_USER",
+ "NOTION_PROXY_URL"
],
"globalDependencies": ["**/.env.*local", "**/.env", ".env", ".env.local", "tsconfig.json"],
"tasks": {