diff --git a/apps/web/.env.example b/apps/web/.env.example index 703891dc..6b33d817 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -10,4 +10,14 @@ QDRANT_API_KEY= QDRANT_URL= VECTOR_DIMENSION=768 # Upto 768 dimensions are supported -CACHE_EXPIRE=1800 # expiration time of the cache memory \ No newline at end of file +CACHE_EXPIRE=1800 # expiration time of the cache memory + +# Notion session cookies from a logged-in notion.so session. Required in production so +# that Notion's Cloudflare protection doesn't 403-block server-side content fetches from +# datacenter IPs. Leave blank for local dev. +NOTION_TOKEN_V2= +NOTION_ACTIVE_USER= + +# Optional: route all Notion API calls through an egress proxy on a clean (non-datacenter) +# IP. Needed if Notion's Cloudflare keeps blocking the server IP. e.g. http://user:pass@host:port +NOTION_PROXY_URL= \ No newline at end of file diff --git a/apps/web/app/api/AddTracks/route.ts b/apps/web/app/api/AddTracks/route.ts index 26b0111d..eca12f71 100644 --- a/apps/web/app/api/AddTracks/route.ts +++ b/apps/web/app/api/AddTracks/route.ts @@ -1,6 +1,6 @@ import { getServerSession } from "next-auth"; import { NextRequest, NextResponse } from "next/server"; -import { NotionAPI } from "notion-client"; +import { fetchNotionPage, getNotionClient } from "../../../lib/notion"; import { authOptions } from "../../../lib/auth"; export async function POST(req: NextRequest) { @@ -11,9 +11,12 @@ export async function POST(req: NextRequest) { } const body = await req.json(); const notionId = body.notionId; - const notion = new NotionAPI(); + const notion = getNotionClient(); try { - const recordMap = await notion.getPage(notionId); + const recordMap = await fetchNotionPage(notion, notionId); + if (!recordMap?.block) { + return NextResponse.json({ message: "Failed to load Notion page" }, { status: 502 }); + } const data = Object.keys(recordMap.block).filter((key) => { const block = recordMap.block[key]; return block?.role !== "none" diff --git a/apps/web/app/pdf/[...pdfId]/page.tsx b/apps/web/app/pdf/[...pdfId]/page.tsx index f723361e..aafc3245 100644 --- a/apps/web/app/pdf/[...pdfId]/page.tsx +++ b/apps/web/app/pdf/[...pdfId]/page.tsx @@ -1,14 +1,13 @@ import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved"; -import { NotionAPI } from "notion-client"; import { redirect } from "next/navigation"; import { Print } from "../../../components/Print"; import { getProblem, getTrack } from "../../../components/utils"; import { LessonView } from "../../../components/LessonView"; import { getServerSession } from "next-auth"; import { authOptions } from "../../../lib/auth"; -import { fetchNotionPage } from "../../../lib/notion"; +import { fetchNotionPage, getNotionClient } from "../../../lib/notion"; -const notion = new NotionAPI(); +const notion = getNotionClient(); export default async function TrackComponent({ params }: { params: { pdfId: string[] } }) { const trackId: string = params.pdfId[0] || ""; diff --git a/apps/web/app/tracks/[...trackIds]/page.tsx b/apps/web/app/tracks/[...trackIds]/page.tsx index 419720a9..d16bc1cd 100644 --- a/apps/web/app/tracks/[...trackIds]/page.tsx +++ b/apps/web/app/tracks/[...trackIds]/page.tsx @@ -1,12 +1,11 @@ import { RedirectToLastSolved } from "../../../components/RedirectToLastSolved"; -import { NotionAPI } from "notion-client"; import { redirect, notFound } from "next/navigation"; import { getAllTracks, getProblem, getTrack } from "../../../components/utils"; import { cache } from "react"; import { LessonView } from "../../../components/LessonView"; -import { fetchNotionPage } from "../../../lib/notion"; +import { fetchNotionPage, getNotionClient } from "../../../lib/notion"; -const notion = new NotionAPI(); +const notion = getNotionClient(); export const dynamic = "auto"; // Dynamic Metadata diff --git a/apps/web/components/NotionRenderer.tsx b/apps/web/components/NotionRenderer.tsx index 5ec17197..c3015dba 100644 --- a/apps/web/components/NotionRenderer.tsx +++ b/apps/web/components/NotionRenderer.tsx @@ -23,6 +23,17 @@ export const NotionRenderer = ({ recordMap }: { recordMap: any }) => { [] ); + if (!recordMap?.block || Object.keys(recordMap.block).length === 0) { + return ( +
+

This lesson is temporarily unavailable

+

+ We couldn't load the content right now. Please refresh in a little while. +

+
+ ); + } + return ( { - let recordMap: any = await notion.getPage(pageId, { fetchMissingBlocks: false }); - recordMap = normalizeRecordMap(recordMap); - - for (let i = 0; i < 10; i++) { - const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]); - if (!missing.length) break; - const fetched = await notion.getBlocks(missing).then((r: any) => r.recordMap.block); - recordMap = normalizeRecordMap({ ...recordMap, block: { ...recordMap.block, ...fetched } }); +// --- Why this file looks the way it does ------------------------------------- +// Notion serves its unofficial `/api/v3` endpoints behind Cloudflare. Requests from our +// k8s datacenter egress IP get hard-blocked with a 403 "Attention Required" Cloudflare +// page on the endpoints notion-client normally uses (`loadPageChunk`, `syncRecordValues`, +// `queryCollection`), which took down every track/problem page with a 500. +// +// Empirically, `loadCachedPageChunkV2` is NOT blocked from the cluster and returns the +// full page recordMap in a single request, so we fetch through that endpoint directly +// (via the client's public `fetch`) instead of `getPage`. We also authenticate with +// NOTION_TOKEN_V2 (private-page access) and aggressively throttle + cache, because the +// block is IP-reputation based: bursts of requests re-trigger a broader Cloudflare block, +// so keeping request volume low is what keeps this endpoint working. +let notionSingleton: NotionAPI | null = null; + +export function getNotionClient(): NotionAPI { + if (notionSingleton) return notionSingleton; + notionSingleton = new NotionAPI({ + // eslint-disable-next-line turbo/no-undeclared-env-vars + authToken: process.env.NOTION_TOKEN_V2 || undefined, + // eslint-disable-next-line turbo/no-undeclared-env-vars + activeUser: process.env.NOTION_ACTIVE_USER || undefined, + }); + return notionSingleton; +} + +// Optional egress proxy. Because Notion's Cloudflare block is IP-reputation based, the +// fully-robust fix is to route requests through a clean/non-datacenter IP. When +// NOTION_PROXY_URL is set (e.g. http://user:pass@host:port), all Notion calls tunnel +// through it — which also unblocks the endpoints loadCachedPageChunkV2 can't cover +// (images via getSignedFileUrls, embedded DBs via queryCollection). Inert when unset. +let gotOptionsCache: { agent: { https: HttpsProxyAgent } } | undefined | null = null; + +function getGotOptions() { + if (gotOptionsCache !== null) return gotOptionsCache; + // eslint-disable-next-line turbo/no-undeclared-env-vars + const proxy = process.env.NOTION_PROXY_URL; + gotOptionsCache = proxy ? { agent: { https: new HttpsProxyAgent(proxy) } } : undefined; + return gotOptionsCache; +} + +// Two-tier cache: `fresh` entries are served within TTL; `stale` entries never expire and +// are the fallback when Notion is unreachable/blocked, so a transient block degrades to +// slightly-stale content instead of a 500. Long TTL keeps refetch volume (and IP-flag +// risk) low; content changes rarely. +const CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes +type CacheEntry = { recordMap: any; ts: number }; +const cache = new Map(); +const staleCache = new Map(); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +// Global throttle across all concurrent renders sharing this process. Notion's Cloudflare +// block is triggered by request bursts, and the PDF route in particular fetches every +// problem of a track in parallel, so we cap concurrency and space requests out. +const MAX_CONCURRENCY = 2; +const MIN_GAP_MS = 250; +let active = 0; +let lastStart = 0; +const waiters: Array<() => void> = []; + +async function acquireSlot() { + if (active >= MAX_CONCURRENCY) { + await new Promise((resolve) => waiters.push(resolve)); + } + active++; + const wait = lastStart + MIN_GAP_MS - Date.now(); + if (wait > 0) await sleep(wait); + lastStart = Date.now(); +} + +function releaseSlot() { + active--; + const next = waiters.shift(); + if (next) next(); +} + +function isForbidden(err: any): boolean { + const status = err?.response?.statusCode ?? err?.response?.status; + return status === 403 || /403/.test(err?.message || ""); +} + +async function loadPageViaCachedChunk(notion: NotionAPI, pageId: string): Promise { + // notion-client's typed `fetch` reuses auth (token_v2 cookie) and error handling. + const res: any = await notion.fetch({ + endpoint: "loadCachedPageChunkV2", + body: { pageId, limit: 100, cursor: { stack: [] }, chunkNumber: 0, verticalColumns: false }, + gotOptions: getGotOptions(), + }); + + const recordMap = res?.recordMap ?? {}; + recordMap.block = normalizeBlocks(recordMap.block); + // react-notion-x expects these maps to exist even when empty. + recordMap.collection = recordMap.collection ?? {}; + recordMap.collection_view = recordMap.collection_view ?? {}; + recordMap.notion_user = recordMap.notion_user ?? {}; + recordMap.collection_query = recordMap.collection_query ?? {}; + recordMap.signed_urls = recordMap.signed_urls ?? {}; + + if (!recordMap.block || Object.keys(recordMap.block).length === 0) { + throw new Error(`Notion page not found "${pageId}"`); + } + + const missing = collectContentBlockIds(recordMap).filter((id) => !recordMap.block[id]); + if (missing.length) { + // loadCachedPageChunkV2 returns the full page tree in practice; if a handful of nested + // blocks are missing we render what we have rather than hitting the (blocked) + // syncRecordValues endpoint. + console.warn(`[notion] ${pageId}: ${missing.length} nested block(s) missing from cached chunk`); } return recordMap; } + +async function fetchWithRetry(notion: NotionAPI, pageId: string): Promise { + const attempts = 4; + let lastErr: unknown; + for (let i = 0; i < attempts; i++) { + await acquireSlot(); + try { + return await loadPageViaCachedChunk(notion, pageId); + } catch (err) { + lastErr = err; + // Back off harder on Cloudflare 403s to let the IP-reputation block cool down. + if (i < attempts - 1) { + const base = isForbidden(err) ? 1500 : 300; + await sleep(base * 2 ** i); + } + } finally { + releaseSlot(); + } + } + throw lastErr; +} + +export async function fetchNotionPage(notion: NotionAPI, pageId: string): Promise { + if (!pageId) return null; + + const cached = cache.get(pageId); + if (cached && Date.now() - cached.ts < CACHE_TTL_MS) { + return cached.recordMap; + } + + try { + const recordMap = await fetchWithRetry(notion, pageId); + cache.set(pageId, { recordMap, ts: Date.now() }); + staleCache.set(pageId, recordMap); + return recordMap; + } catch (err) { + // Serve the last good version if we have one, so an upstream Notion/Cloudflare failure + // doesn't 500 the whole page. + const stale = staleCache.get(pageId); + if (stale) { + console.error(`[notion] fetch failed for ${pageId}, serving stale content:`, (err as Error)?.message); + return stale; + } + console.error(`[notion] fetch failed for ${pageId} with no cached fallback:`, (err as Error)?.message); + return null; + } +} diff --git a/apps/web/lib/search.ts b/apps/web/lib/search.ts index c100c230..0a0bacc9 100644 --- a/apps/web/lib/search.ts +++ b/apps/web/lib/search.ts @@ -1,5 +1,5 @@ "use server"; -import { NotionAPI } from "notion-client"; +import { fetchNotionPage, getNotionClient } from "./notion"; import { getTrack } from "../components/utils"; import { GoogleGenerativeAI } from "@google/generative-ai"; import { QdrantClient } from "@qdrant/js-client-rest"; @@ -16,13 +16,13 @@ const model = genAI.getGenerativeModel({ model: "embedding-001"}); const VECTOR_SIZE = parseInt(process.env.VECTOR_SIZE!) || 768; export async function scrapeData({ trackId }: { trackId: string }) { - const notion = new NotionAPI(); + const notion = getNotionClient(); const track = await getTrack(trackId); const data = await Promise.all( track?.problems.map(async (problem: any) => { const notionDocId = problem.notionDocId; - const notionPage = await notion.getPage(notionDocId); - const titles = Object.values(notionPage.block) + const notionPage = await fetchNotionPage(notion, notionDocId); + const titles = Object.values(notionPage?.block ?? {}) .map((block) => { const title = block?.value?.properties?.title; if (title && title[0] && title[0][0]) { diff --git a/apps/web/package.json b/apps/web/package.json index e4ec22a3..fc7e6d96 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -23,6 +23,7 @@ "clsx": "^2.1.0", "date-fns": "^3.6.0", "framer-motion": "^11.3.30", + "https-proxy-agent": "^7.0.6", "next": "^14.0.4", "next-auth": "^4.24.7", "next-themes": "^0.2.1", diff --git a/turbo.json b/turbo.json index 7dabf435..57404986 100644 --- a/turbo.json +++ b/turbo.json @@ -12,7 +12,10 @@ "GOOGLEAI_URL", "VECTOR_SIZE", "NEXTAUTH_SECRET", - "CACHE_EXPIRE" + "CACHE_EXPIRE", + "NOTION_TOKEN_V2", + "NOTION_ACTIVE_USER", + "NOTION_PROXY_URL" ], "globalDependencies": ["**/.env.*local", "**/.env", ".env", ".env.local", "tsconfig.json"], "tasks": {