From 403221360307a83fd824599800a83e536620ae2e Mon Sep 17 00:00:00 2001 From: mountain Date: Tue, 21 Jul 2026 20:34:44 +0800 Subject: [PATCH 1/2] feat(web): render extracted images inline in the document reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #197: long/short source docs embed `![image](sources/images/...)` references whose files were extracted at ingest but never displayed (they showed as a literal "!image"). Now they render inline in the reader. - Backend: GET /api/v1/document/image serves wiki/sources/images/** — narrowed to the images dir + raster suffixes with a traversal guard (read-only; cannot reach wiki pages/skills/arbitrary files). - MarkdownView: gain an optional image token, ENABLED ONLY when a resolveImageSrc prop is passed (the doc reader). Without it the `![](…)` token is omitted from the regex, so chat/wiki rendering is byte-for-byte unchanged. A new AuthedImage fetches the API path as a blob (the bearer token can't ride on ; mirrors artifacts.ts) and revokes the object URL on unmount. - Doc reader: passes a resolver that normalizes both source path conventions (wiki-root `sources/images/...` and note-relative `images/...`) to the image endpoint; external/data URLs stay literal text. Images are now visible in the UI. (LLM answers are still image-blind — ingestion does no caption/OCR — which is a separate pipeline change.) Backend adds serve/traversal/suffix/404/auth tests (1251 passed). Verified in a browser: the Soochow Securities report's figures render inline. --- frontend/src/components/MarkdownView.tsx | 103 +++++++++++++++++++---- frontend/src/pages/KbDetail.tsx | 20 ++++- openkb/api_documents_router.py | 31 ++++++- tests/test_api_documents.py | 74 ++++++++++++++++ 4 files changed, 209 insertions(+), 19 deletions(-) diff --git a/frontend/src/components/MarkdownView.tsx b/frontend/src/components/MarkdownView.tsx index b323fe3f..3d44486e 100644 --- a/frontend/src/components/MarkdownView.tsx +++ b/frontend/src/components/MarkdownView.tsx @@ -1,6 +1,7 @@ import React, { useEffect, useId, useRef, useState } from 'react' import katex from 'katex' import 'katex/dist/katex.min.css' +import { fetchAsBlobUrl } from '@/api/client' import { useTheme } from '@/lib/theme' /** Guard for [text](url) links: only render a real anchor for http(s) or @@ -31,11 +32,19 @@ const isSafeUrl = (u: string) => { * wikilink target is NOT recursed). Single `*`/`_` obey a minimal left/right * flanking rule so intraword runs (`a_b_c`, `2*3*4`) stay literal. */ -function inline(text: string, onWikiLink?: (target: string) => void): React.ReactNode[] { +function inline( + text: string, + onWikiLink?: (target: string) => void, + resolveImageSrc?: (rawSrc: string) => string | null, +): React.ReactNode[] { const parts: React.ReactNode[] = [] // Bold is non-greedy (`.+?`) so it tolerates an inner opposite-emphasis - // (`**a *b* c**`); italic keeps a bounded `[^*]+`/`[^_]+` class. - const re = /(\[\[[^\]]+\]\]|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g + // (`**a *b* c**`); italic keeps a bounded `[^*]+`/`[^_]+` class. The image + // alternative `![alt](url)` is included ONLY when a resolveImageSrc is given + // (the document reader), so chat/wiki rendering stays byte-for-byte unchanged. + const re = resolveImageSrc + ? /(\[\[[^\]]+\]\]|!\[[^\]]*\]\([^)]+\)|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g + : /(\[\[[^\]]+\]\]|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g let last = 0, m: RegExpExecArray | null, k = 0 while ((m = re.exec(text))) { if (m.index > last) parts.push(text.slice(last, m.index)) @@ -85,11 +94,28 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac ) } else if (tok.startsWith('**')) { // Recurse so nested markup (e.g. `**[docs](url)**`) resolves. slice is strictly shorter. - parts.push({inline(tok.slice(2, -2), onWikiLink)}) + parts.push({inline(tok.slice(2, -2), onWikiLink, resolveImageSrc)}) } else if (tok.startsWith('~~')) { - parts.push({inline(tok.slice(2, -2), onWikiLink)}) + parts.push({inline(tok.slice(2, -2), onWikiLink, resolveImageSrc)}) } else if (tok.startsWith('`')) { parts.push({tok.slice(1, -1)}) + } else if (tok.startsWith('![')) { + // Markdown image — only reached when resolveImageSrc is provided (the + // regex omits this token otherwise). resolveImageSrc maps a KB-relative + // source path to an authed API path, or null for external/data URLs + // (left as literal text). Cannot collide with the link/wikilink branches: + // those match tokens starting with '[', this one starts with '!['. + const im = /^!\[([^\]]*)\]\(([^)]+)\)$/.exec(tok) + const alt = im ? im[1] : '' + const raw = im ? im[2].trim() : '' + const apiPath = raw ? (resolveImageSrc?.(raw) ?? null) : null + parts.push( + apiPath ? ( + + ) : ( + {tok} + ), + ) } else if (tok.startsWith('[')) { // Inline link [text](url). `[[…]]` was already consumed above, so any // `[` reaching here is a genuine link. Only emit an anchor when the URL @@ -109,7 +135,7 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac rel="noopener noreferrer" className="text-accent-brand hover:underline" > - {inline(label, onWikiLink)} + {inline(label, onWikiLink, resolveImageSrc)} ) : ( {tok} @@ -135,7 +161,7 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac re.lastIndex = m.index + 1 continue } - parts.push({inline(innerEm, onWikiLink)}) + parts.push({inline(innerEm, onWikiLink, resolveImageSrc)}) } last = m.index + tok.length } @@ -216,14 +242,59 @@ function MermaidBlock({ code }: { code: string }) { ) } +/** Render a bearer-authed KB image. The API token can't ride on ``, + * so we fetch the API path as a blob (mirrors `artifacts.ts`), show it, and + * revoke the object URL on unmount / src change. On failure it renders nothing + * rather than a broken-image glyph; while loading it shows a muted placeholder. */ +function AuthedImage({ apiPath, alt }: { apiPath: string; alt: string }) { + const [url, setUrl] = useState(null) + const [failed, setFailed] = useState(false) + useEffect(() => { + let cancelled = false + let objectUrl: string | null = null + setUrl(null) + setFailed(false) + fetchAsBlobUrl(apiPath) + .then((u) => { + if (cancelled) { + URL.revokeObjectURL(u) + return + } + objectUrl = u + setUrl(u) + }) + .catch(() => { + if (!cancelled) setFailed(true) + }) + return () => { + cancelled = true + if (objectUrl) URL.revokeObjectURL(objectUrl) + } + }, [apiPath]) + if (failed) return null + if (!url) return + return ( + {alt} + ) +} + export default function MarkdownView({ source, onWikiLink, + resolveImageSrc, }: { source: string /** Navigate to a `[[target]]` wikilink's page. Omit to render plain, * non-interactive tokens (no `cursor-pointer` implying a dead click). */ onWikiLink?: (target: string) => void + /** Map a Markdown image's raw src to an authed API path (or null to leave it + * as text). Only the document reader passes this; without it, `![](…)` is + * not tokenized as an image, so chat/wiki rendering is unchanged. */ + resolveImageSrc?: (rawSrc: string) => string | null }) { const lines = source.split('\n') const out: React.ReactNode[] = [] @@ -240,7 +311,7 @@ export default function MarkdownView({ {list.map((li, i) => (
  • - {inline(li, onWikiLink)} + {inline(li, onWikiLink, resolveImageSrc)}
    • ))} , @@ -254,7 +325,7 @@ export default function MarkdownView({
      {olist.map((li, i) => (
    1. - {inline(li, onWikiLink)} + {inline(li, onWikiLink, resolveImageSrc)}
      2. ))}
    , @@ -382,7 +453,7 @@ export default function MarkdownView({ key={c} className={`border border-[hsl(var(--glass-border))] bg-muted/50 px-3 py-1.5 font-semibold text-foreground ${alignOf(c)}`} > - {inline(h, onWikiLink)} + {inline(h, onWikiLink, resolveImageSrc)} ))} @@ -395,7 +466,7 @@ export default function MarkdownView({ key={c} className={`border border-[hsl(var(--glass-border))] px-3 py-1.5 text-muted-foreground ${alignOf(c)}`} > - {inline(r[c] ?? '', onWikiLink)} + {inline(r[c] ?? '', onWikiLink, resolveImageSrc)} ))} @@ -422,11 +493,11 @@ export default function MarkdownView({ } flushBlocks() if (!line.trim()) { out.push(
    ); continue } - if (line.startsWith('### ')) out.push(

    {inline(line.slice(4), onWikiLink)}

    ) - else if (line.startsWith('## ')) out.push(

    {inline(line.slice(3), onWikiLink)}

    ) - else if (line.startsWith('# ')) out.push(

    {inline(line.slice(2), onWikiLink)}

    ) - else if (line.startsWith('> ')) out.push(
    {inline(line.slice(2), onWikiLink)}
    ) - else out.push(

    {inline(line, onWikiLink)}

    ) + if (line.startsWith('### ')) out.push(

    {inline(line.slice(4), onWikiLink, resolveImageSrc)}

    ) + else if (line.startsWith('## ')) out.push(

    {inline(line.slice(3), onWikiLink, resolveImageSrc)}

    ) + else if (line.startsWith('# ')) out.push(

    {inline(line.slice(2), onWikiLink, resolveImageSrc)}

    ) + else if (line.startsWith('> ')) out.push(
    {inline(line.slice(2), onWikiLink, resolveImageSrc)}
    ) + else out.push(

    {inline(line, onWikiLink, resolveImageSrc)}

    ) } flushBlocks() return
    {out}
    diff --git a/frontend/src/pages/KbDetail.tsx b/frontend/src/pages/KbDetail.tsx index c4c5935e..4c667a88 100644 --- a/frontend/src/pages/KbDetail.tsx +++ b/frontend/src/pages/KbDetail.tsx @@ -1081,11 +1081,27 @@ function DocumentsPane({ } }, [kb, openHash, docReloadSeq]) + // Map a source image ref to the authed image endpoint. Long-doc JSON stores + // wiki-root-relative `sources/images/...`; short-doc MD uses note-relative + // `images/...` — normalize both to a wiki-relative path. Non-matching refs + // (external / data URLs) return null and render as plain text. + const resolveDocImageSrc = useCallback( + (rawSrc: string): string | null => { + let rel: string | null = null + if (rawSrc.startsWith('sources/images/')) rel = rawSrc + else if (rawSrc.startsWith('images/')) rel = `sources/${rawSrc}` + if (!rel) return null + return `/api/v1/document/image?kb=${encodeURIComponent(kb)}&path=${encodeURIComponent(rel)}` + }, + [kb], + ) // Parse Markdown once per fetched source (stable cache ref → no re-parse). const readerBody = useMemo( () => - docSource && docSource.content.trim() ? : null, - [docSource], + docSource && docSource.content.trim() ? ( + + ) : null, + [docSource, resolveDocImageSrc], ) const readerEmpty = docSource != null && docSource.content.trim().length === 0 diff --git a/openkb/api_documents_router.py b/openkb/api_documents_router.py index d8c84cd1..77d0a852 100644 --- a/openkb/api_documents_router.py +++ b/openkb/api_documents_router.py @@ -8,7 +8,8 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import FileResponse from starlette.concurrency import run_in_threadpool from openkb.api_helpers import _resolve_kb, require_bearer_token @@ -17,6 +18,9 @@ documents_router = APIRouter() +# Raster types the image extractor produces; SVG is excluded (inline-script risk). +_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".webp") + @documents_router.post("/api/v1/document/source", response_model=DocumentSourceResponse) async def document_source_endpoint( @@ -33,3 +37,28 @@ async def document_source_endpoint( if result is None: raise HTTPException(status_code=404, detail="Document source not found.") return DocumentSourceResponse(**result) + + +@documents_router.get("/api/v1/document/image") +async def document_image_endpoint( + kb: str = Query(...), + path: str = Query(..., min_length=1), + _: None = Depends(require_bearer_token), +) -> FileResponse: + """Serve an extracted document image (``wiki/sources/images/**``). + + ``path`` is the image reference from the source text, resolved relative to + the KB's ``wiki/`` dir (source text stores ``sources/images//...``). + Narrowed to the images dir + raster suffixes with a traversal guard, so this + read-only sink can never serve wiki pages, skills, or arbitrary files. + """ + kb_dir = _resolve_kb(kb) + images_root = (kb_dir / "wiki" / "sources" / "images").resolve() + full = (kb_dir / "wiki" / path).resolve() + if not full.is_relative_to(images_root): + raise HTTPException(status_code=400, detail="Invalid image path.") + if full.suffix.lower() not in _IMAGE_SUFFIXES: + raise HTTPException(status_code=400, detail="Only extracted images are served.") + if not full.is_file(): + raise HTTPException(status_code=404, detail="Image not found.") + return FileResponse(full) diff --git a/tests/test_api_documents.py b/tests/test_api_documents.py index 861a6edf..0a13c1b8 100644 --- a/tests/test_api_documents.py +++ b/tests/test_api_documents.py @@ -186,3 +186,77 @@ def test_document_source_requires_auth(monkeypatch, kb_dir): resp = client.post("/api/v1/document/source", json={"kb": "test-kb", "hash": "h1"}) assert resp.status_code == 401 + + +def test_document_image_serves_extracted_image(monkeypatch, kb_dir): + client = _client(monkeypatch) + kb = _use_named_kb(monkeypatch, kb_dir) + img_dir = kb_dir / "wiki" / "sources" / "images" / "doc" + img_dir.mkdir(parents=True) + (img_dir / "p1_img1.png").write_bytes(b"\x89PNG\r\n\x1a\nfake-bytes") + + resp = client.get( + "/api/v1/document/image", + params={"kb": kb, "path": "sources/images/doc/p1_img1.png"}, + headers=_auth(), + ) + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("image/") + assert resp.content == b"\x89PNG\r\n\x1a\nfake-bytes" + + +def test_document_image_rejects_traversal(monkeypatch, kb_dir): + """A path escaping wiki/sources/images (even to another .png) is rejected by + the containment guard before the suffix check.""" + client = _client(monkeypatch) + kb = _use_named_kb(monkeypatch, kb_dir) + (kb_dir / "wiki" / "sources" / "evil.png").write_bytes(b"x") # inside sources/, outside images/ + + resp = client.get( + "/api/v1/document/image", + params={"kb": kb, "path": "sources/images/../evil.png"}, + headers=_auth(), + ) + + assert resp.status_code == 400 + + +def test_document_image_rejects_non_image_suffix(monkeypatch, kb_dir): + client = _client(monkeypatch) + kb = _use_named_kb(monkeypatch, kb_dir) + d = kb_dir / "wiki" / "sources" / "images" / "doc" + d.mkdir(parents=True) + (d / "notes.md").write_text("secret", encoding="utf-8") + + resp = client.get( + "/api/v1/document/image", + params={"kb": kb, "path": "sources/images/doc/notes.md"}, + headers=_auth(), + ) + + assert resp.status_code == 400 + + +def test_document_image_missing_404(monkeypatch, kb_dir): + client = _client(monkeypatch) + kb = _use_named_kb(monkeypatch, kb_dir) + + resp = client.get( + "/api/v1/document/image", + params={"kb": kb, "path": "sources/images/doc/nope.png"}, + headers=_auth(), + ) + + assert resp.status_code == 404 + + +def test_document_image_requires_auth(monkeypatch, kb_dir): + client = _client(monkeypatch) + _use_named_kb(monkeypatch, kb_dir) + + resp = client.get( + "/api/v1/document/image", params={"kb": "test-kb", "path": "sources/images/doc/p.png"} + ) + + assert resp.status_code == 401 From 849b999cbb15d56f1f1fd48ceac9b2acf9046fe0 Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 22 Jul 2026 11:13:51 +0800 Subject: [PATCH 2/2] fix(web): address code-review findings on inline image rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Image URL regex now tolerates one level of balanced parens, so an image whose path contains ')' (a legacy/cloud doc_name with ASCII parentheses, e.g. 'report (1)') is no longer truncated to a broken path + stray text [#1]. - The image endpoint sets an explicit media type per suffix instead of letting FileResponse guess: a .webp is served as image/webp even where mimetypes has no webp entry (would otherwise be text/plain → a blob-loaded refuses to render it) [#2]. Adds a webp media-type test. - AuthedImage shows a muted dashed placeholder (the alt) on load failure instead of rendering nothing, so a missing image is visible rather than silently dropped [#3]. Verified: the balanced-paren regex extracts full paths for 'report (1)' and fullwidth-paren names; backend 1252 passed; frontend build green. --- frontend/src/components/MarkdownView.tsx | 18 +++++++++++++----- openkb/api_documents_router.py | 19 +++++++++++++++---- tests/test_api_documents.py | 20 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/MarkdownView.tsx b/frontend/src/components/MarkdownView.tsx index 3d44486e..0e01314f 100644 --- a/frontend/src/components/MarkdownView.tsx +++ b/frontend/src/components/MarkdownView.tsx @@ -43,7 +43,7 @@ function inline( // alternative `![alt](url)` is included ONLY when a resolveImageSrc is given // (the document reader), so chat/wiki rendering stays byte-for-byte unchanged. const re = resolveImageSrc - ? /(\[\[[^\]]+\]\]|!\[[^\]]*\]\([^)]+\)|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g + ? /(\[\[[^\]]+\]\]|!\[[^\]]*\]\((?:[^()]|\([^()]*\))*\)|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g : /(\[\[[^\]]+\]\]|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g let last = 0, m: RegExpExecArray | null, k = 0 while ((m = re.exec(text))) { @@ -105,7 +105,7 @@ function inline( // source path to an authed API path, or null for external/data URLs // (left as literal text). Cannot collide with the link/wikilink branches: // those match tokens starting with '[', this one starts with '!['. - const im = /^!\[([^\]]*)\]\(([^)]+)\)$/.exec(tok) + const im = /^!\[([^\]]*)\]\(((?:[^()]|\([^()]*\))*)\)$/.exec(tok) const alt = im ? im[1] : '' const raw = im ? im[2].trim() : '' const apiPath = raw ? (resolveImageSrc?.(raw) ?? null) : null @@ -244,8 +244,9 @@ function MermaidBlock({ code }: { code: string }) { /** Render a bearer-authed KB image. The API token can't ride on ``, * so we fetch the API path as a blob (mirrors `artifacts.ts`), show it, and - * revoke the object URL on unmount / src change. On failure it renders nothing - * rather than a broken-image glyph; while loading it shows a muted placeholder. */ + * revoke the object URL on unmount / src change. While loading it shows a muted + * placeholder; on failure it shows a small dashed chip (the alt) so a missing + * image is visible, not silently dropped. */ function AuthedImage({ apiPath, alt }: { apiPath: string; alt: string }) { const [url, setUrl] = useState(null) const [failed, setFailed] = useState(false) @@ -271,7 +272,14 @@ function AuthedImage({ apiPath, alt }: { apiPath: string; alt: string }) { if (objectUrl) URL.revokeObjectURL(objectUrl) } }, [apiPath]) - if (failed) return null + // Surface a failed load (missing file, network, expired token) with a muted + // dashed placeholder showing the alt, rather than silently rendering nothing. + if (failed) + return ( + + {alt} + + ) if (!url) return return ( `` then refuses to render. +_IMAGE_MEDIA_TYPES = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", +} @documents_router.post("/api/v1/document/source", response_model=DocumentSourceResponse) @@ -57,8 +67,9 @@ async def document_image_endpoint( full = (kb_dir / "wiki" / path).resolve() if not full.is_relative_to(images_root): raise HTTPException(status_code=400, detail="Invalid image path.") - if full.suffix.lower() not in _IMAGE_SUFFIXES: + media_type = _IMAGE_MEDIA_TYPES.get(full.suffix.lower()) + if media_type is None: raise HTTPException(status_code=400, detail="Only extracted images are served.") if not full.is_file(): raise HTTPException(status_code=404, detail="Image not found.") - return FileResponse(full) + return FileResponse(full, media_type=media_type) diff --git a/tests/test_api_documents.py b/tests/test_api_documents.py index 0a13c1b8..dd8b90c3 100644 --- a/tests/test_api_documents.py +++ b/tests/test_api_documents.py @@ -206,6 +206,26 @@ def test_document_image_serves_extracted_image(monkeypatch, kb_dir): assert resp.content == b"\x89PNG\r\n\x1a\nfake-bytes" +def test_document_image_sets_explicit_media_type(monkeypatch, kb_dir): + """Media type is set from the suffix (not guessed), so a .webp is served as + image/webp even on Pythons whose mimetypes lacks a webp entry (which would + otherwise degrade to text/plain and break a blob-loaded ).""" + client = _client(monkeypatch) + kb = _use_named_kb(monkeypatch, kb_dir) + img_dir = kb_dir / "wiki" / "sources" / "images" / "doc" + img_dir.mkdir(parents=True) + (img_dir / "p1_img1.webp").write_bytes(b"RIFFfake") + + resp = client.get( + "/api/v1/document/image", + params={"kb": kb, "path": "sources/images/doc/p1_img1.webp"}, + headers=_auth(), + ) + + assert resp.status_code == 200 + assert resp.headers["content-type"] == "image/webp" + + def test_document_image_rejects_traversal(monkeypatch, kb_dir): """A path escaping wiki/sources/images (even to another .png) is rejected by the containment guard before the suffix check."""