security: harden error handling, token exposure, URL ingest, and deps

Author: mdavistffhrtporg

backend/app/chunking.py

@@ -14,7 +14,8 @@ def normalize(text: str) -> str:
     text = text.replace("\u00a0", " ")
     text = text.replace("\r\n", "\n").replace("\r", "\n")
     text = re.sub(r"[ \t]+", " ", text)
-    text = re.sub(r"[ \t]+\n", "\n", text)
+    # Trim trailing horizontal whitespace per line without regex backtracking risk.
+    text = "\n".join(line.rstrip(" \t") for line in text.split("\n"))
     text = re.sub(r"\n{3,}", "\n\n", text)
     return text.strip()
 

backend/app/ingest.py

@@ -168,6 +168,10 @@ def _validate_remote_url_sync(url: str) -> None:
         raise RuntimeError("Only http/https URLs are allowed.")
     if not parsed.hostname:
         raise RuntimeError("URL must include a hostname.")
+    if parsed.username or parsed.password:
+        raise RuntimeError("URLs with embedded credentials are not allowed.")
+    if parsed.port and parsed.port not in (80, 443):
+        raise RuntimeError("Only standard ports 80/443 are allowed for URL ingest.")
     if _is_blocked_host_label(parsed.hostname):
         raise RuntimeError("Local hostnames are blocked for URL ingest.")
 
@@ -373,6 +377,7 @@ async def _download_html_with_diagnostics(url: str) -> Tuple[str, Dict[str, Any]
 
     async with httpx.AsyncClient(follow_redirects=False, timeout=URL_TIMEOUT) as client:
         while True:
+            current_host = (urlparse(current_url).hostname or "").strip().lower().rstrip(".")
             if current_url in seen_urls:
                 raise RuntimeError("Redirect loop detected.")
             seen_urls.add(current_url)
@@ -399,7 +404,11 @@ async def _download_html_with_diagnostics(url: str) -> Tuple[str, Dict[str, Any]
             if _is_redirect_status(response.status_code):
                 if redirects >= MAX_URL_REDIRECTS:
                     raise RuntimeError(f"Too many redirects (>{MAX_URL_REDIRECTS}).")
-                current_url = _resolve_redirect_url(str(response.url), response.headers.get("location", ""))
+                next_url = _resolve_redirect_url(str(response.url), response.headers.get("location", ""))
+                next_host = (urlparse(next_url).hostname or "").strip().lower().rstrip(".")
+                if next_host != current_host:
+                    raise RuntimeError("Cross-host redirects are blocked for URL ingest.")
+                current_url = next_url
                 redirects += 1
                 diagnostics["redirect_count"] = redirects
                 diagnostics["final_url"] = current_url

backend/flashcard_store.py

@@ -6,6 +6,7 @@
 import secrets
 import threading
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from app.atomic_io import atomic_write_json, path_lock
@@ -20,9 +21,19 @@ def _safe_owner(owner: str) -> str:
     return owner[:128]
 
 
+def _resolve_under_base(base_dir: str, name: str) -> str:
+    base = Path(base_dir).resolve()
+    candidate = (base / name).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError as e:
+        raise ValueError("path_outside_base_dir") from e
+    return str(candidate)
+
+
 def _path(owner: str, base_dir: str = DEFAULT_DIR) -> str:
     os.makedirs(base_dir, exist_ok=True)
-    return os.path.join(base_dir, f"{_safe_owner(owner)}.json")
+    return _resolve_under_base(base_dir, f"{_safe_owner(owner)}.json")
 
 
 def _now_dt() -> datetime:

backend/mcp_server.py

@@ -191,11 +191,12 @@ async def sift_generate(
             out = await anyio.to_thread.run_sync(lambda: generate_with_claude(prompt, model=model or "claude-3-5-sonnet-latest"))
         else:
             out = await anyio.to_thread.run_sync(lambda: generate_with_openai(prompt, model=model or "gpt-4.1-mini"))
-    except Exception as e:
+    except Exception:
         logger.exception("mcp_sift_generate_failed owner=%s provider=%s", owner, provider)
         return {
             "ok": False,
-            "error": str(e),
+            "error": "generation_failed",
+            "message": "Generation failed for the selected provider.",
             "hint": "If you're using Codex, call `search` and let Codex generate the final response.",
             "sources": res["results"],
         }

backend/quiz_store.py

@@ -5,6 +5,7 @@
 import re
 import secrets
 import threading
+from pathlib import Path
 from typing import Any, Dict, List
 
 from app.atomic_io import atomic_write_json, path_lock
@@ -19,9 +20,19 @@ def _safe_owner(owner: str) -> str:
     return owner[:128]
 
 
+def _resolve_under_base(base_dir: str, name: str) -> str:
+    base = Path(base_dir).resolve()
+    candidate = (base / name).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError as e:
+        raise ValueError("path_outside_base_dir") from e
+    return str(candidate)
+
+
 def _path(owner: str, base_dir: str = DEFAULT_DIR) -> str:
     os.makedirs(base_dir, exist_ok=True)
-    return os.path.join(base_dir, f"{_safe_owner(owner)}.json")
+    return _resolve_under_base(base_dir, f"{_safe_owner(owner)}.json")
 
 
 def load_attempts(owner: str, base_dir: str = DEFAULT_DIR) -> List[Dict[str, Any]]:

backend/requirements.txt

@@ -2,19 +2,19 @@ fastapi==0.115.8
 uvicorn[standard]==0.30.6
 pydantic>=2.10.1
 pydantic-settings>=2.6.1
-python-multipart==0.0.9
+python-multipart==0.0.22
 
 openai==1.61.0
 
 chromadb==0.5.5
 
 beautifulsoup4==4.12.3
 httpx==0.27.0
-pypdf==4.3.1
+pypdf==6.7.3
 tiktoken==0.12.0
-mcp==1.2.0
+mcp==1.23.0
 anthropic==0.45.2
-jinja2==3.1.4
+jinja2==3.1.6
 pillow>=10.4.0
 pytesseract>=0.3.10
 pdf2image>=1.17.0

backend/session_store.py

@@ -4,6 +4,7 @@
 import os
 import re
 import threading
+from pathlib import Path
 from typing import Any, Dict, List
 
 from app.atomic_io import atomic_write_json, path_lock
@@ -18,9 +19,19 @@ def _safe_owner(owner: str) -> str:
     return owner[:128]
 
 
+def _resolve_under_base(base_dir: str, name: str) -> str:
+    base = Path(base_dir).resolve()
+    candidate = (base / name).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError as e:
+        raise ValueError("path_outside_base_dir") from e
+    return str(candidate)
+
+
 def session_path(owner: str, base_dir: str = DEFAULT_DIR) -> str:
     os.makedirs(base_dir, exist_ok=True)
-    return os.path.join(base_dir, f"{_safe_owner(owner)}.json")
+    return _resolve_under_base(base_dir, f"{_safe_owner(owner)}.json")
 
 
 def load_session(owner: str, base_dir: str = DEFAULT_DIR) -> List[Dict[str, Any]]:

backend/source_store.py

@@ -5,6 +5,7 @@
 import secrets
 import threading
 from glob import glob
+from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from app.atomic_io import atomic_write_json, path_lock
@@ -19,13 +20,23 @@ def _safe_owner(owner: str) -> str:
     return owner[:128]
 
 
+def _resolve_under_base(base_dir: str, *parts: str) -> str:
+    base = Path(base_dir).resolve()
+    candidate = (base.joinpath(*parts)).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError as e:
+        raise ValueError("path_outside_base_dir") from e
+    return str(candidate)
+
+
 def _owner_manifest_path(owner: str, base_dir: str = DEFAULT_DIR) -> str:
     os.makedirs(base_dir, exist_ok=True)
-    return os.path.join(base_dir, f"{_safe_owner(owner)}.json")
+    return _resolve_under_base(base_dir, f"{_safe_owner(owner)}.json")
 
 
 def _owner_files_dir(owner: str, base_dir: str = DEFAULT_DIR) -> str:
-    path = os.path.join(base_dir, "_files", _safe_owner(owner))
+    path = _resolve_under_base(base_dir, "_files", _safe_owner(owner))
     os.makedirs(path, exist_ok=True)
     return path
 
@@ -185,7 +196,8 @@ def batch_mutate_items(
 
 def write_text_blob(owner: str, source_id: str, text: str, base_dir: str = DEFAULT_DIR) -> str:
     files_dir = _owner_files_dir(owner, base_dir)
-    path = os.path.join(files_dir, f"{source_id}.txt")
+    safe_source_id = re.sub(r"[^a-zA-Z0-9._-]+", "_", (source_id or "").strip())[:120] or "source"
+    path = _resolve_under_base(files_dir, f"{safe_source_id}.txt")
     with _LOCK:
         parent = os.path.dirname(path)
         if parent:
@@ -195,18 +207,21 @@ def write_text_blob(owner: str, source_id: str, text: str, base_dir: str = DEFAU
     return path
 
 
-def read_text_blob(path: str) -> str:
+def read_text_blob(path: str, base_dir: str = DEFAULT_DIR) -> str:
     try:
-        with open(path, "r", encoding="utf-8") as f:
+        resolved = Path(path or "").resolve()
+        resolved.relative_to(Path(base_dir).resolve())
+        with open(resolved, "r", encoding="utf-8") as f:
             return f.read()
     except Exception:
         return ""
 
 
 def write_binary_blob(owner: str, source_id: str, filename: str, data: bytes, base_dir: str = DEFAULT_DIR) -> str:
     files_dir = _owner_files_dir(owner, base_dir)
+    safe_source_id = re.sub(r"[^a-zA-Z0-9._-]+", "_", (source_id or "").strip())[:120] or "source"
     safe = re.sub(r"[^a-zA-Z0-9._-]+", "_", (filename or "upload").strip())[:180] or "upload"
-    path = os.path.join(files_dir, f"{source_id}__{safe}")
+    path = _resolve_under_base(files_dir, f"{safe_source_id}__{safe}")
     with _LOCK:
         parent = os.path.dirname(path)
         if parent:
@@ -216,9 +231,13 @@ def write_binary_blob(owner: str, source_id: str, filename: str, data: bytes, ba
     return path
 
 
-def remove_file(path: str) -> None:
+def remove_file(path: str, base_dir: str = DEFAULT_DIR) -> None:
     try:
-        if path and os.path.exists(path):
-            os.remove(path)
+        if not path:
+            return
+        resolved = Path(path).resolve()
+        resolved.relative_to(Path(base_dir).resolve())
+        if os.path.exists(resolved):
+            os.remove(resolved)
     except Exception:
         pass

backend/study_store.py

@@ -5,6 +5,7 @@
 import re
 import secrets
 import threading
+from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from app.atomic_io import atomic_write_json, path_lock
@@ -19,9 +20,19 @@ def _safe_owner(owner: str) -> str:
     return owner[:128]
 
 
+def _resolve_under_base(base_dir: str, name: str) -> str:
+    base = Path(base_dir).resolve()
+    candidate = (base / name).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError as e:
+        raise ValueError("path_outside_base_dir") from e
+    return str(candidate)
+
+
 def library_path(owner: str, base_dir: str = DEFAULT_DIR) -> str:
     os.makedirs(base_dir, exist_ok=True)
-    return os.path.join(base_dir, f"{_safe_owner(owner)}.json")
+    return _resolve_under_base(base_dir, f"{_safe_owner(owner)}.json")
 
 
 def load_library(owner: str, base_dir: str = DEFAULT_DIR) -> List[Dict[str, Any]]:

backend/templates/chat.html

@@ -849,7 +849,11 @@
         const res = await fetch(u);
         if (!(await ensureAuth(res))) return;
         if (!res.ok) {
-          librarySelectListEl.innerHTML = `<div class="hint">Failed to load library sources.</div>`;
+          librarySelectListEl.innerHTML = "";
+          const hint = document.createElement("div");
+          hint.className = "hint";
+          hint.textContent = "Failed to load library sources.";
+          librarySelectListEl.appendChild(hint);
           return;
         }
         const data = await res.json();