fix: make dense-embedding truncation observable and configurable

.env.example

@@ -1,5 +1,10 @@
 # Embeddings provider — one of: jina | jina-api | voyage | openai | ollama
 EMBEDDINGS_PROVIDER=jina
+# Max chars of a symbol's dense-embedding text (preamble + signature + docstring +
+# source). Oversized symbols are truncated and a WARNING is logged. Keep conservative
+# for self-hosted Jina TEI (it errors on inputs over the model token limit); raise it
+# (e.g. 24000) for voyage/openai/jina-api, which trim oversized inputs server-side.
+EMBEDDING_MAX_CHARS=6000
 
 # Jina Code V2 via HuggingFace TEI (default)
 JINA_URL=http://localhost:8087
@@ -39,4 +44,4 @@ MCP_PORT=8090
 CONFIG_PATH=./config.yaml
 
 # GitHub token — requires repo:read (or contents:read for fine-grained tokens)
-GITHUB_TOKEN=
+GITHUB_TOKEN=

.gitignore

@@ -1,6 +1,9 @@
 # Environment
 .env
 
+# AI
+.ai/
+
 # Python
 .venv/
 __pycache__/

docs/configuration.md

@@ -100,6 +100,7 @@ Used when `EMBEDDINGS_PROVIDER=ollama`. Requires a running [Ollama](https://olla
 | `GITHUB_TOKEN` | `""` | GitHub personal access token. Required for all indexing operations. Without it, GitHub API calls return 403. |
 | `CONFIG_PATH` | `./config.yaml` | Path to the services config file. Relative to the working directory at server start. |
 | `GIT_HISTORY_MAX_COMMITS` | `500` | Maximum number of commits fetched per service for git history indexing. |
+| `EMBEDDING_MAX_CHARS` | `6000` | Max characters of a symbol's dense-embedding text (preamble + signature + docstring + source). Oversized symbols are truncated (with a logged `WARNING`). Keep conservative for self-hosted Jina TEI, which errors on inputs over the model's token limit; raise it (e.g. `~24000`) for `voyage`/`openai`/`jina-api`, which trim oversized inputs server-side and accept ~8k–32k tokens. ~3–4 chars ≈ 1 token for code. |
 
 ---
 

docs/ingestion.md

@@ -88,12 +88,14 @@ Annotations: @PostMapping, @Transactional
 Processes an order and emits an OrderPlaced event...  ← docstring (first 300 chars)
 
 def process_order(self, order: Order) -> Result:     ← signature
-    # full source code...                            ← source (up to 6,000 chars)
+    # full source code...                            ← source (fills remaining budget)
 ```
 
 Metadata included (when present): language, symbol type, name, parent class, service name, package, Spring stereotypes, HTTP method/route, annotations (first 8), Lombok annotations, React.memo flag, docstring.
 
-Source is hard-truncated at **6,000 characters** (~1,500 tokens). Longer symbols are truncated with a `// ... (truncated)` marker.
+The whole embedding text (preamble + signature + docstring + source) is budgeted to **`EMBEDDING_MAX_CHARS`** (default **6,000** chars, ~1,500 tokens). The metadata preamble and signature consume the budget first; the source fills whatever remains. When a symbol's source exceeds its budget it is truncated with a `// ... (truncated)` marker **and a `WARNING` is logged** (naming the symbol and file) so the loss is observable. Raise `EMBEDDING_MAX_CHARS` for providers that accept larger inputs — see [configuration.md](configuration.md).
+
+> **Future direction — sub-chunking (deferred):** Truncation still drops the tail of a genuinely oversized symbol (e.g. a 1,000-line class) from the *dense* vector; BM25 and the stored payload keep the full source. A complete fix would split such symbols into overlapping windows and emit multiple dense points each. That was deferred because it requires a chunk index in the point ID (`store/qdrant.py`, `_symbol_point_id`) and search-result dedup so sub-chunks of one symbol don't crowd results. Watch the truncation warnings to judge whether it's worth it.
 
 #### Sparse: `_build_bm25_text`
 
@@ -193,4 +195,4 @@ All `CodeSymbol` fields are stored verbatim, plus:
 
 **GitHub Trees truncation** — Very large repositories may have their tree response silently truncated by the GitHub API. The pipeline logs a warning but does not retry or paginate to recover the missing entries.
 
-**Hardcoded truncation limit** — The 6,000-character source truncation in `_build_embedding_text` is a constant (`_MAX_EMBEDDING_CHARS`). It cannot be adjusted via configuration.
+**Dense-embedding truncation drops the tail of huge symbols** — The embedding text is budgeted to `EMBEDDING_MAX_CHARS` (configurable, default 6,000). A symbol larger than the budget has its tail excluded from the *dense* vector (BM25 and the stored source keep everything), so semantic search over that tail relies on BM25 alone. Truncation now emits a `WARNING`; the deferred sub-chunking fix is described in the [Embedding Text Construction](#5-embedding-text-construction) section above.

server/config.py

@@ -71,6 +71,12 @@ class Settings(BaseSettings):
     )
     git_history_max_commits: int = Field(default=500, alias="GIT_HISTORY_MAX_COMMITS")
 
+    # Max characters of a symbol's dense-embedding text. Budgeted against the WHOLE text
+    # (metadata preamble + signature + docstring + source), not just source. Kept
+    # conservative by default because local TEI (jina.py) does not trim oversized inputs
+    # server-side; raise it for providers that do (Voyage/OpenAI/Jina-API).
+    embedding_max_chars: int = Field(default=6000, alias="EMBEDDING_MAX_CHARS")
+
     mcp_transport: Literal["streamable-http", "sse", "stdio"] = Field(
         default="streamable-http", alias="MCP_TRANSPORT"
     )

server/indexer/pipeline.py

@@ -21,8 +21,6 @@
 
 logger = logging.getLogger(__name__)
 
-_MAX_EMBEDDING_CHARS = 6000  # ~1500 tokens
-
 
 @dataclass
 class ProgressEvent:
@@ -33,7 +31,12 @@ class ProgressEvent:
     service: str
 
 
-def _build_embedding_text(symbol: CodeSymbol, service_name: str) -> str:
+def _build_embedding_text(
+    symbol: CodeSymbol, service_name: str, max_chars: int | None = None
+) -> str:
+    if max_chars is None:
+        max_chars = settings.embedding_max_chars
+
     lines = []
 
     lang_display = {"java": "Java", "python": "Python", "typescript": "TypeScript"}.get(
@@ -83,9 +86,23 @@ def _build_embedding_text(symbol: CodeSymbol, service_name: str) -> str:
     if symbol.signature:
         lines.append(symbol.signature)
 
+    # Budget the source against the WHOLE embedding text so the total stays under the
+    # model's token limit — the metadata preamble and signature also consume the budget.
+    header = "\n".join(lines)
     source = symbol.source or ""
-    if len(source) > _MAX_EMBEDDING_CHARS:
-        source = source[:_MAX_EMBEDDING_CHARS] + "\n// ... (truncated)"
+    budget = max(max_chars - len(header) - 1, 0)  # -1 for the newline before source
+    if len(source) > budget:
+        logger.warning(
+            "Truncating embedding source for %s `%s` in %s: %d -> %d chars "
+            "(EMBEDDING_MAX_CHARS=%d); the tail is absent from the dense vector.",
+            symbol.symbol_type,
+            symbol.name,
+            symbol.file_path,
+            len(source),
+            budget,
+            max_chars,
+        )
+        source = source[:budget] + "\n// ... (truncated)"
     lines.append(source)
 
     return "\n".join(lines)

tests/test_pipeline.py

@@ -1,8 +1,25 @@
 from __future__ import annotations
 
+import logging
+
 from server.indexer.pipeline import _build_bm25_text, _build_embedding_text
 from server.parser.base import CodeSymbol
 
+_TRUNCATION_MARKER = "// ... (truncated)"
+
+
+def _sym_with_source(source: str, docstring: str = "") -> CodeSymbol:
+    return CodeSymbol(
+        name="fn",
+        symbol_type="function",
+        language="python",
+        source=source,
+        file_path="svc/mod.py",
+        start_line=1,
+        end_line=1,
+        docstring=docstring,
+    )
+
 
 def _sym(docstring: str) -> CodeSymbol:
     return CodeSymbol(
@@ -73,3 +90,40 @@ def test_bm25_text_excludes_preamble() -> None:
     dense = _build_embedding_text(sym, "orders-service")
     assert "Java method" in dense
     assert "Java method" not in bm25
+
+
+def test_small_source_not_truncated(caplog) -> None:
+    sym = _sym_with_source("def fn(): return 1")
+    with caplog.at_level(logging.WARNING):
+        text = _build_embedding_text(sym, "svc", max_chars=6000)
+    assert _TRUNCATION_MARKER not in text
+    assert "def fn(): return 1" in text
+    assert not caplog.records
+
+
+def test_oversized_source_truncated_and_logged(caplog) -> None:
+    sym = _sym_with_source("x" * 10_000)
+    with caplog.at_level(logging.WARNING):
+        text = _build_embedding_text(sym, "svc", max_chars=6000)
+    assert _TRUNCATION_MARKER in text
+    # Whole text is bounded by max_chars (plus the short marker), not just the source.
+    assert len(text) <= 6000 + len(_TRUNCATION_MARKER) + 1
+    assert any("Truncating embedding source" in r.message for r in caplog.records)
+
+
+def test_preamble_counts_against_budget() -> None:
+    # Same source and max_chars, but a long preamble (docstring, capped at 300 chars)
+    # eats into the budget, tipping a source that fits bare over the limit.
+    source = "y" * 5_800
+    bare = _build_embedding_text(_sym_with_source(source), "svc", max_chars=6000)
+    with_preamble = _build_embedding_text(
+        _sym_with_source(source, docstring="d" * 300), "svc", max_chars=6000
+    )
+    assert _TRUNCATION_MARKER not in bare
+    assert _TRUNCATION_MARKER in with_preamble
+
+
+def test_max_chars_is_configurable() -> None:
+    sym = _sym_with_source("z" * 5_000)
+    assert _TRUNCATION_MARKER not in _build_embedding_text(sym, "svc", max_chars=6000)
+    assert _TRUNCATION_MARKER in _build_embedding_text(sym, "svc", max_chars=1000)