Refrag testing

.env

@@ -51,7 +51,7 @@ USE_TREE_SITTER=1
 
 
 # Hybrid/rerank quick-win defaults (can override via flags)
-HYBRID_EXPAND=1
+HYBRID_EXPAND=0
 HYBRID_PER_PATH=1
 # Increased symbol boost to prioritize function/class definitions
 HYBRID_SYMBOL_BOOST=0.35
@@ -72,7 +72,7 @@ MEMORY_MCP_TIMEOUT=6
 LLM_PROVIDER=ollama
 OLLAMA_HOST=http://host.docker.internal:11434
 LLM_EXPAND_MODEL=phi3:mini
-LLM_EXPAND_MAX=4
+LLM_EXPAND_MAX=0
 # PRF defaults (enabled by default)
 PRF_ENABLED=1
 
@@ -86,14 +86,14 @@ HYBRID_MINI_WEIGHT=1.0
 # Micro-chunking controls (token-based)
 # Smaller chunks = more precise retrieval (8 tokens ~= 2-3 lines of code)
 INDEX_MICRO_CHUNKS=1
-MICRO_CHUNK_TOKENS=8
-MICRO_CHUNK_STRIDE=4
+MICRO_CHUNK_TOKENS=24
+MICRO_CHUNK_STRIDE=48
 # Optional: gate-first using mini vectors to prefilter dense search
 REFRAG_GATE_FIRST=1
 REFRAG_CANDIDATES=200
 
 # Output shaping for micro spans
-# Conservative settings for small 1.5B model (Qwen2.5-Coder-1.5B)
+# Conservative settings
 MICRO_OUT_MAX_SPANS=3
 MICRO_MERGE_LINES=4
 MICRO_BUDGET_TOKENS=1500
@@ -123,7 +123,7 @@ REFRAG_SOFT_SCALE=1.0
 
 
 # Operational safeguards and timeouts
-MAX_MICRO_CHUNKS_PER_FILE=150
+MAX_MICRO_CHUNKS_PER_FILE=500
 QDRANT_TIMEOUT=20
 MEMORY_AUTODETECT=1
 MEMORY_COLLECTION_TTL_SECS=300

README.md

@@ -30,7 +30,7 @@ INDEX_MICRO_CHUNKS=1 MAX_MICRO_CHUNKS_PER_FILE=200 make reset-dev-dual
 
 - You can skip the decoder; it’s feature-flagged off by default.
 ### Switch decoder model (llama.cpp)
-- Default tiny model: Qwen2.5 Coder 1.5B (GGUF)
+- Default tiny model: Granite 4.0 Micro (Q4_K_M GGUF)
 - Change the model by overriding Make vars (downloads to ./models/model.gguf):
 ```bash
 LLAMACPP_MODEL_URL="https://huggingface.co/ORG/MODEL/resolve/main/model.gguf" \
@@ -44,19 +44,26 @@ Alternative (compose only)
 HOST_INDEX_PATH="$(pwd)" FASTMCP_INDEXER_PORT=8001 docker compose up -d qdrant mcp mcp_indexer indexer watcher
 ```
 
+### Recommended development flow
+1. Bring the stack up with the reset target that matches your client (`make reset-dev`, `make reset-dev-codex`, or `make reset-dev-dual`).
+2. When you need a clean ingest (after large edits or when the `qdrant_status` tool/`make qdrant-status` reports zero points), run `make reindex-hard`. This clears `.codebase/cache.json` before recreating the collection so unchanged files cannot be skipped.
+3. Confirm collection health with `make qdrant-status` (calls the MCP router to print counts and timestamps).
+4. Iterate using search helpers such as `make hybrid ARGS="--query 'async file watcher'"` or invoke the MCP tools directly from your client.
+
 ### Make targets (quick reference)
 - reset-dev: SSE stack on 8000/8001; seeds Qdrant, downloads tokenizer + tiny llama.cpp model, reindexes, brings up memory + indexer + watcher
 - reset-dev-codex: RMCP stack on 8002/8003; same seeding + bring-up for Codex/Qodo
 - reset-dev-dual: SSE + RMCP together (8000/8001 and 8002/8003)
 - up / down / logs / ps: Docker Compose lifecycle helpers
-- index / reindex: Index current repo; reindex recreates the collection first
+- index / reindex / reindex-hard: Index current repo; `reindex` recreates the collection; `reindex-hard` also clears the local cache so unchanged files are re-uploaded
 - index-here / index-path: Index arbitrary host path without cloning into this repo
 - watch: Watch-and-reindex on file changes
 - warm / health: Warm caches and run health checks
 - hybrid / rerank: Example hybrid search + reranker helper
 - setup-reranker / rerank-local / quantize-reranker: Manage ONNX reranker assets and local runs
 - prune / prune-path: Remove stale points (missing files or hash mismatch)
 - llama-model / tokenizer: Fetch tiny GGUF model and tokenizer.json
+- qdrant-status / qdrant-list / qdrant-prune / qdrant-index-root: Convenience wrappers that route through the MCP bridge to inspect or maintain collections
 
 ## Index another codebase (outside this repo)
 
@@ -144,6 +151,7 @@ Ports
 | HOST_INDEX_PATH | Host path mounted at /work in containers | current repo (.) |
 | QDRANT_URL | Qdrant base URL | container: http://qdrant:6333; local: http://localhost:6333 |
 | INDEX_MICRO_CHUNKS | Enable token-based micro-chunking | 0 (off) |
+| HYBRID_EXPAND | Enable heuristic multi-query expansion | 0 (off) |
 | MAX_MICRO_CHUNKS_PER_FILE | Cap micro-chunks per file | 200 |
 | TOKENIZER_URL | HF tokenizer.json URL (for Make download) | n/a (use Make target) |
 | TOKENIZER_PATH | Local path where tokenizer is saved (Make) | models/tokenizer.json |
@@ -163,6 +171,7 @@ Ports
 | FASTMCP_HTTP_PORT | Memory RMCP host port mapping | 8002 |
 | FASTMCP_INDEXER_HTTP_PORT | Indexer RMCP host port mapping | 8003 |
 | FASTMCP_HEALTH_PORT | Health port (memory/indexer) | memory: 18000; indexer: 18001 |
+| LLM_EXPAND_MAX | Max alternate queries generated via LLM | 0 |
 
 ## Running tests
 
@@ -218,7 +227,8 @@ curl -sI http://localhost:8001/sse | head -n1
 4) Single command to index + search
 ```bash
 # Fresh index of your repo and a quick hybrid example
-make reindex
+make reindex-hard
+make qdrant-status
 make hybrid ARGS="--query 'async file watcher' --limit 5 --include-snippet"
 ```
 

scripts/hybrid_search.py

@@ -1348,7 +1348,7 @@ def _norm_under(u: str | None) -> str | None:
     # Build query list (LLM-assisted first, then synonym expansion)
     qlist = list(clean_queries)
     try:
-        llm_max = int(os.environ.get("LLM_EXPAND_MAX", "4") or 4)
+        llm_max = int(os.environ.get("LLM_EXPAND_MAX", "0") or 0)
     except (ValueError, TypeError):
         llm_max = 4
     _llm_more = _llm_expand_queries(qlist, eff_language, max_new=llm_max)
@@ -2338,12 +2338,12 @@ def main():
     ap.add_argument("--under", type=str, default=None)
     ap.add_argument("--kind", type=str, default=None)
     ap.add_argument("--symbol", type=str, default=None)
-    # Expansion enabled by default; allow disabling via --no-expand or HYBRID_EXPAND=0
+    # Expansion disabled by default; enable via --expand or HYBRID_EXPAND=1
     ap.add_argument(
         "--expand",
         dest="expand",
         action="store_true",
-        default=_env_truthy(os.environ.get("HYBRID_EXPAND"), True),
+        default=_env_truthy(os.environ.get("HYBRID_EXPAND"), False),
         help="Enable simple query expansion",
     )
     ap.add_argument(

scripts/mcp_indexer_server.py

@@ -1470,7 +1470,7 @@ def _to_str_list(x):
                     path_regex=path_regex or None,
                     path_glob=(path_globs or None),
                     not_glob=(not_globs or None),
-                    expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower()
+                    expand=str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower()
                     in {"1", "true", "yes", "on"},
                     model=model,
                 )
@@ -1553,7 +1553,7 @@ def _to_str_list(x):
                     path_regex=path_regex or None,
                     path_glob=(path_globs or None),
                     not_glob=(not_globs or None),
-                    expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1", "true", "yes", "on"},
+                    expand=str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1", "true", "yes", "on"},
                     model=model,
                 )
                 json_lines = items
@@ -3404,7 +3404,7 @@ def _add_query(q: str):
         path_regex=(path_regex or kwargs.get("path_regex") or None),
         path_glob=(eff_path_glob or None),
         not_glob=eff_not_glob,
-        expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}),
+        expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1","true","yes","on"}),
         model=model,
     )
     if os.environ.get("DEBUG_CONTEXT_ANSWER"):
@@ -3444,7 +3444,7 @@ def _add_query(q: str):
                     path_regex=(path_regex or kwargs.get("path_regex") or None),
                     path_glob=(eff_path_glob or None),
                     not_glob=eff_not_glob,
-                    expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}),
+                    expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1","true","yes","on"}),
                     model=model,
                 )
                 def _ikey(it: Dict[str, Any]):
@@ -3471,7 +3471,7 @@ def _ikey(it: Dict[str, Any]):
                         path_regex=(path_regex or kwargs.get("path_regex") or None),
                         path_glob=(eff_path_glob or None),
                         not_glob=eff_not_glob,
-                        expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}),
+                        expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1","true","yes","on"}),
                         model=model,
                     )
 
@@ -3627,7 +3627,7 @@ def _ok_lang(it: Dict[str, Any]) -> bool:
                 expand=False
                 if did_local_expand
                 else (
-                    str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower()
+                    str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower()
                     in {"1", "true", "yes", "on"}
                 ),
                 model=model,
@@ -3647,7 +3647,7 @@ def _ok_lang(it: Dict[str, Any]) -> bool:
                 path_regex=None,
                 path_glob=None,
                 not_glob=eff_not_glob,
-                expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1", "true", "yes", "on"}),
+                expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1", "true", "yes", "on"}),
                 model=model,
             )
 
@@ -4672,7 +4672,7 @@ def _k(s: Dict[str, Any]):
             path_regex=None,
             path_glob=None,
             not_glob=eff_not_glob,
-            expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}),
+            expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1","true","yes","on"}),
             model=model,
         )
     except Exception: