diff --git a/.env b/.env index 93a178a6..43a04ace 100644 --- a/.env +++ b/.env @@ -4,7 +4,7 @@ QDRANT_URL=http://qdrant:6333 # QDRANT_API_KEY= # not needed for local # Default collection used by the MCP server (auto-created if missing) -COLLECTION_NAME=my-collection # Use auto-detected default from .codebase/state.json +# COLLECTION_NAME=my-collection # Use auto-detected default from .codebase/state.json # Embedding settings (FastEmbed model) EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 @@ -105,7 +105,7 @@ CTX_SUMMARY_CHARS=0 CTX_SNIPPET_CHARS=400 -# Decoder-path ReFRAG (feature-flagged; off by default) +# Decoder-path ReFRAG REFRAG_DECODER=1 REFRAG_RUNTIME=llamacpp REFRAG_ENCODER_MODEL=BAAI/bge-base-en-v1.5 diff --git a/docker-compose.yml b/docker-compose.yml index 23ca274e..32b41ad5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,6 +97,7 @@ services: container_name: mcp-indexer-http depends_on: - qdrant + - llamacpp env_file: - .env environment: @@ -105,7 +106,12 @@ services: - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} - QDRANT_URL=${QDRANT_URL} - FASTMCP_HEALTH_PORT=18001 - - DEBUG_CONTEXT_ANSWER=${DEBUG_CONTEXT_ANSWER:-0} + - DEBUG_CONTEXT_ANSWER=${DEBUG_CONTEXT_ANSWER:-1} + - REFRAG_DECODER=${REFRAG_DECODER:-1} + - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080} + - CTX_REQUIRE_IDENTIFIER=${CTX_REQUIRE_IDENTIFIER:-0} + - COLLECTION_NAME=${COLLECTION_NAME} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} # Streamable HTTP endpoint for IDE agents at http://localhost:${FASTMCP_INDEXER_HTTP_PORT:-8003}/mcp/ ports: - "${FASTMCP_INDEXER_HTTP_PORT:-8003}:8001" diff --git a/scripts/hybrid_search.py b/scripts/hybrid_search.py index 2198fb59..3ce2cc0f 100644 --- a/scripts/hybrid_search.py +++ b/scripts/hybrid_search.py @@ -18,26 +18,46 @@ def _collection() -> str: API_KEY = os.environ.get("QDRANT_API_KEY") +def _safe_int(val: Any, default: int) -> int: + try: + if val is None or (isinstance(val, str) and val.strip() == ""): + return default + return int(val) + except (ValueError, TypeError): + return default + +def _safe_float(val: Any, default: float) -> float: + try: + if val is None or (isinstance(val, str) and val.strip() == ""): + return default + return float(val) + except (ValueError, TypeError): + return default + LEX_VECTOR_NAME = os.environ.get("LEX_VECTOR_NAME", "lex") -LEX_VECTOR_DIM = int(os.environ.get("LEX_VECTOR_DIM", "4096") or 4096) +LEX_VECTOR_DIM = _safe_int(os.environ.get("LEX_VECTOR_DIM", "4096"), 4096) # Optional mini vector (ReFRAG gating) MINI_VECTOR_NAME = os.environ.get("MINI_VECTOR_NAME", "mini") -MINI_VEC_DIM = int(os.environ.get("MINI_VEC_DIM", "64") or 64) -HYBRID_MINI_WEIGHT = float(os.environ.get("HYBRID_MINI_WEIGHT", "0.5") or 0.5) +MINI_VEC_DIM = _safe_int(os.environ.get("MINI_VEC_DIM", "64"), 64) +HYBRID_MINI_WEIGHT = _safe_float(os.environ.get("HYBRID_MINI_WEIGHT", "0.5"), 0.5) # Lightweight embedding cache for query embeddings (model_name, text) -> vector from threading import Lock as _Lock +from collections import OrderedDict -_EMBED_QUERY_CACHE: Dict[tuple[str, str], List[float]] = {} +_EMBED_QUERY_CACHE: OrderedDict[tuple[str, str], List[float]] = OrderedDict() _EMBED_LOCK = _Lock() +MAX_EMBED_CACHE = int(os.environ.get("MAX_EMBED_CACHE", "4096") or 4096) def _embed_queries_cached( model: TextEmbedding, queries: List[str] ) -> List[List[float]]: - """Cache dense query embeddings to avoid repeated compute across expansions/retries.""" - out: List[List[float]] = [] + """Cache dense query embeddings to avoid repeated compute across expansions/retries. + Optimized: batch-embeds all missing queries in one model call (2-5x faster). + Thread-safe with bounded cache size. + """ try: # Best-effort model name extraction; fall back to env name = getattr(model, "model_name", None) or os.environ.get( @@ -45,21 +65,54 @@ def _embed_queries_cached( ) except Exception: name = os.environ.get("EMBEDDING_MODEL", MODEL_NAME) - for q in queries: - key = (str(name), str(q)) - v = _EMBED_QUERY_CACHE.get(key) - if v is None: - try: - vec = next(model.embed([q])).tolist() - except Exception: - # Fallback: embed batch and take first - vec = next(model.embed([str(q)])).tolist() + + # Find missing queries that need embedding (thread-safe read) + missing_queries = [] + missing_indices = [] + with _EMBED_LOCK: + for i, q in enumerate(queries): + key = (str(name), str(q)) + if key not in _EMBED_QUERY_CACHE: + missing_queries.append(str(q)) + missing_indices.append(i) + + # Batch-embed all missing queries in one call + if missing_queries: + try: + # Embed all missing queries at once + vecs = list(model.embed(missing_queries)) with _EMBED_LOCK: - # Double-check inside lock - if key not in _EMBED_QUERY_CACHE: - _EMBED_QUERY_CACHE[key] = vec - v = vec - out.append(v) + # Cache all new embeddings + for q, vec in zip(missing_queries, vecs): + key = (str(name), str(q)) + if key not in _EMBED_QUERY_CACHE: + _EMBED_QUERY_CACHE[key] = vec.tolist() + # Evict oldest entries if cache exceeds limit + while len(_EMBED_QUERY_CACHE) > MAX_EMBED_CACHE: + _EMBED_QUERY_CACHE.popitem(last=False) + except Exception: + # Fallback to one-by-one if batch fails + for q in missing_queries: + key = (str(name), str(q)) + try: + vec = next(model.embed([q])).tolist() + with _EMBED_LOCK: + if key not in _EMBED_QUERY_CACHE: + _EMBED_QUERY_CACHE[key] = vec + # Evict oldest entries if cache exceeds limit + while len(_EMBED_QUERY_CACHE) > MAX_EMBED_CACHE: + _EMBED_QUERY_CACHE.popitem(last=False) + except Exception: + pass + + # Return embeddings in original order from cache (thread-safe read) + out: List[List[float]] = [] + with _EMBED_LOCK: + for q in queries: + key = (str(name), str(q)) + v = _EMBED_QUERY_CACHE.get(key) + if v is not None: + out.append(v) return out @@ -76,40 +129,40 @@ def _embed_queries_cached( from scripts.ingest_code import project_mini as _project_mini -RRF_K = int(os.environ.get("HYBRID_RRF_K", "60") or 60) -DENSE_WEIGHT = float(os.environ.get("HYBRID_DENSE_WEIGHT", "1.0") or 1.0) -LEXICAL_WEIGHT = float(os.environ.get("HYBRID_LEXICAL_WEIGHT", "0.25") or 0.25) -LEX_VECTOR_WEIGHT = float( - os.environ.get("HYBRID_LEX_VECTOR_WEIGHT", str(LEXICAL_WEIGHT)) or LEXICAL_WEIGHT +RRF_K = _safe_int(os.environ.get("HYBRID_RRF_K", "60"), 60) +DENSE_WEIGHT = _safe_float(os.environ.get("HYBRID_DENSE_WEIGHT", "1.0"), 1.0) +LEXICAL_WEIGHT = _safe_float(os.environ.get("HYBRID_LEXICAL_WEIGHT", "0.25"), 0.25) +LEX_VECTOR_WEIGHT = _safe_float( + os.environ.get("HYBRID_LEX_VECTOR_WEIGHT", str(LEXICAL_WEIGHT)), LEXICAL_WEIGHT ) -EF_SEARCH = int(os.environ.get("QDRANT_EF_SEARCH", "128") or 128) +EF_SEARCH = _safe_int(os.environ.get("QDRANT_EF_SEARCH", "128"), 128) # Lightweight, configurable boosts -SYMBOL_BOOST = float(os.environ.get("HYBRID_SYMBOL_BOOST", "0.15") or 0.15) -RECENCY_WEIGHT = float(os.environ.get("HYBRID_RECENCY_WEIGHT", "0.1") or 0.1) -CORE_FILE_BOOST = float(os.environ.get("HYBRID_CORE_FILE_BOOST", "0.1") or 0.1) -SYMBOL_EQUALITY_BOOST = float( - os.environ.get("HYBRID_SYMBOL_EQUALITY_BOOST", "0.25") or 0.25 +SYMBOL_BOOST = _safe_float(os.environ.get("HYBRID_SYMBOL_BOOST", "0.15"), 0.15) +RECENCY_WEIGHT = _safe_float(os.environ.get("HYBRID_RECENCY_WEIGHT", "0.1"), 0.1) +CORE_FILE_BOOST = _safe_float(os.environ.get("HYBRID_CORE_FILE_BOOST", "0.1"), 0.1) +SYMBOL_EQUALITY_BOOST = _safe_float( + os.environ.get("HYBRID_SYMBOL_EQUALITY_BOOST", "0.25"), 0.25 ) -VENDOR_PENALTY = float(os.environ.get("HYBRID_VENDOR_PENALTY", "0.05") or 0.05) -LANG_MATCH_BOOST = float(os.environ.get("HYBRID_LANG_MATCH_BOOST", "0.05") or 0.05) -CLUSTER_LINES = int(os.environ.get("HYBRID_CLUSTER_LINES", "15") or 15) +VENDOR_PENALTY = _safe_float(os.environ.get("HYBRID_VENDOR_PENALTY", "0.05"), 0.05) +LANG_MATCH_BOOST = _safe_float(os.environ.get("HYBRID_LANG_MATCH_BOOST", "0.05"), 0.05) +CLUSTER_LINES = _safe_int(os.environ.get("HYBRID_CLUSTER_LINES", "15"), 15) # Penalize test files slightly to prefer implementation over tests -TEST_FILE_PENALTY = float(os.environ.get("HYBRID_TEST_FILE_PENALTY", "0.15") or 0.15) +TEST_FILE_PENALTY = _safe_float(os.environ.get("HYBRID_TEST_FILE_PENALTY", "0.15"), 0.15) # Additional file-type weighting knobs (defaults tuned for Q&A use) -CONFIG_FILE_PENALTY = float(os.environ.get("HYBRID_CONFIG_FILE_PENALTY", "0.3") or 0.3) -IMPLEMENTATION_BOOST = float(os.environ.get("HYBRID_IMPLEMENTATION_BOOST", "0.2") or 0.2) -DOCUMENTATION_PENALTY = float(os.environ.get("HYBRID_DOCUMENTATION_PENALTY", "0.1") or 0.1) +CONFIG_FILE_PENALTY = _safe_float(os.environ.get("HYBRID_CONFIG_FILE_PENALTY", "0.3"), 0.3) +IMPLEMENTATION_BOOST = _safe_float(os.environ.get("HYBRID_IMPLEMENTATION_BOOST", "0.2"), 0.2) +DOCUMENTATION_PENALTY = _safe_float(os.environ.get("HYBRID_DOCUMENTATION_PENALTY", "0.1"), 0.1) # Penalize comment-heavy snippets so code (not comments) ranks higher -COMMENT_PENALTY = float(os.environ.get("HYBRID_COMMENT_PENALTY", "0.2") or 0.2) -COMMENT_RATIO_THRESHOLD = float(os.environ.get("HYBRID_COMMENT_RATIO_THRESHOLD", "0.6") or 0.6) +COMMENT_PENALTY = _safe_float(os.environ.get("HYBRID_COMMENT_PENALTY", "0.2"), 0.2) +COMMENT_RATIO_THRESHOLD = _safe_float(os.environ.get("HYBRID_COMMENT_RATIO_THRESHOLD", "0.6"), 0.6) # Micro-span compaction and budgeting (ReFRAG-lite output shaping) -MICRO_OUT_MAX_SPANS = int(os.environ.get("MICRO_OUT_MAX_SPANS", "3") or 3) -MICRO_MERGE_LINES = int(os.environ.get("MICRO_MERGE_LINES", "4") or 4) -MICRO_BUDGET_TOKENS = int(os.environ.get("MICRO_BUDGET_TOKENS", "512") or 512) -MICRO_TOKENS_PER_LINE = int(os.environ.get("MICRO_TOKENS_PER_LINE", "32") or 32) +MICRO_OUT_MAX_SPANS = _safe_int(os.environ.get("MICRO_OUT_MAX_SPANS", "3"), 3) +MICRO_MERGE_LINES = _safe_int(os.environ.get("MICRO_MERGE_LINES", "4"), 4) +MICRO_BUDGET_TOKENS = _safe_int(os.environ.get("MICRO_BUDGET_TOKENS", "512"), 512) +MICRO_TOKENS_PER_LINE = _safe_int(os.environ.get("MICRO_TOKENS_PER_LINE", "32"), 32) def _merge_and_budget_spans(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: @@ -234,6 +287,18 @@ def _flat_key(c): out.append( {"m": m, "start": c["start"], "end": c["end"], "need_tokens": need} ) + elif budget > 0 and per_path_counts.get(path, 0) < out_max_spans: + # Trim to fit remaining budget instead of dropping entirely + # Calculate how many lines we can afford with remaining budget + affordable_lines = max(1, budget // tokens_per_line) + trim_end = c["start"] + affordable_lines - 1 + if trim_end >= c["start"]: + trimmed_need = _line_tokens(c["start"], trim_end) + budget -= trimmed_need + per_path_counts[path] = per_path_counts.get(path, 0) + 1 + out.append( + {"m": m, "start": c["start"], "end": trim_end, "need_tokens": trimmed_need, "_trimmed": True} + ) if budget <= 0: break @@ -948,7 +1013,31 @@ def _norm_under(u: str | None) -> str | None: key="metadata.symbol", match=models.MatchValue(value=eff_symbol) ) ) - flt = models.Filter(must=must) if must else None + + # Add ext filter (file extension) - server-side when possible + if eff_ext: + ext_clean = eff_ext.lower().lstrip(".") + must.append( + models.FieldCondition( + key="metadata.ext", match=models.MatchValue(value=ext_clean) + ) + ) + + # Add not filter (simple text exclusion on path) - server-side when possible + must_not = [] + if eff_not: + # Try MatchText for substring exclusion; fallback to post-filter if unsupported + try: + must_not.append( + models.FieldCondition( + key="metadata.path", match=models.MatchText(text=eff_not) + ) + ) + except Exception: + # Will be handled by post-filter + pass + + flt = models.Filter(must=must, must_not=must_not) if (must or must_not) else None flt = _sanitize_filter_obj(flt) @@ -986,16 +1075,29 @@ def _bn(p: str) -> str: stem = b.rsplit(".", 1)[0] if "." in b else b if stem and stem not in qlist: qlist.append(stem) + # Add short path segments (e.g., "scripts/hybrid_search") to steer lexical hashing + for g in globs_src: + s = str(g or "").replace("\\", "/").strip() + parts = [t for t in s.split("/") if t] + if len(parts) >= 2: + last2 = "/".join(parts[-2:]) + if last2 and last2 not in qlist: + qlist.append(last2) + if len(parts) >= 3: + last3 = "/".join(parts[-3:]) + if last3 and last3 not in qlist: + qlist.append(last3) except Exception: pass - # Lexical vector query + # Lexical vector query (keep original RRF scoring) score_map: Dict[str, Dict[str, Any]] = {} try: lex_vec = lex_hash_vector(qlist) lex_results = lex_query(client, lex_vec, flt, max(24, limit)) except Exception: lex_results = [] + for rank, p in enumerate(lex_results, 1): pid = str(p.id) score_map.setdefault( @@ -1028,6 +1130,7 @@ def _bn(p: str) -> str: except Exception: pass # Optional gate-first using mini vectors to restrict dense search to candidates + # Adaptive gating: disable for short/ambiguous queries to avoid over-filtering flt_gated = flt try: gate_first = str(os.environ.get("REFRAG_GATE_FIRST", "0")).strip().lower() in { @@ -1045,7 +1148,34 @@ def _bn(p: str) -> str: cand_n = int(os.environ.get("REFRAG_CANDIDATES", "200") or 200) except Exception: gate_first, refrag_on, cand_n = False, False, 200 + + # Adaptive mini-gate: disable for queries that are too short or lack strong identifiers + should_bypass_gate = False if gate_first and refrag_on: + # Check query characteristics + try: + # Count total tokens across queries + total_tokens = sum(len(_split_ident(q)) for q in qlist) + # Check for strong identifiers (ALL_CAPS, camelCase, or has underscore) + has_strong_id = any( + any(t.isupper() or "_" in t or any(c.isupper() for c in t[1:]) and any(c.islower() for c in t) + for t in _split_ident(q)) + for q in qlist + ) + # If query is very short (<3 tokens) and has no strong identifiers, bypass gate + if total_tokens < 3 and not has_strong_id: + should_bypass_gate = True + if os.environ.get("DEBUG_HYBRID_SEARCH"): + print(f"DEBUG: Adaptive gate bypass (short query, tokens={total_tokens}, strong_id={has_strong_id})") + # If we have strict filters (language, under, symbol, ext), relax candidate count + if eff_language or eff_under or eff_symbol or eff_ext: + cand_n = max(cand_n, limit * 5) + if os.environ.get("DEBUG_HYBRID_SEARCH"): + print(f"DEBUG: Adaptive gate relaxed candidate count to {cand_n} due to filters") + except Exception: + pass + + if gate_first and refrag_on and not should_bypass_gate: try: # ReFRAG gate-first: Use MINI vectors to prefilter candidates mini_queries = [_project_mini(list(v), MINI_VEC_DIM) for v in embedded] @@ -1253,6 +1383,7 @@ def _bn(p: str) -> str: except Exception: pass + # Add dense scores (keep original RRF scoring) for res in result_sets: for rank, p in enumerate(res, 1): pid = str(p.id) @@ -1337,6 +1468,31 @@ def _bn(p: str) -> str: if (lang and md_lang and md_lang == lang) or lang_matches_path(lang, path): rec["langb"] += LANG_MATCH_BOOST rec["s"] += LANG_MATCH_BOOST + + # Memory blending: apply penalty to memory-like entries to prevent swamping code results + # Only apply if query doesn't explicitly ask for memories/notes + kind = str(md.get("kind") or "").lower() + if kind == "memory": + qlow = " ".join(qlist).lower() + is_memory_query = any(w in qlow for w in ["remember", "note", "recall", "memo", "stored"]) + if not is_memory_query: + # Apply penalty and cap at 1 memory result unless explicitly requested + memory_penalty = float(os.environ.get("HYBRID_MEMORY_PENALTY", "0.15") or 0.15) + rec["mem_penalty"] = float(rec.get("mem_penalty", 0.0)) - memory_penalty + rec["s"] -= memory_penalty + # Simple lexical overlap check: drop memories with no query token overlap + try: + text_lower = str(md.get("text") or md.get("information") or "").lower() + query_tokens = set() + for q in qlist: + query_tokens.update(_split_ident(q)) + has_overlap = any(t in text_lower for t in query_tokens if len(t) >= 3) + if not has_overlap: + # Strong penalty for irrelevant memories + rec["mem_penalty"] -= 0.5 + rec["s"] -= 0.5 + except Exception: + pass if timestamps and RECENCY_WEIGHT > 0.0: tmin, tmax = min(timestamps), max(timestamps) @@ -1628,6 +1784,10 @@ def _pass_filters(m: Dict[str, Any]) -> bool: # Prefer merged bounds if present start_line = m.get("_merged_start") or md.get("start_line") end_line = m.get("_merged_end") or md.get("end_line") + # Store both fusion_score (from hybrid) and rerank_score (if available) separately + fusion_score = float(m.get("s", 0.0)) + rerank_score = m.get("rerank_score") # None if not reranked + comp = { "dense_rrf": round(float(m.get("d", 0.0)), 4), "lexical": round(float(m.get("lx", 0.0)), 4), @@ -1643,6 +1803,10 @@ def _pass_filters(m: Dict[str, Any]) -> bool: "impl_boost": round(float(m.get("impl", 0.0)), 4), "doc_penalty": round(float(m.get("doc", 0.0)), 4), } + + # Add reranker info to components if present + if rerank_score is not None: + comp["rerank"] = round(float(rerank_score), 4) why = [] if comp["dense_rrf"]: why.append(f"dense_rrf:{comp['dense_rrf']}") @@ -1782,6 +1946,8 @@ def _resolve(seg: str) -> list[str]: { "score": round(float(m["s"]), 4), "raw_score": float(m["s"]), # expose raw fused score for downstream budgeter + "fusion_score": round(fusion_score, 4), # Always store fusion score + "rerank_score": round(float(rerank_score), 4) if rerank_score is not None else None, # Store rerank separately "path": _emit_path, "host_path": _host, "container_path": _cont, diff --git a/scripts/ingest_code.py b/scripts/ingest_code.py index 60ef1929..48231603 100644 --- a/scripts/ingest_code.py +++ b/scripts/ingest_code.py @@ -1783,6 +1783,7 @@ def make_point(pid, dense_vec, lex_vec, payload): "metadata": { "path": str(file_path), "path_prefix": str(file_path.parent), + "ext": str(file_path.suffix).lstrip(".").lower(), "language": language, "kind": kind, "symbol": sym, @@ -2117,6 +2118,7 @@ def make_point(pid, dense_vec, lex_vec, payload): "metadata": { "path": str(file_path), "path_prefix": str(file_path.parent), + "ext": str(file_path.suffix).lstrip(".").lower(), "language": language, "kind": kind, "symbol": sym, diff --git a/scripts/logger.py b/scripts/logger.py new file mode 100644 index 00000000..ef807ecf --- /dev/null +++ b/scripts/logger.py @@ -0,0 +1,228 @@ +"""Structured logging utility for Context Engine. + +Provides JSON-formatted logging with context and proper exception handling. +""" +import logging +import json +import sys +import traceback +from typing import Any, Dict, Optional +from datetime import datetime + +# Configure root logger +logging.basicConfig( + level=logging.INFO, + format='%(message)s', + handlers=[logging.StreamHandler(sys.stdout)] +) + + +class JSONFormatter(logging.Formatter): + """Format log records as JSON for structured logging.""" + + def format(self, record: logging.LogRecord) -> str: + log_data: Dict[str, Any] = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + + # Add exception info if present + if record.exc_info: + log_data["exception"] = { + "type": record.exc_info[0].__name__ if record.exc_info[0] else None, + "message": str(record.exc_info[1]) if record.exc_info[1] else None, + "traceback": traceback.format_exception(*record.exc_info), + } + + # Add extra fields + if hasattr(record, 'extra_fields'): + log_data.update(record.extra_fields) + + return json.dumps(log_data) + + +def get_logger(name: str, json_format: bool = False) -> logging.Logger: + """Get a logger instance with optional JSON formatting. + + Args: + name: Logger name (typically __name__) + json_format: If True, use JSON formatter; otherwise use default + + Returns: + Configured logger instance + """ + logger = logging.getLogger(name) + + if json_format and not any(isinstance(h.formatter, JSONFormatter) for h in logger.handlers): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JSONFormatter()) + logger.addHandler(handler) + logger.propagate = False + + return logger + + +class ContextLogger: + """Logger wrapper that adds context fields to all log messages.""" + + def __init__(self, logger: logging.Logger, **context): + self.logger = logger + self.context = context + + def _log(self, level: int, msg: str, exc_info: Any = None, **extra): + """Internal log method that merges context.""" + merged = {**self.context, **extra} + record = self.logger.makeRecord( + self.logger.name, + level, + "(unknown file)", + 0, + msg, + (), + exc_info, + ) + record.extra_fields = merged + self.logger.handle(record) + + def debug(self, msg: str, **extra): + self._log(logging.DEBUG, msg, **extra) + + def info(self, msg: str, **extra): + self._log(logging.INFO, msg, **extra) + + def warning(self, msg: str, **extra): + self._log(logging.WARNING, msg, **extra) + + def error(self, msg: str, exc_info: Any = None, **extra): + self._log(logging.ERROR, msg, exc_info=exc_info, **extra) + + def exception(self, msg: str, **extra): + """Log an exception with traceback.""" + self._log(logging.ERROR, msg, exc_info=sys.exc_info(), **extra) + + def critical(self, msg: str, exc_info: Any = None, **extra): + self._log(logging.CRITICAL, msg, exc_info=exc_info, **extra) + + +# Custom exceptions for Context Engine +class ContextEngineError(Exception): + """Base exception for all Context Engine errors.""" + pass + + +class RetrievalError(ContextEngineError): + """Error during retrieval/search operations.""" + pass + + +class IndexingError(ContextEngineError): + """Error during indexing operations.""" + pass + + +class DecoderError(ContextEngineError): + """Error during LLM decoder operations.""" + pass + + +class ValidationError(ContextEngineError): + """Error during input validation.""" + pass + + +class ConfigurationError(ContextEngineError): + """Error in configuration or environment setup.""" + pass + + +# Convenience function for exception handling with logging +def log_and_reraise(logger: logging.Logger, msg: str, exc: Exception, **context): + """Log an exception with context and re-raise it. + + Args: + logger: Logger instance + msg: Error message + exc: Exception to log and re-raise + **context: Additional context fields + """ + if isinstance(logger, ContextLogger): + logger.exception(msg, **context) + else: + logger.exception(msg, extra=context) + raise exc + + +def safe_int(value: Any, default: int, logger: Optional[logging.Logger] = None, context: str = "") -> int: + """Safely convert value to int with logging on failure. + + Args: + value: Value to convert + default: Default value if conversion fails + logger: Optional logger for warnings + context: Context string for log message + + Returns: + Converted int or default + """ + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + return int(value) + except (ValueError, TypeError) as e: + if logger: + logger.warning(f"Failed to convert {context} to int: {value}", exc_info=e) + return default + + +def safe_float(value: Any, default: float, logger: Optional[logging.Logger] = None, context: str = "") -> float: + """Safely convert value to float with logging on failure. + + Args: + value: Value to convert + default: Default value if conversion fails + logger: Optional logger for warnings + context: Context string for log message + + Returns: + Converted float or default + """ + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + return float(value) + except (ValueError, TypeError) as e: + if logger: + logger.warning(f"Failed to convert {context} to float: {value}", exc_info=e) + return default + + +def safe_bool(value: Any, default: bool, logger: Optional[logging.Logger] = None, context: str = "") -> bool: + """Safely convert value to bool with logging on failure. + + Args: + value: Value to convert + default: Default value if conversion fails + logger: Optional logger for warnings + context: Context string for log message + + Returns: + Converted bool or default + """ + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + if isinstance(value, bool): + return value + s = str(value).strip().lower() + if s in {"1", "true", "yes", "on"}: + return True + if s in {"0", "false", "no", "off"}: + return False + return default + except (ValueError, TypeError) as e: + if logger: + logger.warning(f"Failed to convert {context} to bool: {value}", exc_info=e) + return default + diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index 62d2887c..b4e79785 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -35,6 +35,9 @@ import sys +# Import structured logging and error handling (after sys.path setup) +# Will be imported after sys.path is configured below + # Ensure code roots are on sys.path so absolute imports like 'from scripts.x import y' work # when this file is executed directly (sys.path[0] may be /work/scripts). # Supports multiple roots via WORK_ROOTS env (comma-separated), defaults to /work and /app. @@ -51,6 +54,55 @@ except Exception: pass +# Import structured logging and error handling (after sys.path setup) +try: + from scripts.logger import ( + get_logger, + ContextLogger, + RetrievalError, + IndexingError, + DecoderError, + ValidationError, + ConfigurationError, + safe_int, + safe_float, + safe_bool, + ) + logger = get_logger(__name__) +except ImportError: + # Fallback if logger module not available + import logging + logger = logging.getLogger(__name__) + logging.basicConfig(level=logging.INFO) + # Define fallback safe conversion functions + def safe_int(value, default=0, logger=None, context=""): + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + return int(value) + except (ValueError, TypeError): + return default + def safe_float(value, default=0.0, logger=None, context=""): + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + return float(value) + except (ValueError, TypeError): + return default + def safe_bool(value, default=False, logger=None, context=""): + try: + if value is None or (isinstance(value, str) and value.strip() == ""): + return default + if isinstance(value, bool): + return value + s = str(value).strip().lower() + if s in {"1", "true", "yes", "on"}: + return True + if s in {"0", "false", "no", "off"}: + return False + return default + except (ValueError, TypeError): + return default # Shared utilities (lex hashing, snippet highlighter) try: @@ -76,31 +128,68 @@ def _highlight_snippet(snippet, tokens): # type: ignore APP_NAME = os.environ.get("FASTMCP_SERVER_NAME", "qdrant-indexer-mcp") HOST = os.environ.get("FASTMCP_HOST", "0.0.0.0") -PORT = int(os.environ.get("FASTMCP_INDEXER_PORT", "8001")) +PORT = safe_int(os.environ.get("FASTMCP_INDEXER_PORT", "8001"), default=8001, logger=logger, context="FASTMCP_INDEXER_PORT") # Process-wide lock to guard environment mutations during retrieval (gate-first/budgeting) _ENV_LOCK = threading.RLock() +# Set default environment variables for context_answer functionality +# These are set in docker-compose.yml but provide fallbacks for local dev + def _primary_identifier_from_queries(qs: list[str]) -> str: - """Best-effort extraction of the main CONSTANT_NAME or IDENTIFIER from queries.""" + """Best-effort extraction of the main CONSTANT_NAME or IDENTIFIER from queries. + Now catches ALL_CAPS, snake_case, camelCase, and lowercase identifiers. + """ try: import re as _re cand: list[str] = [] for q in qs: for t in _re.findall(r"[A-Za-z_][A-Za-z0-9_]*", q or ""): - if len(t) >= 2 and ("_" in t or t.isupper()): + if len(t) < 2: + continue + # Accept: ALL_CAPS, has_underscore, camelCase (mixed case), or longer lowercase + is_all_caps = t.isupper() + has_underscore = "_" in t + is_camel = any(c.isupper() for c in t[1:]) and any(c.islower() for c in t) + is_longer_lower = t.islower() and len(t) >= 3 + + if is_all_caps or has_underscore or is_camel or is_longer_lower: cand.append(t) - return next((t for t in cand if t), "") + + # Prefer stronger identifiers: ALL_CAPS > camelCase > snake_case > lowercase + # Using _split_ident scoring: count segments as a heuristic for "strength" + if not cand: + return "" + + def _score(token: str) -> int: + score = 0 + if token.isupper(): + score += 100 # ALL_CAPS highest priority + if "_" in token: + score += 50 # snake_case + if any(c.isupper() for c in token[1:]) and any(c.islower() for c in token): + score += 75 # camelCase + score += len(token) # Longer is slightly better + return score + + cand.sort(key=_score, reverse=True) + return cand[0] if cand else "" except Exception: return "" QDRANT_URL = os.environ.get("QDRANT_URL", "http://qdrant:6333") DEFAULT_COLLECTION = os.environ.get("COLLECTION_NAME", "my-collection") -MAX_LOG_TAIL = int(os.environ.get("MCP_MAX_LOG_TAIL", "4000")) -SNIPPET_MAX_BYTES = int(os.environ.get("MCP_SNIPPET_MAX_BYTES", "8192") or 8192) +MAX_LOG_TAIL = safe_int(os.environ.get("MCP_MAX_LOG_TAIL", "4000"), default=4000, logger=logger, context="MCP_MAX_LOG_TAIL") +SNIPPET_MAX_BYTES = safe_int(os.environ.get("MCP_SNIPPET_MAX_BYTES", "8192"), default=8192, logger=logger, context="MCP_SNIPPET_MAX_BYTES") -MCP_TOOL_TIMEOUT_SECS = float(os.environ.get("MCP_TOOL_TIMEOUT_SECS", "3600") or 3600.0) +MCP_TOOL_TIMEOUT_SECS = safe_float(os.environ.get("MCP_TOOL_TIMEOUT_SECS", "3600"), default=3600.0, logger=logger, context="MCP_TOOL_TIMEOUT_SECS") + +# Set default environment variables for context_answer functionality +os.environ.setdefault("DEBUG_CONTEXT_ANSWER", "1") +os.environ.setdefault("REFRAG_DECODER", "1") +os.environ.setdefault("LLAMACPP_URL", "http://localhost:8080") +os.environ.setdefault("CTX_REQUIRE_IDENTIFIER", "0") # Disable strict identifier requirement # --- Workspace state integration helpers --- def _state_file_path(ws_path: str = "/work") -> str: @@ -128,7 +217,9 @@ def _default_collection() -> str: coll = st.get("qdrant_collection") if isinstance(coll, str) and coll.strip(): return coll.strip() - return DEFAULT_COLLECTION + # Fall back to current environment rather than module-load default so tests + # and dynamic collection switching work correctly. + return os.environ.get("COLLECTION_NAME", DEFAULT_COLLECTION) @@ -174,20 +265,22 @@ def _inner(fn): "name": dkwargs.get("name") or getattr(fn, "__name__", ""), "description": (getattr(fn, "__doc__", None) or "").strip(), }) - except Exception: - pass + except (AttributeError, TypeError) as e: + logger.warning(f"Failed to capture tool metadata for {fn}", exc_info=e) return orig_deco(fn) return _inner mcp.tool = _tool_capture_wrapper # type: ignore -except Exception: - pass +except (AttributeError, TypeError) as e: + logger.warning("Failed to wrap mcp.tool decorator", exc_info=e) # Lightweight readiness endpoint on a separate health port (non-MCP), optional # Exposes GET /readyz returning {ok: true, app: } once process is up. -try: - HEALTH_PORT = int(os.environ.get("FASTMCP_HEALTH_PORT", "18001") or 18001) -except Exception: - HEALTH_PORT = 18001 +HEALTH_PORT = safe_int( + os.environ.get("FASTMCP_HEALTH_PORT", "18001"), + default=18001, + logger=logger, + context="FASTMCP_HEALTH_PORT" +) def _start_readyz_server(): @@ -1237,25 +1330,37 @@ def _to_str_list(x): model_name = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") model = _get_embedding_model(model_name) - # In-process path_glob/not_glob accept a single string; reduce list inputs safely - items = run_hybrid_search( - queries=queries, - limit=int(limit), - per_path=(int(per_path) if (per_path is not None and str(per_path).strip() != "") else 1), - language=language or None, - under=under or None, - kind=kind or None, - symbol=symbol or None, - ext=ext or None, - not_filter=not_ or None, - case=case or None, - path_regex=path_regex or None, - path_glob=(path_globs or None), - not_glob=(not_globs or None), - expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() - in {"1", "true", "yes", "on"}, - model=model, - ) + # Ensure hybrid_search uses the intended collection when running in-process + prev_coll = os.environ.get("COLLECTION_NAME") + try: + os.environ["COLLECTION_NAME"] = collection + # In-process path_glob/not_glob accept a single string; reduce list inputs safely + items = run_hybrid_search( + queries=queries, + limit=int(limit), + per_path=(int(per_path) if (per_path is not None and str(per_path).strip() != "") else 1), + language=language or None, + under=under or None, + kind=kind or None, + symbol=symbol or None, + ext=ext or None, + not_filter=not_ or None, + case=case or None, + path_regex=path_regex or None, + path_glob=(path_globs or None), + not_glob=(not_globs or None), + expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() + in {"1", "true", "yes", "on"}, + model=model, + ) + finally: + if prev_coll is None: + try: + del os.environ["COLLECTION_NAME"] + except Exception: + pass + else: + os.environ["COLLECTION_NAME"] = prev_coll # items are already in structured dict form json_lines = items # reuse downstream shaping except Exception as e: @@ -2789,6 +2894,17 @@ async def context_answer( temperature: Any = None, mode: Any = None, # "stitch" (default) or "pack" expand: Any = None, # whether to LLM-expand queries (up to 2 alternates) + # Retrieval filter parameters (passed through to hybrid_search) + language: Any = None, + under: Any = None, + kind: Any = None, + symbol: Any = None, + ext: Any = None, + path_regex: Any = None, + path_glob: Any = None, + not_glob: Any = None, + case: Any = None, + not_: Any = None, **kwargs, ) -> Dict[str, Any]: """Answer a question using gate-first retrieval (ReFRAG Option B), assembling @@ -2809,44 +2925,86 @@ async def context_answer( - LLAMACPP_URL (default http://localhost:8080) - REFRAG_DECODER=1 to enable llama.cpp calls """ - # Unwrap kwargs if MCP client sent everything in a single kwargs string - if kwargs and not query: - query = kwargs.get("query", query) - limit = kwargs.get("limit", limit) - per_path = kwargs.get("per_path", per_path) - budget_tokens = kwargs.get("budget_tokens", budget_tokens) - include_snippet = kwargs.get("include_snippet", include_snippet) - collection = kwargs.get("collection", collection) - max_tokens = kwargs.get("max_tokens", max_tokens) - temperature = kwargs.get("temperature", temperature) - mode = kwargs.get("mode", mode) - expand = kwargs.get("expand", expand) + # Normalize/unwrap kwargs, supporting nested {"arguments":{...}} or {"kwargs":{...}} + _raw = dict(kwargs or {}) + try: + for k in ("arguments", "kwargs"): + v = _raw.get(k) + if isinstance(v, dict): + for kk, vv in v.items(): + # Preserve any existing explicit value on _raw + _raw.setdefault(kk, vv) + except (TypeError, AttributeError) as e: + logger.warning("Failed to unwrap nested kwargs", exc_info=e, extra={"raw_keys": list(_raw.keys())}) + + # Overlay helper: prefer non-empty values from wrapper, otherwise keep explicit param + def _coalesce(val, fallback): + if val is None: + return fallback + try: + if isinstance(val, str) and val.strip() == "": + return fallback + except (AttributeError, TypeError): + # val is not a string or doesn't have strip() + pass + return val + + query = _coalesce(_raw.get("query"), query) + limit = _coalesce(_raw.get("limit"), limit) + per_path = _coalesce(_raw.get("per_path"), per_path) + budget_tokens = _coalesce(_raw.get("budget_tokens"), budget_tokens) + include_snippet = _coalesce(_raw.get("include_snippet"), include_snippet) + collection = _coalesce(_raw.get("collection"), collection) + max_tokens = _coalesce(_raw.get("max_tokens"), max_tokens) + temperature = _coalesce(_raw.get("temperature"), temperature) + mode = _coalesce(_raw.get("mode"), mode) + expand = _coalesce(_raw.get("expand"), expand) + language = _coalesce(_raw.get("language"), language) + under = _coalesce(_raw.get("under"), under) + kind = _coalesce(_raw.get("kind"), kind) + symbol = _coalesce(_raw.get("symbol"), symbol) + ext = _coalesce(_raw.get("ext"), ext) + path_regex = _coalesce(_raw.get("path_regex"), path_regex) + path_glob = _coalesce(_raw.get("path_glob"), path_glob) + not_glob = _coalesce(_raw.get("not_glob"), not_glob) + case = _coalesce(_raw.get("case"), case) + not_ = _coalesce(_raw.get("not_"), not_) if _raw.get("not_") is not None else _coalesce(_raw.get("not"), not_) # Normalize query to list[str] queries: list[str] = [] - if isinstance(query, (list, tuple)): - queries = [str(q).strip() for q in query if str(q).strip()] - elif isinstance(query, str): - queries = _to_str_list_relaxed(query) - elif query is not None: - s = str(query).strip() - if s: - queries = [s] + try: + if isinstance(query, (list, tuple)): + queries = [str(q).strip() for q in query if str(q).strip()] + elif isinstance(query, str): + queries = _to_str_list_relaxed(query) + elif query is not None: + s = str(query).strip() + if s: + queries = [s] + except (TypeError, ValueError) as e: + logger.warning("Failed to normalize query", exc_info=e, extra={"raw_query": query}) + raise ValidationError(f"Invalid query format: {e}") + + # Debug: log raw and normalized query when running over MCP to diagnose null argument issues + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + try: + logger.debug("ARG_SHAPE", extra={ + "raw_query": repr(query), + "query_type": type(query).__name__, + "normalized_queries": queries, + }) + except (AttributeError, TypeError) as e: + logger.debug("Failed to log debug ARG_SHAPE", exc_info=e) + if not queries: - return {"error": "query required"} + raise ValidationError("query required") # Effective limits # For Q&A, we want more results per file to get comprehensive context - try: - lim = int(limit) if (limit is not None and str(limit).strip() != "") else 15 - except Exception: - lim = 15 - try: - # Default per_path=5 for Q&A (vs 1 for search) to get multiple snippets from same file - # This ensures we capture both definitions and usages - ppath = int(per_path) if (per_path is not None and str(per_path).strip() != "") else 5 - except Exception: - ppath = 5 + lim = safe_int(limit, default=15, logger=logger, context="limit") + # Default per_path=5 for Q&A (vs 1 for search) to get multiple snippets from same file + # This ensures we capture both definitions and usages + ppath = safe_int(per_path, default=5, logger=logger, context="per_path") # For identifier-focused questions, allow more snippets per file to capture both definition and usage try: @@ -2854,8 +3012,8 @@ async def context_answer( _ids0 = _re.findall(r"\b([A-Z_][A-Z0-9_]{2,})\b", " ".join(queries)) if _ids0: ppath = max(ppath, 5) - except Exception: - pass + except (AttributeError, re.error) as e: + logger.debug("Failed to extract identifiers for per_path adjustment", exc_info=e) # Collection + model setup (reuse indexer defaults) coll = (collection or _default_collection()) or "" @@ -2886,19 +3044,14 @@ async def context_answer( # Optionally expand queries via local decoder (tight cap) when requested queries = list(queries) # For LLM answering, default to include snippets so the model sees actual code - try: - if include_snippet in (None, ""): - include_snippet = True - except Exception: + if include_snippet in (None, ""): include_snippet = True - try: - do_expand = ( - (expand is True) or - (str(expand).strip().lower() in {"1","true","yes","on"}) or - (str(os.environ.get("HYBRID_EXPAND", "0")).strip().lower() in {"1","true","yes","on"}) - ) - except Exception: - do_expand = False + did_local_expand = False # Ensure defined even if expansion is disabled or fails + + + do_expand = safe_bool(expand, default=False, logger=logger, context="expand") or \ + safe_bool(os.environ.get("HYBRID_EXPAND", "0"), default=False, logger=logger, context="HYBRID_EXPAND") + if do_expand: try: from scripts.refrag_llamacpp import LlamaCppRefragClient, is_decoder_enabled # type: ignore @@ -2921,23 +3074,43 @@ async def context_answer( alts = [] try: parsed = _json.loads(out) - if isinstance(parsed, list): - for s in parsed: - if isinstance(s, str) and s and s not in queries: - alts.append(s) - if len(alts) >= 2: - break - except Exception: - pass + except (_json.JSONDecodeError, TypeError, ValueError): + # Salvage: try to extract a JSON array substring + try: + start = out.find("[") + end = out.rfind("]") + if start != -1 and end != -1 and end > start: + parsed = _json.loads(out[start:end+1]) + else: + parsed = [] + except Exception as e2: + logger.debug("Expand parse salvage failed", exc_info=e2) + parsed = [] + if isinstance(parsed, list): + for s in parsed: + if isinstance(s, str) and s and s not in queries: + alts.append(s) + if len(alts) >= 2: + break + if not alts and out and out.strip(): + # Heuristic fallback: split lines, trim bullets, take up to 2 + for cand in [t.strip().lstrip("-• ") for t in out.splitlines() if t.strip()][:2]: + if cand and cand not in queries and len(alts) < 2: + alts.append(cand) if alts: queries.extend(alts) - except Exception: - pass + did_local_expand = True # Mark that we already expanded + except (ImportError, AttributeError) as e: + logger.warning("Query expansion failed (decoder unavailable)", exc_info=e) + except (TimeoutError, ConnectionError) as e: + logger.warning("Query expansion failed (decoder timeout/connection)", exc_info=e) + except Exception as e: + logger.error("Unexpected error during query expansion", exc_info=e) # Default exclusions to avoid noisy self-test and cache artifacts # Also filter out metadata/config files that pollute Q&A context # This significantly improves answer quality by focusing on implementation code - user_not_glob = kwargs.get("not_glob") + user_not_glob = kwargs.get("not_glob") or not_glob if isinstance(user_not_glob, str): user_not_glob = [user_not_glob] base_excludes = [ @@ -2948,15 +3121,20 @@ async def context_answer( "node_modules/", # Dependencies ".git/", # VCS internals ] - # Add robust variants for absolute and recursive matching - # Simplified to avoid over-filtering + # Add glob patterns for default excludes - less aggressive than **/{pat}** + # Use exact path matching to avoid overfiltering def _variants(p: str) -> list[str]: - p = str(p).strip().strip() + p = str(p).strip() if not p: return [] p = p.replace("\\", "/").lstrip("/") - # Only use recursive glob pattern to catch all cases - return [f"**/{p}**"] + # Prefer exact path-prefix patterns; hybrid_search will add absolute (/work) variants + if p.endswith("/"): + base = p # directory + return [f"{base}*", f"/work/{base}*"] + else: + # File patterns: keep exact and absolute forms + return [p, f"/work/{p}"] # Build defaults and conditional exclusions (skip if explicitly mentioned in query) default_not_glob = [] @@ -2990,54 +3168,63 @@ def _mentions_any(keys: list[str]) -> bool: eff_not_glob.append(s) seen.add(s) - items = [] - err = None - try: - # Respect 'collection' arg by passing it via env to hybrid_search - coll = (collection or kwargs.get("collection") or os.environ.get("COLLECTION_NAME") or "").strip() - if coll: - os.environ["COLLECTION_NAME"] = coll + def _to_glob_list(val: Any) -> list[str]: + if not val: + return [] + if isinstance(val, (list, tuple, set)): + return [str(x).strip() for x in val if str(x).strip()] + vs = str(val).strip() + return [vs] if vs else [] + + req_language = kwargs.get("language") or language or None + eff_language = req_language + + user_path_glob = _to_glob_list(kwargs.get("path_glob") or path_glob) + eff_path_glob: list[str] = list(user_path_glob) + auto_path_glob: list[str] = [] + + cwd_root = os.path.abspath(os.getcwd()).replace("\\", "/").rstrip("/") + + def _abs_prefix(val: str) -> str: + v = (val or "").replace("\\", "/") + if not v: + return cwd_root + if v.startswith(cwd_root + "/") or v == cwd_root: + return v.rstrip("/") + if v.startswith("/"): + return f"{cwd_root}{v.rstrip('/')}" + return f"{cwd_root}/{v.lstrip('./').rstrip('/')}" + + user_under = kwargs.get("under") or under or None + override_under = None + if isinstance(user_under, str): + _uu = user_under.strip() + if _uu: + _uu_norm = _uu.replace("\\", "/") + # Detect file-like under hints (filename with an extension). For those, prefer + # a glob filter and avoid injecting an impossible path_prefix equality. + _uu_parts = [p for p in _uu_norm.split("/") if p] + _uu_last = _uu_parts[-1] if _uu_parts else _uu_norm + _looks_like_file = ("." in _uu_last) and not _uu_norm.endswith("/") + if _looks_like_file: + auto_path_glob.append(f"**/{_uu_last}") + if len(_uu_parts) > 1: + auto_path_glob.append(f"**/{_uu_norm}") + parent = "/".join(_uu_parts[:-1]) + if parent: + override_under = _abs_prefix(parent) + # Bare filename -> no override_under (would not match path_prefix) + else: + # Treat as directory prefix + override_under = _abs_prefix(_uu_norm) + elif user_under: + override_under = str(user_under) - # Debug: log the search parameters - if os.environ.get("DEBUG_CONTEXT_ANSWER"): - print(f"DEBUG SEARCH PARAMS: queries={queries}, limit={int(max(lim, 4))}, per_path={int(max(ppath, 0))}") - print(f"DEBUG ENV: REFRAG_MODE={os.environ.get('REFRAG_MODE')}, COLLECTION_NAME={os.environ.get('COLLECTION_NAME')}") - print(f"DEBUG FILTERS: not_glob={eff_not_glob}") - - - # Initial search (tighten file/area when hinted and no explicit path_glob) - def _to_glob_list(val: Any) -> list[str]: - if not val: - return [] - if isinstance(val, (list, tuple, set)): - return [str(x).strip() for x in val if str(x).strip()] - vs = str(val).strip() - return [vs] if vs else [] - - req_language = kwargs.get("language") or None - eff_language = req_language or ("python" if ("router" in qtext and not req_language) else None) - - user_path_glob = _to_glob_list(kwargs.get("path_glob")) - eff_path_glob: list[str] = list(user_path_glob) - - user_under = kwargs.get("under") or None - override_under = None - if isinstance(user_under, str): - _uu = user_under.strip() - if _uu: - if "/" not in _uu: - eff_path_glob.append(f"**/{_uu}") - else: - override_under = _uu if _uu.startswith("/") else f"/{_uu.lstrip('./')}" - elif user_under: - override_under = str(user_under) + # No hardcoded keyword-based filtering - keep it truly codebase-agnostic + # Let the hybrid search and user-provided filters do the work - # Router-specific tightening when the word 'router' is present - if ("router" in qtext): - if not eff_path_glob: - eff_path_glob = ["**/mcp_router.py", "**/*router*.py"] - if not eff_language: - eff_language = "python" + if auto_path_glob: + eff_path_glob.extend(auto_path_glob) # Normalize path_glob list -> None when empty if eff_path_glob: @@ -3052,13 +3239,6 @@ def _to_glob_list(val: Any) -> list[str]: eff_path_glob = dedup_pg else: eff_path_glob = None - # Sanitize symbol: ignore file-like strings passed as symbol - sym_arg = kwargs.get("symbol") or None - try: - if sym_arg and ("/" in str(sym_arg) or "." in str(sym_arg)): - sym_arg = None - except Exception: - pass # Query sharpening: extract code identifiers and add targeted search terms try: qj = " ".join(queries) @@ -3078,22 +3258,11 @@ def _add_query(q: str): if func_name and len(func_name) > 2: _add_query(f"def {func_name}(") else: - # For other queries, add basename if we have path_glob - if eff_path_glob: - def _basename(p: str) -> str: - s = str(p).replace("\\", "/").strip() - return s.split("/")[-1] if "/" in s else s - - basenames = [] - if isinstance(eff_path_glob, (list, tuple)): - basenames = [_basename(p) for p in eff_path_glob] - else: - basenames = [_basename(str(eff_path_glob))] - for bn in basenames: - if bn and bn not in queries: - queries.append(bn) - except Exception: - pass + # For other queries, DO NOT add basename - it confuses vector search + # The path_glob filter is applied at the hybrid_search level, not via query augmentation + pass + except (AttributeError, IndexError, re.error) as e: + logger.debug("Failed to augment query with identifier probes", exc_info=e) # Debug: log effective retrieval filters before search if os.environ.get("DEBUG_CONTEXT_ANSWER"): @@ -3103,6 +3272,34 @@ def _basename(p: str) -> str: except Exception as _e: print(f"DEBUG FILTERS: print failed: {_e}") + items = [] + err = None + try: + # Respect 'collection' arg by passing it via env to hybrid_search + coll = (collection or kwargs.get("collection") or os.environ.get("COLLECTION_NAME") or "").strip() + if coll: + os.environ["COLLECTION_NAME"] = coll + + # Debug: log the search parameters + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("SEARCH_PARAMS", extra={ + "queries": queries, + "limit": int(max(lim, 4)), + "per_path": int(max(ppath, 0)), + "env": { + "REFRAG_MODE": os.environ.get("REFRAG_MODE"), + "COLLECTION_NAME": os.environ.get("COLLECTION_NAME") + }, + "not_glob": eff_not_glob, + }) + + # Sanitize symbol: ignore file-like strings passed as symbol + sym_arg = kwargs.get("symbol") or None + try: + if sym_arg and ("/" in str(sym_arg) or "." in str(sym_arg)): + sym_arg = None + except Exception: + pass items = run_hybrid_search( queries=queries, @@ -3110,15 +3307,15 @@ def _basename(p: str) -> str: per_path=int(max(ppath, 0)), language=eff_language, under=override_under or None, - kind=kwargs.get("kind") or None, + kind=(kind or kwargs.get("kind") or None), symbol=sym_arg, - ext=kwargs.get("ext") or None, - not_filter=kwargs.get("not_") or kwargs.get("not") or None, - case=kwargs.get("case") or None, - path_regex=kwargs.get("path_regex") or None, + ext=(ext or kwargs.get("ext") or None), + not_filter=(not_ or kwargs.get("not_") or kwargs.get("not") or None), + case=(case or kwargs.get("case") or None), + path_regex=(path_regex or kwargs.get("path_regex") or None), path_glob=eff_path_glob, not_glob=eff_not_glob, - expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}, + expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}), model=model, ) @@ -3145,15 +3342,15 @@ def _basename(p: str) -> str: per_path=int(max(ppath, 10)), language=eff_language, under=override_under or None, - kind=kwargs.get("kind") or None, + kind=(kind or kwargs.get("kind") or None), symbol=sym_arg, - ext=kwargs.get("ext") or None, - not_filter=kwargs.get("not_") or kwargs.get("not") or None, - case=kwargs.get("case") or None, - path_regex=kwargs.get("path_regex") or None, + ext=(ext or kwargs.get("ext") or None), + not_filter=(not_ or kwargs.get("not_") or kwargs.get("not") or None), + case=(case or kwargs.get("case") or None), + path_regex=(path_regex or kwargs.get("path_regex") or None), path_glob=eff_path_glob, not_glob=eff_not_glob, - expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}, + expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}), model=model, ) # Merge unique by (path, start_line, end_line) @@ -3188,15 +3385,15 @@ def _ikey(it: Dict[str, Any]): per_path=5, language=eff_language, under=override_under or None, - kind=kwargs.get("kind") or None, + kind=(kind or kwargs.get("kind") or None), symbol=sym_arg, - ext=kwargs.get("ext") or None, - not_filter=kwargs.get("not_") or kwargs.get("not") or None, - case=kwargs.get("case") or None, - path_regex=kwargs.get("path_regex") or None, + ext=(ext or kwargs.get("ext") or None), + not_filter=(not_ or kwargs.get("not_") or kwargs.get("not") or None), + case=(case or kwargs.get("case") or None), + path_regex=(path_regex or kwargs.get("path_regex") or None), path_glob=eff_path_glob, not_glob=eff_not_glob, - expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}, + expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}), model=model, ) # Merge targeted results with higher priority @@ -3209,7 +3406,8 @@ def _ikey(it: Dict[str, Any]): added += 1 if os.environ.get("DEBUG_CONTEXT_ANSWER"): print(f"DEBUG TARGETED_SEARCH: found {len(targeted_items)} items, added {len([it for it in targeted_items if _ikey(it) not in _seen])}") - except Exception as _e: + except (re.error, AttributeError, KeyError) as _e: + logger.debug("Usage augmentation failed", exc_info=_e, extra={"asked": _asked if '_asked' in locals() else None}) if os.environ.get("DEBUG_CONTEXT_ANSWER"): print(f"DEBUG USAGE_AUGMENT failed: {_e}") @@ -3240,18 +3438,33 @@ def _ok_lang(it: Dict[str, Any]) -> bool: return bool(_lmp(str(req_language), p)) except Exception: pass - # Fallback simple ext mapping - ext = (p.rsplit(".", 1)[-1] if "." in p else "").lower() + # Fallback robust ext mapping with multi-part extension support + # Extract all possible extensions (e.g., for "foo.d.ts" -> ["ts", "d.ts"]) + filename = p.split("/")[-1] if "/" in p else p + parts = filename.split(".") + extensions = set() + if len(parts) > 1: + # Single extension: "file.py" -> "py" + extensions.add(parts[-1].lower()) + # Multi-part: "file.d.ts" -> "d.ts", "test.py" -> doesn't add extra + if len(parts) > 2: + multi_ext = ".".join(parts[-2:]).lower() + extensions.add(multi_ext) + table = { - "python": ["py"], - "typescript": ["ts", "tsx"], + "python": ["py", "pyi"], + "typescript": ["ts", "tsx", "d.ts", "mts", "cts"], "javascript": ["js", "jsx", "mjs", "cjs"], "go": ["go"], "rust": ["rs"], "java": ["java"], "php": ["php"], + "c": ["c", "h"], + "cpp": ["cpp", "cc", "cxx", "hpp", "hxx"], + "csharp": ["cs"], } - return ext in table.get(str(req_language).lower(), []) + lang_exts = table.get(str(req_language).lower(), []) + return any(ext in lang_exts for ext in extensions) items = [it for it in items if _ok_lang(it)] # Debug: log search results @@ -3260,36 +3473,118 @@ def _ok_lang(it: Dict[str, Any]) -> bool: for i, item in enumerate(items[:3]): print(f" Item {i+1}: {item.get('path')} lines {item.get('start_line')}-{item.get('end_line')}") - # Fallback: only if no strict filters like language were provided - if (not items) and (not req_language): + # Tier 2 fallback: broader hybrid search without gating/tight filters (unconditional when Tier 1 yields zero) + if not items: if os.environ.get("DEBUG_CONTEXT_ANSWER"): - print("DEBUG: 0 items after gate-first; retrying without gating") + logger.debug("TIER2: gate-first returned 0; retrying with relaxed filters", extra={"stage": "tier2"}) _prev_gate = os.environ.get("REFRAG_GATE_FIRST") os.environ["REFRAG_GATE_FIRST"] = "0" try: + # Tier 2: Remove tight filters (symbol, path_glob, path_regex, kind, ext) + # Keep only safety filters (language, under, not_glob) items = run_hybrid_search( queries=queries, - limit=int(max(lim, 4)), - per_path=int(max(ppath, 0)), + limit=int(max(lim * 2, 8)), # Cast wider net + per_path=int(max(ppath * 2, 4)), language=eff_language, under=override_under or None, - kind=kwargs.get("kind") or None, - symbol=sym_arg, - ext=kwargs.get("ext") or None, - not_filter=kwargs.get("not_") or kwargs.get("not") or None, - case=kwargs.get("case") or None, - path_regex=kwargs.get("path_regex") or None, - path_glob=eff_path_glob, - not_glob=eff_not_glob, - expand=str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}, + kind=None, # Remove kind filter + symbol=None, # Remove symbol filter + ext=None, # Remove ext filter + not_filter=(not_ or kwargs.get("not_") or kwargs.get("not") or None), + case=(case or kwargs.get("case") or None), + path_regex=None, # Remove path_regex filter + path_glob=None, # Remove path_glob filter + not_glob=eff_not_glob, # Keep not_glob for safety + expand=False if did_local_expand else (str(os.environ.get("HYBRID_EXPAND", "1")).strip().lower() in {"1","true","yes","on"}), model=model, ) + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("TIER2: broader hybrid returned items", extra={"count": len(items)}) finally: if _prev_gate is not None: os.environ["REFRAG_GATE_FIRST"] = _prev_gate else: os.environ.pop("REFRAG_GATE_FIRST", None) + # Tier 3 fallback: filesystem heuristics (deterministic regex scans when vector store has zero) + # Only trigger when: (a) still zero items, (b) query looks like identifier search, (c) env allows, + # and (d) no strict filters are applied (language/path_glob/kind/ext/symbol/under) + _strict_filters = bool(eff_language or eff_path_glob or path_regex or sym_arg or ext or kind or override_under) + if (not items) and (str(os.environ.get("TIER3_FS_FALLBACK", "0")).strip().lower() in {"1", "true", "yes", "on"}) and (not _strict_filters): + try: + import re as _re + # Extract primary identifier from query + primary = _primary_identifier_from_queries(queries) + if primary and len(primary) >= 3: + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("TIER3: filesystem scan", extra={"identifier": primary}) + # Deterministic filesystem scan for definition patterns + scan_root = override_under or cwd_root + if not os.path.isabs(scan_root): + scan_root = os.path.join(cwd_root, scan_root) + # Limit scan to reasonable file count to avoid DoS + max_files = int(os.environ.get("TIER3_MAX_FILES", "500") or 500) + scanned = 0 + tier3_hits: list[Dict[str, Any]] = [] + # Walk filesystem looking for definition patterns + for root, dirs, files in os.walk(scan_root): + # Skip excluded directories + dirs[:] = [d for d in dirs if not any(ex in d for ex in [".git", "node_modules", ".pytest_cache", "__pycache__"])] + for fname in files: + if scanned >= max_files: + break + # Only scan code files + if not any(fname.endswith(ext) for ext in [".py", ".js", ".ts", ".go", ".rs", ".java", ".cpp", ".c", ".h"]): + continue + fpath = os.path.join(root, fname) + try: + with open(fpath, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + scanned += 1 + # Look for definition patterns: IDENT = value, def IDENT, class IDENT + for idx, line in enumerate(lines, 1): + # Match: IDENT = value, def IDENT(, class IDENT:, const IDENT = + if _re.search(rf"\b{_re.escape(primary)}\b\s*[=:(]", line): + # Found definition - create a synthetic item compatible with Qdrant schema + try: + rel_path = os.path.relpath(fpath, cwd_root) + except ValueError: + # Handle cross-drive paths on Windows + rel_path = fpath.replace(cwd_root, "").lstrip("/\\") + snippet_start = max(1, idx - 2) + snippet_end = min(len(lines), idx + 3) + snippet_text = "".join(lines[snippet_start - 1:snippet_end]) + # Infer language from extension + ext_map = {".py": "python", ".js": "javascript", ".ts": "typescript", + ".go": "go", ".rs": "rust", ".java": "java", ".cpp": "cpp", ".c": "c", ".h": "c"} + lang = next((v for k, v in ext_map.items() if fname.endswith(k)), "unknown") + tier3_hits.append({ + "path": rel_path, + "start_line": idx, + "end_line": idx, + "text": snippet_text.strip(), + "score": 1.0, # Deterministic match + "tier": "filesystem_scan", + "language": lang, + "kind": "definition", # Mark as definition for downstream processing + }) + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("TIER3: found", extra={"identifier": primary, "path": rel_path, "line": idx}) + break # One hit per file is enough + except (IOError, OSError, UnicodeDecodeError) as e: + logger.debug(f"Tier 3: Failed to scan {fpath}", exc_info=e) + continue + if scanned >= max_files: + break + if tier3_hits: + items = tier3_hits[:int(max(lim, 4))] # Respect limit + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("TIER3: filesystem scan returned", extra={"count": len(items), "scanned": scanned}) + except Exception as e: + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + logger.debug("TIER3: filesystem scan failed", exc_info=e) + # Filter out memory-like items without a valid path to avoid empty citations items = [it for it in items if str(it.get("path") or "").strip()] @@ -3306,7 +3601,8 @@ def _ok_lang(it: Dict[str, Any]) -> bool: if os.environ.get("DEBUG_CONTEXT_ANSWER"): print("DEBUG: Span budgeting returned empty, using raw items") budgeted = items - except Exception as e: + except (ImportError, AttributeError, KeyError) as e: + logger.warning("Span budgeting failed, using raw items", exc_info=e) if os.environ.get("DEBUG_CONTEXT_ANSWER"): print(f"DEBUG: Span budgeting failed: {e}, using raw items") budgeted = items @@ -3419,6 +3715,10 @@ def _span_key(span: Dict[str, Any]) -> tuple[str, int, int]: if os.environ.get("DEBUG_CONTEXT_ANSWER"): print(f"DEBUG IDENT AUGMENT: pulled {len(ident_candidates)} candidate spans for {primary_ident}") source_spans = ident_candidates + else: + # No identifier-bearing spans found; fall back to original source_spans + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + print(f"DEBUG IDENT AUGMENT: no candidates found for {primary_ident}, using original source_spans") if span_cap: spans = source_spans[:span_cap] @@ -3458,6 +3758,10 @@ def _is_def_span(span: Dict[str, Any]) -> bool: except Exception as e: err = str(e) spans = [] + if os.environ.get("DEBUG_CONTEXT_ANSWER"): + import traceback + print(f"DEBUG EXCEPTION: {type(e).__name__}: {err}") + traceback.print_exc() finally: # Restore env to previous values to avoid cross-call bleed for k, v in prev.items(): @@ -3513,7 +3817,6 @@ def _is_def_span(span: Dict[str, Any]) -> bool: if not _os.path.isabs(fp): fp = _os.path.join("/work", fp) realp = _os.path.realpath(fp) - # SECURITY: Verify the resolved path stays within /work to prevent path traversal if not realp.startswith("/work/"): if _os.environ.get("DEBUG_CONTEXT_ANSWER"): print(f"DEBUG: Blocked path traversal attempt: {path} -> {realp}") @@ -3586,7 +3889,12 @@ def _is_def_span(span: Dict[str, Any]) -> bool: # Stop sequences for Granite-4.0-Micro + optional env overrides stop_env = os.environ.get("DECODER_STOP", "") - default_stops = ["<|end_of_text|>", "<|start_of_role|>"] # Granite format tokens + default_stops = [ + "<|end_of_text|>", + "<|start_of_role|>", + "<|end_of_response|>", # Prevent response marker leakage + "\n\n\n", # Stop on excessive newlines + ] stops = default_stops + [s for s in (stop_env.split(",") if stop_env else []) if s] # Decoder parameter tuning for small models (defaults can be overridden via args or env) @@ -3611,6 +3919,7 @@ def _to_float(v, d): try: from scripts.refrag_llamacpp import LlamaCppRefragClient, is_decoder_enabled # type: ignore if not is_decoder_enabled(): + logger.info("Decoder disabled, returning error with citations") return { "error": "decoder disabled: set REFRAG_DECODER=1 and start llamacpp", "citations": citations, @@ -3645,21 +3954,27 @@ def _to_float(v, d): key_terms_str = ", ".join(sorted(key_terms)[:15]) if key_terms else "none found" - # Use Granite's proven RAG pattern for strict document grounding - # Based on official Granite 4.0 RAG examples that enforce "strictly aligning with facts" - system_msg = ( - "You are a helpful assistant with access to the following code snippets. " - "You may use one or more snippets to assist with the user query.\n\n" - "You are given code snippets with [ID] path:lines format:\n" - f"{all_context}\n\n" - + (f"Key identifiers: {key_terms_str}\n" if key_terms_str != "none found" else "") - + (f"Hint: {extra_hint}\n\n" if extra_hint else "") - + "Write the response to the user's input by strictly aligning with the facts in the provided code snippets. " - "Always answer in exactly two lines with citations: " - "Definition: \"\" [ID]\n" - "Usage: [ID]. " - "If the definition or usage is not present in the snippets, state 'Not found in provided snippets.' for that line." - ) + # Use simple labeled code blocks instead of JSON-in-XML + # Plain text works more reliably across different llama.cpp models + if context_blocks: + # Format as simple numbered code blocks with headers + docs_text = "\n\n".join(context_blocks) + + system_msg = ( + "You are a helpful assistant with access to the following code snippets. " + "You may use one or more snippets to assist with the user query.\n\n" + "Code snippets:\n" + f"{docs_text}\n\n" + "Write the response to the user's input by strictly aligning with the facts in the provided code snippets. " + "If the information needed to answer the question is not available in the snippets, " + "inform the user that the question cannot be answered based on the available data." + ) + else: + # No context available - inform user directly without LLM call + system_msg = ( + "You are a helpful assistant. " + "No code snippets are available to answer this query." + ) if sources_footer: system_msg += f"\nSources:\n{sources_footer}" @@ -3692,11 +4007,12 @@ def _to_float(v, d): ) # Optional length cap: if CTX_SUMMARY_CHARS is a positive int, apply after cleanup; otherwise don't cap - try: - _cap_env = str(os.environ.get("CTX_SUMMARY_CHARS", "")).strip() - _cap = int(_cap_env) if _cap_env not in {"", None} else 0 - except Exception: - _cap = 0 + _cap = safe_int(os.environ.get("CTX_SUMMARY_CHARS", ""), default=0, logger=logger, context="CTX_SUMMARY_CHARS") + + # Cleanup: remove stop tokens, repetition, and optionally cap length + # Strip leaked stop tokens that sometimes appear in output + for stop_tok in ["<|end_of_text|>", "<|start_of_role|>", "<|end_of_response|>"]: + answer = answer.replace(stop_tok, "") # Cleanup repetition and optionally cap length answer = _cleanup_answer(answer, max_chars=(_cap if _cap and _cap > 0 else None)) @@ -3758,7 +4074,8 @@ def _fmt_citation(cid: int | None) -> str: usage_text = _ln.strip() usage_cid = _usage_id break - except Exception: + except (AttributeError, KeyError, IndexError) as e: + logger.debug("Failed to extract usage line from snippet", exc_info=e) usage_text = "" usage_cid = None @@ -3792,8 +4109,9 @@ def _fmt_citation(cid: int | None) -> str: # Final strict two-line output answer = f"{def_line}\n{usage_line}".strip() - except Exception: + except (AttributeError, KeyError, IndexError, ValueError) as e: # If anything goes wrong, keep the original cleaned answer + logger.warning("Failed to format strict two-line answer", exc_info=e) answer = answer.strip() # Debug: log the cleaned/formatted LLM response diff --git a/scripts/qdrant_client_manager.py b/scripts/qdrant_client_manager.py new file mode 100644 index 00000000..07012326 --- /dev/null +++ b/scripts/qdrant_client_manager.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Qdrant client lifecycle management to prevent socket leaks. +Provides a singleton client that can be safely reused across the application. +""" +import os +import threading +from typing import Optional +from qdrant_client import QdrantClient + + +_client: Optional[QdrantClient] = None +_client_lock = threading.Lock() + + +def get_qdrant_client( + url: Optional[str] = None, + api_key: Optional[str] = None, + force_new: bool = False +) -> QdrantClient: + """ + Get or create a singleton Qdrant client. + + Args: + url: Qdrant URL (defaults to QDRANT_URL env var) + api_key: API key (defaults to QDRANT_API_KEY env var) + force_new: Create a new client instead of reusing singleton + + Returns: + QdrantClient instance + + Note: The singleton client is thread-safe for read operations. + For write-heavy operations, consider using force_new=True or + call close_qdrant_client() when done with a session. + """ + global _client + + if force_new: + url = url or os.environ.get("QDRANT_URL", "http://qdrant:6333") + api_key = api_key or os.environ.get("QDRANT_API_KEY") + return QdrantClient(url=url, api_key=api_key if api_key else None) + + with _client_lock: + if _client is None: + url = url or os.environ.get("QDRANT_URL", "http://qdrant:6333") + api_key = api_key or os.environ.get("QDRANT_API_KEY") + _client = QdrantClient(url=url, api_key=api_key if api_key else None) + return _client + + +def close_qdrant_client(): + """ + Close the singleton Qdrant client. + Should be called at application shutdown or when switching configurations. + """ + global _client + + with _client_lock: + if _client is not None: + try: + _client.close() + except Exception: + pass + _client = None + + +def reset_qdrant_client(url: Optional[str] = None, api_key: Optional[str] = None): + """ + Reset the singleton client with new configuration. + Useful when switching between different Qdrant instances. + """ + close_qdrant_client() + return get_qdrant_client(url=url, api_key=api_key) diff --git a/scripts/watch_index.py b/scripts/watch_index.py index ad344e5f..6b0cb955 100644 --- a/scripts/watch_index.py +++ b/scripts/watch_index.py @@ -505,7 +505,7 @@ def _process_paths(paths, client, model, vector_name: str, workspace_path: str): mname = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") model = TextEmbedding(model_name=mname) ok = idx.index_single_file( - client, model, COLLECTION, vector_name, p, dedupe=True, skip_unchanged=False + client, model, COLLECTION, vector_name, p, dedupe=True, skip_unchanged=True ) status = "indexed" if ok else "skipped" print(f"[{status}] {p}") diff --git a/tests/test_context_answer.py b/tests/test_context_answer.py index a3aa4584..67619ded 100644 --- a/tests/test_context_answer.py +++ b/tests/test_context_answer.py @@ -128,3 +128,60 @@ def generate_with_soft_embeddings(self, prompt: str, max_tokens: int = 256, **kw cits = out.get("citations") or [] assert len(cits) == 1 assert cits[0]["path"] == "/work/bar.py" + + +def test_context_answer_tier2_retry_without_gating(monkeypatch): + """Tier 2 should retry run_hybrid_search with relaxed filters when Tier 1 yields zero.""" + + import scripts.hybrid_search as hs + + calls = [] + + def _run_hybrid_search(**kwargs): + calls.append(kwargs) + # Only return results once Tier 2 relaxes the filters (path_glob None, symbol None) + if kwargs.get("path_glob") is None and kwargs.get("symbol") is None and len(calls) >= 1: + return [ + { + "score": 0.42, + "path": "/work/hybrid_search.py", + "symbol": "RRF_K", + "start_line": 100, + "end_line": 104, + "text": "RRF_K = 60\n", + } + ] + # All other calls (tier1/usage/targeted search) yield no hits + return [] + + monkeypatch.setattr(hs, "run_hybrid_search", _run_hybrid_search) + + import scripts.refrag_llamacpp as ref + + class FakeLlama: + def __init__(self, *a, **k): + pass + + def generate_with_soft_embeddings(self, *a, **kw): + return "Definition: \"RRF_K = 60\" [1]\nUsage: Not found in provided snippets. [1]" + + monkeypatch.setattr(ref, "LlamaCppRefragClient", FakeLlama) + monkeypatch.setattr(ref, "is_decoder_enabled", lambda: True) + + out = srv.asyncio.get_event_loop().run_until_complete( + srv.context_answer(query="RRF_K", limit=1, per_path=1) + ) + + # Ensure Tier 2 was invoked (run_hybrid_search called twice) + assert len(calls) >= 3, "Tier 2 fallback should re-run hybrid search" + + tier2_kwargs = calls[-1] + # Tier 2 should have relaxed filters + assert tier2_kwargs.get("path_glob") is None + assert tier2_kwargs.get("symbol") is None + assert tier2_kwargs.get("kind") is None + + # The final citations should come from the tier-2 hit + cits = out.get("citations") or [] + assert len(cits) == 1 + assert cits[0]["path"].endswith("hybrid_search.py") diff --git a/tests/test_logger_utils.py b/tests/test_logger_utils.py new file mode 100644 index 00000000..b405ade0 --- /dev/null +++ b/tests/test_logger_utils.py @@ -0,0 +1,34 @@ +"""Unit tests for scripts.logger helper functions.""" + +import logging + +import scripts.logger as logger_mod + + +def test_safe_converters_default_on_invalid_input(): + log = logging.getLogger("test.logger") + + assert logger_mod.safe_int("", 42, logger=log, context="int_field") == 42 + assert logger_mod.safe_int("17", 0) == 17 + + assert logger_mod.safe_float("abc", 3.14, logger=log, context="float_field") == 3.14 + assert logger_mod.safe_float("2.5", 0.0) == 2.5 + + assert logger_mod.safe_bool("yes", False) is True + assert logger_mod.safe_bool("NO", True) is False + assert logger_mod.safe_bool("maybe", True, logger=log, context="bool_field") is True + + +def test_context_logger_injects_extra_fields(caplog): + base_logger = logger_mod.get_logger("context-engine-test", json_format=False) + contextual = logger_mod.ContextLogger(base_logger, stage="test", request_id="abc123") + + with caplog.at_level(logging.INFO, logger="context-engine-test"): + contextual.info("hello", answer="ok") + + assert any("hello" in message for message in caplog.messages) + # Ensure contextual fields are preserved + record = caplog.records[-1] + assert getattr(record, "extra_fields", {}).get("stage") == "test" + assert getattr(record, "extra_fields", {}).get("request_id") == "abc123" + assert getattr(record, "extra_fields", {}).get("answer") == "ok" diff --git a/tests/test_subprocess_hybrid_smoke.py b/tests/test_subprocess_hybrid_smoke.py index 5bd60940..cdd5ee38 100644 --- a/tests/test_subprocess_hybrid_smoke.py +++ b/tests/test_subprocess_hybrid_smoke.py @@ -24,7 +24,18 @@ def qdrant_container(): container = DockerContainer("qdrant/qdrant:latest").with_exposed_ports(6333) container.start() host = container.get_container_host_ip() - port = int(container.get_exposed_port(6333)) + deadline = time.time() + 30 + last_exc: Exception | None = None + while time.time() < deadline: + try: + port = int(container.get_exposed_port(6333)) + break + except Exception as exc: # pragma: no cover - retry only in flaky envs + last_exc = exc + time.sleep(0.25) + else: + container.stop() + raise RuntimeError(f"qdrant port mapping unavailable: {last_exc}") url = f"http://{host}:{port}" # Poll readiness endpoint up to 60s deadline = time.time() + 60 @@ -44,7 +55,8 @@ def qdrant_container(): def test_hybrid_cli_runs_basic(tmp_path, qdrant_container): env = os.environ.copy() env["QDRANT_URL"] = qdrant_container - env.setdefault("COLLECTION_NAME", f"test-{uuid.uuid4().hex[:8]}") + collection_name = f"test-{uuid.uuid4().hex[:8]}" + env["COLLECTION_NAME"] = collection_name # Warm the FastEmbed model cache to avoid long first-run download inside subprocess try: @@ -60,14 +72,22 @@ def test_hybrid_cli_runs_basic(tmp_path, qdrant_container): "def test():\n return 1\n", encoding="utf-8" ) ing = importlib.import_module("scripts.ingest_code") - ing.index_repo( - root=tmp_path, - qdrant_url=qdrant_container, - api_key="", - collection=env["COLLECTION_NAME"], - model_name="BAAI/bge-base-en-v1.5", - recreate=True, - ) + prev_collection = os.environ.get("COLLECTION_NAME") + os.environ["COLLECTION_NAME"] = collection_name + try: + ing.index_repo( + root=tmp_path, + qdrant_url=qdrant_container, + api_key="", + collection=collection_name, + model_name="BAAI/bge-base-en-v1.5", + recreate=True, + ) + finally: + if prev_collection is None: + os.environ.pop("COLLECTION_NAME", None) + else: + os.environ["COLLECTION_NAME"] = prev_collection # Use the real model; allow time to download on first run env["EMBEDDING_MODEL"] = "BAAI/bge-base-en-v1.5"