Add DummySearchProvider Fallback and Settings Improvements

.env.example

@@ -4,4 +4,10 @@ VECTOR_DB_ENV=
 POSTGRES_URI=
 SECRET_KEY=
 # Comma-separated list of allowed CORS origins, e.g. http://localhost:3000,https://yourdomain.com
-CORS_ALLOW_ORIGINS=
+CORS_ALLOW_ORIGINS=
+
+# Web search configuration
+WEB_SEARCH_ENGINE=tavily
+TAVILY_API_KEY=your_tavily_api_key_here
+MAX_FETCH_CONCURRENCY=4
+DEFAULT_TOP_K_RESULTS=8

.gitignore

@@ -1,6 +1,7 @@
 authormaton/
 experimentalCode/.env
 .env
+
 # Ignore Python cache
 
 __pycache__/

api/indexing_router.py

@@ -1,15 +1,12 @@
 """
 Indexing router for /internal/index endpoint.
 """
-from fastapi import APIRouter, HTTPException, status, Request
+from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 from config.settings import settings
-from services.vector_db_service import VectorDBService
+from services.vector_db_service import VectorDBClient as VectorDBService
 from services.embedding_service import embed_texts_batched
 from services.chunking_service import chunk_text
-from services.parsing_service import extract_text_from_pdf, extract_text_from_docx
-import logging
-import os
 
 router = APIRouter(prefix="/internal", tags=["internal"])
 

api/main.py

@@ -46,8 +46,10 @@ def read_root():
 # Register routers
 from api.endpoints.upload import router as upload_router
 from api.endpoints.internal import router as internal_router
+from api.endpoints.web_answering import router as web_answering_router
 app.include_router(upload_router, prefix="/upload")
 app.include_router(internal_router)
+app.include_router(web_answering_router, prefix="/internal", tags=["websearch"])
 app.include_router(indexing_router)
 
 @app.get("/health")

config/settings.py

@@ -5,7 +5,8 @@
 """
 import os
 from pydantic_settings import BaseSettings
-from pydantic import SecretStr, ValidationError
+from pydantic import SecretStr, ValidationError, Field
+from typing import Optional
 import sys
 try:
     from dotenv import load_dotenv
@@ -24,6 +25,13 @@ class Settings(BaseSettings):
     embedding_dimension: int = 3072
     embed_batch_size: int = 128
     max_upload_mb: int = 25
+
+    # Web search settings
+    web_search_engine: str = os.environ.get("WEB_SEARCH_ENGINE", "dummy")  # Default to dummy provider if not specified
+    tavily_api_key: Optional[SecretStr] = None
+    bing_api_key: Optional[SecretStr] = None
+    max_fetch_concurrency: int = 4
+    default_top_k_results: int = 8
 
 try:
     settings = Settings()

requirements.txt

@@ -4,7 +4,7 @@ python-multipart>=0.0.6,<1.0.0
 fastapi>=0.110.0,<1.0.0
 uvicorn[standard]>=0.29.0,<1.0.0
 pytest>=8.2.0,<9.0.0
-httpx>=0.27.0,<1.0.0
+httpx[http2]>=0.27.0,<1.0.0
 python-dotenv>=1.0.0,<2.0.0
 PyPDF2>=3.0.0,<4.0.0
 requests>=2.31.0,<3.0.0
@@ -13,4 +13,6 @@ pinecone-client>=3.0.0,<4.0.0
 weaviate-client>=4.4.0,<5.0.0
 transformers>=4.40.0,<5.0.0
 torch>=2.2.0,<3.0.0
-pydantic>=2.6.0,<3.0.0
+pydantic>=2.6.0,<3.0.0
+trafilatura>=1.6.0,<2.0.0
+numpy>=1.26.0,<2.0.0

services/vector_db_service.py

@@ -47,6 +47,27 @@ def upsert_vectors(self, vectors: List[List[float]], ids: List[str]):
         if not self.index:
             raise RuntimeError("Index is not initialized. Call create_index first.")
         self.index.upsert(vectors=[(id, vec) for id, vec in zip(ids, vectors)])
+
+    def upsert(self, namespace, ids, vectors, metadata=None):
+        """
+        Upsert vectors into the index, ensuring index is created and metadata is validated.
+        """
+        if self.index is None:
+            self.create_index()
+        if not (len(ids) == len(vectors)):
+            raise ValueError("ids and vectors must have the same length")
+        if metadata is not None and len(metadata) != len(ids):
+            raise ValueError("metadata length must match ids/vectors length")
+        items = []
+        for i, (id_, vector) in enumerate(zip(ids, vectors)):
+            item = {
+                "id": id_,
+                "values": vector
+            }
+            if metadata is not None:
+                item["metadata"] = metadata[i]
+            items.append(item)
+        self.index.upsert(vectors=items, namespace=namespace)
 
     def query(self, vector: List[float], top_k: int = 5):
         if not self.index: