dep on pydantic v2 but import from pydantic.v1 (#498)

* dep on pydantic v2 but import from pydantic.v1

* cleanups

* make langroid.pydantic_v1

* use langroid.pydantic_v1 namespace

* pydantic >=1 < 3

* rm lancedb from "all" extras; lancedb pydantic err info

* pytest.yml re-run w pydantic 1.10.15 (to pass lancedb etc tests)

* pytest.yml 3rd try should be gpt-4

* pytest.yml continue on err

* pytest.yml install pydantic v1 using poetry

* use dummy langroid_pydantic_v1 constraint lib

* pytest syntax

* examples - fix pydantic imports

* update example

.github/workflows/pytest.yml

@@ -86,7 +86,7 @@ jobs:
         python -m pip install --upgrade pip
         
         poetry install \
-          -E "lancedb momento meilisearch chromadb hf-embeddings unstructured pdf-parsers docx sql" \
+          -E "momento meilisearch chromadb hf-embeddings unstructured pdf-parsers docx sql" \
           --with dev
 
     - name: Dump dependencies to file
@@ -110,6 +110,7 @@ jobs:
       continue-on-error: true
 
     - name: Retry FAILED tests ONLY with gpt-4 instead of gpt-4o (no coverage)
+      id: second_test
       if: steps.first_test.outcome == 'failure'
       run: |
         poetry run pytest --m gpt-4 \
@@ -119,6 +120,23 @@ jobs:
           --lf --last-failed-no-failures none \
           tests/main tests/extras/test_hf_embeddings.py
         poetry run coverage report
+      continue-on-error: true
+
+    - name: Install Langroid with "lancedb" extra which constrains to Pydantic 1.x
+      run: |
+        poetry install \
+        -E "lancedb momento meilisearch chromadb hf-embeddings unstructured pdf-parsers docx sql" \
+        --with dev
+    - name: Retry FAILED tests ONLY with "lancedb" extra (=> pydantic 1.10.15), gpt-4
+      if: steps.second_test.outcome == 'failure'
+      run: |
+        poetry run pytest --m gpt-4 \
+          --first-test-file=tests/main/test_task_inf_loop.py \
+          --first-test-file=tests/main/test_task.py \
+          --first-test-file=tests/main/test_lance_doc_chat_agent.py \
+          --lf --last-failed-no-failures none \
+          tests/main tests/extras/test_hf_embeddings.py
+        poetry run coverage report
 
     - name: Generate XML coverage report
       run: |

Makefile

@@ -35,8 +35,16 @@ stubs:
 	@poetry run stubgen -p langroid -o stubs
 	@echo "Stubs generated in the 'stubs' directory"
 
+.PHONY: fix-pydantic
+
+# Entry to replace pydantic imports in specified directories
+fix-pydantic:
+	@echo "Fixing pydantic imports..."
+	@chmod +x scripts/fix-pydantic-imports.sh
+	@./scripts/fix-pydantic-imports.sh
+
 .PHONY: check
-check: lint type-check
+check: fix-pydantic lint type-check
 
 .PHONY: tests
 tests:

examples/basic/chat-tool-function.py

@@ -13,7 +13,7 @@
 
 import langroid as lr
 import langroid.language_models as lm
-from pydantic import BaseModel, Field
+from langroid.pydantic_v1 import BaseModel, Field
 from fire import Fire
 
 # define a nested structure for Company information

examples/basic/fn-call-local-simple.py

@@ -20,7 +20,7 @@
 from typing import List
 import fire
 
-from pydantic import BaseModel, Field
+from langroid.pydantic_v1 import BaseModel, Field
 import langroid as lr
 from langroid.utils.configuration import settings
 from langroid.agent.tool_message import ToolMessage

examples/chainlit/extract-then-chat.py

@@ -14,7 +14,7 @@
 """
 
 from langroid import ChatDocument
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import os
 

examples/chainlit/multi-extract-3.py

@@ -32,7 +32,7 @@
 """
 
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import json
 import os

examples/chainlit/multi-extract.py

@@ -31,7 +31,7 @@
 """
 
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import json
 import os

examples/docqa/chat-multi-extract-3.py

@@ -31,7 +31,7 @@
 
 import typer
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import json
 import os

examples/docqa/chat-multi-extract-local.py

@@ -35,7 +35,7 @@
 
 import typer
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List, Optional
 import json
 import os

examples/docqa/chat-search.py

@@ -25,7 +25,7 @@
 from rich import print
 from rich.prompt import Prompt
 
-from pydantic import BaseSettings
+from langroid.pydantic_v1 import BaseSettings
 import langroid.language_models as lm
 from langroid.agent.tool_message import ToolMessage
 from langroid.agent.chat_agent import ChatAgent, ChatDocument
@@ -37,9 +37,7 @@
 from langroid.agent.task import Task
 from langroid.utils.constants import NO_ANSWER
 from langroid.utils.configuration import set_global, Settings
-
-app = typer.Typer()
-
+from fire import Fire
 
 class RelevantExtractsTool(ToolMessage):
     request = "relevant_extracts"
@@ -117,12 +115,20 @@ def relevant_search_extracts(self, msg: RelevantSearchExtractsTool) -> str:
         return "\n".join(str(e) for e in extracts)
 
 
-class CLIOptions(BaseSettings):
-    fn_api: bool = False
-    model: str = ""
+def main(
+    debug: bool = False,
+    nocache: bool = False,
+    model: str = "",
+    fn_api: bool = True,
+) -> None:
 
+    set_global(
+        Settings(
+            debug=debug,
+            cache=not nocache,
+        )
+    )
 
-def chat(opts: CLIOptions) -> None:
     print(
         """
         [blue]Welcome to the Internet Search chatbot!
@@ -145,7 +151,7 @@ def chat(opts: CLIOptions) -> None:
     system_msg = re.sub("you are", "", system_msg, flags=re.IGNORECASE)
 
     llm_config = lm.OpenAIGPTConfig(
-        chat_model=opts.model or lm.OpenAIChatModel.GPT4o,
+        chat_model=model or lm.OpenAIChatModel.GPT4o,
         # or, other possibilities for example:
         # "litellm/bedrock/anthropic.claude-instant-v1"
         # "ollama/llama2"
@@ -155,8 +161,8 @@ def chat(opts: CLIOptions) -> None:
     )
 
     config = DocChatAgentConfig(
-        use_functions_api=opts.fn_api,
-        use_tools=not opts.fn_api,
+        use_functions_api=fn_api,
+        use_tools=not fn_api,
         llm=llm_config,
         system_message=f"""
         {system_msg} You will try your best to answer my questions,
@@ -210,26 +216,5 @@ def chat(opts: CLIOptions) -> None:
     task.run("Can you help me answer some questions, possibly using web search?")
 
 
-@app.command()
-def main(
-    debug: bool = typer.Option(False, "--debug", "-d", help="debug mode"),
-    nocache: bool = typer.Option(False, "--nocache", "-nc", help="don't use cache"),
-    model: str = typer.Option("", "--model", "-m", help="model name"),
-    fn_api: bool = typer.Option(False, "--fn_api", "-f", help="use functions api"),
-) -> None:
-    cli_opts = CLIOptions(
-        fn_api=fn_api,
-        model=model,
-    )
-
-    set_global(
-        Settings(
-            debug=debug,
-            cache=not nocache,
-        )
-    )
-    chat(cli_opts)
-
-
 if __name__ == "__main__":
-    app()
+    Fire(main)

examples/docqa/chat_multi_extract.py

@@ -33,7 +33,7 @@
 
 import typer
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import json
 import os

examples/docqa/extract-then-chat.py

@@ -14,7 +14,7 @@
 """
 
 from langroid import ChatDocument
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import os
 from fire import Fire

examples/docqa/filter-multi-doc-auto.py

@@ -24,7 +24,7 @@
 from rich.prompt import Prompt
 import os
 
-from pydantic import Field
+from langroid.pydantic_v1 import Field
 import langroid as lr
 import langroid.language_models as lm
 from langroid.agent.special.doc_chat_agent import DocChatAgentConfig

examples/docqa/filter-multi-doc-manual.py

@@ -24,7 +24,7 @@
 from rich.prompt import Prompt
 import os
 
-from pydantic import Field
+from langroid.pydantic_v1 import Field
 import langroid as lr
 import langroid.language_models as lm
 from langroid.agent.special.doc_chat_agent import DocChatAgentConfig

examples/docqa/filter-multi-doc-query-plan.py

@@ -15,7 +15,7 @@
 import os
 import json
 
-from pydantic import Field
+from langroid.pydantic_v1 import Field
 import langroid as lr
 import langroid.language_models as lm
 from langroid import ChatDocument

examples/docqa/oai-multi-extract.py

@@ -14,7 +14,7 @@
 
 import typer
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 import json
 import os
 

examples/extract/capitals.py

@@ -7,7 +7,7 @@
 """
 
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 from typing import List
 import langroid as lr
 

examples/extract/extract.py

@@ -17,7 +17,7 @@
 import typer
 from typing import List
 from rich import print
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 
 from langroid.agent.chat_agent import ChatAgent, ChatAgentConfig
 from langroid.agent.task import Task

examples/quick-start/chat-agent-tool.py

@@ -17,7 +17,7 @@
 
 import typer
 from rich import print
-from pydantic import BaseSettings
+from langroid.pydantic_v1 import BaseSettings
 
 import langroid as lr
 

langroid/agent/base.py

@@ -20,7 +20,6 @@
     no_type_check,
 )
 
-from pydantic import BaseSettings, ValidationError, validator
 from rich import print
 from rich.console import Console
 from rich.markup import escape
@@ -41,6 +40,7 @@
 from langroid.parsing.parse_json import extract_top_level_json
 from langroid.parsing.parser import Parser, ParsingConfig
 from langroid.prompts.prompts_config import PromptsConfig
+from langroid.pydantic_v1 import BaseSettings, ValidationError, validator
 from langroid.utils.configuration import settings
 from langroid.utils.constants import NO_ANSWER
 from langroid.utils.output import status

langroid/agent/callbacks/chainlit.py

@@ -7,9 +7,8 @@
 import textwrap
 from typing import Any, Callable, Dict, List, Literal, Optional, no_type_check
 
-from pydantic import BaseSettings
-
 from langroid.exceptions import LangroidImportError
+from langroid.pydantic_v1 import BaseSettings
 
 try:
     import chainlit as cl

langroid/agent/chat_document.py

@@ -2,8 +2,6 @@
 from enum import Enum
 from typing import List, Optional, Union
 
-from pydantic import BaseModel, Extra
-
 from langroid.agent.tool_message import ToolMessage
 from langroid.language_models.base import (
     LLMFunctionCall,
@@ -15,6 +13,7 @@
 from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.agent_chats import parse_message
 from langroid.parsing.parse_json import extract_top_level_json, top_level_json_field
+from langroid.pydantic_v1 import BaseModel, Extra
 from langroid.utils.output.printing import shorten_text
 
 
@@ -48,7 +47,7 @@ class ChatDocMetaData(DocMetaData):
     block: None | Entity = None
     sender_name: str = ""
     recipient: str = ""
-    usage: Optional[LLMTokenUsage]
+    usage: Optional[LLMTokenUsage] = None
     cached: bool = False
     displayed: bool = False
     has_citation: bool = False

langroid/agent/openai_assistant.py

@@ -15,7 +15,6 @@
 )
 from openai.types.beta.threads import Message, Run
 from openai.types.beta.threads.runs import RunStep
-from pydantic import BaseModel
 from rich import print
 
 from langroid.agent.chat_agent import ChatAgent, ChatAgentConfig
@@ -27,6 +26,7 @@
     OpenAIGPT,
     OpenAIGPTConfig,
 )
+from langroid.pydantic_v1 import BaseModel
 from langroid.utils.configuration import settings
 from langroid.utils.system import generate_user_id, update_hash
 

langroid/agent/special/doc_chat_agent.py

@@ -1313,7 +1313,7 @@ def answer_from_docs(self, query: str) -> ChatDocument:
             meta.update(extracts[0].metadata)
             return ChatDocument(
                 content="\n\n".join([e.content for e in extracts]),
-                metadata=ChatDocMetaData(**meta),
+                metadata=ChatDocMetaData(**meta),  # type: ignore
             )
         response = self.get_summary_answer(query, extracts)
 

langroid/agent/special/lance_rag/query_planner_agent.py

@@ -191,7 +191,7 @@ def handle_message_fallback(
             # save result, to be used in query_plan_feedback()
             self.result = msg.content
             # assemble QueryPlanAnswerTool...
-            query_plan_answer_tool = QueryPlanAnswerTool(
+            query_plan_answer_tool = QueryPlanAnswerTool(  # type: ignore
                 plan=self.curr_query_plan,
                 answer=self.result,
             )

langroid/agent/special/lance_tools.py

@@ -1,8 +1,7 @@
 import logging
 
-from pydantic import BaseModel
-
 from langroid.agent.tool_message import ToolMessage
+from langroid.pydantic_v1 import BaseModel
 
 logger = logging.getLogger(__name__)