feat: Update QuivrRAG and run_evaluation.py files

.gitignore

@@ -84,3 +84,4 @@ backend/application.log.*
 backend/score.json
 backend/modules/assistant/ito/utils/simple.pdf
 backend/modules/sync/controller/credentials.json
+backend/.env.test

backend/modules/brain/integrations/Multi_Contract/Brain.py

@@ -0,0 +1,206 @@
+import datetime
+from operator import itemgetter
+from typing import List
+
+from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate
+from langchain_community.chat_models import ChatLiteLLM
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
+from langchain_core.pydantic_v1 import BaseModel as BaseModelV1
+from langchain_core.pydantic_v1 import Field as FieldV1
+from langchain_core.runnables import RunnableLambda, RunnablePassthrough
+from langchain_openai import ChatOpenAI
+from logger import get_logger
+from modules.brain.knowledge_brain_qa import KnowledgeBrainQA
+
+logger = get_logger(__name__)
+
+
+class cited_answer(BaseModelV1):
+    """Answer the user question based only on the given sources, and cite the sources used."""
+
+    thoughts: str = FieldV1(
+        ...,
+        description="""Description of the thought process, based only on the given sources. 
+        Cite the text as much as possible and give the document name it appears in. In the format : 'Doc_name states : cited_text'. Be the most 
+        procedural as possible.""",
+    )
+    answer: str = FieldV1(
+        ...,
+        description="The answer to the user question, which is based only on the given sources.",
+    )
+    citations: List[int] = FieldV1(
+        ...,
+        description="The integer IDs of the SPECIFIC sources which justify the answer.",
+    )
+
+    thoughts: str = FieldV1(
+        ...,
+        description="Explain shortly what you did to find the answer and what you used by citing the sources by their name.",
+    )
+    followup_questions: List[str] = FieldV1(
+        ...,
+        description="Generate up to 3 follow-up questions that could be asked based on the answer given or context provided.",
+    )
+
+
+# First step is to create the Rephrasing Prompt
+_template = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. Keep as much details as possible from previous messages. Keep entity names and all.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone question:"""
+CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(_template)
+
+# Next is the answering prompt
+
+template_answer = """
+Context:
+{context}
+
+User Question: {question}
+Answer:
+"""
+
+today_date = datetime.datetime.now().strftime("%B %d, %Y")
+
+system_message_template = (
+    f"Your name is Quivr. You're a helpful assistant. Today's date is {today_date}."
+)
+
+system_message_template += """
+When answering use markdown neat.
+Answer in a concise and clear manner.
+Use the following pieces of context from files provided by the user to answer the users.
+Answer in the same language as the user question.
+If you don't know the answer with the context provided from the files, just say that you don't know, don't try to make up an answer.
+Don't cite the source id in the answer objects, but you can use the source to answer the question.
+You have access to the files to answer the user question (limited to first 20 files):
+{files}
+
+If not None, User instruction to follow to answer: {custom_instructions}
+Don't cite the source id in the answer objects, but you can use the source to answer the question.
+"""
+
+
+ANSWER_PROMPT = ChatPromptTemplate.from_messages(
+    [
+        SystemMessagePromptTemplate.from_template(system_message_template),
+        HumanMessagePromptTemplate.from_template(template_answer),
+    ]
+)
+
+
+# How we format documents
+
+DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template(
+    template="Source: {index} \n {page_content}"
+)
+
+
+class MultiContractBrain(KnowledgeBrainQA):
+    """
+    The MultiContract class integrates advanced conversational retrieval and language model chains
+    to provide comprehensive and context-aware responses to user queries.
+
+    It leverages a combination of document retrieval, question condensation, and document-based
+    question answering to generate responses that are informed by a wide range of knowledge sources.
+    """
+
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        """
+        Initializes the MultiContract class with specific configurations.
+
+        Args:
+            **kwargs: Arbitrary keyword arguments.
+        """
+        super().__init__(
+            **kwargs,
+        )
+
+    def get_chain(self):
+
+        list_files_array = (
+            self.knowledge_qa.knowledge_service.get_all_knowledge_in_brain(
+                self.brain_id
+            )
+        )  # pyright: ignore reportPrivateUsage=none
+
+        list_files_array = [file.file_name for file in list_files_array]
+        # Max first 10 files
+        if len(list_files_array) > 20:
+            list_files_array = list_files_array[:20]
+
+        list_files = "\n".join(list_files_array) if list_files_array else "None"
+
+        retriever_doc = self.knowledge_qa.get_retriever()
+
+        loaded_memory = RunnablePassthrough.assign(
+            chat_history=RunnableLambda(
+                lambda x: self.filter_history(x["chat_history"]),
+            ),
+            question=lambda x: x["question"],
+        )
+
+        api_base = None
+        if self.brain_settings.ollama_api_base_url and self.model.startswith("ollama"):
+            api_base = self.brain_settings.ollama_api_base_url
+
+        standalone_question = {
+            "standalone_question": {
+                "question": lambda x: x["question"],
+                "chat_history": itemgetter("chat_history"),
+            }
+            | CONDENSE_QUESTION_PROMPT
+            | ChatLiteLLM(temperature=0, model=self.model, api_base=api_base)
+            | StrOutputParser(),
+        }
+
+        knowledge_qa = self.knowledge_qa
+        prompt_custom_user = knowledge_qa.prompt_to_use()
+        prompt_to_use = "None"
+        if prompt_custom_user:
+            prompt_to_use = prompt_custom_user.content
+
+        # Now we retrieve the documents
+        retrieved_documents = {
+            "docs": itemgetter("standalone_question") | retriever_doc,
+            "question": lambda x: x["standalone_question"],
+            "custom_instructions": lambda x: prompt_to_use,
+        }
+
+        final_inputs = {
+            "context": lambda x: self.knowledge_qa._combine_documents(x["docs"]),
+            "question": itemgetter("question"),
+            "custom_instructions": itemgetter("custom_instructions"),
+            "files": lambda x: list_files,
+        }
+        llm = ChatLiteLLM(
+            max_tokens=self.max_tokens,
+            model=self.model,
+            temperature=self.temperature,
+            api_base=api_base,
+        )  # pyright: ignore reportPrivateUsage=none
+        if self.model_compatible_with_function_calling(self.model):
+
+            # And finally, we do the part that returns the answers
+            llm_function = ChatOpenAI(
+                max_tokens=self.max_tokens,
+                model=self.model,
+                temperature=self.temperature,
+            )
+            llm = llm_function.bind_tools(
+                [cited_answer],
+                tool_choice="cited_answer",
+            )
+
+        answer = {
+            "answer": final_inputs | ANSWER_PROMPT | llm,
+            "docs": itemgetter("docs"),
+        }
+
+        return loaded_memory | standalone_question | retrieved_documents | answer

backend/modules/brain/knowledge_brain_qa.py

@@ -331,7 +331,9 @@ def generate_answer(
     async def generate_stream(
         self, chat_id: UUID, question: ChatQuestion, save_answer: bool = True
     ) -> AsyncIterable:
-        conversational_qa_chain = self.knowledge_qa.get_chain()
+        conversational_qa_chain = (
+            self.get_chain() if self.get_chain() else self.knowledge_qa.get_chain()
+        )
         transformed_history, streamed_chat_history = (
             self.initialize_streamed_chat_history(chat_id, question)
         )

backend/modules/brain/rags/quivr_rag.py

@@ -24,6 +24,7 @@
 from models.settings import get_supabase_client
 from modules.brain.service.brain_service import BrainService
 from modules.chat.service.chat_service import ChatService
+from modules.knowledge.repository.knowledges import Knowledges
 from modules.prompt.service.get_prompt_to_use import get_prompt_to_use
 from pydantic import BaseModel, ConfigDict
 from pydantic_settings import BaseSettings
@@ -36,6 +37,12 @@
 class cited_answer(BaseModelV1):
     """Answer the user question based only on the given sources, and cite the sources used."""
 
+    thoughts: str = FieldV1(
+        ...,
+        description="""Description of the thought process, based only on the given sources. 
+        Cite the text as much as possible and give the document name it appears in. In the format : 'Doc_name states : cited_text'. Be the most 
+        procedural as possible.""",
+    )
     answer: str = FieldV1(
         ...,
         description="The answer to the user question, which is based only on the given sources.",
@@ -47,7 +54,7 @@ class cited_answer(BaseModelV1):
 
     thoughts: str = FieldV1(
         ...,
-        description="Explain shortly what you did to generate the answer. Explain any assumptions you made, and why you made them.",
+        description="Explain shortly what you did to find the answer and what you used by citing the sources by their name.",
     )
     followup_questions: List[str] = FieldV1(
         ...,
@@ -86,6 +93,10 @@ class cited_answer(BaseModelV1):
 Use the following pieces of context from files provided by the user to answer the users.
 Answer in the same language as the user question.
 If you don't know the answer with the context provided from the files, just say that you don't know, don't try to make up an answer.
+Don't cite the source id in the answer objects, but you can use the source to answer the question.
+You have access to the files to answer the user question (limited to first 20 files):
+{files}
+
 If not None, User instruction to follow to answer: {custom_instructions}
 Don't cite the source id in the answer objects, but you can use the source to answer the question.
 """
@@ -128,7 +139,6 @@ class QuivrRAG(BaseModel):
 
     # Instantiate settings
     brain_settings: BaseSettings = BrainSettings()
-
     # Default class attributes
     model: str = None  # pyright: ignore reportPrivateUsage=none
     temperature: float = 0.1
@@ -137,6 +147,7 @@ class QuivrRAG(BaseModel):
     max_tokens: int = 2000  # Output length
     max_input: int = 2000
     streaming: bool = False
+    knowledge_service: Knowledges = None
 
     @property
     def embeddings(self):
@@ -205,6 +216,7 @@ def __init__(
         self.brain_id = brain_id
         self.chat_id = chat_id
         self.streaming = streaming
+        self.knowledge_service = Knowledges()
 
     def _create_supabase_client(self) -> Client:
         return get_supabase_client()
@@ -235,7 +247,9 @@ def _create_llm(
 
         api_base = None
         if self.brain_settings.ollama_api_base_url and model.startswith("ollama"):
-            api_base = self.brain_settings.ollama_api_base_url
+            api_base = (
+                self.brain_settings.ollama_api_base_url  # pyright: ignore reportPrivateUsage=none
+            )
 
         return ChatLiteLLM(
             temperature=temperature,
@@ -245,7 +259,7 @@ def _create_llm(
             verbose=False,
             callbacks=callbacks,
             api_base=api_base,
-        )
+        )  # pyright: ignore reportPrivateUsage=none
 
     def _combine_documents(
         self, docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n"
@@ -294,11 +308,23 @@ def filter_history(
         return chat_history
 
     def get_chain(self):
+
+        list_files_array = self.knowledge_service.get_all_knowledge_in_brain(
+            self.brain_id
+        )  # pyright: ignore reportPrivateUsage=none
+
+        list_files_array = [file.file_name for file in list_files_array]
+        # Max first 10 files
+        if len(list_files_array) > 20:
+            list_files_array = list_files_array[:20]
+
+        list_files = "\n".join(list_files_array) if list_files_array else "None"
+
         compressor = None
         if os.getenv("COHERE_API_KEY"):
-            compressor = CohereRerank(top_n=10)
+            compressor = CohereRerank(top_n=20)
         else:
-            compressor = FlashrankRerank(model="ms-marco-TinyBERT-L-2-v2", top_n=10)
+            compressor = FlashrankRerank(model="ms-marco-TinyBERT-L-2-v2", top_n=20)
 
         retriever_doc = self.get_retriever()
         compression_retriever = ContextualCompressionRetriever(
@@ -342,6 +368,7 @@ def get_chain(self):
             "context": lambda x: self._combine_documents(x["docs"]),
             "question": itemgetter("question"),
             "custom_instructions": itemgetter("custom_instructions"),
+            "files": lambda x: list_files,
         }
         llm = ChatLiteLLM(
             max_tokens=self.max_tokens,

backend/modules/chat/controller/chat/brainful_chat.py

@@ -1,8 +1,8 @@
 from logger import get_logger
-from modules.brain.api_brain_qa import APIBrainQA
 from modules.brain.entity.brain_entity import BrainType, RoleEnum
 from modules.brain.integrations.Big.Brain import BigBrain
 from modules.brain.integrations.GPT4.Brain import GPT4Brain
+from modules.brain.integrations.Multi_Contract.Brain import MultiContractBrain
 from modules.brain.integrations.Notion.Brain import NotionBrain
 from modules.brain.integrations.Proxy.Brain import ProxyBrain
 from modules.brain.integrations.Self.Brain import SelfBrain
@@ -47,6 +47,7 @@
     "doc": KnowledgeBrainQA,
     "proxy": ProxyBrain,
     "self": SelfBrain,
+    "multi-contract": MultiContractBrain,
 }
 
 brain_service = BrainService()
@@ -82,23 +83,6 @@ def get_answer_generator(
                 user_email=user_email,
             )
 
-        if brain.brain_type == BrainType.API:
-            brain_definition = api_brain_definition_service.get_api_brain_definition(
-                brain.brain_id
-            )
-            return APIBrainQA(
-                chat_id=chat_id,
-                temperature=temperature,
-                brain_id=str(brain.brain_id),
-                streaming=streaming,
-                prompt_id=prompt_id,
-                user_id=user_id,
-                raw=(brain_definition.raw if brain_definition else None),
-                jq_instructions=(
-                    brain_definition.jq_instructions if brain_definition else None
-                ),
-                user_email=user_email,
-            )
         if brain.brain_type == BrainType.INTEGRATION:
             integration_brain = integration_brain_description_service.get_integration_description_by_user_brain_id(
                 brain.brain_id, user_id