004-RAG_Retrieval_Eval/004-RAG_Retrieval_Eval_LlamaIndex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b2114dd-1c70-4c9e-834d-dc808819cf04",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n",
    "from llama_index.node_parser import SimpleNodeParser\n",
    "from llama_index.schema import Document\n",
    "\n",
    "# LLM\n",
    "from llama_index.llms import Anthropic\n",
    "\n",
    "# Embeddings\n",
    "from llama_index.embeddings import OpenAIEmbedding, HuggingFaceEmbedding, CohereEmbedding\n",
    "\n",
    "# Retrievers\n",
    "from llama_index.retrievers import (\n",
    "    BaseRetriever,\n",
    "    VectorIndexRetriever,\n",
    ")\n",
    "\n",
    "# Rerankers\n",
    "from llama_index.indices.query.schema import QueryBundle, QueryType\n",
    "from llama_index.schema import NodeWithScore\n",
    "# from llama_index.indices.postprocessor.cohere_rerank import CohereRerank\n",
    "from llama_index.indices.postprocessor import SentenceTransformerRerank\n",
    "from llama_index.finetuning.embeddings.common import EmbeddingQAFinetuneDataset\n",
    "\n",
    "# Evaluator\n",
    "from llama_index.evaluation import (\n",
    "    generate_question_context_pairs,\n",
    "    EmbeddingQAFinetuneDataset,\n",
    ")\n",
    "from llama_index.evaluation import RetrieverEvaluator\n",
    "\n",
    "from typing import List\n",
    "import pandas as pd\n",
    "\n",
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6907da95-ce06-41ef-bd5b-6cdf6caade67",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"sciq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9e8079f-ec1a-4f62-8bdc-b793421e7711",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_parser = SimpleNodeParser.from_defaults(chunk_size=512)\n",
    "\n",
    "corpus = []\n",
    "filtered_queries = []\n",
    "counter = 0\n",
    "for train_row in dataset[\"train\"]:\n",
    "    if len(train_row[\"support\"].strip()) == 0:\n",
    "        continue\n",
    "    current_document = Document(text=train_row[\"support\"])\n",
    "    if len(node_parser.get_nodes_from_documents([current_document])) == 1:\n",
    "        corpus.append(train_row[\"support\"])\n",
    "        filtered_queries.append(train_row[\"question\"])\n",
    "        counter += 1\n",
    "    if counter == 500:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "14fdcf4c-ec28-4b07-ba04-3627c0d7b18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [Document(text=c) for c in corpus]\n",
    "nodes = node_parser.get_nodes_from_documents(documents)\n",
    "for idx, node in enumerate(nodes):\n",
    "    node.id_ = f\"corpus_{idx}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c72a666c-47a4-48a1-b892-6f67570d2a5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "queries_dict = {f\"query_{index}\":filtered_queries[index] for index in range(counter)}\n",
    "corpus_dict = {f\"corpus_{index}\":corpus[index] for index in range(counter)}\n",
    "relevant_docs_dict = {f\"query_{index}\":[f\"corpus_{index}\"] for index in range(counter)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87a6b8aa-ddc3-4081-b7f0-16b262415556",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_dataset = EmbeddingQAFinetuneDataset(\n",
    "    queries=queries_dict,\n",
    "    corpus=corpus_dict,\n",
    "    relevant_docs=relevant_docs_dict\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "79c6cae3-96e0-4041-a4f4-17363a07f7c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c959337089745288edd88617267d421",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aebc4c94cff04abdaf107adb654f9c10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "configuration_bert.py:   0%|          | 0.00/8.24k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new version of the following files was downloaded from https://huggingface.co/jinaai/jina-bert-implementation:\n",
      "- configuration_bert.py\n",
      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef56cd80c07e43e1a4f96ae2dadc58c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modeling_bert.py:   0%|          | 0.00/97.5k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new version of the following files was downloaded from https://huggingface.co/jinaai/jina-bert-implementation:\n",
      "- modeling_bert.py\n",
      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2ab6ce2869d490d84ce352434686926",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/65.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db5c8442c5204897a610ba2ea2b0d636",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/373 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c49bf9f62814cc2a8ae114d0129b9d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ece13c4568a54e008c1854f5045f59d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/712k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a2730ea9f944890b9f95c034f9847f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24543ebb8db64b4eacb7461cd83cea61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/373 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d99e7d2479f7484a8189bf3c5c519749",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "799d1a92297e4fac85cebf89d1562ef8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/712k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "408a1ec175094059b20f1263481de908",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b138a7ae582f49cbb835b8555c79dd7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "069f467a58c145389aff3f3f8f183ad9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/275M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f9bcb9b49d24b08aebeeb6c5c6437cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/373 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f51600da2f9d432c9c325c4cd9173f9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0f3e0fc787f463491f213f4fd0c3101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/712k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff56d19b07bc49dd9a4679139a4397d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "946864deadd044de92968975baec3b4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/373 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c842476057d446ea13f011f8c96b275",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71a49d602d454e3a91ba6fb3d965a5fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/712k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3396ca51182e4e77ad67cf628e59052d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a4915bd2b554712b51dac02e0f1ed12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/801 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c6bbd3962d1418f87f7958975ea67ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/2.24G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6ab6991eb604aca9feed623c4505a5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/443 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c984f21274e4d25aef68703e95f6e8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1657d356feee4a87a5c5689233f4ab6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab2515f822e2421393e101e71fab38c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/279 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Define all embeddings and rerankers\n",
    "EMBEDDINGS = {\n",
    "    \"bge-large\": HuggingFaceEmbedding(model_name='BAAI/bge-large-en'), # You can use mean pooling by addin pooling='mean' parameter\n",
    "    \"JinaAI-Small\": HuggingFaceEmbedding(model_name='jinaai/jina-embeddings-v2-small-en', pooling='mean', trust_remote_code=True),\n",
    "    \"JinaAI-Base\": HuggingFaceEmbedding(model_name='jinaai/jina-embeddings-v2-base-en', pooling='mean', trust_remote_code=True),\n",
    "}\n",
    "\n",
    "RERANKERS = {\n",
    "    \"WithoutReranker\": \"None\",\n",
    "    \"bge-reranker-base\": SentenceTransformerRerank(model=\"BAAI/bge-reranker-base\", top_n=5),\n",
    "    \"bge-reranker-large\": SentenceTransformerRerank(model=\"BAAI/bge-reranker-large\", top_n=5)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4e5e227c-936a-48d0-b6ec-2e321b8280fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_results(embedding_name, reranker_name, eval_results):\n",
    "    \"\"\"Display results from evaluate.\"\"\"\n",
    "\n",
    "    metric_dicts = []\n",
    "    for eval_result in eval_results:\n",
    "        metric_dict = eval_result.metric_vals_dict\n",
    "        metric_dicts.append(metric_dict)\n",
    "\n",
    "    full_df = pd.DataFrame(metric_dicts)\n",
    "\n",
    "    hit_rate = full_df[\"hit_rate\"].mean()\n",
    "    mrr = full_df[\"mrr\"].mean()\n",
    "\n",
    "    metric_df = pd.DataFrame(\n",
    "        {\"Embedding\": [embedding_name], \"Reranker\": [reranker_name], \"hit_rate\": [hit_rate], \"mrr\": [mrr]}\n",
    "    )\n",
    "\n",
    "    return metric_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "36c5c4f5-198a-46b1-813d-17b6dea23286",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Evaluation for Embedding Model: bge-large\n",
      "LLM is explicitly disabled. Using MockLLM.\n",
      "Running Evaluation for Embedding Model: bge-large and Reranker: WithoutReranker\n",
      "Running Evaluation for Embedding Model: bge-large and Reranker: bge-reranker-base\n",
      "Running Evaluation for Embedding Model: bge-large and Reranker: bge-reranker-large\n",
      "Running Evaluation for Embedding Model: JinaAI-Small\n",
      "LLM is explicitly disabled. Using MockLLM.\n",
      "Running Evaluation for Embedding Model: JinaAI-Small and Reranker: WithoutReranker\n",
      "Running Evaluation for Embedding Model: JinaAI-Small and Reranker: bge-reranker-base\n",
      "Running Evaluation for Embedding Model: JinaAI-Small and Reranker: bge-reranker-large\n",
      "Running Evaluation for Embedding Model: JinaAI-Base\n",
      "LLM is explicitly disabled. Using MockLLM.\n",
      "Running Evaluation for Embedding Model: JinaAI-Base and Reranker: WithoutReranker\n",
      "Running Evaluation for Embedding Model: JinaAI-Base and Reranker: bge-reranker-base\n",
      "Running Evaluation for Embedding Model: JinaAI-Base and Reranker: bge-reranker-large\n"
     ]
    }
   ],
   "source": [
    "results_df = pd.DataFrame()\n",
    "\n",
    "# Loop over embeddings\n",
    "for embed_name, embed_model in EMBEDDINGS.items():\n",
    "\n",
    "    print(f\"Running Evaluation for Embedding Model: {embed_name}\")\n",
    "\n",
    "    service_context = ServiceContext.from_defaults(llm=None, embed_model=embed_model)\n",
    "    vector_index = VectorStoreIndex(nodes, service_context=service_context)\n",
    "\n",
    "    vector_retriever = VectorIndexRetriever(index=vector_index, similarity_top_k=5, service_context=service_context)\n",
    "\n",
    "    # Loop over rerankers\n",
    "    for rerank_name, reranker in RERANKERS.items():\n",
    "\n",
    "        print(f\"Running Evaluation for Embedding Model: {embed_name} and Reranker: {rerank_name}\")\n",
    "\n",
    "        # Define Retriever\n",
    "        class CustomRetriever(BaseRetriever):\n",
    "            \"\"\"Custom retriever that performs both Vector search and Knowledge Graph search\"\"\"\n",
    "\n",
    "            def __init__(\n",
    "                self,\n",
    "                vector_retriever: VectorIndexRetriever,\n",
    "            ) -> None:\n",
    "                \"\"\"Init params.\"\"\"\n",
    "\n",
    "                self._vector_retriever = vector_retriever\n",
    "\n",
    "            def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:\n",
    "                \"\"\"Retrieve nodes given query.\"\"\"\n",
    "\n",
    "                retrieved_nodes = self._vector_retriever.retrieve(query_bundle)\n",
    "\n",
    "                if reranker != 'None':\n",
    "                    retrieved_nodes = reranker.postprocess_nodes(retrieved_nodes, query_bundle)\n",
    "                else:\n",
    "                    retrieved_nodes = retrieved_nodes[:5]\n",
    "\n",
    "                return retrieved_nodes\n",
    "\n",
    "            async def _aretrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:\n",
    "                \"\"\"Asynchronously retrieve nodes given query.\n",
    "\n",
    "                Implemented by the user.\n",
    "\n",
    "                \"\"\"\n",
    "                return self._retrieve(query_bundle)\n",
    "\n",
    "            async def aretrieve(self, str_or_query_bundle: QueryType) -> List[NodeWithScore]:\n",
    "                if isinstance(str_or_query_bundle, str):\n",
    "                    str_or_query_bundle = QueryBundle(str_or_query_bundle)\n",
    "                return await self._aretrieve(str_or_query_bundle)\n",
    "\n",
    "        custom_retriever = CustomRetriever(vector_retriever)\n",
    "\n",
    "        retriever_evaluator = RetrieverEvaluator.from_metric_names(\n",
    "            [\"mrr\", \"hit_rate\"], retriever=custom_retriever\n",
    "        )\n",
    "        eval_results = await retriever_evaluator.aevaluate_dataset(qa_dataset)\n",
    "\n",
    "        current_df = display_results(embed_name, rerank_name, eval_results)\n",
    "        results_df = pd.concat([results_df, current_df], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09ba173c-63bf-487c-b520-d9965d8f5cf9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Embedding            Reranker  hit_rate       mrr\n",
      "0     bge-large     WithoutReranker     0.970  0.934400\n",
      "1     bge-large   bge-reranker-base     0.970  0.938900\n",
      "2     bge-large  bge-reranker-large     0.970  0.935333\n",
      "3  JinaAI-Small     WithoutReranker     0.968  0.914100\n",
      "4  JinaAI-Small   bge-reranker-base     0.968  0.938067\n",
      "5  JinaAI-Small  bge-reranker-large     0.968  0.937833\n",
      "6   JinaAI-Base     WithoutReranker     0.972  0.926133\n",
      "7   JinaAI-Base   bge-reranker-base     0.972  0.941667\n",
      "8   JinaAI-Base  bge-reranker-large     0.972  0.948000\n"
     ]
    }
   ],
   "source": [
    "# Display final results\n",
    "print(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf52b6a-415d-4ccc-a92c-942f907b1cf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#       Embedding            Reranker  hit_rate       mrr\n",
    "# 0     bge-large     WithoutReranker     0.970  0.934400\n",
    "# 1     bge-large   bge-reranker-base     0.970  0.938900\n",
    "# 2     bge-large  bge-reranker-large     0.970  0.935333\n",
    "# 3  JinaAI-Small     WithoutReranker     0.968  0.914100\n",
    "# 4  JinaAI-Small   bge-reranker-base     0.968  0.938067\n",
    "# 5  JinaAI-Small  bge-reranker-large     0.968  0.937833\n",
    "# 6   JinaAI-Base     WithoutReranker     0.972  0.926133\n",
    "# 7   JinaAI-Base   bge-reranker-base     0.972  0.941667\n",
    "# 8   JinaAI-Base  bge-reranker-large     0.972  0.948000"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}