hw-03.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0a983b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from elasticsearch import Elasticsearch\n",
    "from tqdm.autonotebook import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "caaa8b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "course_name = 'machine-learning-zoomcamp'\n",
    "user_question = \"I just discovered the course. Can I still join it?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1348edd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main'\n",
    "\n",
    "def get_json(endpoint):\n",
    "  docs_url = f'{base_url}/{endpoint}?raw=1'\n",
    "  docs_response = requests.get(docs_url)\n",
    "  return docs_response.json()\n",
    "\n",
    "def get_csv(endpoint):\n",
    "  docs_url = f'{base_url}/{endpoint}?raw=1'\n",
    "  return pd.read_csv(docs_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b887563",
   "metadata": {},
   "source": [
    "# Question 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "c6589ed7-2a77-4d2e-b0d2-69654dfaa64f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'multi-qa-distilbert-cos-v1'\n",
    "\n",
    "embedding_model = SentenceTransformer(model_name)\n",
    "embedding_vector = embedding_model.encode(user_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "92a88bf6-3552-43d7-a7ff-ad7ee391c299",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.078222655"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_vector[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9041307",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Question 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "696d80cc-d6e2-4ce7-8651-c0470f1d4e2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "375"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents = get_json('03-vector-search/eval/documents-with-ids.json')\n",
    "\n",
    "def filterCourse(q):\n",
    "    return q['course'] == course_name\n",
    "\n",
    "filtered_documents = list(filter(filterCourse, documents))\n",
    "len(filtered_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3c87bdc1-2861-4557-b49e-b1bee7856587",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = []\n",
    "\n",
    "for document in filtered_documents:\n",
    "    qa_text = f\"{document['question']} {document['text']}\"\n",
    "    embedding = embedding_model.encode(qa_text)\n",
    "    embeddings.append(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "28948be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5a185fe9-6004-4584-8880-3432ccd7bfa8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(375, 768)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0a83c99",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Question 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "075c643c-4bed-4bab-9ed0-1dadf1405dd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = X.dot(embedding_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b9e60484-45f4-4b47-9ea6-6b1ae4dd085c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6506573"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b157562",
   "metadata": {},
   "source": [
    "# Question 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "25b53c05-1059-43ea-b538-4d9157bfc971",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VectorSearchEngine():\n",
    "    def __init__(self, documents, embeddings):\n",
    "        self.documents = documents\n",
    "        self.embeddings = embeddings\n",
    "\n",
    "    def search(self, v_query, num_results=10):\n",
    "        scores = self.embeddings.dot(v_query)\n",
    "        idx = np.argpartition(-scores, num_results)[:num_results]\n",
    "        return [self.documents[i] for i in idx]\n",
    "\n",
    "## usage\n",
    "# search_engine = VectorSearchEngine(documents=documents, embeddings=X)\n",
    "# search_engine.search(embedding_vector, num_results=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "21cfd3ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ground_truth = get_csv('03-vector-search/eval/ground-truth-data.csv')\n",
    "df_ground_truth = df_ground_truth[df_ground_truth.course == course_name]\n",
    "ground_truth = df_ground_truth.to_dict(orient='records')\n",
    "\n",
    "search_engine = VectorSearchEngine(documents=filtered_documents, embeddings=X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "37342a14-c673-4360-b812-151d163ed35e",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_results = [];\n",
    "\n",
    "for record in ground_truth:\n",
    "    query = record['question']\n",
    "    v_query = embedding_model.encode(query)\n",
    "    current_results = search_engine.search(v_query, num_results=5)\n",
    "    q4_results.append(current_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "d5afd3ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_hits = 0\n",
    "\n",
    "for idx, arr in enumerate(q4_results):\n",
    "    record = ground_truth[idx]\n",
    "    for result in arr:\n",
    "        if record['document'] == result['id']:\n",
    "            q4_hits += 1\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "99ff0487",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1720\n",
      "0.9398907103825137\n"
     ]
    }
   ],
   "source": [
    "print(q4_hits)\n",
    "print(q4_hits / len(ground_truth))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0280592e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Question 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "77179820",
   "metadata": {},
   "outputs": [],
   "source": [
    "es_client = Elasticsearch('http://localhost:9200')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "279d6a98",
   "metadata": {},
   "outputs": [],
   "source": [
    "index_name = \"documents\"\n",
    "\n",
    "index_settings = {\n",
    "    \"settings\": {\n",
    "        \"number_of_shards\": 1,\n",
    "        \"number_of_replicas\": 0\n",
    "    },\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"text\": {\"type\": \"text\"},\n",
    "            \"section\": {\"type\": \"text\"},\n",
    "            \"question\": {\"type\": \"text\"},\n",
    "            \"course\": {\"type\": \"keyword\"} ,\n",
    "            \"text_vector\": {\n",
    "                \"type\": \"dense_vector\",\n",
    "                \"dims\": 768,\n",
    "                \"index\": True,\n",
    "                \"similarity\": \"cosine\"\n",
    "            },\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "90d7d100",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'documents'})"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# es_client.indices.delete(index=index_name, ignore_unavailable=True)\n",
    "es_client.indices.create(index=index_name, body=index_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "0bc82e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 375/375 [00:09<00:00, 40.18it/s]\n"
     ]
    }
   ],
   "source": [
    "for doc in tqdm(filtered_documents):\n",
    "    es_client.index(index=index_name, document=doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "0f361649",
   "metadata": {},
   "outputs": [],
   "source": [
    "def es_search(query):\n",
    "    search_query = {\n",
    "        \"size\": 5,\n",
    "        \"query\": {\n",
    "            \"bool\": {\n",
    "                \"must\": { \"multi_match\": { \"query\": query } }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "\n",
    "    return es_client.search(index=index_name, body=search_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "f194c6a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = es_search(user_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "27f0ff2a-df31-4a97-9f99-50dd9d1c6de4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_index': 'documents',\n",
       " '_id': 'x5nKoZABHBbT7Y1hrOH1',\n",
       " '_score': 25.625399,\n",
       " '_source': {'text': 'Yes, you can. You won’t be able to submit some of the homeworks, but you can still take part in the course.\\nIn order to get a certificate, you need to submit 2 out of 3 course projects and review 3 peers’ Projects by the deadline. It means that if you join the course at the end of November and manage to work on two projects, you will still be eligible for a certificate.',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'The course has already started. Can I still join it?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'id': 'ee58a693'}}"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response['hits']['hits'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f4f270b",
   "metadata": {},
   "source": [
    "# Question 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "aebad1fd-38a8-45ba-8a0b-3e8e0be5cec4",
   "metadata": {},
   "outputs": [],
   "source": [
    "q6_results = [];\n",
    "\n",
    "for record in ground_truth:\n",
    "    query = record['question']\n",
    "    current_results = es_search(query)\n",
    "    q6_results.append(current_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "bd8496b7-4b94-471e-968e-2134055421ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "q6_hits = 0\n",
    "\n",
    "for idx, arr in enumerate(q6_results):\n",
    "    record = ground_truth[idx]\n",
    "    for result in arr['hits']['hits']:\n",
    "        if record['document'] == result['_source']['id']:\n",
    "            q6_hits += 1\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "c12a39cb-a926-4214-9852-74adea9823d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1662\n",
      "0.9081967213114754\n"
     ]
    }
   ],
   "source": [
    "print(q6_hits)\n",
    "print(q6_hits / len(ground_truth))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}