01-intro/homework/homework.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "from elasticsearch import Elasticsearch\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "es_client = Elasticsearch(\"http://localhost:9200\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'documents' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdocuments\u001b[49m[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mkeys()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'documents' is not defined"
     ]
    }
   ],
   "source": [
    "documents[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_settings = {\n",
    "    \"settings\": {\n",
    "        \"number_of_shards\": 1,\n",
    "        \"number_of_replicas\": 0\n",
    "    },\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"text\": {\"type\": \"text\"},\n",
    "            \"section\": {\"type\": \"text\"},\n",
    "            \"question\": {\"type\": \"text\"},\n",
    "            \"course\": {\"type\": \"keyword\"} \n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "index_name = \"course-questions\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "BadRequestError",
     "evalue": "BadRequestError(400, 'resource_already_exists_exception', 'index [course-questions/JmfjoxHdSUug387zN66sWw] already exists')",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mBadRequestError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mes_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindices\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_settings\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/utils.py:446\u001b[0m, in \u001b[0;36m_rewrite_parameters.<locals>.wrapper.<locals>.wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m    444\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapi\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/indices.py:553\u001b[0m, in \u001b[0;36mIndicesClient.create\u001b[0;34m(self, index, aliases, error_trace, filter_path, human, mappings, master_timeout, pretty, settings, timeout, wait_for_active_shards, body)\u001b[0m\n\u001b[1;32m    551\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m __body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    552\u001b[0m     __headers[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent-type\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[return-value]\u001b[39;49;00m\n\u001b[1;32m    554\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPUT\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    555\u001b[0m \u001b[43m    \u001b[49m\u001b[43m__path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__query\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    557\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__headers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    558\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__body\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    559\u001b[0m \u001b[43m    \u001b[49m\u001b[43mendpoint_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mindices.create\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    560\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_parts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__path_parts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    561\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:423\u001b[0m, in \u001b[0;36mNamespacedClient.perform_request\u001b[0;34m(self, method, path, params, headers, body, endpoint_id, path_parts)\u001b[0m\n\u001b[1;32m    410\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mperform_request\u001b[39m(\n\u001b[1;32m    411\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    412\u001b[0m     method: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    421\u001b[0m     \u001b[38;5;66;03m# Use the internal clients .perform_request() implementation\u001b[39;00m\n\u001b[1;32m    422\u001b[0m     \u001b[38;5;66;03m# so we take advantage of their transport options.\u001b[39;00m\n\u001b[0;32m--> 423\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    424\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    425\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    426\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    427\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    428\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    429\u001b[0m \u001b[43m        \u001b[49m\u001b[43mendpoint_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    430\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpath_parts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath_parts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    431\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:271\u001b[0m, in \u001b[0;36mBaseClient.perform_request\u001b[0;34m(self, method, path, params, headers, body, endpoint_id, path_parts)\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mperform_request\u001b[39m(\n\u001b[1;32m    256\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    257\u001b[0m     method: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    264\u001b[0m     path_parts: Optional[Mapping[\u001b[38;5;28mstr\u001b[39m, Any]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    265\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ApiResponse[Any]:\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_otel\u001b[38;5;241m.\u001b[39mspan(\n\u001b[1;32m    267\u001b[0m         method,\n\u001b[1;32m    268\u001b[0m         endpoint_id\u001b[38;5;241m=\u001b[39mendpoint_id,\n\u001b[1;32m    269\u001b[0m         path_parts\u001b[38;5;241m=\u001b[39mpath_parts \u001b[38;5;129;01mor\u001b[39;00m {},\n\u001b[1;32m    270\u001b[0m     ) \u001b[38;5;28;01mas\u001b[39;00m otel_span:\n\u001b[0;32m--> 271\u001b[0m         response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_perform_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    272\u001b[0m \u001b[43m            \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    273\u001b[0m \u001b[43m            \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    274\u001b[0m \u001b[43m            \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    275\u001b[0m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    276\u001b[0m \u001b[43m            \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    277\u001b[0m \u001b[43m            \u001b[49m\u001b[43motel_span\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43motel_span\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    278\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    279\u001b[0m         otel_span\u001b[38;5;241m.\u001b[39mset_elastic_cloud_metadata(response\u001b[38;5;241m.\u001b[39mmeta\u001b[38;5;241m.\u001b[39mheaders)\n\u001b[1;32m    280\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:352\u001b[0m, in \u001b[0;36mBaseClient._perform_request\u001b[0;34m(self, method, path, params, headers, body, otel_span)\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mKeyError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m):\n\u001b[1;32m    350\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 352\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HTTP_EXCEPTIONS\u001b[38;5;241m.\u001b[39mget(meta\u001b[38;5;241m.\u001b[39mstatus, ApiError)(\n\u001b[1;32m    353\u001b[0m         message\u001b[38;5;241m=\u001b[39mmessage, meta\u001b[38;5;241m=\u001b[39mmeta, body\u001b[38;5;241m=\u001b[39mresp_body\n\u001b[1;32m    354\u001b[0m     )\n\u001b[1;32m    356\u001b[0m \u001b[38;5;66;03m# 'X-Elastic-Product: Elasticsearch' should be on every 2XX response.\u001b[39;00m\n\u001b[1;32m    357\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verified_elasticsearch:\n\u001b[1;32m    358\u001b[0m     \u001b[38;5;66;03m# If the header is set we mark the server as verified.\u001b[39;00m\n",
      "\u001b[0;31mBadRequestError\u001b[0m: BadRequestError(400, 'resource_already_exists_exception', 'index [course-questions/JmfjoxHdSUug387zN66sWw] already exists')"
     ]
    }
   ],
   "source": [
    "es_client.indices.create(index=index_name, body=index_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/948 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "BadRequestError",
     "evalue": "BadRequestError(400, 'illegal_argument_exception', 'failed to parse setting [timeout] with value [30] as a time value: unit is missing or unrecognized')",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mBadRequestError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m doc \u001b[38;5;129;01min\u001b[39;00m tqdm(documents):\n\u001b[0;32m----> 2\u001b[0m     \u001b[43mes_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdocument\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdoc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m30\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/utils.py:446\u001b[0m, in \u001b[0;36m_rewrite_parameters.<locals>.wrapper.<locals>.wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m    444\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapi\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/__init__.py:2424\u001b[0m, in \u001b[0;36mElasticsearch.index\u001b[0;34m(self, index, document, body, id, error_trace, filter_path, human, if_primary_term, if_seq_no, op_type, pipeline, pretty, refresh, require_alias, routing, timeout, version, version_type, wait_for_active_shards)\u001b[0m\n\u001b[1;32m   2422\u001b[0m __body \u001b[38;5;241m=\u001b[39m document \u001b[38;5;28;01mif\u001b[39;00m document \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m body\n\u001b[1;32m   2423\u001b[0m __headers \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maccept\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent-type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m}\n\u001b[0;32m-> 2424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[return-value]\u001b[39;49;00m\n\u001b[1;32m   2425\u001b[0m \u001b[43m    \u001b[49m\u001b[43m__method\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2426\u001b[0m \u001b[43m    \u001b[49m\u001b[43m__path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2427\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__query\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2428\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__headers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2429\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__body\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2430\u001b[0m \u001b[43m    \u001b[49m\u001b[43mendpoint_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mindex\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2431\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_parts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__path_parts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2432\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:271\u001b[0m, in \u001b[0;36mBaseClient.perform_request\u001b[0;34m(self, method, path, params, headers, body, endpoint_id, path_parts)\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mperform_request\u001b[39m(\n\u001b[1;32m    256\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    257\u001b[0m     method: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    264\u001b[0m     path_parts: Optional[Mapping[\u001b[38;5;28mstr\u001b[39m, Any]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    265\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ApiResponse[Any]:\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_otel\u001b[38;5;241m.\u001b[39mspan(\n\u001b[1;32m    267\u001b[0m         method,\n\u001b[1;32m    268\u001b[0m         endpoint_id\u001b[38;5;241m=\u001b[39mendpoint_id,\n\u001b[1;32m    269\u001b[0m         path_parts\u001b[38;5;241m=\u001b[39mpath_parts \u001b[38;5;129;01mor\u001b[39;00m {},\n\u001b[1;32m    270\u001b[0m     ) \u001b[38;5;28;01mas\u001b[39;00m otel_span:\n\u001b[0;32m--> 271\u001b[0m         response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_perform_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    272\u001b[0m \u001b[43m            \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    273\u001b[0m \u001b[43m            \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    274\u001b[0m \u001b[43m            \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    275\u001b[0m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    276\u001b[0m \u001b[43m            \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    277\u001b[0m \u001b[43m            \u001b[49m\u001b[43motel_span\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43motel_span\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    278\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    279\u001b[0m         otel_span\u001b[38;5;241m.\u001b[39mset_elastic_cloud_metadata(response\u001b[38;5;241m.\u001b[39mmeta\u001b[38;5;241m.\u001b[39mheaders)\n\u001b[1;32m    280\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/Projects/llm-24/.llm-24-venv/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:352\u001b[0m, in \u001b[0;36mBaseClient._perform_request\u001b[0;34m(self, method, path, params, headers, body, otel_span)\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mKeyError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m):\n\u001b[1;32m    350\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 352\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HTTP_EXCEPTIONS\u001b[38;5;241m.\u001b[39mget(meta\u001b[38;5;241m.\u001b[39mstatus, ApiError)(\n\u001b[1;32m    353\u001b[0m         message\u001b[38;5;241m=\u001b[39mmessage, meta\u001b[38;5;241m=\u001b[39mmeta, body\u001b[38;5;241m=\u001b[39mresp_body\n\u001b[1;32m    354\u001b[0m     )\n\u001b[1;32m    356\u001b[0m \u001b[38;5;66;03m# 'X-Elastic-Product: Elasticsearch' should be on every 2XX response.\u001b[39;00m\n\u001b[1;32m    357\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verified_elasticsearch:\n\u001b[1;32m    358\u001b[0m     \u001b[38;5;66;03m# If the header is set we mark the server as verified.\u001b[39;00m\n",
      "\u001b[0;31mBadRequestError\u001b[0m: BadRequestError(400, 'illegal_argument_exception', 'failed to parse setting [timeout] with value [30] as a time value: unit is missing or unrecognized')"
     ]
    }
   ],
   "source": [
    "for doc in tqdm(documents):\n",
    "    es_client.index(index=index_name, document=doc, timeout=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"How do I execute a command in a running docker container?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "body = {\n",
    "    \"size\": 5,\n",
    "    \"query\": {\n",
    "        \"bool\": {\n",
    "            \"must\": {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": query,\n",
    "                    \"fields\": [\"question^4\", \"text\"],\n",
    "                    \"type\": \"best_fields\"\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "response = es_client.search(index=index_name, body=body)\n",
    "response[\"hits\"][\"hits\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'query' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m      1\u001b[0m body \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m      2\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msize\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m3\u001b[39m,\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquery\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m      4\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbool\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m      5\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmust\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m      6\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmulti_match\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[0;32m----> 7\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquery\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[43mquery\u001b[49m,\n\u001b[1;32m      8\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfields\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquestion^4\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m      9\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbest_fields\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     10\u001b[0m                 }\n\u001b[1;32m     11\u001b[0m             },\n\u001b[1;32m     12\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfilter\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m     13\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mterm\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m     14\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcourse\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmachine-learning-zoomcamp\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     15\u001b[0m                 }\n\u001b[1;32m     16\u001b[0m             }\n\u001b[1;32m     17\u001b[0m         }\n\u001b[1;32m     18\u001b[0m     }\n\u001b[1;32m     19\u001b[0m }\n",
      "\u001b[0;31mNameError\u001b[0m: name 'query' is not defined"
     ]
    }
   ],
   "source": [
    "body = {\n",
    "    \"size\": 3,\n",
    "    \"query\": {\n",
    "        \"bool\": {\n",
    "            \"must\": {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": query,\n",
    "                    \"fields\": [\"question^4\", \"text\"],\n",
    "                    \"type\": \"best_fields\"\n",
    "                }\n",
    "            },\n",
    "            \"filter\": {\n",
    "                \"term\": {\n",
    "                    \"course\": \"machine-learning-zoomcamp\"\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "response = es_client.search(index=index_name, body=body)\n",
    "response[\"hits\"][\"hits\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}