01-intro/HW1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "483b129f-6cca-495d-a8f8-8c1f5db91fd3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99bd96b5-5766-49fe-87eb-60a7768842a7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38106a9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#HOMEWORK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cb4c985",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Question 1\n",
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17841476",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests in /home/codespace/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages (2.32.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages (from requests) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages (from requests) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/codespace/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages (from requests) (2.2.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages (from requests) (2024.6.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7831f0fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Question 2\n",
    "##ADDING documents to Elastic Search\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "for doc in tqdm(documents):\n",
    "    es.index(index=index_name, document=doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3caebca",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Question 3\n",
    "\n",
    "user_question = \"How do I execute a command in a running docker container?\"\n",
    "es = Elasticsearch(\"http://localhost:9200\")\n",
    "\n",
    "# ^3 indicates that if user question matches the question in doc, it is given more priority when it matches with text or section\n",
    "search_query = {\n",
    "    \"size\": 5,\n",
    "    \"query\": {\n",
    "        \"bool\": {\n",
    "            \"must\": {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": user_question,\n",
    "                    \"fields\": [\"question^4\", \"text\", \"section\"], \n",
    "                    \"type\": \"best_fields\"\n",
    "                }\n",
    "            },\n",
    "            \"filter\": { \n",
    "                \"term\": {\n",
    "                    \"course\": \"data-engineering-zoomcamp\"\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b795489",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = es.search(index=index_name, body=search_query)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2d17232",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['total', 'max_score', 'hits'])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "response['hits'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b5bb2d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total': {'value': 407, 'relation': 'eq'},\n",
       " 'max_score': 75.54128,\n",
       " 'hits': [{'_index': 'course-questions',\n",
       "   '_id': '8f7fUpABu1U5aAdqyGKa',\n",
       "   '_score': 75.54128,\n",
       "   '_source': {'text': 'In case running pgcli  locally causes issues or you do not want to install it locally you can use it running in a Docker container instead.\\nBelow the usage with values used in the videos of the course for:\\nnetwork name (docker network)\\npostgres related variables for pgcli\\nHostname\\nUsername\\nPort\\nDatabase name\\n$ docker run -it --rm --network pg-network ai2ys/dockerized-pgcli:4.0.1\\n175dd47cda07:/# pgcli -h pg-database -U root -p 5432 -d ny_taxi\\nPassword for root:\\nServer: PostgreSQL 16.1 (Debian 16.1-1.pgdg120+1)\\nVersion: 4.0.1\\nHome: http://pgcli.com\\nroot@pg-database:ny_taxi> \\\\dt\\n+--------+------------------+-------+-------+\\n| Schema | Name             | Type  | Owner |\\n|--------+------------------+-------+-------|\\n| public | yellow_taxi_data | table | root  |\\n+--------+------------------+-------+-------+\\nSELECT 1\\nTime: 0.009s\\nroot@pg-database:ny_taxi>',\n",
       "    'section': 'Module 1: Docker and Terraform',\n",
       "    'question': 'PGCLI - running in a Docker container',\n",
       "    'course': 'data-engineering-zoomcamp'}},\n",
       "  {'_index': 'course-questions',\n",
       "   '_id': 'EP7fUpABu1U5aAdq4mS2',\n",
       "   '_score': 43.922554,\n",
       "   '_source': {'text': 'You can check the version of your local spark using spark-submit --version. In the build.sh file of the Python folder, make sure that SPARK_VERSION matches your local version. Similarly, make sure the pyspark you pip installed also matches this version.',\n",
       "    'section': 'Module 6: streaming with kafka',\n",
       "    'question': 'How do I check compatibility of local and container Spark versions?',\n",
       "    'course': 'data-engineering-zoomcamp'}},\n",
       "  {'_index': 'course-questions',\n",
       "   '_id': 'jP7fUpABu1U5aAdqvmKf',\n",
       "   '_score': 38.684105,\n",
       "   '_source': {'text': \"There are 3 Zoom Camps in a year, as of 2024. However, they are for separate courses:\\nData-Engineering (Jan - Apr)\\nMLOps (May - Aug)\\nMachine Learning (Sep - Jan)\\nThere's only one Data-Engineering Zoomcamp “live” cohort per year, for the certification. Same as for the other Zoomcamps.\\nThey follow pretty much the same schedule for each cohort per zoomcamp. For Data-Engineering it is (generally) from Jan-Apr of the year. If you’re not interested in the Certificate, you can take any zoom camps at any time, at your own pace, out of sync with any “live” cohort.\",\n",
       "    'section': 'General course-related questions',\n",
       "    'question': 'Course - how many Zoomcamps in a year?',\n",
       "    'course': 'data-engineering-zoomcamp'}},\n",
       "  {'_index': 'course-questions',\n",
       "   '_id': 'wP7fUpABu1U5aAdqw2K1',\n",
       "   '_score': 38.33403,\n",
       "   '_source': {'text': \"It is recommended by the Docker do\\n[Windows 10 / 11 Home Edition] If you're running a Home Edition, you can still make it work with WSL2 (Windows Subsystem for Linux) by following the tutorial here\\nIf even after making sure your WSL2 (or Hyper-V) is set up accordingly, Docker remains stuck, you can try the option to Reset to Factory Defaults or do a fresh install.\",\n",
       "    'section': 'Module 1: Docker and Terraform',\n",
       "    'question': 'Should I run docker commands from the windows file system or a file system of a Linux distribution in WSL?',\n",
       "    'course': 'data-engineering-zoomcamp'}},\n",
       "  {'_index': 'course-questions',\n",
       "   '_id': 'kv7fUpABu1U5aAdqv2JC',\n",
       "   '_score': 35.94081,\n",
       "   '_source': {'text': \"No, you can only get a certificate if you finish the course with a “live” cohort. We don't award certificates for the self-paced mode. The reason is you need to peer-review capstone(s) after submitting a project. You can only peer-review projects at the time the course is running.\",\n",
       "    'section': 'General course-related questions',\n",
       "    'question': 'Certificate - Can I follow the course in a self-paced mode and get a certificate?',\n",
       "    'course': 'data-engineering-zoomcamp'}}]}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "response['hits']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc1c53ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': 'In case running pgcli  locally causes issues or you do not want to install it locally you can use it running in a Docker container instead.\\nBelow the usage with values used in the videos of the course for:\\nnetwork name (docker network)\\npostgres related variables for pgcli\\nHostname\\nUsername\\nPort\\nDatabase name\\n$ docker run -it --rm --network pg-network ai2ys/dockerized-pgcli:4.0.1\\n175dd47cda07:/# pgcli -h pg-database -U root -p 5432 -d ny_taxi\\nPassword for root:\\nServer: PostgreSQL 16.1 (Debian 16.1-1.pgdg120+1)\\nVersion: 4.0.1\\nHome: http://pgcli.com\\nroot@pg-database:ny_taxi> \\\\dt\\n+--------+------------------+-------+-------+\\n| Schema | Name             | Type  | Owner |\\n|--------+------------------+-------+-------|\\n| public | yellow_taxi_data | table | root  |\\n+--------+------------------+-------+-------+\\nSELECT 1\\nTime: 0.009s\\nroot@pg-database:ny_taxi>',\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'PGCLI - running in a Docker container',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'You can check the version of your local spark using spark-submit --version. In the build.sh file of the Python folder, make sure that SPARK_VERSION matches your local version. Similarly, make sure the pyspark you pip installed also matches this version.',\n",
       "  'section': 'Module 6: streaming with kafka',\n",
       "  'question': 'How do I check compatibility of local and container Spark versions?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"There are 3 Zoom Camps in a year, as of 2024. However, they are for separate courses:\\nData-Engineering (Jan - Apr)\\nMLOps (May - Aug)\\nMachine Learning (Sep - Jan)\\nThere's only one Data-Engineering Zoomcamp “live” cohort per year, for the certification. Same as for the other Zoomcamps.\\nThey follow pretty much the same schedule for each cohort per zoomcamp. For Data-Engineering it is (generally) from Jan-Apr of the year. If you’re not interested in the Certificate, you can take any zoom camps at any time, at your own pace, out of sync with any “live” cohort.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - how many Zoomcamps in a year?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"It is recommended by the Docker do\\n[Windows 10 / 11 Home Edition] If you're running a Home Edition, you can still make it work with WSL2 (Windows Subsystem for Linux) by following the tutorial here\\nIf even after making sure your WSL2 (or Hyper-V) is set up accordingly, Docker remains stuck, you can try the option to Reset to Factory Defaults or do a fresh install.\",\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'Should I run docker commands from the windows file system or a file system of a Linux distribution in WSL?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"No, you can only get a certificate if you finish the course with a “live” cohort. We don't award certificates for the self-paced mode. The reason is you need to peer-review capstone(s) after submitting a project. You can only peer-review projects at the time the course is running.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Certificate - Can I follow the course in a self-paced mode and get a certificate?',\n",
       "  'course': 'data-engineering-zoomcamp'}]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "documents = [hit['_source'] for hit in response['hits']['hits']]\n",
    "documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2938d51e",
   "metadata": {},
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "Connection error caused by: ConnectionError(Connection error caused by: NewConnectionError(<urllib3.connection.HTTPConnection object at 0x7a8f22b14ee0>: Failed to establish a new connection: [Errno 111] Connection refused))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)\n",
      "Cell \u001b[0;32mIn[80], line 27\u001b[0m\n",
      "\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# ^3 indicates that if user question matches the question in doc, it is given more priority when it matches with text or section\u001b[39;00m\n",
      "\u001b[1;32m      8\u001b[0m search_query \u001b[38;5;241m=\u001b[39m {\n",
      "\u001b[1;32m      9\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msize\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m3\u001b[39m,\n",
      "\u001b[1;32m     10\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquery\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m     25\u001b[0m     }\n",
      "\u001b[1;32m     26\u001b[0m }\n",
      "\u001b[0;32m---> 27\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_query\u001b[49m\u001b[43m)\u001b[49m\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elasticsearch/_sync/client/utils.py:446\u001b[0m, in \u001b[0;36m_rewrite_parameters.<locals>.wrapper.<locals>.wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;32m    443\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n",
      "\u001b[1;32m    444\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapi\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elasticsearch/_sync/client/__init__.py:4119\u001b[0m, in \u001b[0;36mElasticsearch.search\u001b[0;34m(self, index, aggregations, aggs, allow_no_indices, allow_partial_search_results, analyze_wildcard, analyzer, batched_reduce_size, ccs_minimize_roundtrips, collapse, default_operator, df, docvalue_fields, error_trace, expand_wildcards, explain, ext, fields, filter_path, force_synthetic_source, from_, highlight, human, ignore_throttled, ignore_unavailable, indices_boost, knn, lenient, max_concurrent_shard_requests, min_compatible_shard_node, min_score, pit, post_filter, pre_filter_shard_size, preference, pretty, profile, q, query, rank, request_cache, rescore, rest_total_hits_as_int, retriever, routing, runtime_mappings, script_fields, scroll, search_after, search_type, seq_no_primary_term, size, slice, sort, source, source_excludes, source_includes, stats, stored_fields, suggest, suggest_field, suggest_mode, suggest_size, suggest_text, terminate_after, timeout, track_scores, track_total_hits, typed_keys, version, body)\u001b[0m\n",
      "\u001b[1;32m   4117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m __body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;32m   4118\u001b[0m     __headers[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent-type\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "\u001b[0;32m-> 4119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[return-value]\u001b[39;49;00m\n",
      "\u001b[1;32m   4120\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPOST\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4121\u001b[0m \u001b[43m    \u001b[49m\u001b[43m__path\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4122\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__query\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4123\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__headers\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4124\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__body\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4125\u001b[0m \u001b[43m    \u001b[49m\u001b[43mendpoint_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msearch\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4126\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_parts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m__path_parts\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m   4127\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:271\u001b[0m, in \u001b[0;36mBaseClient.perform_request\u001b[0;34m(self, method, path, params, headers, body, endpoint_id, path_parts)\u001b[0m\n",
      "\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mperform_request\u001b[39m(\n",
      "\u001b[1;32m    256\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n",
      "\u001b[1;32m    257\u001b[0m     method: \u001b[38;5;28mstr\u001b[39m,\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    264\u001b[0m     path_parts: Optional[Mapping[\u001b[38;5;28mstr\u001b[39m, Any]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n",
      "\u001b[1;32m    265\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ApiResponse[Any]:\n",
      "\u001b[1;32m    266\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_otel\u001b[38;5;241m.\u001b[39mspan(\n",
      "\u001b[1;32m    267\u001b[0m         method,\n",
      "\u001b[1;32m    268\u001b[0m         endpoint_id\u001b[38;5;241m=\u001b[39mendpoint_id,\n",
      "\u001b[1;32m    269\u001b[0m         path_parts\u001b[38;5;241m=\u001b[39mpath_parts \u001b[38;5;129;01mor\u001b[39;00m {},\n",
      "\u001b[1;32m    270\u001b[0m     ) \u001b[38;5;28;01mas\u001b[39;00m otel_span:\n",
      "\u001b[0;32m--> 271\u001b[0m         response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_perform_request\u001b[49m\u001b[43m(\u001b[49m\n",
      "\u001b[1;32m    272\u001b[0m \u001b[43m            \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    273\u001b[0m \u001b[43m            \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    274\u001b[0m \u001b[43m            \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    275\u001b[0m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    276\u001b[0m \u001b[43m            \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    277\u001b[0m \u001b[43m            \u001b[49m\u001b[43motel_span\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43motel_span\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    278\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;32m    279\u001b[0m         otel_span\u001b[38;5;241m.\u001b[39mset_elastic_cloud_metadata(response\u001b[38;5;241m.\u001b[39mmeta\u001b[38;5;241m.\u001b[39mheaders)\n",
      "\u001b[1;32m    280\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elasticsearch/_sync/client/_base.py:316\u001b[0m, in \u001b[0;36mBaseClient._perform_request\u001b[0;34m(self, method, path, params, headers, body, otel_span)\u001b[0m\n",
      "\u001b[1;32m    313\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[1;32m    314\u001b[0m     target \u001b[38;5;241m=\u001b[39m path\n",
      "\u001b[0;32m--> 316\u001b[0m meta, resp_body \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\n",
      "\u001b[1;32m    317\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    318\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    319\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_headers\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    320\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    321\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request_timeout\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    322\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_max_retries\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    323\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretry_on_status\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_on_status\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    324\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretry_on_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_retry_on_timeout\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    325\u001b[0m \u001b[43m    \u001b[49m\u001b[43mclient_meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_client_meta\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    326\u001b[0m \u001b[43m    \u001b[49m\u001b[43motel_span\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43motel_span\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    327\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;32m    329\u001b[0m \u001b[38;5;66;03m# HEAD with a 404 is returned as a normal response\u001b[39;00m\n",
      "\u001b[1;32m    330\u001b[0m \u001b[38;5;66;03m# since this is used as an 'exists' functionality.\u001b[39;00m\n",
      "\u001b[1;32m    331\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (method \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHEAD\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m meta\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m404\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m (\n",
      "\u001b[1;32m    332\u001b[0m     \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m meta\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m299\u001b[39m\n",
      "\u001b[1;32m    333\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    337\u001b[0m     )\n",
      "\u001b[1;32m    338\u001b[0m ):\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elastic_transport/_transport.py:342\u001b[0m, in \u001b[0;36mTransport.perform_request\u001b[0;34m(self, method, target, body, headers, max_retries, retry_on_status, retry_on_timeout, request_timeout, client_meta, otel_span)\u001b[0m\n",
      "\u001b[1;32m    340\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[1;32m    341\u001b[0m     otel_span\u001b[38;5;241m.\u001b[39mset_node_metadata(node\u001b[38;5;241m.\u001b[39mhost, node\u001b[38;5;241m.\u001b[39mport, node\u001b[38;5;241m.\u001b[39mbase_url, target)\n",
      "\u001b[0;32m--> 342\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperform_request\u001b[49m\u001b[43m(\u001b[49m\n",
      "\u001b[1;32m    343\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    344\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    345\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_body\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    346\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_headers\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    347\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_timeout\u001b[49m\u001b[43m,\u001b[49m\n",
      "\u001b[1;32m    348\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;32m    349\u001b[0m     _logger\u001b[38;5;241m.\u001b[39minfo(\n",
      "\u001b[1;32m    350\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m [status:\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m duration:\u001b[39m\u001b[38;5;132;01m%.3f\u001b[39;00m\u001b[38;5;124ms]\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "\u001b[1;32m    351\u001b[0m         \u001b[38;5;241m%\u001b[39m (\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    357\u001b[0m         )\n",
      "\u001b[1;32m    358\u001b[0m     )\n",
      "\u001b[1;32m    360\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m method \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHEAD\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/llm-rag-J7PTIVi-/lib/python3.10/site-packages/elastic_transport/_node/_http_urllib3.py:202\u001b[0m, in \u001b[0;36mUrllib3HttpNode.perform_request\u001b[0;34m(self, method, target, body, headers, request_timeout)\u001b[0m\n",
      "\u001b[1;32m    194\u001b[0m         err \u001b[38;5;241m=\u001b[39m \u001b[38;5;167;01mConnectionError\u001b[39;00m(\u001b[38;5;28mstr\u001b[39m(e), errors\u001b[38;5;241m=\u001b[39m(e,))\n",
      "\u001b[1;32m    195\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_log_request(\n",
      "\u001b[1;32m    196\u001b[0m         method\u001b[38;5;241m=\u001b[39mmethod,\n",
      "\u001b[1;32m    197\u001b[0m         target\u001b[38;5;241m=\u001b[39mtarget,\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    200\u001b[0m         exception\u001b[38;5;241m=\u001b[39merr,\n",
      "\u001b[1;32m    201\u001b[0m     )\n",
      "\u001b[0;32m--> 202\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m err \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[1;32m    204\u001b[0m meta \u001b[38;5;241m=\u001b[39m ApiResponseMeta(\n",
      "\u001b[1;32m    205\u001b[0m     node\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig,\n",
      "\u001b[1;32m    206\u001b[0m     duration\u001b[38;5;241m=\u001b[39mduration,\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    209\u001b[0m     headers\u001b[38;5;241m=\u001b[39mresponse_headers,\n",
      "\u001b[1;32m    210\u001b[0m )\n",
      "\u001b[1;32m    211\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_log_request(\n",
      "\u001b[1;32m    212\u001b[0m     method\u001b[38;5;241m=\u001b[39mmethod,\n",
      "\u001b[1;32m    213\u001b[0m     target\u001b[38;5;241m=\u001b[39mtarget,\n",
      "\u001b[0;32m   (...)\u001b[0m\n",
      "\u001b[1;32m    217\u001b[0m     response\u001b[38;5;241m=\u001b[39mdata,\n",
      "\u001b[1;32m    218\u001b[0m )\n",
      "\n",
      "\u001b[0;31mConnectionError\u001b[0m: Connection error caused by: ConnectionError(Connection error caused by: NewConnectionError(<urllib3.connection.HTTPConnection object at 0x7a8f22b14ee0>: Failed to establish a new connection: [Errno 111] Connection refused))"
     ]
    }
   ],
   "source": [
    "#Question 4\n",
    "\n",
    "#Question 3\n",
    "\n",
    "user_question = \"How do I execute a command in a running docker container?\"\n",
    "\n",
    "# ^3 indicates that if user question matches the question in doc, it is given more priority when it matches with text or section\n",
    "search_query = {\n",
    "    \"size\": 3,\n",
    "    \"query\": {\n",
    "        \"bool\": {\n",
    "            \"must\": {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": user_question,\n",
    "                    \"fields\": [\"question^4\", \"text\", \"section\"], \n",
    "                    \"type\": \"best_fields\"\n",
    "                }\n",
    "            },\n",
    "            \"filter\": { \n",
    "                \"term\": {\n",
    "                    \"course\": \"machine-learning-zoomcamp\"\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "response = es.search(index=index_name, body=search_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73802f81",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'ChatCompletion' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)\n",
      "Cell \u001b[0;32mIn[73], line 1\u001b[0m\n",
      "\u001b[0;32m----> 1\u001b[0m documents \u001b[38;5;241m=\u001b[39m [hit[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_source\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m hit \u001b[38;5;129;01min\u001b[39;00m \u001b[43mresponse\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhits\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhits\u001b[39m\u001b[38;5;124m'\u001b[39m]]\n",
      "\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m doc \u001b[38;5;129;01min\u001b[39;00m documents:\n",
      "\u001b[1;32m      3\u001b[0m     \u001b[38;5;28mprint\u001b[39m(doc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mquestion\u001b[39m\u001b[38;5;124m'\u001b[39m])\n",
      "\n",
      "\u001b[0;31mTypeError\u001b[0m: 'ChatCompletion' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "documents = [hit['_source'] for hit in response['hits']['hits']]\n",
    "for doc in documents:\n",
    "    print(doc['question'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b2ad6b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Question 5\n",
    "context_template = \"\"\"\n",
    "Q: {question}\n",
    "A: {text}\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bcdc5be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: How do I debug a docker container?\n",
      "A: Launch the container image in interactive mode and overriding the entrypoint, so that it starts a bash command.\n",
      "docker run -it --entrypoint bash <image>\n",
      "If the container is already running, execute a command in the specific container:\n",
      "docker ps (find the container-id)\n",
      "docker exec -it <container-id> bash\n",
      "(Marcos MJD)\n",
      "\n",
      "Q: How do I copy files from my local machine to docker container?\n",
      "A: You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\n",
      "To copy a file or directory from your local machine into a running Docker container, you can use the `docker cp command`. The basic syntax is as follows:\n",
      "docker cp /path/to/local/file_or_directory container_id:/path/in/container\n",
      "Hrithik Kumar Advani\n",
      "\n",
      "Q: How do I copy files from a different folder into docker container’s working directory?\n",
      "A: You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\n",
      "In the Dockerfile, you can provide the folder containing the files that you want to copy over. The basic syntax is as follows:\n",
      "COPY [\"src/predict.py\", \"models/xgb_model.bin\", \"./\"]\t\t\t\t\t\t\t\t\t\t\tGopakumar Gopinathan\n",
      "\n",
      "Q: The input device is not a TTY when running docker in interactive mode (Running Docker on Windows in GitBash)\n",
      "A: $ docker exec -it 1e5a1b663052 bash\n",
      "the input device is not a TTY.  If you are using mintty, try prefixing the command with 'winpty'\n",
      "Fix:\n",
      "winpty docker exec -it 1e5a1b663052 bash\n",
      "A TTY is a terminal interface that supports escape sequences, moving the cursor around, etc.\n",
      "Winpty is a Windows software package providing an interface similar to a Unix pty-master for communicating with Windows console programs.\n",
      "More info on terminal, shell, console applications hi and so on:\n",
      "https://conemu.github.io/en/TerminalVsShell.html\n",
      "(Marcos MJD)\n",
      "\n",
      "Q: How to fix error after running the Docker run command\n",
      "A: Solution\n",
      "This error was because there was another instance of gunicorn running. So I thought of removing this along with the zoomcamp_test image. However, it didn’t let me remove the orphan container. So I did the following\n",
      "Running the following commands\n",
      "docker ps -a <to list all docker containers>\n",
      "docker images <to list images>\n",
      "docker stop <container ID>\n",
      "docker rm <container ID>\n",
      "docker rmi image\n",
      "I rebuilt the Docker image, and ran it once again; this time it worked correctly and I was able to serve the test script to the endpoint.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "context = \"\"\n",
    "\n",
    "for doc in documents:\n",
    "    doc_str = f\"Q: {doc['question']}\\nA: {doc['text']}\\n\\n\"\n",
    "    context += doc_str\n",
    "\n",
    "context = context.strip()\n",
    "print(context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4890a29c",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_question = \"How do I execute a command in a running docker container?\"\n",
    "\n",
    "prompt_template = f\"\"\"\n",
    "You're a course teaching assistant. Answer the QUESTION based on the CONTEXT from the FAQ database.\n",
    "Use only the facts from the CONTEXT when answering the QUESTION.\n",
    "\n",
    "QUESTION: {user_question}\n",
    "\n",
    "CONTEXT:\n",
    "{context}\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4036ba52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1462"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "len(prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9139defc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Question 6\n",
    "# !pip install tiktoken\n",
    "import tiktoken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05022d98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "325"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo-0125\")\n",
    "encoded_tokens = encoding.encode(prompt_template)\n",
    "len(encoded_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "390862a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "322"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoding = tiktoken.encoding_for_model(\"gpt-4o\")\n",
    "encoded_tokens = encoding.encode(prompt_template)\n",
    "len(encoded_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4744b264",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'To execute a command in a running docker container, you can use the following command:\\ndocker exec -it <container-id> bash'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "client = OpenAI()\n",
    "response = client.chat.completions.create(\n",
    "    model=\"gpt-3.5-turbo-0125\",\n",
    "    messages=[{\"role\": \"user\", \"content\": prompt_template}]\n",
    ")\n",
    "answer = response.choices[0].message.content\n",
    "answer"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}