01-intro/Homework_01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9217184e-e0c7-43fb-872e-035c99f059ef",
   "metadata": {},
   "source": [
    "## Home work\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c110f5fc-4c2a-42d9-a084-8e488b53c564",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-06-30 04:53:40--  https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.py\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.108.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 3832 (3.7K) [text/plain]\n",
      "Saving to: ‘minsearch.py.2’\n",
      "\n",
      "minsearch.py.2      100%[===================>]   3.74K  --.-KB/s    in 0s      \n",
      "\n",
      "2024-06-30 04:53:41 (18.7 MB/s) - ‘minsearch.py.2’ saved [3832/3832]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "59d27aaa-a2c3-4fec-b716-a9c3f79e1b5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import minsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b19b3057-1a24-487e-8b44-17b882c1ff58",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "818b1da9-41af-4655-be43-1fbbdfdbb359",
   "metadata": {},
   "outputs": [],
   "source": [
    "index= minsearch.Index(\n",
    "    text_fields=[\"question\", \"text\", \"section\"],\n",
    "    keyword_fields=[\"course\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b3f90143-c750-4327-8c13-7ea69931c726",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<minsearch.Index at 0x7c5b4999acb0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.fit(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1fe6d750-9bfc-48cf-b793-b957315781cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': 'In case running pgcli  locally causes issues or you do not want to install it locally you can use it running in a Docker container instead.\\nBelow the usage with values used in the videos of the course for:\\nnetwork name (docker network)\\npostgres related variables for pgcli\\nHostname\\nUsername\\nPort\\nDatabase name\\n$ docker run -it --rm --network pg-network ai2ys/dockerized-pgcli:4.0.1\\n175dd47cda07:/# pgcli -h pg-database -U root -p 5432 -d ny_taxi\\nPassword for root:\\nServer: PostgreSQL 16.1 (Debian 16.1-1.pgdg120+1)\\nVersion: 4.0.1\\nHome: http://pgcli.com\\nroot@pg-database:ny_taxi> \\\\dt\\n+--------+------------------+-------+-------+\\n| Schema | Name             | Type  | Owner |\\n|--------+------------------+-------+-------|\\n| public | yellow_taxi_data | table | root  |\\n+--------+------------------+-------+-------+\\nSELECT 1\\nTime: 0.009s\\nroot@pg-database:ny_taxi>',\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'PGCLI - running in a Docker container',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'Sometimes, when you try to restart a docker image configured with a network name, the above message appears. In this case, use the following command with the appropriate container name:\\n>>> If the container is running state, use docker stop <container_name>\\n>>> then, docker rm pg-database\\nOr use docker start instead of docker run in order to restart the docker image without removing it.',\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'Docker - Error response from daemon: Conflict. The container name \"pg-database\" is already in use by container “xxx”.  You have to remove (or rename) that container to be able to reuse that name.',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"You may have this error:\\nRetrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.u\\nrllib3.connection.HTTPSConnection object at 0x7efe331cf790>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')':\\n/simple/pandas/\\nPossible solution might be:\\n$ winpty docker run -it --dns=8.8.8.8 --entrypoint=bash python:3.9\",\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'Docker - Cannot pip install on Docker container (Windows)',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"Make sure you're able to start the Docker daemon, and check the issue immediately down below:\\nAnd don’t forget to update the wsl in powershell the  command is wsl –update\",\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'Docker - Cannot connect to Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'Whenever a `docker pull is performed (either manually or by `docker-compose up`), it attempts to fetch the given image name (pgadmin4, for the example above) from a repository (dbpage).\\nIF the repository is public, the fetch and download happens without any issue whatsoever.\\nFor instance:\\ndocker pull postgres:13\\ndocker pull dpage/pgadmin4\\nBE ADVISED:\\n\\nThe Docker Images we\\'ll be using throughout the Data Engineering Zoomcamp are all public (except when or if explicitly said otherwise by the instructors or co-instructors).\\n\\nMeaning: you are NOT required to perform a docker login to fetch them. \\n\\nSo if you get the message above saying \"docker login\\': denied: requested access to the resource is denied. That is most likely due to a typo in your image name:\\n\\nFor instance:\\n$ docker pull dbpage/pgadmin4\\nWill throw that exception telling you \"repository does not exist or may require \\'docker login\\'\\nError response from daemon: pull access denied for dbpage/pgadmin4, repository does not exist or \\nmay require \\'docker login\\': denied: requested access to the resource is denied\\nBut that actually happened because the actual image is dpage/pgadmin4 and NOT dbpage/pgadmin4\\nHow to fix it:\\n$ docker pull dpage/pgadmin4\\nEXTRA NOTES:\\nIn the real world, occasionally, when you\\'re working for a company or closed organisation, the Docker image you\\'re trying to fetch might be under a private repo that your DockerHub Username was granted access to.\\nFor which cases, you must first execute:\\n$ docker login\\nFill in the details of your username and password.\\nAnd only then perform the `docker pull` against that private repository\\nWhy am I encountering a \"permission denied\" error when creating a PostgreSQL Docker container for the New York Taxi Database with a mounted volume on macOS M1?\\nIssue Description:\\nWhen attempting to run a Docker command similar to the one below:\\ndocker run -it \\\\\\n-e POSTGRES_USER=\"root\" \\\\\\n-e POSTGRES_PASSWORD=\"root\" \\\\\\n-e POSTGRES_DB=\"ny_taxi\" \\\\\\n-v $(pwd)/ny_taxi_postgres_data:/var/lib/postgresql/data \\\\\\n-p 5432:5432 \\\\mount\\npostgres:13\\nYou encounter the error message:\\ndocker: Error response from daemon: error while creating mount source path \\'/path/to/ny_taxi_postgres_data\\': chown /path/to/ny_taxi_postgres_data: permission denied.\\nSolution:\\n1- Stop Rancher Desktop:\\nIf you are using Rancher Desktop and face this issue, stop Rancher Desktop to resolve compatibility problems.\\n2- Install Docker Desktop:\\nInstall Docker Desktop, ensuring that it is properly configured and has the required permissions.\\n2-Retry Docker Command:\\nRun the Docker command again after switching to Docker Desktop. This step resolves compatibility issues on some systems.\\nNote: The issue occurred because Rancher Desktop was in use. Switching to Docker Desktop resolves compatibility problems and allows for the successful creation of PostgreSQL containers with mounted volumes for the New York Taxi Database on macOS M1.',\n",
       "  'section': 'Module 1: Docker and Terraform',\n",
       "  'question': 'Docker - docker pull dbpage',\n",
       "  'course': 'data-engineering-zoomcamp'}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#boost_dict el 3 indica que la question es 3 veces mas importante que text y section\n",
    "query = \"How do I execute a command in a running docker container?\"\n",
    "\n",
    "filter_dict = {\"course\": \"data-engineering-zoomcamp\"}\n",
    "boost_dict = {\"question\": 3.0, \"text\": 1.0, \"section\": 1.0}\n",
    "\n",
    "results = index.search(query, filter_dict, boost_dict, num_results= 5)\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "00656a30-d363-49c8-8a8e-f5b2626bcfe8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in /usr/local/python/3.10.13/lib/python3.10/site-packages (1.35.7)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/codespace/.local/lib/python3.10/site-packages (from openai) (4.4.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/codespace/.local/lib/python3.10/site-packages (from openai) (0.27.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from openai) (2.7.4)\n",
      "Requirement already satisfied: sniffio in /home/codespace/.local/lib/python3.10/site-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/codespace/.local/lib/python3.10/site-packages (from openai) (4.12.0)\n",
      "Requirement already satisfied: idna>=2.8 in /home/codespace/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/codespace/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
      "Requirement already satisfied: certifi in /home/codespace/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/codespace/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/codespace/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai) (2.18.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e0611133-2c6f-4b96-ba69-57e5fe7b40dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c6f32a6-9f5a-4073-8831-022fef0ee60f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "api_key = os.getenv('OPENAI_API_KEY')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "73090ec1-b2ff-44eb-b15f-47b95e56dbbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "client= OpenAI(api_key=api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "aa2ef975-8f10-4e12-995b-d535cc450909",
   "metadata": {},
   "outputs": [],
   "source": [
    "query= \"How do I execute a command in a running docker container?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "86d0b507-6227-4099-ad11-7e55227e4686",
   "metadata": {},
   "outputs": [],
   "source": [
    "response= client.chat.completions.create(\n",
    "    model= 'gpt-4o',\n",
    "    messages= [{\"role\": \"user\", \"content\": query}]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ad3bf1c6-25c0-4d1c-8d58-32f27bcfb436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"To execute a command in a running Docker container, you can use the `docker exec` command. This command allows you to run a new command in the context of an existing container. Here are the basic steps:\\n\\n1. **Identify the Container ID or Name:** First, you need to identify the container where you want to execute the command. You can do this with the `docker ps` command, which lists all running containers.\\n\\n    ```sh\\n    docker ps\\n    ```\\n\\n    This will output a list of running containers, including their IDs and names.\\n\\n2. **Execute the Command:** Once you have the container ID or name, you can use the `docker exec` command to run the desired command inside the container. The general syntax is:\\n\\n    ```sh\\n    docker exec [OPTIONS] CONTAINER COMMAND [ARG...]\\n    ```\\n\\n    - `OPTIONS`: You can use various options such as `-it` for interactive terminal.\\n    - `CONTAINER`: The ID or name of the running container.\\n    - `COMMAND`: The command you want to execute.\\n    - `[ARG...]`: (Optional) Arguments for the command.\\n\\n    For example, to open a bash shell in an interactive mode inside a container named `my_container`:\\n\\n    ```sh\\n    docker exec -it my_container /bin/bash\\n    ```\\n\\n    If you simply want to run a non-interactive command, you can do so without `-it`. For example, to list the contents of the `/var` directory inside the container:\\n\\n    ```sh\\n    docker exec my_container ls /var\\n    ```\\n\\n### Examples\\n\\n1. **Run an Interactive Shell:**\\n\\n    ```sh\\n    docker exec -it <container_id_or_name> /bin/bash\\n    ```\\n\\n    or if the container uses `sh` instead of `bash`:\\n\\n    ```sh\\n    docker exec -it <container_id_or_name> /bin/sh\\n    ```\\n\\n2. **Run a Single Command:**\\n\\n    ```sh\\n    docker exec <container_id_or_name> ps aux\\n    ```\\n\\n3. **Run a Command with Arguments:**\\n\\n    ```sh\\n    docker exec <container_id_or_name> ls -la /var/log\\n    ```\\n\\n### Important Options\\n\\n- `-i`: Keep STDIN open even if not attached.\\n- `-t`: Allocate a pseudo-TTY.\\n- `-u`: Specify the username or UID to use.\\n\\n### Tips\\n\\n- Make sure that the command you're trying to run is available inside the container.\\n- For `root` access, ensure you have appropriate permissions or use the `-u root` option if required.\\n\\n### Summary\\n\\nUsing `docker exec` is a versatile way to interact with a running container or to run specific commands within it. It is especially useful for debugging, maintenance, and administrative tasks.\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a8cae211-e969-4fc5-9653-377535f7ee5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_template= \"\"\"\n",
    "You're a teaching assistant. Answer the QUESTION based on the CONTEXT.\n",
    "Use only the facts form the CONTEXT when answering the QUESTION.\n",
    "If the CONTEXT don't have the answer, output NONE\n",
    "\n",
    "QUESTION: {question}\n",
    "CONTEXT: {context}\n",
    "\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5a7adb00-f2ff-4023-8dd6-c89d8f7c54d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "context= \"\"\n",
    "\n",
    "for doc in results:\n",
    "    context= context + f\"section: {doc['section']}\\nquestion: {doc['question']}\\nanswer: {doc['text']}\\n\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bd2a675e-b43f-4305-87d1-1f9de585b2ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "section: Module 1: Docker and Terraform\n",
      "question: PGCLI - running in a Docker container\n",
      "answer: In case running pgcli  locally causes issues or you do not want to install it locally you can use it running in a Docker container instead.\n",
      "Below the usage with values used in the videos of the course for:\n",
      "network name (docker network)\n",
      "postgres related variables for pgcli\n",
      "Hostname\n",
      "Username\n",
      "Port\n",
      "Database name\n",
      "$ docker run -it --rm --network pg-network ai2ys/dockerized-pgcli:4.0.1\n",
      "175dd47cda07:/# pgcli -h pg-database -U root -p 5432 -d ny_taxi\n",
      "Password for root:\n",
      "Server: PostgreSQL 16.1 (Debian 16.1-1.pgdg120+1)\n",
      "Version: 4.0.1\n",
      "Home: http://pgcli.com\n",
      "root@pg-database:ny_taxi> \\dt\n",
      "+--------+------------------+-------+-------+\n",
      "| Schema | Name             | Type  | Owner |\n",
      "|--------+------------------+-------+-------|\n",
      "| public | yellow_taxi_data | table | root  |\n",
      "+--------+------------------+-------+-------+\n",
      "SELECT 1\n",
      "Time: 0.009s\n",
      "root@pg-database:ny_taxi>\n",
      "\n",
      "section: Module 1: Docker and Terraform\n",
      "question: Docker - Error response from daemon: Conflict. The container name \"pg-database\" is already in use by container “xxx”.  You have to remove (or rename) that container to be able to reuse that name.\n",
      "answer: Sometimes, when you try to restart a docker image configured with a network name, the above message appears. In this case, use the following command with the appropriate container name:\n",
      ">>> If the container is running state, use docker stop <container_name>\n",
      ">>> then, docker rm pg-database\n",
      "Or use docker start instead of docker run in order to restart the docker image without removing it.\n",
      "\n",
      "section: Module 1: Docker and Terraform\n",
      "question: Docker - Cannot pip install on Docker container (Windows)\n",
      "answer: You may have this error:\n",
      "Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.u\n",
      "rllib3.connection.HTTPSConnection object at 0x7efe331cf790>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')':\n",
      "/simple/pandas/\n",
      "Possible solution might be:\n",
      "$ winpty docker run -it --dns=8.8.8.8 --entrypoint=bash python:3.9\n",
      "\n",
      "section: Module 1: Docker and Terraform\n",
      "question: Docker - Cannot connect to Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\n",
      "answer: Make sure you're able to start the Docker daemon, and check the issue immediately down below:\n",
      "And don’t forget to update the wsl in powershell the  command is wsl –update\n",
      "\n",
      "section: Module 1: Docker and Terraform\n",
      "question: Docker - docker pull dbpage\n",
      "answer: Whenever a `docker pull is performed (either manually or by `docker-compose up`), it attempts to fetch the given image name (pgadmin4, for the example above) from a repository (dbpage).\n",
      "IF the repository is public, the fetch and download happens without any issue whatsoever.\n",
      "For instance:\n",
      "docker pull postgres:13\n",
      "docker pull dpage/pgadmin4\n",
      "BE ADVISED:\n",
      "\n",
      "The Docker Images we'll be using throughout the Data Engineering Zoomcamp are all public (except when or if explicitly said otherwise by the instructors or co-instructors).\n",
      "\n",
      "Meaning: you are NOT required to perform a docker login to fetch them. \n",
      "\n",
      "So if you get the message above saying \"docker login': denied: requested access to the resource is denied. That is most likely due to a typo in your image name:\n",
      "\n",
      "For instance:\n",
      "$ docker pull dbpage/pgadmin4\n",
      "Will throw that exception telling you \"repository does not exist or may require 'docker login'\n",
      "Error response from daemon: pull access denied for dbpage/pgadmin4, repository does not exist or \n",
      "may require 'docker login': denied: requested access to the resource is denied\n",
      "But that actually happened because the actual image is dpage/pgadmin4 and NOT dbpage/pgadmin4\n",
      "How to fix it:\n",
      "$ docker pull dpage/pgadmin4\n",
      "EXTRA NOTES:\n",
      "In the real world, occasionally, when you're working for a company or closed organisation, the Docker image you're trying to fetch might be under a private repo that your DockerHub Username was granted access to.\n",
      "For which cases, you must first execute:\n",
      "$ docker login\n",
      "Fill in the details of your username and password.\n",
      "And only then perform the `docker pull` against that private repository\n",
      "Why am I encountering a \"permission denied\" error when creating a PostgreSQL Docker container for the New York Taxi Database with a mounted volume on macOS M1?\n",
      "Issue Description:\n",
      "When attempting to run a Docker command similar to the one below:\n",
      "docker run -it \\\n",
      "-e POSTGRES_USER=\"root\" \\\n",
      "-e POSTGRES_PASSWORD=\"root\" \\\n",
      "-e POSTGRES_DB=\"ny_taxi\" \\\n",
      "-v $(pwd)/ny_taxi_postgres_data:/var/lib/postgresql/data \\\n",
      "-p 5432:5432 \\mount\n",
      "postgres:13\n",
      "You encounter the error message:\n",
      "docker: Error response from daemon: error while creating mount source path '/path/to/ny_taxi_postgres_data': chown /path/to/ny_taxi_postgres_data: permission denied.\n",
      "Solution:\n",
      "1- Stop Rancher Desktop:\n",
      "If you are using Rancher Desktop and face this issue, stop Rancher Desktop to resolve compatibility problems.\n",
      "2- Install Docker Desktop:\n",
      "Install Docker Desktop, ensuring that it is properly configured and has the required permissions.\n",
      "2-Retry Docker Command:\n",
      "Run the Docker command again after switching to Docker Desktop. This step resolves compatibility issues on some systems.\n",
      "Note: The issue occurred because Rancher Desktop was in use. Switching to Docker Desktop resolves compatibility problems and allows for the successful creation of PostgreSQL containers with mounted volumes for the New York Taxi Database on macOS M1.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(context)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39348bf9-c22b-43f4-a3c8-3893af8c57cf",
   "metadata": {},
   "source": [
    "## Elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "942b487f-8e85-4b96-bbf8-258d3bc25516",
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "07e87bb6-1e40-46b3-b4b3-6eea3852fc43",
   "metadata": {},
   "outputs": [],
   "source": [
    "es_client= Elasticsearch('http://localhost:9200')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1a1e34b7-f268-413f-ab37-f14ce1dbf976",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'name': '3c64b60885c6', 'cluster_name': 'docker-cluster', 'cluster_uuid': '2p2VNxtwRRK8q7zLdzJMyw', 'version': {'number': '8.4.3', 'build_flavor': 'default', 'build_type': 'docker', 'build_hash': '42f05b9372a9a4a470db3b52817899b99a76ee73', 'build_date': '2022-10-04T07:17:24.662462378Z', 'build_snapshot': False, 'lucene_version': '9.3.0', 'minimum_wire_compatibility_version': '7.17.0', 'minimum_index_compatibility_version': '7.0.0'}, 'tagline': 'You Know, for Search'})"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es_client.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "383fbfb7-9982-4f48-86d9-f030041e8c47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'course-questions2'})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_settings= {\n",
    "    \"settings\": {\n",
    "        \"number_of_shards\": 1,\n",
    "        \"number_of_replicas\": 0\n",
    "    },\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"text\": {\"type\": \"text\"},\n",
    "            \"section\": {\"type\": \"text\"},\n",
    "            \"question\": {\"type\": \"text\"},\n",
    "            \"course\": {\"type\": \"keyword\"} \n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "index_name= 'course-questions2'\n",
    "\n",
    "es_client.indices.create(index= index_name, body= index_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fbe26243-15a8-44f9-b0e3-01228ec3468e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
       " 'section': 'General course-related questions',\n",
       " 'question': 'Course - When will the course start?',\n",
       " 'course': 'data-engineering-zoomcamp'}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "63508739-3f2f-4edd-8e39-88d43c60e576",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ea6ee2dc-d6df-4b5d-a576-489186ec595a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d170562b587047c88cace679def9fc2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/948 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for doc in tqdm(documents):\n",
    "    es_client.index(index= index_name, document=doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d83eb7da-6302-4915-b491-2edbfbf607bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "query= \"How do I execute a command in a running docker container?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "aab400f2-4bea-4396-a27b-698e0cc1727a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#def elastic_search(query):\n",
    "search_query= {\n",
    "    \"size\": 5,\n",
    "    \"query\": {\n",
    "        \"bool\": {\n",
    "            \"must\": {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": query,\n",
    "                    \"fields\": [\"question^4\", \"text\"],\n",
    "                    \"type\": \"best_fields\"\n",
    "                }\n",
    "            },\n",
    "            \"filter\": {\n",
    "                \"term\": {\n",
    "                    \"course\": \"machine-learning-zoomcamp\"\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "response= es_client.search(index= index_name, body= search_query)\n",
    "result_docs= []\n",
    "\n",
    "for hit in response['hits']['hits']:\n",
    "    result_docs.append(hit['_source'])\n",
    "#return result_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "05feae55-d940-46e4-a99f-c99d9a964854",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': 'Launch the container image in interactive mode and overriding the entrypoint, so that it starts a bash command.\\ndocker run -it --entrypoint bash <image>\\nIf the container is already running, execute a command in the specific container:\\ndocker ps (find the container-id)\\ndocker exec -it <container-id> bash\\n(Marcos MJD)',\n",
       "  'section': '5. Deploying Machine Learning Models',\n",
       "  'question': 'How do I debug a docker container?',\n",
       "  'course': 'machine-learning-zoomcamp'},\n",
       " {'text': \"You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\\nTo copy a file or directory from your local machine into a running Docker container, you can use the `docker cp command`. The basic syntax is as follows:\\ndocker cp /path/to/local/file_or_directory container_id:/path/in/container\\nHrithik Kumar Advani\",\n",
       "  'section': '5. Deploying Machine Learning Models',\n",
       "  'question': 'How do I copy files from my local machine to docker container?',\n",
       "  'course': 'machine-learning-zoomcamp'},\n",
       " {'text': 'You can copy files from your local machine into a Docker container using the docker cp command. Here\\'s how to do it:\\nIn the Dockerfile, you can provide the folder containing the files that you want to copy over. The basic syntax is as follows:\\nCOPY [\"src/predict.py\", \"models/xgb_model.bin\", \"./\"]\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\tGopakumar Gopinathan',\n",
       "  'section': '5. Deploying Machine Learning Models',\n",
       "  'question': 'How do I copy files from a different folder into docker container’s working directory?',\n",
       "  'course': 'machine-learning-zoomcamp'},\n",
       " {'text': \"$ docker exec -it 1e5a1b663052 bash\\nthe input device is not a TTY.  If you are using mintty, try prefixing the command with 'winpty'\\nFix:\\nwinpty docker exec -it 1e5a1b663052 bash\\nA TTY is a terminal interface that supports escape sequences, moving the cursor around, etc.\\nWinpty is a Windows software package providing an interface similar to a Unix pty-master for communicating with Windows console programs.\\nMore info on terminal, shell, console applications hi and so on:\\nhttps://conemu.github.io/en/TerminalVsShell.html\\n(Marcos MJD)\",\n",
       "  'section': '5. Deploying Machine Learning Models',\n",
       "  'question': 'The input device is not a TTY when running docker in interactive mode (Running Docker on Windows in GitBash)',\n",
       "  'course': 'machine-learning-zoomcamp'},\n",
       " {'text': 'Solution\\nThis error was because there was another instance of gunicorn running. So I thought of removing this along with the zoomcamp_test image. However, it didn’t let me remove the orphan container. So I did the following\\nRunning the following commands\\ndocker ps -a <to list all docker containers>\\ndocker images <to list images>\\ndocker stop <container ID>\\ndocker rm <container ID>\\ndocker rmi image\\nI rebuilt the Docker image, and ran it once again; this time it worked correctly and I was able to serve the test script to the endpoint.',\n",
       "  'section': '5. Deploying Machine Learning Models',\n",
       "  'question': 'How to fix error after running the Docker run command',\n",
       "  'course': 'machine-learning-zoomcamp'}]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "d58fe636-320b-453a-997d-8a792110735c",
   "metadata": {},
   "outputs": [],
   "source": [
    "context_entries = [\n",
    "    {\n",
    "        \"question\": \"How do I debug a docker container?\",\n",
    "        \"text\": \"Launch the container image in interactive mode and overriding the entrypoint, so that it starts a bash command.\\ndocker run -it --entrypoint bash <image>\\nIf the container is already running, execute a command in the specific container:\\ndocker ps (find the container-id)\\ndocker exec -it <container-id> bash\\n(Marcos MJD)\"\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"How do I copy files from my local machine to docker container?\",\n",
    "        \"text\": \"You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\\nTo copy a file or directory from your local machine into a running Docker container, you can use the `docker cp command`. The basic syntax is as follows:\\ndocker cp /path/to/local/file_or_directory container_id:/path/in/container\\nHrithik Kumar Advani\"\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"How do I copy files from a different folder into docker container’s working directory?\",\n",
    "        \"text\": \"You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\\nIn the Dockerfile, you can provide the folder containing the files that you want to copy over. The basic syntax is as follows:\\nCOPY [\\\"src/predict.py\\\", \\\"models/xgb_model.bin\\\", \\\"./\\\"]\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\tGopakumar Gopinathan\"\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"The input device is not a TTY when running docker in interactive mode (Running Docker on Windows in GitBash)\",\n",
    "        \"text\": \"$ docker exec -it 1e5a1b663052 bash\\nthe input device is not a TTY.  If you are using mintty, try prefixing the command with 'winpty'\\nFix:\\nwinpty docker exec -it 1e5a1b663052 bash\\nA TTY is a terminal interface that supports escape sequences, moving the cursor around, etc.\\nWinpty is a Windows software package providing an interface similar to a Unix pty-master for communicating with Windows console programs.\\nMore info on terminal, shell, console applications hi and so on:\\nhttps://conemu.github.io/en/TerminalVsShell.html\\n(Marcos MJD)\"\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"How to fix error after running the Docker run command\",\n",
    "        \"text\": \"Solution\\nThis error was because there was another instance of gunicorn running. So I thought of removing this along with the zoomcamp_test image. However, it didn’t let me remove the orphan container. So I did the following\\nRunning the following commands\\ndocker ps -a <to list all docker containers>\\ndocker images <to list images>\\ndocker stop <container ID>\\ndocker rm <container ID>\\ndocker rmi image\\nI rebuilt the Docker image, and ran it once again; this time it worked correctly and I was able to serve the test script to the endpoint.\"\n",
    "    }\n",
    "]\n",
    "\n",
    "context = \"\\n\\n\".join([f\"Q: {entry['question']}\\nA: {entry['text']}\" for entry in context_entries])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "5ba02804-4f38-4bbe-9fef-da0be4a9461b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You're a course teaching assistant. Answer the QUESTION based on the CONTEXT from the FAQ database.\n",
      "Use only the facts from the CONTEXT when answering the QUESTION.\n",
      "\n",
      "QUESTION: How do I execute a command in a running docker container?\n",
      "\n",
      "CONTEXT:\n",
      "Q: How do I debug a docker container?\n",
      "A: Launch the container image in interactive mode and overriding the entrypoint, so that it starts a bash command.\n",
      "docker run -it --entrypoint bash <image>\n",
      "If the container is already running, execute a command in the specific container:\n",
      "docker ps (find the container-id)\n",
      "docker exec -it <container-id> bash\n",
      "(Marcos MJD)\n",
      "\n",
      "Q: How do I copy files from my local machine to docker container?\n",
      "A: You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\n",
      "To copy a file or directory from your local machine into a running Docker container, you can use the `docker cp command`. The basic syntax is as follows:\n",
      "docker cp /path/to/local/file_or_directory container_id:/path/in/container\n",
      "Hrithik Kumar Advani\n",
      "\n",
      "Q: How do I copy files from a different folder into docker container’s working directory?\n",
      "A: You can copy files from your local machine into a Docker container using the docker cp command. Here's how to do it:\n",
      "In the Dockerfile, you can provide the folder containing the files that you want to copy over. The basic syntax is as follows:\n",
      "COPY [\"src/predict.py\", \"models/xgb_model.bin\", \"./\"]\t\t\t\t\t\t\t\t\t\t\tGopakumar Gopinathan\n",
      "\n",
      "Q: The input device is not a TTY when running docker in interactive mode (Running Docker on Windows in GitBash)\n",
      "A: $ docker exec -it 1e5a1b663052 bash\n",
      "the input device is not a TTY.  If you are using mintty, try prefixing the command with 'winpty'\n",
      "Fix:\n",
      "winpty docker exec -it 1e5a1b663052 bash\n",
      "A TTY is a terminal interface that supports escape sequences, moving the cursor around, etc.\n",
      "Winpty is a Windows software package providing an interface similar to a Unix pty-master for communicating with Windows console programs.\n",
      "More info on terminal, shell, console applications hi and so on:\n",
      "https://conemu.github.io/en/TerminalVsShell.html\n",
      "(Marcos MJD)\n",
      "\n",
      "Q: How to fix error after running the Docker run command\n",
      "A: Solution\n",
      "This error was because there was another instance of gunicorn running. So I thought of removing this along with the zoomcamp_test image. However, it didn’t let me remove the orphan container. So I did the following\n",
      "Running the following commands\n",
      "docker ps -a <to list all docker containers>\n",
      "docker images <to list images>\n",
      "docker stop <container ID>\n",
      "docker rm <container ID>\n",
      "docker rmi image\n",
      "I rebuilt the Docker image, and ran it once again; this time it worked correctly and I was able to serve the test script to the endpoint.\n"
     ]
    }
   ],
   "source": [
    "question = \"How do I execute a command in a running docker container?\"\n",
    "\n",
    "prompt = f\"\"\"\n",
    "You're a course teaching assistant. Answer the QUESTION based on the CONTEXT from the FAQ database.\n",
    "Use only the facts from the CONTEXT when answering the QUESTION.\n",
    "\n",
    "QUESTION: {question}\n",
    "\n",
    "CONTEXT:\n",
    "{context}\n",
    "\"\"\".strip()\n",
    "\n",
    "print(prompt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "2cdb1e9b-ed31-48f1-a6ab-2c14e147e593",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2714\n"
     ]
    }
   ],
   "source": [
    "# prompt len\n",
    "prompt_length = len(prompt)\n",
    "print(prompt_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "aad44282-3ad7-41ef-a0d2-7159f6e160fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tiktoken\n",
      "  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
      "Collecting regex>=2022.1.18 (from tiktoken)\n",
      "  Downloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.26.0 in /home/codespace/.local/lib/python3.10/site-packages (from tiktoken) (2.32.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2024.2.2)\n",
      "Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (775 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m775.1/775.1 kB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: regex, tiktoken\n",
      "Successfully installed regex-2024.5.15 tiktoken-0.7.0\n"
     ]
    }
   ],
   "source": [
    "!pip install tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "e12f38bb-65ce-4823-8158-726f6a2d01a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "fd2fbaa8-0a90-42b3-b5ba-ff3e76667242",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding = tiktoken.encoding_for_model(\"gpt-4o\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "80e10263-bac5-4003-96a9-07c8642bc149",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 621\n"
     ]
    }
   ],
   "source": [
    "# modify the prompt and calculate the number of tokens\n",
    "tokenized_prompt = encoding.encode(prompt)\n",
    "num_tokens = len(tokenized_prompt)\n",
    "print(f\"Number of tokens: {num_tokens}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82abf505-5f9e-4d52-9be3-9dde64db31f0",
   "metadata": {},
   "source": [
    "### Bonus: calculating the costs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "8f92df55-623a-4133-9253-a4dd3fa923e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cost= (num_tokens*0.005)/1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "aebd0e7d-6e20-401c-9f8e-df2fe3b7c143",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost of tokens: 0.003105$\n"
     ]
    }
   ],
   "source": [
    "print(f\"Cost of tokens: {cost}$\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ea6e860-44e1-4da6-8935-e8f0bb754a2b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}