zCamp/adi_llm_01-intro.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"lcQJ7najgWZp"},"source":["# Start"],"id":"lcQJ7najgWZp"},{"cell_type":"code","metadata":{"id":"n-d5JfUQgVWb","executionInfo":{"status":"ok","timestamp":1719476931187,"user_tz":-330,"elapsed":1244,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}}},"source":["from IPython import InteractiveShell\n","InteractiveShell.ast_node_interactivity = 'all'"],"execution_count":1,"outputs":[],"id":"n-d5JfUQgVWb"},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KJ9sDXn1XOQ-","executionInfo":{"status":"ok","timestamp":1719476973609,"user_tz":-330,"elapsed":38174,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"129125e6-600f-4d0c-a24f-c42f754e0790"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"id":"KJ9sDXn1XOQ-"},{"cell_type":"code","source":["%pwd\n","%cd drive/My Drive/Works/X-ops/llm/zCamp\n","!python --version"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":72},"id":"r5MBVs1xaJlm","executionInfo":{"status":"ok","timestamp":1719476977201,"user_tz":-330,"elapsed":19,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"314dbe15-a151-42f6-a615-10780aa1cb8f"},"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'/content'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":3},{"output_type":"stream","name":"stdout","text":["/content/drive/My Drive/Works/X-ops/llm/zCamp\n","Python 3.10.12\n"]}],"id":"r5MBVs1xaJlm"},{"cell_type":"code","source":["# GPU info in Colab\n","!nvidia-smi -L"],"metadata":{"id":"jP6y7oyiXfv6","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1715944152548,"user_tz":-330,"elapsed":599,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"6d1ebf3d-b0bb-43bb-c83c-4693c3e6dce6"},"id":"jP6y7oyiXfv6","execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/bin/bash: line 1: nvidia-smi: command not found\n"]}]},{"cell_type":"markdown","id":"8116e19d","metadata":{"id":"8116e19d"},"source":["# 💻 Installation - Environment"]},{"cell_type":"code","execution_count":4,"id":"d7142a33","metadata":{"id":"d7142a33","outputId":"c9b4b1de-f4bb-410a-d65b-85b001677da8","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1719477006227,"user_tz":-330,"elapsed":19450,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m384.6/384.6 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.5/64.5 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}],"source":["# INSTALLING REQUIRED packages\n","!pip install elasticsearch==8.4.3 -q\n","!pip install tiktoken -q"]},{"cell_type":"code","source":["%%bash\n","\n","rm -rf elasticsearch*\n","wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.4.3-linux-x86_64.tar.gz\n","tar -xzf elasticsearch-8.4.3-linux-x86_64.tar.gz\n","sudo chown -R daemon:daemon elasticsearch-8.4.3/\n","umount /sys/fs/cgroup\n","apt install cgroup-tools"],"metadata":{"id":"52qMtgcAogT-"},"id":"52qMtgcAogT-","execution_count":null,"outputs":[]},{"cell_type":"code","source":["%%bash --bg\n","\n","sudo -H -u daemon /content/drive/My Drive/Works/X-ops/llm/zCamp/elasticsearch-8.4.3/bin/elasticsearch"],"metadata":{"id":"Tq7RQSZKo5iD","executionInfo":{"status":"ok","timestamp":1719477607648,"user_tz":-330,"elapsed":734,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}}},"id":"Tq7RQSZKo5iD","execution_count":18,"outputs":[]},{"cell_type":"code","source":["# importing required packages $\n","import time\n","time.sleep(60)"],"metadata":{"id":"Nkg1dOrVIsBX","executionInfo":{"status":"ok","timestamp":1719477674303,"user_tz":-330,"elapsed":60568,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}}},"id":"Nkg1dOrVIsBX","execution_count":19,"outputs":[]},{"cell_type":"code","source":["!ps -ef | grep elastic"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"quIN4X-DpSCT","executionInfo":{"status":"ok","timestamp":1719477680563,"user_tz":-330,"elapsed":463,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"cb284262-50f9-4030-d066-4321a24b05ac"},"id":"quIN4X-DpSCT","execution_count":20,"outputs":[{"output_type":"stream","name":"stdout","text":["root        4094     613  0 08:41 ?        00:00:00 /bin/bash -c ps -ef | grep elastic\n","root        4096    4094  0 08:41 ?        00:00:00 grep elastic\n"]}]},{"cell_type":"code","source":["!curl --cacert /content/drive/My Drive/Works/X-ops/llm/zCamp/elasticsearch-8.4.3/config/certs/http_ca.crt -u elastic -H 'Content-Type: application/json' -XGET https://localhost:9200/?pretty=true"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JnkK09RMpd8D","executionInfo":{"status":"ok","timestamp":1719477732308,"user_tz":-330,"elapsed":48336,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"b4b99ad5-d23e-4d98-b864-9de977593ad6"},"id":"JnkK09RMpd8D","execution_count":21,"outputs":[{"output_type":"stream","name":"stdout","text":["Enter host password for user 'elastic':\n","curl: (6) Could not resolve host: Drive\n","curl: (7) Failed to connect to localhost port 9200 after 0 ms: Connection refused\n"]}]},{"cell_type":"markdown","source":["# Elastic Search in Colab!!"],"metadata":{"id":"QG3gVGS_63Ch"},"id":"QG3gVGS_63Ch"},{"cell_type":"code","source":["# Getting datasets into working Environment - 1: $\n","data_df = pd.read_parquet('https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-03.parquet')\n","data_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":226},"id":"OcIOnNlJ6_NW","executionInfo":{"status":"ok","timestamp":1717052573840,"user_tz":-330,"elapsed":3571,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"a7d0af96-372a-4188-dae2-3c5acf861073"},"id":"OcIOnNlJ6_NW","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n","0         2  2023-03-01 00:06:43   2023-03-01 00:16:43              1.0   \n","1         2  2023-03-01 00:08:25   2023-03-01 00:39:30              2.0   \n","2         1  2023-03-01 00:15:04   2023-03-01 00:29:26              0.0   \n","3         1  2023-03-01 00:49:37   2023-03-01 01:01:05              1.0   \n","4         2  2023-03-01 00:08:04   2023-03-01 00:11:06              1.0   \n","\n","   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n","0           0.00         1.0                  N           238            42   \n","1          12.40         1.0                  N           138           231   \n","2           3.30         1.0                  N           140           186   \n","3           2.90         1.0                  N           140            43   \n","4           1.23         1.0                  N            79           137   \n","\n","   payment_type  fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n","0             2          8.6    1.0      0.5        0.00           0.0   \n","1             1         52.7    6.0      0.5       12.54           0.0   \n","2             1         18.4    3.5      0.5        4.65           0.0   \n","3             1         15.6    3.5      0.5        4.10           0.0   \n","4             1          7.2    1.0      0.5        2.44           0.0   \n","\n","   improvement_surcharge  total_amount  congestion_surcharge  Airport_fee  \n","0                    1.0         11.10                   0.0         0.00  \n","1                    1.0         76.49                   2.5         1.25  \n","2                    1.0         28.05                   2.5         0.00  \n","3                    1.0         24.70                   2.5         0.00  \n","4                    1.0         14.64                   2.5         0.00  "],"text/html":["\n","  <div id=\"df-f1505677-7213-444e-9617-29f0bbe33af0\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>VendorID</th>\n","      <th>tpep_pickup_datetime</th>\n","      <th>tpep_dropoff_datetime</th>\n","      <th>passenger_count</th>\n","      <th>trip_distance</th>\n","      <th>RatecodeID</th>\n","      <th>store_and_fwd_flag</th>\n","      <th>PULocationID</th>\n","      <th>DOLocationID</th>\n","      <th>payment_type</th>\n","      <th>fare_amount</th>\n","      <th>extra</th>\n","      <th>mta_tax</th>\n","      <th>tip_amount</th>\n","      <th>tolls_amount</th>\n","      <th>improvement_surcharge</th>\n","      <th>total_amount</th>\n","      <th>congestion_surcharge</th>\n","      <th>Airport_fee</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2</td>\n","      <td>2023-03-01 00:06:43</td>\n","      <td>2023-03-01 00:16:43</td>\n","      <td>1.0</td>\n","      <td>0.00</td>\n","      <td>1.0</td>\n","      <td>N</td>\n","      <td>238</td>\n","      <td>42</td>\n","      <td>2</td>\n","      <td>8.6</td>\n","      <td>1.0</td>\n","      <td>0.5</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>11.10</td>\n","      <td>0.0</td>\n","      <td>0.00</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>2023-03-01 00:08:25</td>\n","      <td>2023-03-01 00:39:30</td>\n","      <td>2.0</td>\n","      <td>12.40</td>\n","      <td>1.0</td>\n","      <td>N</td>\n","      <td>138</td>\n","      <td>231</td>\n","      <td>1</td>\n","      <td>52.7</td>\n","      <td>6.0</td>\n","      <td>0.5</td>\n","      <td>12.54</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>76.49</td>\n","      <td>2.5</td>\n","      <td>1.25</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1</td>\n","      <td>2023-03-01 00:15:04</td>\n","      <td>2023-03-01 00:29:26</td>\n","      <td>0.0</td>\n","      <td>3.30</td>\n","      <td>1.0</td>\n","      <td>N</td>\n","      <td>140</td>\n","      <td>186</td>\n","      <td>1</td>\n","      <td>18.4</td>\n","      <td>3.5</td>\n","      <td>0.5</td>\n","      <td>4.65</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>28.05</td>\n","      <td>2.5</td>\n","      <td>0.00</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>1</td>\n","      <td>2023-03-01 00:49:37</td>\n","      <td>2023-03-01 01:01:05</td>\n","      <td>1.0</td>\n","      <td>2.90</td>\n","      <td>1.0</td>\n","      <td>N</td>\n","      <td>140</td>\n","      <td>43</td>\n","      <td>1</td>\n","      <td>15.6</td>\n","      <td>3.5</td>\n","      <td>0.5</td>\n","      <td>4.10</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>24.70</td>\n","      <td>2.5</td>\n","      <td>0.00</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>2</td>\n","      <td>2023-03-01 00:08:04</td>\n","      <td>2023-03-01 00:11:06</td>\n","      <td>1.0</td>\n","      <td>1.23</td>\n","      <td>1.0</td>\n","      <td>N</td>\n","      <td>79</td>\n","      <td>137</td>\n","      <td>1</td>\n","      <td>7.2</td>\n","      <td>1.0</td>\n","      <td>0.5</td>\n","      <td>2.44</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>14.64</td>\n","      <td>2.5</td>\n","      <td>0.00</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f1505677-7213-444e-9617-29f0bbe33af0')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-f1505677-7213-444e-9617-29f0bbe33af0 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-f1505677-7213-444e-9617-29f0bbe33af0');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-cf2d6c5b-62bc-4dab-9161-291e966d456e\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cf2d6c5b-62bc-4dab-9161-291e966d456e')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-cf2d6c5b-62bc-4dab-9161-291e966d456e button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"data_df"}},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["data_df.info() # Q1-Ans"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tTMAlFh0-eTQ","executionInfo":{"status":"ok","timestamp":1717052578427,"user_tz":-330,"elapsed":383,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"9a8c3224-feac-49b8-9e6a-7f1db07567e6"},"id":"tTMAlFh0-eTQ","execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 3403766 entries, 0 to 3403765\n","Data columns (total 19 columns):\n"," #   Column                 Dtype         \n","---  ------                 -----         \n"," 0   VendorID               int32         \n"," 1   tpep_pickup_datetime   datetime64[us]\n"," 2   tpep_dropoff_datetime  datetime64[us]\n"," 3   passenger_count        float64       \n"," 4   trip_distance          float64       \n"," 5   RatecodeID             float64       \n"," 6   store_and_fwd_flag     object        \n"," 7   PULocationID           int32         \n"," 8   DOLocationID           int32         \n"," 9   payment_type           int64         \n"," 10  fare_amount            float64       \n"," 11  extra                  float64       \n"," 12  mta_tax                float64       \n"," 13  tip_amount             float64       \n"," 14  tolls_amount           float64       \n"," 15  improvement_surcharge  float64       \n"," 16  total_amount           float64       \n"," 17  congestion_surcharge   float64       \n"," 18  Airport_fee            float64       \n","dtypes: datetime64[us](2), float64(12), int32(3), int64(1), object(1)\n","memory usage: 454.5+ MB\n"]}]},{"cell_type":"markdown","source":["## EDA"],"metadata":{"id":"JU4fs-P0wpo7"},"id":"JU4fs-P0wpo7"},{"cell_type":"markdown","source":["* Do some basic EDA on 1 month (jan-2023) data !\n","* Use the findings to Pre-process any upcoming data !"],"metadata":{"id":"QOgUoparCgnS"},"id":"QOgUoparCgnS"},{"cell_type":"code","source":["# Checking for features with NULLs !\n","data_df.isnull().sum()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2Nw_1EUqlHWZ","executionInfo":{"status":"ok","timestamp":1715946203656,"user_tz":-330,"elapsed":753,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"ab54f090-e7db-47e2-d59b-1af5fcc122be"},"id":"2Nw_1EUqlHWZ","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["VendorID                     0\n","tpep_pickup_datetime         0\n","tpep_dropoff_datetime        0\n","passenger_count          71743\n","trip_distance                0\n","RatecodeID               71743\n","store_and_fwd_flag       71743\n","PULocationID                 0\n","DOLocationID                 0\n","payment_type                 0\n","fare_amount                  0\n","extra                        0\n","mta_tax                      0\n","tip_amount                   0\n","tolls_amount                 0\n","improvement_surcharge        0\n","total_amount                 0\n","congestion_surcharge     71743\n","airport_fee              71743\n","dtype: int64"]},"metadata":{},"execution_count":22}]},{"cell_type":"code","source":["# Check in categoricals!!\n","data_df.store_and_fwd_flag.nunique(), data_df.payment_type.nunique(),data_df.VendorID.nunique()\n","data_df.RatecodeID.nunique(),data_df.PULocationID.nunique(),data_df.DOLocationID.nunique()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"C4punKCzy7Le","executionInfo":{"status":"ok","timestamp":1715946591791,"user_tz":-330,"elapsed":1281,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"3d76b829-dc13-445f-82ec-b6eec40702e4"},"id":"C4punKCzy7Le","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(2, 5, 2)"]},"metadata":{},"execution_count":25},{"output_type":"execute_result","data":{"text/plain":["(7, 257, 261)"]},"metadata":{},"execution_count":25}]},{"cell_type":"code","source":["# Examining Categories!!\n","data_df.store_and_fwd_flag.value_counts(), data_df.payment_type.value_counts(), data_df.VendorID.value_counts(), data_df.RatecodeID.value_counts()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pY9nHIFxVMXb","executionInfo":{"status":"ok","timestamp":1715946621653,"user_tz":-330,"elapsed":663,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"c96d0f2d-1323-45da-94a0-c0f7f4522b6c"},"id":"pY9nHIFxVMXb","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(store_and_fwd_flag\n"," N    2975020\n"," Y      20003\n"," Name: count, dtype: int64,\n"," payment_type\n"," 1    2411462\n"," 2     532241\n"," 0      71743\n"," 4      33297\n"," 3      18023\n"," Name: count, dtype: int64,\n"," VendorID\n"," 2    2239399\n"," 1     827367\n"," Name: count, dtype: int64,\n"," RatecodeID\n"," 1.0     2839305\n"," 2.0      114239\n"," 5.0       15043\n"," 99.0      13106\n"," 3.0        8958\n"," 4.0        4366\n"," 6.0           6\n"," Name: count, dtype: int64)"]},"metadata":{},"execution_count":26}]},{"cell_type":"code","source":["# Examining some Numerical features with NULLs !\n","data_df[data_df.passenger_count.isnull()].sample(10)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":383},"id":"maJcbCP0ws23","executionInfo":{"status":"ok","timestamp":1715946429009,"user_tz":-330,"elapsed":9,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"55c5d309-6862-4fae-fa34-772b5fe62ec3"},"id":"maJcbCP0ws23","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["         VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n","3003269         2  2023-01-04 18:33:10   2023-01-04 18:55:05              NaN   \n","2998162         1  2023-01-01 18:13:47   2023-01-01 18:22:18              NaN   \n","3032429         1  2023-01-17 21:21:13   2023-01-17 21:34:53              NaN   \n","3025942         2  2023-01-14 19:14:00   2023-01-14 19:32:00              NaN   \n","2995278         2  2023-01-01 00:55:48   2023-01-01 00:59:14              NaN   \n","3047259         1  2023-01-23 19:40:14   2023-01-23 19:53:59              NaN   \n","3056189         2  2023-01-27 09:42:36   2023-01-27 09:52:43              NaN   \n","3042192         2  2023-01-21 16:58:00   2023-01-21 17:09:00              NaN   \n","2999413         2  2023-01-02 18:43:00   2023-01-02 18:47:00              NaN   \n","3056582         1  2023-01-27 13:12:00   2023-01-27 13:30:04              NaN   \n","\n","         trip_distance  RatecodeID store_and_fwd_flag  PULocationID  \\\n","3003269           5.01         NaN               None           262   \n","2998162           2.00         NaN               None           142   \n","3032429           3.40         NaN               None            90   \n","3025942           2.41         NaN               None            79   \n","2995278           0.61         NaN               None           107   \n","3047259           2.20         NaN               None           230   \n","3056189           1.36         NaN               None           114   \n","3042192           0.96         NaN               None           249   \n","2999413           0.95         NaN               None           186   \n","3056582           1.70         NaN               None           107   \n","\n","         DOLocationID  payment_type  fare_amount  extra  mta_tax  tip_amount  \\\n","3003269            90             0        28.58    0.0      0.5        6.52   \n","2998162           236             0        10.70    0.0      0.5        1.47   \n","3032429           236             0        16.30    1.0      0.5        2.13   \n","3025942           161             0        14.82    0.0      0.5        4.76   \n","2995278           170             0        12.80    0.0      0.5        3.36   \n","3047259           239             0        14.90    2.5      0.5        4.28   \n","3056189            90             0        14.40    0.0      0.5        2.00   \n","3042192            79             0        13.67    0.0      0.5        3.53   \n","2999413           234             0        12.80    0.0      0.5        3.36   \n","3056582           231             0        17.00    0.0      0.5        4.20   \n","\n","         tolls_amount  improvement_surcharge  total_amount  \\\n","3003269           0.0                    1.0         39.10   \n","2998162           0.0                    1.0         16.17   \n","3032429           0.0                    1.0         23.43   \n","3025942           0.0                    1.0         23.58   \n","2995278           0.0                    1.0         20.16   \n","3047259           0.0                    1.0         25.68   \n","3056189           0.0                    1.0         20.40   \n","3042192           0.0                    1.0         21.20   \n","2999413           0.0                    1.0         20.16   \n","3056582           0.0                    1.0         25.20   \n","\n","         congestion_surcharge  airport_fee  \n","3003269                   NaN          NaN  \n","2998162                   NaN          NaN  \n","3032429                   NaN          NaN  \n","3025942                   NaN          NaN  \n","2995278                   NaN          NaN  \n","3047259                   NaN          NaN  \n","3056189                   NaN          NaN  \n","3042192                   NaN          NaN  \n","2999413                   NaN          NaN  \n","3056582                   NaN          NaN  "],"text/html":["\n","  <div id=\"df-3170870d-d366-4574-8d04-845cad477b5c\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>VendorID</th>\n","      <th>tpep_pickup_datetime</th>\n","      <th>tpep_dropoff_datetime</th>\n","      <th>passenger_count</th>\n","      <th>trip_distance</th>\n","      <th>RatecodeID</th>\n","      <th>store_and_fwd_flag</th>\n","      <th>PULocationID</th>\n","      <th>DOLocationID</th>\n","      <th>payment_type</th>\n","      <th>fare_amount</th>\n","      <th>extra</th>\n","      <th>mta_tax</th>\n","      <th>tip_amount</th>\n","      <th>tolls_amount</th>\n","      <th>improvement_surcharge</th>\n","      <th>total_amount</th>\n","      <th>congestion_surcharge</th>\n","      <th>airport_fee</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>3003269</th>\n","      <td>2</td>\n","      <td>2023-01-04 18:33:10</td>\n","      <td>2023-01-04 18:55:05</td>\n","      <td>NaN</td>\n","      <td>5.01</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>262</td>\n","      <td>90</td>\n","      <td>0</td>\n","      <td>28.58</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>6.52</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>39.10</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>2998162</th>\n","      <td>1</td>\n","      <td>2023-01-01 18:13:47</td>\n","      <td>2023-01-01 18:22:18</td>\n","      <td>NaN</td>\n","      <td>2.00</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>142</td>\n","      <td>236</td>\n","      <td>0</td>\n","      <td>10.70</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>1.47</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>16.17</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3032429</th>\n","      <td>1</td>\n","      <td>2023-01-17 21:21:13</td>\n","      <td>2023-01-17 21:34:53</td>\n","      <td>NaN</td>\n","      <td>3.40</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>90</td>\n","      <td>236</td>\n","      <td>0</td>\n","      <td>16.30</td>\n","      <td>1.0</td>\n","      <td>0.5</td>\n","      <td>2.13</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>23.43</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3025942</th>\n","      <td>2</td>\n","      <td>2023-01-14 19:14:00</td>\n","      <td>2023-01-14 19:32:00</td>\n","      <td>NaN</td>\n","      <td>2.41</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>79</td>\n","      <td>161</td>\n","      <td>0</td>\n","      <td>14.82</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>4.76</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>23.58</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>2995278</th>\n","      <td>2</td>\n","      <td>2023-01-01 00:55:48</td>\n","      <td>2023-01-01 00:59:14</td>\n","      <td>NaN</td>\n","      <td>0.61</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>107</td>\n","      <td>170</td>\n","      <td>0</td>\n","      <td>12.80</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>3.36</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>20.16</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3047259</th>\n","      <td>1</td>\n","      <td>2023-01-23 19:40:14</td>\n","      <td>2023-01-23 19:53:59</td>\n","      <td>NaN</td>\n","      <td>2.20</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>230</td>\n","      <td>239</td>\n","      <td>0</td>\n","      <td>14.90</td>\n","      <td>2.5</td>\n","      <td>0.5</td>\n","      <td>4.28</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>25.68</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3056189</th>\n","      <td>2</td>\n","      <td>2023-01-27 09:42:36</td>\n","      <td>2023-01-27 09:52:43</td>\n","      <td>NaN</td>\n","      <td>1.36</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>114</td>\n","      <td>90</td>\n","      <td>0</td>\n","      <td>14.40</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>2.00</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>20.40</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3042192</th>\n","      <td>2</td>\n","      <td>2023-01-21 16:58:00</td>\n","      <td>2023-01-21 17:09:00</td>\n","      <td>NaN</td>\n","      <td>0.96</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>249</td>\n","      <td>79</td>\n","      <td>0</td>\n","      <td>13.67</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>3.53</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>21.20</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>2999413</th>\n","      <td>2</td>\n","      <td>2023-01-02 18:43:00</td>\n","      <td>2023-01-02 18:47:00</td>\n","      <td>NaN</td>\n","      <td>0.95</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>186</td>\n","      <td>234</td>\n","      <td>0</td>\n","      <td>12.80</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>3.36</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>20.16</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3056582</th>\n","      <td>1</td>\n","      <td>2023-01-27 13:12:00</td>\n","      <td>2023-01-27 13:30:04</td>\n","      <td>NaN</td>\n","      <td>1.70</td>\n","      <td>NaN</td>\n","      <td>None</td>\n","      <td>107</td>\n","      <td>231</td>\n","      <td>0</td>\n","      <td>17.00</td>\n","      <td>0.0</td>\n","      <td>0.5</td>\n","      <td>4.20</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>25.20</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3170870d-d366-4574-8d04-845cad477b5c')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-3170870d-d366-4574-8d04-845cad477b5c button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-3170870d-d366-4574-8d04-845cad477b5c');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-49aace3b-b3c2-41f0-9579-190e064936f7\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-49aace3b-b3c2-41f0-9579-190e064936f7')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-49aace3b-b3c2-41f0-9579-190e064936f7 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","repr_error":"0"}},"metadata":{},"execution_count":24}]},{"cell_type":"code","source":["# EDA for duration variable !\n","len(data_df[data_df['tpep_dropoff_datetime'] < data_df['tpep_pickup_datetime']])\n","\n","# Creating duration variable & examining it !\n","data_df['duration'] = data_df['tpep_dropoff_datetime'] - data_df['tpep_pickup_datetime']\n","data_df['duration'] = data_df['duration'].apply(lambda dur: round(dur.total_seconds() / 60 , 2))\n","\n","round(data_df['duration'].describe(),2) # Q2 - Ans"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PVeAIKaCZbbw","executionInfo":{"status":"ok","timestamp":1715949895816,"user_tz":-330,"elapsed":11970,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"2215484d-a15d-4971-cb8f-279db040942a"},"id":"PVeAIKaCZbbw","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["3"]},"metadata":{},"execution_count":36},{"output_type":"execute_result","data":{"text/plain":["count    3066766.00\n","mean          15.67\n","std           42.59\n","min          -29.20\n","25%            7.12\n","50%           11.52\n","75%           18.30\n","max        10029.18\n","Name: duration, dtype: float64"]},"metadata":{},"execution_count":36}]},{"cell_type":"code","source":["# Outliers in duration variable !\n","data_df = data_df[(data_df.duration >= 1) & (data_df.duration <= 60)]\n","round(len(data_df)/3066766,2)*100 # Q3 - Ans"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-uUDyulqkZkB","executionInfo":{"status":"ok","timestamp":1715950623901,"user_tz":-330,"elapsed":580,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"9f457bb5-7284-497a-fe68-e04e952fb4ef"},"id":"-uUDyulqkZkB","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["98.0"]},"metadata":{},"execution_count":39}]},{"cell_type":"code","source":["# Business selected features for the Base Model\n","cat_features = ['PULocationID', 'DOLocationID']\n","data_df[cat_features] = data_df[cat_features].astype(str)\n","\n","# Dictionary Vectorizer & X_train data\n","dVec = DictVectorizer()\n","\n","feature_dicts = data_df[cat_features].to_dict(orient = 'records')\n","X_train = dVec.fit_transform(feature_dicts)\n","X_train # Q5 - Ans"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EqL--8NFodi2","executionInfo":{"status":"ok","timestamp":1715952146077,"user_tz":-330,"elapsed":27141,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"351a7cde-833e-4611-dfb1-0f42da3c971f"},"id":"EqL--8NFodi2","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<3009173x515 sparse matrix of type '<class 'numpy.float64'>'\n","\twith 6018346 stored elements in Compressed Sparse Row format>"]},"metadata":{},"execution_count":40}]},{"cell_type":"code","source":["# Obtaining target and creating target vector !\n","y_train = data_df['duration'].values.copy()\n","len(y_train)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VrLhoATAuKrM","executionInfo":{"status":"ok","timestamp":1715952402052,"user_tz":-330,"elapsed":451,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"15c44262-f5ba-4a1f-8d25-917b4ebb21b3"},"id":"VrLhoATAuKrM","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["3009173"]},"metadata":{},"execution_count":42}]},{"cell_type":"code","source":["# Training a Simple Linear Reg Model !\n","\n","lr = LinearRegression()\n","jan_model = lr.fit(X_train, y_train)\n","y_pred = jan_model.predict(X_train)\n","\n","# Training RMSE: Q5 - Ans !\n","mean_squared_error(y_train, y_pred, squared = False)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8CRcUsa7vRXu","executionInfo":{"status":"ok","timestamp":1715953917393,"user_tz":-330,"elapsed":50478,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"9368812d-b14a-4d7b-e292-d749943b6f6d"},"id":"8CRcUsa7vRXu","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["7.649261716283177"]},"metadata":{},"execution_count":44}]},{"cell_type":"code","source":["# Data Process & Prep Function !\n","def prep_data(data_source):\n","  df = pd.read_parquet(data_source)\n","  # Creating duration variable & examining it !\n","  df['duration'] = df['tpep_dropoff_datetime'] - df['tpep_pickup_datetime']\n","  df['duration'] = df['duration'].apply(lambda dur: round(dur.total_seconds() / 60 , 2))\n","  df = df[(df.duration >= 1) & (df.duration <= 60)]\n","  # cat-features from EDA\n","  cat_features = ['PULocationID', 'DOLocationID']\n","  df[cat_features] = df[cat_features].astype(str)\n","\n","  return df"],"metadata":{"id":"HJUQ2bNazild"},"id":"HJUQ2bNazild","execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Preparing validation Data using Mar-2023 data\n","df_val = prep_data(data_source = 'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-03.parquet')\n","len(df_val)"],"metadata":{"id":"BuZFNmB-5lbU","executionInfo":{"status":"ok","timestamp":1717052892613,"user_tz":-330,"elapsed":19935,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"e7249eca-b59f-44e0-8a42-0149e97a1ae9"},"id":"BuZFNmB-5lbU","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["3316216"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["# Business selected features for model validation !\n","cat_features = ['PULocationID', 'DOLocationID']\n","\n","feature_dicts = df_val[cat_features].to_dict(orient = 'records')\n","X_val = dVec.transform(feature_dicts)\n","\n","# Obtaining target and creating target vector for Validation !\n","y_val = df_val['duration'].values.copy()"],"metadata":{"id":"PBH5CLCB6omU"},"id":"PBH5CLCB6omU","execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Use the Trained model of Jan to Predict for validation Data !\n","y_pred = jan_model.predict(X_val)\n","\n","# Validation RMSE: Q6 - Ans !\n","mean_squared_error(y_val, y_pred, squared = False)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6Voi-3nq7Ykz","executionInfo":{"status":"ok","timestamp":1715955854171,"user_tz":-330,"elapsed":534,"user":{"displayName":"ADITYA KISHOREJONNALAGADDA","userId":"12527884344016978796"}},"outputId":"354dd6b5-e8e5-41e2-de04-2cbd47d02cd5"},"id":"6Voi-3nq7Ykz","execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["7.811819639415271"]},"metadata":{},"execution_count":48}]}],"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"colab":{"provenance":[{"file_id":"https://github.com/pycaret/pycaret/blob/master/tutorials/Tutorial%20-%20Regression.ipynb","timestamp":1690215004408}],"collapsed_sections":["lcQJ7najgWZp"]}},"nbformat":4,"nbformat_minor":5}