missing_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "f-string: unmatched '[' (feature_engineering.py, line 69)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[1;36m(most recent call last)\u001b[0m:\n",
      "\u001b[0m  File \u001b[0;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\venv\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3526\u001b[0m in \u001b[0;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\u001b[0m\n",
      "\u001b[1;36m  Cell \u001b[1;32mIn[1], line 6\u001b[1;36m\n\u001b[1;33m    from src.features.feature_engineering import create_time_features_from_date\u001b[1;36m\n",
      "\u001b[1;36m  File \u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\src\\features\\feature_engineering.py:69\u001b[1;36m\u001b[0m\n\u001b[1;33m    print(f\"Index: {idx}, Timestamp: {train_observed.loc[idx, \"date_forecast\"]}\")\u001b[0m\n\u001b[1;37m                                                               ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m f-string: unmatched '['\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "from src.data.data_fetcher import get_raw_data\n",
    "from src.features.feature_engineering import create_time_features_from_date\n",
    "plt.style.use('ggplot')\n",
    "pd.set_option('display.max_columns', 200)\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "from src.data.data_fetcher import get_all_features, get_raw_data\n",
    "from src.features.feature_engineering import prepare_data\n",
    "from src.features.preprocess_data import get_preprocessed_test_data, fetch_preprocessed_data\n",
    "pd.set_option('display.max_columns', 200)\n",
    "\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "%pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hei2\n",
      "time                              0\n",
      "pv_measurement                    0\n",
      "date_forecast                     0\n",
      "absolute_humidity_2m:gm3          0\n",
      "air_density_2m:kgm3               0\n",
      "clear_sky_energy_1h:J             0\n",
      "clear_sky_rad:W                   0\n",
      "cloud_base_agl:m                  0\n",
      "dew_or_rime:idx                   0\n",
      "dew_point_2m:K                    0\n",
      "diffuse_rad:W                     0\n",
      "diffuse_rad_1h:J                  0\n",
      "direct_rad:W                      0\n",
      "direct_rad_1h:J                   0\n",
      "effective_cloud_cover:p           0\n",
      "fresh_snow_12h:cm                 0\n",
      "fresh_snow_1h:cm                  0\n",
      "fresh_snow_24h:cm                 0\n",
      "fresh_snow_3h:cm                  0\n",
      "fresh_snow_6h:cm                  0\n",
      "is_day:idx                        0\n",
      "is_in_shadow:idx                  0\n",
      "msl_pressure:hPa                  0\n",
      "precip_5min:mm                    0\n",
      "precip_type_5min:idx              0\n",
      "pressure_100m:hPa                 0\n",
      "pressure_50m:hPa                  0\n",
      "prob_rime:p                       0\n",
      "rain_water:kgm2                   0\n",
      "relative_humidity_1000hPa:p       0\n",
      "sfc_pressure:hPa                  0\n",
      "snow_depth:cm                     0\n",
      "snow_drift:idx                    0\n",
      "snow_melt_10min:mm                0\n",
      "snow_water:kgm2                   0\n",
      "sun_azimuth:d                     0\n",
      "sun_elevation:d                   0\n",
      "super_cooled_liquid_water:kgm2    0\n",
      "t_1000hPa:K                       0\n",
      "total_cloud_cover:p               0\n",
      "visibility:m                      0\n",
      "wind_speed_10m:ms                 0\n",
      "wind_speed_u_10m:ms               0\n",
      "wind_speed_v_10m:ms               0\n",
      "wind_speed_w_1000hPa:ms           0\n",
      "location_a                        0\n",
      "location_b                        0\n",
      "location_c                        0\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pv_measurement</th>\n",
       "      <th>date_forecast</th>\n",
       "      <th>absolute_humidity_2m:gm3</th>\n",
       "      <th>air_density_2m:kgm3</th>\n",
       "      <th>clear_sky_energy_1h:J</th>\n",
       "      <th>clear_sky_rad:W</th>\n",
       "      <th>cloud_base_agl:m</th>\n",
       "      <th>dew_or_rime:idx</th>\n",
       "      <th>dew_point_2m:K</th>\n",
       "      <th>diffuse_rad:W</th>\n",
       "      <th>diffuse_rad_1h:J</th>\n",
       "      <th>direct_rad:W</th>\n",
       "      <th>direct_rad_1h:J</th>\n",
       "      <th>effective_cloud_cover:p</th>\n",
       "      <th>is_day:idx</th>\n",
       "      <th>is_in_shadow:idx</th>\n",
       "      <th>precip_5min:mm</th>\n",
       "      <th>precip_type_5min:idx</th>\n",
       "      <th>pressure_50m:hPa</th>\n",
       "      <th>prob_rime:p</th>\n",
       "      <th>rain_water:kgm2</th>\n",
       "      <th>relative_humidity_1000hPa:p</th>\n",
       "      <th>sun_azimuth:d</th>\n",
       "      <th>super_cooled_liquid_water:kgm2</th>\n",
       "      <th>t_1000hPa:K</th>\n",
       "      <th>total_cloud_cover:p</th>\n",
       "      <th>visibility:m</th>\n",
       "      <th>wind_speed_10m:ms</th>\n",
       "      <th>wind_speed_u_10m:ms</th>\n",
       "      <th>wind_speed_v_10m:ms</th>\n",
       "      <th>wind_speed_w_1000hPa:ms</th>\n",
       "      <th>location_a</th>\n",
       "      <th>location_b</th>\n",
       "      <th>location_c</th>\n",
       "      <th>sin_day_of_year</th>\n",
       "      <th>cos_day_of_year</th>\n",
       "      <th>sin_hour</th>\n",
       "      <th>cos_hour</th>\n",
       "      <th>sun_product</th>\n",
       "      <th>modified_solar_elevation</th>\n",
       "      <th>effective_radiation</th>\n",
       "      <th>time_since_prediction</th>\n",
       "      <th>cloud_ratio</th>\n",
       "      <th>cloud_cover_over_30%</th>\n",
       "      <th>sun_addition</th>\n",
       "      <th>is_freezing</th>\n",
       "      <th>is_snow</th>\n",
       "      <th>is_rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-09-10 04:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2021-09-10 04:00:00</td>\n",
       "      <td>8.500</td>\n",
       "      <td>1.24250</td>\n",
       "      <td>3.581750e+02</td>\n",
       "      <td>1.275000</td>\n",
       "      <td>1215.050049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>281.524994</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>1296.25000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>92.125000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1003.900024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>96.574997</td>\n",
       "      <td>75.323997</td>\n",
       "      <td>0.075</td>\n",
       "      <td>282.150024</td>\n",
       "      <td>99.800003</td>\n",
       "      <td>22414.726562</td>\n",
       "      <td>2.075</td>\n",
       "      <td>-1.850</td>\n",
       "      <td>-0.950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.929634</td>\n",
       "      <td>-0.368484</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.923096</td>\n",
       "      <td>1</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-11-21 16:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2020-11-21 16:00:00</td>\n",
       "      <td>4.675</td>\n",
       "      <td>1.23925</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2067.925049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>272.924988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>91.949997</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>981.574951</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.949997</td>\n",
       "      <td>252.160492</td>\n",
       "      <td>0.000</td>\n",
       "      <td>279.774994</td>\n",
       "      <td>98.974998</td>\n",
       "      <td>45661.750000</td>\n",
       "      <td>3.800</td>\n",
       "      <td>1.100</td>\n",
       "      <td>3.625</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.638384</td>\n",
       "      <td>0.769718</td>\n",
       "      <td>-0.866025</td>\n",
       "      <td>-5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.929022</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-06-17 08:00:00</td>\n",
       "      <td>4779.28</td>\n",
       "      <td>2022-06-17 08:00:00</td>\n",
       "      <td>9.950</td>\n",
       "      <td>1.21400</td>\n",
       "      <td>2.273937e+06</td>\n",
       "      <td>683.724976</td>\n",
       "      <td>793.400024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>284.325012</td>\n",
       "      <td>120.974998</td>\n",
       "      <td>452943.59375</td>\n",
       "      <td>440.375000</td>\n",
       "      <td>1365512.750</td>\n",
       "      <td>63.400002</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1009.150024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>122.494499</td>\n",
       "      <td>0.000</td>\n",
       "      <td>286.899994</td>\n",
       "      <td>74.800003</td>\n",
       "      <td>20910.275391</td>\n",
       "      <td>1.250</td>\n",
       "      <td>-1.225</td>\n",
       "      <td>-0.200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.265563</td>\n",
       "      <td>-0.964094</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-5.000000e-01</td>\n",
       "      <td>53274.363281</td>\n",
       "      <td>0.647212</td>\n",
       "      <td>0.600506</td>\n",
       "      <td>0</td>\n",
       "      <td>0.847594</td>\n",
       "      <td>1</td>\n",
       "      <td>561.349976</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-08-02 18:00:00</td>\n",
       "      <td>320.54</td>\n",
       "      <td>2022-08-02 18:00:00</td>\n",
       "      <td>8.900</td>\n",
       "      <td>1.20000</td>\n",
       "      <td>5.981147e+05</td>\n",
       "      <td>109.099998</td>\n",
       "      <td>2517.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>282.649994</td>\n",
       "      <td>62.075001</td>\n",
       "      <td>290013.87500</td>\n",
       "      <td>20.674999</td>\n",
       "      <td>158899.625</td>\n",
       "      <td>80.550003</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>994.174988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>290.874756</td>\n",
       "      <td>0.000</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>45820.773438</td>\n",
       "      <td>3.975</td>\n",
       "      <td>-2.900</td>\n",
       "      <td>2.750</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.499069</td>\n",
       "      <td>-0.866562</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.836970e-16</td>\n",
       "      <td>1283.400635</td>\n",
       "      <td>0.167035</td>\n",
       "      <td>0.265667</td>\n",
       "      <td>0</td>\n",
       "      <td>0.805500</td>\n",
       "      <td>1</td>\n",
       "      <td>82.750000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-11-19 04:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2021-11-19 04:00:00</td>\n",
       "      <td>5.700</td>\n",
       "      <td>1.25000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>487.700012</td>\n",
       "      <td>0.0</td>\n",
       "      <td>275.549988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>988.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.574997</td>\n",
       "      <td>89.191750</td>\n",
       "      <td>0.650</td>\n",
       "      <td>277.075012</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>3899.100098</td>\n",
       "      <td>2.375</td>\n",
       "      <td>-0.250</td>\n",
       "      <td>2.325</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.677240</td>\n",
       "      <td>0.735762</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 time  pv_measurement       date_forecast  \\\n",
       "0 2021-09-10 04:00:00            0.00 2021-09-10 04:00:00   \n",
       "1 2020-11-21 16:00:00            0.00 2020-11-21 16:00:00   \n",
       "2 2022-06-17 08:00:00         4779.28 2022-06-17 08:00:00   \n",
       "3 2022-08-02 18:00:00          320.54 2022-08-02 18:00:00   \n",
       "4 2021-11-19 04:00:00            0.00 2021-11-19 04:00:00   \n",
       "\n",
       "   absolute_humidity_2m:gm3  air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
       "0                     8.500              1.24250           3.581750e+02   \n",
       "1                     4.675              1.23925           0.000000e+00   \n",
       "2                     9.950              1.21400           2.273937e+06   \n",
       "3                     8.900              1.20000           5.981147e+05   \n",
       "4                     5.700              1.25000           0.000000e+00   \n",
       "\n",
       "   clear_sky_rad:W  cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  \\\n",
       "0         1.275000       1215.050049              0.0      281.524994   \n",
       "1         0.000000       2067.925049              0.0      272.924988   \n",
       "2       683.724976        793.400024              0.0      284.325012   \n",
       "3       109.099998       2517.000000              0.0      282.649994   \n",
       "4         0.000000        487.700012              0.0      275.549988   \n",
       "\n",
       "   diffuse_rad:W  diffuse_rad_1h:J  direct_rad:W  direct_rad_1h:J  \\\n",
       "0       0.725000        1296.25000      0.000000            0.000   \n",
       "1       0.000000           0.00000      0.000000            0.000   \n",
       "2     120.974998      452943.59375    440.375000      1365512.750   \n",
       "3      62.075001      290013.87500     20.674999       158899.625   \n",
       "4       0.000000           0.00000      0.000000            0.000   \n",
       "\n",
       "   effective_cloud_cover:p  is_day:idx  is_in_shadow:idx  precip_5min:mm  \\\n",
       "0                92.125000        0.25               1.0             0.0   \n",
       "1                91.949997        0.00               1.0             0.0   \n",
       "2                63.400002        1.00               0.0             0.0   \n",
       "3                80.550003        1.00               0.0             0.0   \n",
       "4               100.000000        0.00               1.0             0.1   \n",
       "\n",
       "   precip_type_5min:idx  pressure_50m:hPa  prob_rime:p  rain_water:kgm2  \\\n",
       "0                   0.0       1003.900024          0.0              0.0   \n",
       "1                   0.0        981.574951          0.0              0.0   \n",
       "2                   0.0       1009.150024          0.0              0.0   \n",
       "3                   0.0        994.174988          0.0              0.0   \n",
       "4                   1.0        988.125000          0.0              0.0   \n",
       "\n",
       "   relative_humidity_1000hPa:p  sun_azimuth:d  super_cooled_liquid_water:kgm2  \\\n",
       "0                    96.574997      75.323997                           0.075   \n",
       "1                    65.949997     252.160492                           0.000   \n",
       "2                    73.000000     122.494499                           0.000   \n",
       "3                    64.000000     290.874756                           0.000   \n",
       "4                    92.574997      89.191750                           0.650   \n",
       "\n",
       "   t_1000hPa:K  total_cloud_cover:p  visibility:m  wind_speed_10m:ms  \\\n",
       "0   282.150024            99.800003  22414.726562              2.075   \n",
       "1   279.774994            98.974998  45661.750000              3.800   \n",
       "2   286.899994            74.800003  20910.275391              1.250   \n",
       "3   289.000000           100.000000  45820.773438              3.975   \n",
       "4   277.075012           100.000000   3899.100098              2.375   \n",
       "\n",
       "   wind_speed_u_10m:ms  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  \\\n",
       "0               -1.850               -0.950                      0.0   \n",
       "1                1.100                3.625                      0.0   \n",
       "2               -1.225               -0.200                      0.0   \n",
       "3               -2.900                2.750                      0.0   \n",
       "4               -0.250                2.325                      0.0   \n",
       "\n",
       "   location_a  location_b  location_c  sin_day_of_year  cos_day_of_year  \\\n",
       "0           0           0           1        -0.929634        -0.368484   \n",
       "1           1           0           0        -0.638384         0.769718   \n",
       "2           1           0           0         0.265563        -0.964094   \n",
       "3           1           0           0        -0.499069        -0.866562   \n",
       "4           0           1           0        -0.677240         0.735762   \n",
       "\n",
       "   sin_hour      cos_hour   sun_product  modified_solar_elevation  \\\n",
       "0  0.866025  5.000000e-01      0.000000                  0.000000   \n",
       "1 -0.866025 -5.000000e-01      0.000000                  0.000000   \n",
       "2  0.866025 -5.000000e-01  53274.363281                  0.647212   \n",
       "3 -1.000000 -1.836970e-16   1283.400635                  0.167035   \n",
       "4  0.866025  5.000000e-01      0.000000                  0.000000   \n",
       "\n",
       "   effective_radiation  time_since_prediction  cloud_ratio  \\\n",
       "0             0.000000                      0     0.923096   \n",
       "1             0.000000                      0     0.929022   \n",
       "2             0.600506                      0     0.847594   \n",
       "3             0.265667                      0     0.805500   \n",
       "4             0.000000                      0     1.000000   \n",
       "\n",
       "   cloud_cover_over_30%  sun_addition  is_freezing  is_snow  is_rain  \n",
       "0                     1      0.725000            0        0        0  \n",
       "1                     1      0.000000            0        0        0  \n",
       "2                     1    561.349976            0        0        0  \n",
       "3                     1     82.750000            0        0        0  \n",
       "4                     1      0.000000            0        0        1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets_A, targets_B, targets_C, X_train_estimated_a, X_train_estimated_b, X_train_estimated_c, X_train_observed_a, X_train_observed_b, X_train_observed_c, X_test_estimated_a, X_test_estimated_b, X_test_estimated_c = get_raw_data()\n",
    "all_features = get_raw_data()\n",
    "\n",
    "\n",
    "train_a, train_b, train_c, X_train_estimated_a, X_train_estimated_b, X_train_estimated_c, X_train_observed_a, X_train_observed_b, X_train_observed_c, X_test_estimated_a, X_test_estimated_b, X_test_estimated_c = get_raw_data()\n",
    "\n",
    "X_train_obs_combined, X_val_obs_combined, y_train_obs_combined, y_val_obs_combined, X_train_est_combined, X_val_est_combined, y_train_est_combined, y_val_est_combined = fetch_preprocessed_data(drop_features=False)\n",
    "x_test_whole = get_preprocessed_test_data()\n",
    "\n",
    "x_whole = pd.concat([X_train_obs_combined, X_val_obs_combined])\n",
    "y_whole = pd.concat([y_train_obs_combined, y_val_obs_combined])\n",
    "x_whole.reset_index(drop=True, inplace=True)\n",
    "y_whole.reset_index(drop=True, inplace=True)\n",
    "\n",
    "x_whole.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pv_measurement</th>\n",
       "      <th>date_forecast</th>\n",
       "      <th>absolute_humidity_2m:gm3</th>\n",
       "      <th>air_density_2m:kgm3</th>\n",
       "      <th>clear_sky_energy_1h:J</th>\n",
       "      <th>clear_sky_rad:W</th>\n",
       "      <th>cloud_base_agl:m</th>\n",
       "      <th>dew_or_rime:idx</th>\n",
       "      <th>dew_point_2m:K</th>\n",
       "      <th>diffuse_rad:W</th>\n",
       "      <th>diffuse_rad_1h:J</th>\n",
       "      <th>direct_rad:W</th>\n",
       "      <th>direct_rad_1h:J</th>\n",
       "      <th>effective_cloud_cover:p</th>\n",
       "      <th>is_day:idx</th>\n",
       "      <th>is_in_shadow:idx</th>\n",
       "      <th>precip_5min:mm</th>\n",
       "      <th>precip_type_5min:idx</th>\n",
       "      <th>pressure_50m:hPa</th>\n",
       "      <th>prob_rime:p</th>\n",
       "      <th>rain_water:kgm2</th>\n",
       "      <th>relative_humidity_1000hPa:p</th>\n",
       "      <th>sun_azimuth:d</th>\n",
       "      <th>super_cooled_liquid_water:kgm2</th>\n",
       "      <th>t_1000hPa:K</th>\n",
       "      <th>total_cloud_cover:p</th>\n",
       "      <th>visibility:m</th>\n",
       "      <th>wind_speed_10m:ms</th>\n",
       "      <th>wind_speed_u_10m:ms</th>\n",
       "      <th>wind_speed_v_10m:ms</th>\n",
       "      <th>wind_speed_w_1000hPa:ms</th>\n",
       "      <th>location_a</th>\n",
       "      <th>location_b</th>\n",
       "      <th>location_c</th>\n",
       "      <th>sin_day_of_year</th>\n",
       "      <th>cos_day_of_year</th>\n",
       "      <th>sin_hour</th>\n",
       "      <th>cos_hour</th>\n",
       "      <th>sun_product</th>\n",
       "      <th>modified_solar_elevation</th>\n",
       "      <th>effective_radiation</th>\n",
       "      <th>time_since_prediction</th>\n",
       "      <th>cloud_ratio</th>\n",
       "      <th>cloud_cover_over_30%</th>\n",
       "      <th>sun_addition</th>\n",
       "      <th>is_freezing</th>\n",
       "      <th>is_snow</th>\n",
       "      <th>is_rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-09-10 04:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2021-09-10 04:00:00</td>\n",
       "      <td>8.500</td>\n",
       "      <td>1.24250</td>\n",
       "      <td>3.581750e+02</td>\n",
       "      <td>1.275000</td>\n",
       "      <td>1215.050049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>281.524994</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>1296.25000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>92.125000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1003.900024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>96.574997</td>\n",
       "      <td>75.323997</td>\n",
       "      <td>0.075</td>\n",
       "      <td>282.150024</td>\n",
       "      <td>99.800003</td>\n",
       "      <td>22414.726562</td>\n",
       "      <td>2.075</td>\n",
       "      <td>-1.850</td>\n",
       "      <td>-0.950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.929634</td>\n",
       "      <td>-0.368484</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.923096</td>\n",
       "      <td>1</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-11-21 16:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2020-11-21 16:00:00</td>\n",
       "      <td>4.675</td>\n",
       "      <td>1.23925</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2067.925049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>272.924988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>91.949997</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>981.574951</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.949997</td>\n",
       "      <td>252.160492</td>\n",
       "      <td>0.000</td>\n",
       "      <td>279.774994</td>\n",
       "      <td>98.974998</td>\n",
       "      <td>45661.750000</td>\n",
       "      <td>3.800</td>\n",
       "      <td>1.100</td>\n",
       "      <td>3.625</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.638384</td>\n",
       "      <td>0.769718</td>\n",
       "      <td>-0.866025</td>\n",
       "      <td>-5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.929022</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-06-17 08:00:00</td>\n",
       "      <td>4779.28</td>\n",
       "      <td>2022-06-17 08:00:00</td>\n",
       "      <td>9.950</td>\n",
       "      <td>1.21400</td>\n",
       "      <td>2.273937e+06</td>\n",
       "      <td>683.724976</td>\n",
       "      <td>793.400024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>284.325012</td>\n",
       "      <td>120.974998</td>\n",
       "      <td>452943.59375</td>\n",
       "      <td>440.375000</td>\n",
       "      <td>1365512.750</td>\n",
       "      <td>63.400002</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1009.150024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>122.494499</td>\n",
       "      <td>0.000</td>\n",
       "      <td>286.899994</td>\n",
       "      <td>74.800003</td>\n",
       "      <td>20910.275391</td>\n",
       "      <td>1.250</td>\n",
       "      <td>-1.225</td>\n",
       "      <td>-0.200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.265563</td>\n",
       "      <td>-0.964094</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-5.000000e-01</td>\n",
       "      <td>53274.363281</td>\n",
       "      <td>0.647212</td>\n",
       "      <td>0.600506</td>\n",
       "      <td>0</td>\n",
       "      <td>0.847594</td>\n",
       "      <td>1</td>\n",
       "      <td>561.349976</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-08-02 18:00:00</td>\n",
       "      <td>320.54</td>\n",
       "      <td>2022-08-02 18:00:00</td>\n",
       "      <td>8.900</td>\n",
       "      <td>1.20000</td>\n",
       "      <td>5.981147e+05</td>\n",
       "      <td>109.099998</td>\n",
       "      <td>2517.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>282.649994</td>\n",
       "      <td>62.075001</td>\n",
       "      <td>290013.87500</td>\n",
       "      <td>20.674999</td>\n",
       "      <td>158899.625</td>\n",
       "      <td>80.550003</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>994.174988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>290.874756</td>\n",
       "      <td>0.000</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>45820.773438</td>\n",
       "      <td>3.975</td>\n",
       "      <td>-2.900</td>\n",
       "      <td>2.750</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.499069</td>\n",
       "      <td>-0.866562</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.836970e-16</td>\n",
       "      <td>1283.400635</td>\n",
       "      <td>0.167035</td>\n",
       "      <td>0.265667</td>\n",
       "      <td>0</td>\n",
       "      <td>0.805500</td>\n",
       "      <td>1</td>\n",
       "      <td>82.750000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-11-19 04:00:00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2021-11-19 04:00:00</td>\n",
       "      <td>5.700</td>\n",
       "      <td>1.25000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>487.700012</td>\n",
       "      <td>0.0</td>\n",
       "      <td>275.549988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>988.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.574997</td>\n",
       "      <td>89.191750</td>\n",
       "      <td>0.650</td>\n",
       "      <td>277.075012</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>3899.100098</td>\n",
       "      <td>2.375</td>\n",
       "      <td>-0.250</td>\n",
       "      <td>2.325</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.677240</td>\n",
       "      <td>0.735762</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 time  pv_measurement       date_forecast  \\\n",
       "0 2021-09-10 04:00:00            0.00 2021-09-10 04:00:00   \n",
       "1 2020-11-21 16:00:00            0.00 2020-11-21 16:00:00   \n",
       "2 2022-06-17 08:00:00         4779.28 2022-06-17 08:00:00   \n",
       "3 2022-08-02 18:00:00          320.54 2022-08-02 18:00:00   \n",
       "4 2021-11-19 04:00:00            0.00 2021-11-19 04:00:00   \n",
       "\n",
       "   absolute_humidity_2m:gm3  air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
       "0                     8.500              1.24250           3.581750e+02   \n",
       "1                     4.675              1.23925           0.000000e+00   \n",
       "2                     9.950              1.21400           2.273937e+06   \n",
       "3                     8.900              1.20000           5.981147e+05   \n",
       "4                     5.700              1.25000           0.000000e+00   \n",
       "\n",
       "   clear_sky_rad:W  cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  \\\n",
       "0         1.275000       1215.050049              0.0      281.524994   \n",
       "1         0.000000       2067.925049              0.0      272.924988   \n",
       "2       683.724976        793.400024              0.0      284.325012   \n",
       "3       109.099998       2517.000000              0.0      282.649994   \n",
       "4         0.000000        487.700012              0.0      275.549988   \n",
       "\n",
       "   diffuse_rad:W  diffuse_rad_1h:J  direct_rad:W  direct_rad_1h:J  \\\n",
       "0       0.725000        1296.25000      0.000000            0.000   \n",
       "1       0.000000           0.00000      0.000000            0.000   \n",
       "2     120.974998      452943.59375    440.375000      1365512.750   \n",
       "3      62.075001      290013.87500     20.674999       158899.625   \n",
       "4       0.000000           0.00000      0.000000            0.000   \n",
       "\n",
       "   effective_cloud_cover:p  is_day:idx  is_in_shadow:idx  precip_5min:mm  \\\n",
       "0                92.125000        0.25               1.0             0.0   \n",
       "1                91.949997        0.00               1.0             0.0   \n",
       "2                63.400002        1.00               0.0             0.0   \n",
       "3                80.550003        1.00               0.0             0.0   \n",
       "4               100.000000        0.00               1.0             0.1   \n",
       "\n",
       "   precip_type_5min:idx  pressure_50m:hPa  prob_rime:p  rain_water:kgm2  \\\n",
       "0                   0.0       1003.900024          0.0              0.0   \n",
       "1                   0.0        981.574951          0.0              0.0   \n",
       "2                   0.0       1009.150024          0.0              0.0   \n",
       "3                   0.0        994.174988          0.0              0.0   \n",
       "4                   1.0        988.125000          0.0              0.0   \n",
       "\n",
       "   relative_humidity_1000hPa:p  sun_azimuth:d  super_cooled_liquid_water:kgm2  \\\n",
       "0                    96.574997      75.323997                           0.075   \n",
       "1                    65.949997     252.160492                           0.000   \n",
       "2                    73.000000     122.494499                           0.000   \n",
       "3                    64.000000     290.874756                           0.000   \n",
       "4                    92.574997      89.191750                           0.650   \n",
       "\n",
       "   t_1000hPa:K  total_cloud_cover:p  visibility:m  wind_speed_10m:ms  \\\n",
       "0   282.150024            99.800003  22414.726562              2.075   \n",
       "1   279.774994            98.974998  45661.750000              3.800   \n",
       "2   286.899994            74.800003  20910.275391              1.250   \n",
       "3   289.000000           100.000000  45820.773438              3.975   \n",
       "4   277.075012           100.000000   3899.100098              2.375   \n",
       "\n",
       "   wind_speed_u_10m:ms  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  \\\n",
       "0               -1.850               -0.950                      0.0   \n",
       "1                1.100                3.625                      0.0   \n",
       "2               -1.225               -0.200                      0.0   \n",
       "3               -2.900                2.750                      0.0   \n",
       "4               -0.250                2.325                      0.0   \n",
       "\n",
       "   location_a  location_b  location_c  sin_day_of_year  cos_day_of_year  \\\n",
       "0           0           0           1        -0.929634        -0.368484   \n",
       "1           1           0           0        -0.638384         0.769718   \n",
       "2           1           0           0         0.265563        -0.964094   \n",
       "3           1           0           0        -0.499069        -0.866562   \n",
       "4           0           1           0        -0.677240         0.735762   \n",
       "\n",
       "   sin_hour      cos_hour   sun_product  modified_solar_elevation  \\\n",
       "0  0.866025  5.000000e-01      0.000000                  0.000000   \n",
       "1 -0.866025 -5.000000e-01      0.000000                  0.000000   \n",
       "2  0.866025 -5.000000e-01  53274.363281                  0.647212   \n",
       "3 -1.000000 -1.836970e-16   1283.400635                  0.167035   \n",
       "4  0.866025  5.000000e-01      0.000000                  0.000000   \n",
       "\n",
       "   effective_radiation  time_since_prediction  cloud_ratio  \\\n",
       "0             0.000000                      0     0.923096   \n",
       "1             0.000000                      0     0.929022   \n",
       "2             0.600506                      0     0.847594   \n",
       "3             0.265667                      0     0.805500   \n",
       "4             0.000000                      0     1.000000   \n",
       "\n",
       "   cloud_cover_over_30%  sun_addition  is_freezing  is_snow  is_rain  \n",
       "0                     1      0.725000            0        0        0  \n",
       "1                     1      0.000000            0        0        0  \n",
       "2                     1    561.349976            0        0        0  \n",
       "3                     1     82.750000            0        0        0  \n",
       "4                     1      0.000000            0        0        1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_whole.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pv_measurement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-06-02 22:00:00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-06-02 23:00:00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-06-03 00:00:00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-06-03 01:00:00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-06-03 02:00:00</td>\n",
       "      <td>19.36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 time  pv_measurement\n",
       "0 2019-06-02 22:00:00            0.00\n",
       "1 2019-06-02 23:00:00            0.00\n",
       "2 2019-06-03 00:00:00            0.00\n",
       "3 2019-06-03 01:00:00            0.00\n",
       "4 2019-06-03 02:00:00           19.36"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_observed_a['date_forecast'] = pd.to_datetime(X_train_observed_a['date_forecast'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Range of indices in targets_A: (0, 34084)\n",
      "Range of indices in X_train_observed_a: (0, 118668)\n"
     ]
    }
   ],
   "source": [
    "# Check the range of indices in both dataframes\n",
    "range_targets_A = (targets_A.index.min(), targets_A.index.max())\n",
    "range_X_train_observed_a = (X_train_observed_a.index.min(), X_train_observed_a.index.max())\n",
    "\n",
    "print(\"Range of indices in targets_A:\", range_targets_A)\n",
    "print(\"Range of indices in X_train_observed_a:\", range_X_train_observed_a)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset for Location A\n",
    "targets_A['time'] = pd.to_datetime(targets_A['time'])\n",
    "targets_A.set_index('time', inplace=True)\n",
    "\n",
    "targets_B['time'] = pd.to_datetime(targets_B['time'])\n",
    "targets_B.set_index('time', inplace=True)\n",
    "\n",
    "targets_C['time'] = pd.to_datetime(targets_C['time'])\n",
    "targets_C.set_index('time', inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['pv_measurement'], dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets_A.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hour\n",
      "0        0.009318\n",
      "1        3.275153\n",
      "2       37.997365\n",
      "3      137.552800\n",
      "4      370.981082\n",
      "5      813.822447\n",
      "6     1414.227812\n",
      "7     2017.630353\n",
      "8     2553.940141\n",
      "9     2892.494259\n",
      "10    3041.032565\n",
      "11    3073.337365\n",
      "12    2888.446776\n",
      "13    2624.679718\n",
      "14    2215.669176\n",
      "15    1766.465412\n",
      "16    1172.741318\n",
      "17     630.691859\n",
      "18     297.967482\n",
      "19     132.128376\n",
      "20      32.066682\n",
      "21       1.135718\n",
      "22       0.001549\n",
      "23       0.000000\n",
      "Name: pv_measurement, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_average_pv_using_date_forecast(targets_df, x_train_observed_df, location_name):\n",
    "    # Ensure 'date_forecast' is in datetime format in both DataFrames\n",
    "\n",
    "    # Now proceed with the merge\n",
    "    combined_df = pd.merge(targets_df, x_train_observed_df, left_index=True, right_on='date_forecast')\n",
    "    \n",
    "    # Extracting Hour Information from the 'date_forecast' column\n",
    "    combined_df[\"hour\"] = combined_df['date_forecast'].dt.hour\n",
    "\n",
    "    # Grouping and Averaging\n",
    "    average_pv_by_hour = combined_df.groupby(\"hour\")[\"pv_measurement\"].mean()\n",
    "\n",
    "    print(average_pv_by_hour)\n",
    "\n",
    "    # Plotting\n",
    "    # average_pv_by_hour.plot(kind=\"bar\", figsize=(10, 6), title=f\"Average PV Measurement for Each Hour of the Day - {location_name}\")\n",
    "    # plt.xlabel(\"Hour of the Day\")\n",
    "    # plt.ylabel(\"Average PV Measurement\")\n",
    "    # plt.grid(axis=\"y\")\n",
    "    # plt.tight_layout()\n",
    "    # plt.show()\n",
    "\n",
    "# For representation purposes, the function's return value\n",
    "\"Function defined. You can now run this function in your environment for the desired results.\"\n",
    "\n",
    "\n",
    "# Extract months from the datetime index\n",
    "months = targets_A.index.month\n",
    "\n",
    "# Filter rows where the month is between April (4) and July (7)\n",
    "filtered_df = targets_A[(months >= 4) & (months <= 7)]\n",
    "# Call the function for each location's dataframes\n",
    "plot_average_pv_using_date_forecast(filtered_df, X_train_observed_a, \"Location A\")\n",
    "# plot_average_pv_using_date_forecast(filtered_df, X_train_observed_b, \"Location B\")\n",
    "# plot_average_pv_using_date_forecast(y_whole, x_whole, \"Location C\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "absolute_humidity_2m:gm3          0\n",
      "air_density_2m:kgm3               0\n",
      "clear_sky_energy_1h:J             0\n",
      "clear_sky_rad:W                   0\n",
      "cloud_base_agl:m                  0\n",
      "dew_or_rime:idx                   0\n",
      "dew_point_2m:K                    0\n",
      "diffuse_rad:W                     0\n",
      "diffuse_rad_1h:J                  0\n",
      "direct_rad:W                      0\n",
      "direct_rad_1h:J                   0\n",
      "effective_cloud_cover:p           0\n",
      "is_day:idx                        0\n",
      "is_in_shadow:idx                  0\n",
      "precip_5min:mm                    0\n",
      "precip_type_5min:idx              0\n",
      "pressure_50m:hPa                  0\n",
      "prob_rime:p                       0\n",
      "rain_water:kgm2                   0\n",
      "relative_humidity_1000hPa:p       0\n",
      "sun_azimuth:d                     0\n",
      "super_cooled_liquid_water:kgm2    0\n",
      "t_1000hPa:K                       0\n",
      "total_cloud_cover:p               0\n",
      "visibility:m                      0\n",
      "wind_speed_10m:ms                 0\n",
      "wind_speed_u_10m:ms               0\n",
      "wind_speed_v_10m:ms               0\n",
      "wind_speed_w_1000hPa:ms           0\n",
      "location_a                        0\n",
      "location_b                        0\n",
      "location_c                        0\n",
      "sin_day_of_year                   0\n",
      "cos_day_of_year                   0\n",
      "sin_hour                          0\n",
      "cos_hour                          0\n",
      "sun_product                       0\n",
      "modified_solar_elevation          0\n",
      "effective_radiation               0\n",
      "time_since_prediction             0\n",
      "cloud_ratio                       0\n",
      "cloud_cover_over_30%              0\n",
      "sun_addition                      0\n",
      "is_freezing                       0\n",
      "is_snow                           0\n",
      "is_rain                           0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(x_test_whole.isna().sum())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Function defined. You can now run this function in your environment for the desired results.'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def calculate_pv_percentage_by_hour(targets_df, x_train_observed_df):\n",
    "    # Ensure the 'date_forecast' column is a Datetime column\n",
    "    x_train_observed_df['date_forecast'] = pd.to_datetime(x_train_observed_df['date_forecast'])\n",
    "\n",
    "    # Merging the DataFrames using 'date_forecast' for x_train_observed_df and index for targets_df\n",
    "    combined_df = pd.merge(targets_df, x_train_observed_df, left_index=True, right_on='date_forecast')\n",
    "    \n",
    "    # Extracting Hour Information from the 'date_forecast' column\n",
    "    combined_df[\"hour\"] = combined_df['date_forecast'].dt.hour\n",
    "\n",
    "    # Grouping and Averaging\n",
    "    average_pv_by_hour = combined_df.groupby(\"hour\")[\"pv_measurement\"].mean()\n",
    "\n",
    "    # Calculate the percentage of the max hour\n",
    "    max_hour_value = average_pv_by_hour.max()\n",
    "    percentage_of_max_hour = (average_pv_by_hour / max_hour_value) * 100\n",
    "\n",
    "    return percentage_of_max_hour\n",
    "\n",
    "# For representation purposes, the function's return value\n",
    "\"Function defined. You can now run this function in your environment for the desired results.\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Location A - Percentage of Max Hour PV:\n",
      "hour\n",
      "0       0.000159\n",
      "1       0.056063\n",
      "2       0.652588\n",
      "3       2.493848\n",
      "4       7.398189\n",
      "5      17.937975\n",
      "6      34.967078\n",
      "7      54.989824\n",
      "8      74.842756\n",
      "9      89.825527\n",
      "10     98.383528\n",
      "11    100.000000\n",
      "12     91.833482\n",
      "13     77.776764\n",
      "14     60.984522\n",
      "15     44.170774\n",
      "16     27.015991\n",
      "17     13.350792\n",
      "18      5.895078\n",
      "19      2.387916\n",
      "20      0.549216\n",
      "21      0.019441\n",
      "22      0.000027\n",
      "23      0.000000\n",
      "Name: pv_measurement, dtype: float64\n",
      "\n",
      "Location B - Percentage of Max Hour PV:\n",
      "hour\n",
      "0       4.961233\n",
      "1       5.281501\n",
      "2       6.013889\n",
      "3       8.548226\n",
      "4      14.711482\n",
      "5      24.720872\n",
      "6      39.861623\n",
      "7      57.147685\n",
      "8      75.389541\n",
      "9      88.561434\n",
      "10     98.307949\n",
      "11    100.000000\n",
      "12     93.088619\n",
      "13     79.528733\n",
      "14     62.269500\n",
      "15     46.851170\n",
      "16     31.756596\n",
      "17     19.487354\n",
      "18     12.163215\n",
      "19      7.510654\n",
      "20      5.419775\n",
      "21      5.068415\n",
      "22      4.931610\n",
      "23      4.979176\n",
      "Name: pv_measurement, dtype: float64\n",
      "\n",
      "Location C - Percentage of Max Hour PV:\n",
      "hour\n",
      "0       0.000000\n",
      "1       0.012426\n",
      "2       0.639997\n",
      "3       2.654954\n",
      "4       6.957383\n",
      "5      15.467274\n",
      "6      31.004741\n",
      "7      49.246710\n",
      "8      67.950833\n",
      "9      84.066161\n",
      "10     95.916515\n",
      "11    100.000000\n",
      "12     95.124567\n",
      "13     84.437387\n",
      "14     63.929220\n",
      "15     46.630094\n",
      "16     30.217786\n",
      "17     16.754661\n",
      "18      7.403894\n",
      "19      2.557334\n",
      "20      0.593961\n",
      "21      0.016499\n",
      "22      0.000000\n",
      "23      0.000000\n",
      "Name: pv_measurement, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "percentage_A = calculate_pv_percentage_by_hour(targets_A, X_train_observed_a)\n",
    "percentage_B = calculate_pv_percentage_by_hour(targets_B, X_train_observed_b)\n",
    "percentage_C = calculate_pv_percentage_by_hour(targets_C, X_train_observed_c)\n",
    "\n",
    "print(\"Location A - Percentage of Max Hour PV:\")\n",
    "print(percentage_A)\n",
    "print(\"\\nLocation B - Percentage of Max Hour PV:\")\n",
    "print(percentage_B)\n",
    "print(\"\\nLocation C - Percentage of Max Hour PV:\")\n",
    "print(percentage_C)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['date_forecast', 'absolute_humidity_2m:gm3', 'air_density_2m:kgm3',\n",
       "       'ceiling_height_agl:m', 'clear_sky_energy_1h:J', 'clear_sky_rad:W',\n",
       "       'cloud_base_agl:m', 'dew_or_rime:idx', 'dew_point_2m:K',\n",
       "       'diffuse_rad:W', 'diffuse_rad_1h:J', 'direct_rad:W', 'direct_rad_1h:J',\n",
       "       'effective_cloud_cover:p', 'elevation:m', 'fresh_snow_12h:cm',\n",
       "       'fresh_snow_1h:cm', 'fresh_snow_24h:cm', 'fresh_snow_3h:cm',\n",
       "       'fresh_snow_6h:cm', 'is_day:idx', 'is_in_shadow:idx',\n",
       "       'msl_pressure:hPa', 'precip_5min:mm', 'precip_type_5min:idx',\n",
       "       'pressure_100m:hPa', 'pressure_50m:hPa', 'prob_rime:p',\n",
       "       'rain_water:kgm2', 'relative_humidity_1000hPa:p', 'sfc_pressure:hPa',\n",
       "       'snow_density:kgm3', 'snow_depth:cm', 'snow_drift:idx',\n",
       "       'snow_melt_10min:mm', 'snow_water:kgm2', 'sun_azimuth:d',\n",
       "       'sun_elevation:d', 'super_cooled_liquid_water:kgm2', 't_1000hPa:K',\n",
       "       'total_cloud_cover:p', 'visibility:m', 'wind_speed_10m:ms',\n",
       "       'wind_speed_u_10m:ms', 'wind_speed_v_10m:ms',\n",
       "       'wind_speed_w_1000hPa:ms'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_observed_a.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train_a.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Rain'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\venv\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3790\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3789\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m-> 3790\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_engine\u001b[39m.\u001b[39;49mget_loc(casted_key)\n\u001b[0;32m   3791\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n",
      "File \u001b[1;32mindex.pyx:152\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mindex.pyx:181\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:7080\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:7088\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Rain'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\missing_data.ipynb Cell 15\u001b[0m line \u001b[0;36m3\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/gunna/Documents/Maskinl%C3%A6ring/Prosjekt/power-predictor/missing_data.ipynb#X20sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m X_train_observed_a\u001b[39m.\u001b[39mhead()\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/gunna/Documents/Maskinl%C3%A6ring/Prosjekt/power-predictor/missing_data.ipynb#X20sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m value_counts \u001b[39m=\u001b[39m X_train_observed_a[\u001b[39m'\u001b[39;49m\u001b[39mRain\u001b[39;49m\u001b[39m'\u001b[39;49m]\u001b[39m.\u001b[39mvalue_counts()\u001b[39m.\u001b[39msort_index()\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/gunna/Documents/Maskinl%C3%A6ring/Prosjekt/power-predictor/missing_data.ipynb#X20sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m value_counts\n",
      "File \u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\venv\\lib\\site-packages\\pandas\\core\\frame.py:3896\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3894\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcolumns\u001b[39m.\u001b[39mnlevels \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m   3895\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getitem_multilevel(key)\n\u001b[1;32m-> 3896\u001b[0m indexer \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcolumns\u001b[39m.\u001b[39;49mget_loc(key)\n\u001b[0;32m   3897\u001b[0m \u001b[39mif\u001b[39;00m is_integer(indexer):\n\u001b[0;32m   3898\u001b[0m     indexer \u001b[39m=\u001b[39m [indexer]\n",
      "File \u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\venv\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3797\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3792\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(casted_key, \u001b[39mslice\u001b[39m) \u001b[39mor\u001b[39;00m (\n\u001b[0;32m   3793\u001b[0m         \u001b[39misinstance\u001b[39m(casted_key, abc\u001b[39m.\u001b[39mIterable)\n\u001b[0;32m   3794\u001b[0m         \u001b[39mand\u001b[39;00m \u001b[39many\u001b[39m(\u001b[39misinstance\u001b[39m(x, \u001b[39mslice\u001b[39m) \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m casted_key)\n\u001b[0;32m   3795\u001b[0m     ):\n\u001b[0;32m   3796\u001b[0m         \u001b[39mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[1;32m-> 3797\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mKeyError\u001b[39;00m(key) \u001b[39mfrom\u001b[39;00m \u001b[39merr\u001b[39;00m\n\u001b[0;32m   3798\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mTypeError\u001b[39;00m:\n\u001b[0;32m   3799\u001b[0m     \u001b[39m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[0;32m   3800\u001b[0m     \u001b[39m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[0;32m   3801\u001b[0m     \u001b[39m#  the TypeError.\u001b[39;00m\n\u001b[0;32m   3802\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Rain'"
     ]
    }
   ],
   "source": [
    "X_train_observed_a.head()\n",
    "\n",
    "value_counts = X_train_observed_a['Rain'].value_counts().sort_index()\n",
    "value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date_calc                         352423\n",
      "date_forecast                          0\n",
      "absolute_humidity_2m:gm3               0\n",
      "air_density_2m:kgm3                    0\n",
      "ceiling_height_agl:m               75543\n",
      "clear_sky_energy_1h:J                  0\n",
      "clear_sky_rad:W                        0\n",
      "cloud_base_agl:m                   31862\n",
      "dew_or_rime:idx                        0\n",
      "dew_point_2m:K                         0\n",
      "diffuse_rad:W                          0\n",
      "diffuse_rad_1h:J                       0\n",
      "direct_rad:W                           0\n",
      "direct_rad_1h:J                        0\n",
      "effective_cloud_cover:p                0\n",
      "elevation:m                            0\n",
      "fresh_snow_12h:cm                      0\n",
      "fresh_snow_1h:cm                       0\n",
      "fresh_snow_24h:cm                      0\n",
      "fresh_snow_3h:cm                       0\n",
      "fresh_snow_6h:cm                       0\n",
      "is_day:idx                             0\n",
      "is_in_shadow:idx                       0\n",
      "msl_pressure:hPa                       0\n",
      "precip_5min:mm                         0\n",
      "precip_type_5min:idx                   0\n",
      "pressure_100m:hPa                      0\n",
      "pressure_50m:hPa                       0\n",
      "prob_rime:p                            0\n",
      "rain_water:kgm2                        0\n",
      "relative_humidity_1000hPa:p            0\n",
      "sfc_pressure:hPa                       0\n",
      "snow_density:kgm3                 388759\n",
      "snow_depth:cm                          0\n",
      "snow_drift:idx                         0\n",
      "snow_melt_10min:mm                     0\n",
      "snow_water:kgm2                        0\n",
      "sun_azimuth:d                          0\n",
      "sun_elevation:d                        0\n",
      "super_cooled_liquid_water:kgm2         0\n",
      "t_1000hPa:K                            0\n",
      "total_cloud_cover:p                    0\n",
      "visibility:m                           0\n",
      "wind_speed_10m:ms                      0\n",
      "wind_speed_u_10m:ms                    0\n",
      "wind_speed_v_10m:ms                    0\n",
      "wind_speed_w_1000hPa:ms                0\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(413791, 47)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine all the datasets except for train_a, train_b, and train_c\n",
    "combined_df = pd.concat([\n",
    "    X_train_estimated_a, X_train_estimated_b, X_train_estimated_c, \n",
    "    X_train_observed_a, X_train_observed_b, X_train_observed_c, \n",
    "    X_test_estimated_a, X_test_estimated_b, X_test_estimated_c\n",
    "])\n",
    "\n",
    "# Calculate the number of NaN values for each column\n",
    "na_count_per_column = combined_df.isna().sum()\n",
    "\n",
    "print(na_count_per_column)\n",
    "\n",
    "combined_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(118669, 46)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_observed_a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date_forecast                          0\n",
       "absolute_humidity_2m:gm3               0\n",
       "air_density_2m:kgm3                    0\n",
       "ceiling_height_agl:m               19923\n",
       "clear_sky_energy_1h:J                  0\n",
       "clear_sky_rad:W                        0\n",
       "cloud_base_agl:m                    8512\n",
       "dew_or_rime:idx                        0\n",
       "dew_point_2m:K                         0\n",
       "diffuse_rad:W                          0\n",
       "diffuse_rad_1h:J                       0\n",
       "direct_rad:W                           0\n",
       "direct_rad_1h:J                        0\n",
       "effective_cloud_cover:p                0\n",
       "elevation:m                            0\n",
       "fresh_snow_12h:cm                      0\n",
       "fresh_snow_1h:cm                       0\n",
       "fresh_snow_24h:cm                      0\n",
       "fresh_snow_3h:cm                       0\n",
       "fresh_snow_6h:cm                       0\n",
       "is_day:idx                             0\n",
       "is_in_shadow:idx                       0\n",
       "msl_pressure:hPa                       0\n",
       "precip_5min:mm                         0\n",
       "precip_type_5min:idx                   0\n",
       "pressure_100m:hPa                      0\n",
       "pressure_50m:hPa                       0\n",
       "prob_rime:p                            0\n",
       "rain_water:kgm2                        0\n",
       "relative_humidity_1000hPa:p            0\n",
       "sfc_pressure:hPa                       0\n",
       "snow_density:kgm3                 107593\n",
       "snow_depth:cm                          0\n",
       "snow_drift:idx                         0\n",
       "snow_melt_10min:mm                     0\n",
       "snow_water:kgm2                        0\n",
       "sun_azimuth:d                          0\n",
       "sun_elevation:d                        0\n",
       "super_cooled_liquid_water:kgm2         0\n",
       "t_1000hPa:K                            0\n",
       "total_cloud_cover:p                    0\n",
       "visibility:m                           0\n",
       "wind_speed_10m:ms                      0\n",
       "wind_speed_u_10m:ms                    0\n",
       "wind_speed_v_10m:ms                    0\n",
       "wind_speed_w_1000hPa:ms                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_observed_c.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\gunna\\Documents\\Maskinlæring\\Prosjekt\\power-predictor\\missing_data.ipynb Cell 10\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/gunna/Documents/Maskinl%C3%A6ring/Prosjekt/power-predictor/missing_data.ipynb#X12sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m df\u001b[39m.\u001b[39misna()\u001b[39m.\u001b[39msum()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.isna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_a: time              0\n",
      "pv_measurement    0\n",
      "dtype: int64\n",
      "train_b: time              0\n",
      "pv_measurement    4\n",
      "dtype: int64\n",
      "train_c: time                 0\n",
      "pv_measurement    6060\n",
      "dtype: int64\n",
      "X_train_estimated_a: date_calc                             0\n",
      "date_forecast                         0\n",
      "absolute_humidity_2m:gm3              0\n",
      "air_density_2m:kgm3                   0\n",
      "ceiling_height_agl:m               3919\n",
      "clear_sky_energy_1h:J                 0\n",
      "clear_sky_rad:W                       0\n",
      "cloud_base_agl:m                   2094\n",
      "dew_or_rime:idx                       0\n",
      "dew_point_2m:K                        0\n",
      "diffuse_rad:W                         0\n",
      "diffuse_rad_1h:J                      0\n",
      "direct_rad:W                          0\n",
      "direct_rad_1h:J                       0\n",
      "effective_cloud_cover:p               0\n",
      "elevation:m                           0\n",
      "fresh_snow_12h:cm                     0\n",
      "fresh_snow_1h:cm                      0\n",
      "fresh_snow_24h:cm                     0\n",
      "fresh_snow_3h:cm                      0\n",
      "fresh_snow_6h:cm                      0\n",
      "is_day:idx                            0\n",
      "is_in_shadow:idx                      0\n",
      "msl_pressure:hPa                      0\n",
      "precip_5min:mm                        0\n",
      "precip_type_5min:idx                  0\n",
      "pressure_100m:hPa                     0\n",
      "pressure_50m:hPa                      0\n",
      "prob_rime:p                           0\n",
      "rain_water:kgm2                       0\n",
      "relative_humidity_1000hPa:p           0\n",
      "sfc_pressure:hPa                      0\n",
      "snow_density:kgm3                 15769\n",
      "snow_depth:cm                         0\n",
      "snow_drift:idx                        0\n",
      "snow_melt_10min:mm                    0\n",
      "snow_water:kgm2                       0\n",
      "sun_azimuth:d                         0\n",
      "sun_elevation:d                       0\n",
      "super_cooled_liquid_water:kgm2        0\n",
      "t_1000hPa:K                           0\n",
      "total_cloud_cover:p                   0\n",
      "visibility:m                          0\n",
      "wind_speed_10m:ms                     0\n",
      "wind_speed_u_10m:ms                   0\n",
      "wind_speed_v_10m:ms                   0\n",
      "wind_speed_w_1000hPa:ms               0\n",
      "dtype: int64\n",
      "X_train_estimated_b: date_calc                             0\n",
      "date_forecast                         0\n",
      "absolute_humidity_2m:gm3              0\n",
      "air_density_2m:kgm3                   0\n",
      "ceiling_height_agl:m               3689\n",
      "clear_sky_energy_1h:J                 0\n",
      "clear_sky_rad:W                       0\n",
      "cloud_base_agl:m                   1963\n",
      "dew_or_rime:idx                       0\n",
      "dew_point_2m:K                        0\n",
      "diffuse_rad:W                         0\n",
      "diffuse_rad_1h:J                      0\n",
      "direct_rad:W                          0\n",
      "direct_rad_1h:J                       0\n",
      "effective_cloud_cover:p               0\n",
      "elevation:m                           0\n",
      "fresh_snow_12h:cm                     0\n",
      "fresh_snow_1h:cm                      0\n",
      "fresh_snow_24h:cm                     0\n",
      "fresh_snow_3h:cm                      0\n",
      "fresh_snow_6h:cm                      0\n",
      "is_day:idx                            0\n",
      "is_in_shadow:idx                      0\n",
      "msl_pressure:hPa                      0\n",
      "precip_5min:mm                        0\n",
      "precip_type_5min:idx                  0\n",
      "pressure_100m:hPa                     0\n",
      "pressure_50m:hPa                      0\n",
      "prob_rime:p                           0\n",
      "rain_water:kgm2                       0\n",
      "relative_humidity_1000hPa:p           0\n",
      "sfc_pressure:hPa                      0\n",
      "snow_density:kgm3                 15713\n",
      "snow_depth:cm                         0\n",
      "snow_drift:idx                        0\n",
      "snow_melt_10min:mm                    0\n",
      "snow_water:kgm2                       0\n",
      "sun_azimuth:d                         0\n",
      "sun_elevation:d                       0\n",
      "super_cooled_liquid_water:kgm2        0\n",
      "t_1000hPa:K                           0\n",
      "total_cloud_cover:p                   0\n",
      "visibility:m                          0\n",
      "wind_speed_10m:ms                     0\n",
      "wind_speed_u_10m:ms                   0\n",
      "wind_speed_v_10m:ms                   0\n",
      "wind_speed_w_1000hPa:ms               0\n",
      "dtype: int64\n",
      "X_train_estimated_c: date_calc                             0\n",
      "date_forecast                         0\n",
      "absolute_humidity_2m:gm3              0\n",
      "air_density_2m:kgm3                   0\n",
      "ceiling_height_agl:m               4596\n",
      "clear_sky_energy_1h:J                 0\n",
      "clear_sky_rad:W                       0\n",
      "cloud_base_agl:m                   2804\n",
      "dew_or_rime:idx                       0\n",
      "dew_point_2m:K                        0\n",
      "diffuse_rad:W                         0\n",
      "diffuse_rad_1h:J                      0\n",
      "direct_rad:W                          0\n",
      "direct_rad_1h:J                       0\n",
      "effective_cloud_cover:p               0\n",
      "elevation:m                           0\n",
      "fresh_snow_12h:cm                     0\n",
      "fresh_snow_1h:cm                      0\n",
      "fresh_snow_24h:cm                     0\n",
      "fresh_snow_3h:cm                      0\n",
      "fresh_snow_6h:cm                      0\n",
      "is_day:idx                            0\n",
      "is_in_shadow:idx                      0\n",
      "msl_pressure:hPa                      0\n",
      "precip_5min:mm                        0\n",
      "precip_type_5min:idx                  0\n",
      "pressure_100m:hPa                     0\n",
      "pressure_50m:hPa                      0\n",
      "prob_rime:p                           0\n",
      "rain_water:kgm2                       0\n",
      "relative_humidity_1000hPa:p           0\n",
      "sfc_pressure:hPa                      0\n",
      "snow_density:kgm3                 13898\n",
      "snow_depth:cm                         0\n",
      "snow_drift:idx                        0\n",
      "snow_melt_10min:mm                    0\n",
      "snow_water:kgm2                       0\n",
      "sun_azimuth:d                         0\n",
      "sun_elevation:d                       0\n",
      "super_cooled_liquid_water:kgm2        0\n",
      "t_1000hPa:K                           0\n",
      "total_cloud_cover:p                   0\n",
      "visibility:m                          0\n",
      "wind_speed_10m:ms                     0\n",
      "wind_speed_u_10m:ms                   0\n",
      "wind_speed_v_10m:ms                   0\n",
      "wind_speed_w_1000hPa:ms               0\n",
      "dtype: int64\n",
      "X_train_observed_a: date_forecast                          0\n",
      "absolute_humidity_2m:gm3               0\n",
      "air_density_2m:kgm3                    0\n",
      "ceiling_height_agl:m               22247\n",
      "clear_sky_energy_1h:J                  0\n",
      "clear_sky_rad:W                        0\n",
      "cloud_base_agl:m                    8066\n",
      "dew_or_rime:idx                        0\n",
      "dew_point_2m:K                         0\n",
      "diffuse_rad:W                          0\n",
      "diffuse_rad_1h:J                       0\n",
      "direct_rad:W                           0\n",
      "direct_rad_1h:J                        0\n",
      "effective_cloud_cover:p                0\n",
      "elevation:m                            0\n",
      "fresh_snow_12h:cm                      0\n",
      "fresh_snow_1h:cm                       0\n",
      "fresh_snow_24h:cm                      0\n",
      "fresh_snow_3h:cm                       0\n",
      "fresh_snow_6h:cm                       0\n",
      "is_day:idx                             0\n",
      "is_in_shadow:idx                       0\n",
      "msl_pressure:hPa                       0\n",
      "precip_5min:mm                         0\n",
      "precip_type_5min:idx                   0\n",
      "pressure_100m:hPa                      0\n",
      "pressure_50m:hPa                       0\n",
      "prob_rime:p                            0\n",
      "rain_water:kgm2                        0\n",
      "relative_humidity_1000hPa:p            0\n",
      "sfc_pressure:hPa                       0\n",
      "snow_density:kgm3                 115945\n",
      "snow_depth:cm                          0\n",
      "snow_drift:idx                         0\n",
      "snow_melt_10min:mm                     0\n",
      "snow_water:kgm2                        0\n",
      "sun_azimuth:d                          0\n",
      "sun_elevation:d                        0\n",
      "super_cooled_liquid_water:kgm2         0\n",
      "t_1000hPa:K                            0\n",
      "total_cloud_cover:p                    0\n",
      "visibility:m                           0\n",
      "wind_speed_10m:ms                      0\n",
      "wind_speed_u_10m:ms                    0\n",
      "wind_speed_v_10m:ms                    0\n",
      "wind_speed_w_1000hPa:ms                0\n",
      "dtype: int64\n",
      "X_train_observed_b: date_forecast                          0\n",
      "absolute_humidity_2m:gm3               0\n",
      "air_density_2m:kgm3                    0\n",
      "ceiling_height_agl:m               18772\n",
      "clear_sky_energy_1h:J                  0\n",
      "clear_sky_rad:W                        0\n",
      "cloud_base_agl:m                    7473\n",
      "dew_or_rime:idx                        0\n",
      "dew_point_2m:K                         0\n",
      "diffuse_rad:W                          0\n",
      "diffuse_rad_1h:J                       0\n",
      "direct_rad:W                           0\n",
      "direct_rad_1h:J                        0\n",
      "effective_cloud_cover:p                0\n",
      "elevation:m                            0\n",
      "fresh_snow_12h:cm                      0\n",
      "fresh_snow_1h:cm                       0\n",
      "fresh_snow_24h:cm                      0\n",
      "fresh_snow_3h:cm                       0\n",
      "fresh_snow_6h:cm                       0\n",
      "is_day:idx                             0\n",
      "is_in_shadow:idx                       0\n",
      "msl_pressure:hPa                       0\n",
      "precip_5min:mm                         0\n",
      "precip_type_5min:idx                   0\n",
      "pressure_100m:hPa                      0\n",
      "pressure_50m:hPa                       0\n",
      "prob_rime:p                            0\n",
      "rain_water:kgm2                        0\n",
      "relative_humidity_1000hPa:p            0\n",
      "sfc_pressure:hPa                       0\n",
      "snow_density:kgm3                 111201\n",
      "snow_depth:cm                          0\n",
      "snow_drift:idx                         0\n",
      "snow_melt_10min:mm                     0\n",
      "snow_water:kgm2                        0\n",
      "sun_azimuth:d                          0\n",
      "sun_elevation:d                        0\n",
      "super_cooled_liquid_water:kgm2         0\n",
      "t_1000hPa:K                            0\n",
      "total_cloud_cover:p                    0\n",
      "visibility:m                           0\n",
      "wind_speed_10m:ms                      0\n",
      "wind_speed_u_10m:ms                    0\n",
      "wind_speed_v_10m:ms                    0\n",
      "wind_speed_w_1000hPa:ms                0\n",
      "dtype: int64\n",
      "X_train_observed_c: date_forecast                          0\n",
      "absolute_humidity_2m:gm3               0\n",
      "air_density_2m:kgm3                    0\n",
      "ceiling_height_agl:m               19923\n",
      "clear_sky_energy_1h:J                  0\n",
      "clear_sky_rad:W                        0\n",
      "cloud_base_agl:m                    8512\n",
      "dew_or_rime:idx                        0\n",
      "dew_point_2m:K                         0\n",
      "diffuse_rad:W                          0\n",
      "diffuse_rad_1h:J                       0\n",
      "direct_rad:W                           0\n",
      "direct_rad_1h:J                        0\n",
      "effective_cloud_cover:p                0\n",
      "elevation:m                            0\n",
      "fresh_snow_12h:cm                      0\n",
      "fresh_snow_1h:cm                       0\n",
      "fresh_snow_24h:cm                      0\n",
      "fresh_snow_3h:cm                       0\n",
      "fresh_snow_6h:cm                       0\n",
      "is_day:idx                             0\n",
      "is_in_shadow:idx                       0\n",
      "msl_pressure:hPa                       0\n",
      "precip_5min:mm                         0\n",
      "precip_type_5min:idx                   0\n",
      "pressure_100m:hPa                      0\n",
      "pressure_50m:hPa                       0\n",
      "prob_rime:p                            0\n",
      "rain_water:kgm2                        0\n",
      "relative_humidity_1000hPa:p            0\n",
      "sfc_pressure:hPa                       0\n",
      "snow_density:kgm3                 107593\n",
      "snow_depth:cm                          0\n",
      "snow_drift:idx                         0\n",
      "snow_melt_10min:mm                     0\n",
      "snow_water:kgm2                        0\n",
      "sun_azimuth:d                          0\n",
      "sun_elevation:d                        0\n",
      "super_cooled_liquid_water:kgm2         0\n",
      "t_1000hPa:K                            0\n",
      "total_cloud_cover:p                    0\n",
      "visibility:m                           0\n",
      "wind_speed_10m:ms                      0\n",
      "wind_speed_u_10m:ms                    0\n",
      "wind_speed_v_10m:ms                    0\n",
      "wind_speed_w_1000hPa:ms                0\n",
      "dtype: int64\n",
      "X_test_estimated_a: date_calc                            0\n",
      "date_forecast                        0\n",
      "absolute_humidity_2m:gm3             0\n",
      "air_density_2m:kgm3                  0\n",
      "ceiling_height_agl:m               793\n",
      "clear_sky_energy_1h:J                0\n",
      "clear_sky_rad:W                      0\n",
      "cloud_base_agl:m                   298\n",
      "dew_or_rime:idx                      0\n",
      "dew_point_2m:K                       0\n",
      "diffuse_rad:W                        0\n",
      "diffuse_rad_1h:J                     0\n",
      "direct_rad:W                         0\n",
      "direct_rad_1h:J                      0\n",
      "effective_cloud_cover:p              0\n",
      "elevation:m                          0\n",
      "fresh_snow_12h:cm                    0\n",
      "fresh_snow_1h:cm                     0\n",
      "fresh_snow_24h:cm                    0\n",
      "fresh_snow_3h:cm                     0\n",
      "fresh_snow_6h:cm                     0\n",
      "is_day:idx                           0\n",
      "is_in_shadow:idx                     0\n",
      "msl_pressure:hPa                     0\n",
      "precip_5min:mm                       0\n",
      "precip_type_5min:idx                 0\n",
      "pressure_100m:hPa                    0\n",
      "pressure_50m:hPa                     0\n",
      "prob_rime:p                          0\n",
      "rain_water:kgm2                      0\n",
      "relative_humidity_1000hPa:p          0\n",
      "sfc_pressure:hPa                     0\n",
      "snow_density:kgm3                 2880\n",
      "snow_depth:cm                        0\n",
      "snow_drift:idx                       0\n",
      "snow_melt_10min:mm                   0\n",
      "snow_water:kgm2                      0\n",
      "sun_azimuth:d                        0\n",
      "sun_elevation:d                      0\n",
      "super_cooled_liquid_water:kgm2       0\n",
      "t_1000hPa:K                          0\n",
      "total_cloud_cover:p                  0\n",
      "visibility:m                         0\n",
      "wind_speed_10m:ms                    0\n",
      "wind_speed_u_10m:ms                  0\n",
      "wind_speed_v_10m:ms                  0\n",
      "wind_speed_w_1000hPa:ms              0\n",
      "dtype: int64\n",
      "X_test_estimated_b: date_calc                            0\n",
      "date_forecast                        0\n",
      "absolute_humidity_2m:gm3             0\n",
      "air_density_2m:kgm3                  0\n",
      "ceiling_height_agl:m               755\n",
      "clear_sky_energy_1h:J                0\n",
      "clear_sky_rad:W                      0\n",
      "cloud_base_agl:m                   277\n",
      "dew_or_rime:idx                      0\n",
      "dew_point_2m:K                       0\n",
      "diffuse_rad:W                        0\n",
      "diffuse_rad_1h:J                     0\n",
      "direct_rad:W                         0\n",
      "direct_rad_1h:J                      0\n",
      "effective_cloud_cover:p              0\n",
      "elevation:m                          0\n",
      "fresh_snow_12h:cm                    0\n",
      "fresh_snow_1h:cm                     0\n",
      "fresh_snow_24h:cm                    0\n",
      "fresh_snow_3h:cm                     0\n",
      "fresh_snow_6h:cm                     0\n",
      "is_day:idx                           0\n",
      "is_in_shadow:idx                     0\n",
      "msl_pressure:hPa                     0\n",
      "precip_5min:mm                       0\n",
      "precip_type_5min:idx                 0\n",
      "pressure_100m:hPa                    0\n",
      "pressure_50m:hPa                     0\n",
      "prob_rime:p                          0\n",
      "rain_water:kgm2                      0\n",
      "relative_humidity_1000hPa:p          0\n",
      "sfc_pressure:hPa                     0\n",
      "snow_density:kgm3                 2880\n",
      "snow_depth:cm                        0\n",
      "snow_drift:idx                       0\n",
      "snow_melt_10min:mm                   0\n",
      "snow_water:kgm2                      0\n",
      "sun_azimuth:d                        0\n",
      "sun_elevation:d                      0\n",
      "super_cooled_liquid_water:kgm2       0\n",
      "t_1000hPa:K                          0\n",
      "total_cloud_cover:p                  0\n",
      "visibility:m                         0\n",
      "wind_speed_10m:ms                    0\n",
      "wind_speed_u_10m:ms                  0\n",
      "wind_speed_v_10m:ms                  0\n",
      "wind_speed_w_1000hPa:ms              0\n",
      "dtype: int64\n",
      "X_test_estimated_c: date_calc                            0\n",
      "date_forecast                        0\n",
      "absolute_humidity_2m:gm3             0\n",
      "air_density_2m:kgm3                  0\n",
      "ceiling_height_agl:m               849\n",
      "clear_sky_energy_1h:J                0\n",
      "clear_sky_rad:W                      0\n",
      "cloud_base_agl:m                   375\n",
      "dew_or_rime:idx                      0\n",
      "dew_point_2m:K                       0\n",
      "diffuse_rad:W                        0\n",
      "diffuse_rad_1h:J                     0\n",
      "direct_rad:W                         0\n",
      "direct_rad_1h:J                      0\n",
      "effective_cloud_cover:p              0\n",
      "elevation:m                          0\n",
      "fresh_snow_12h:cm                    0\n",
      "fresh_snow_1h:cm                     0\n",
      "fresh_snow_24h:cm                    0\n",
      "fresh_snow_3h:cm                     0\n",
      "fresh_snow_6h:cm                     0\n",
      "is_day:idx                           0\n",
      "is_in_shadow:idx                     0\n",
      "msl_pressure:hPa                     0\n",
      "precip_5min:mm                       0\n",
      "precip_type_5min:idx                 0\n",
      "pressure_100m:hPa                    0\n",
      "pressure_50m:hPa                     0\n",
      "prob_rime:p                          0\n",
      "rain_water:kgm2                      0\n",
      "relative_humidity_1000hPa:p          0\n",
      "sfc_pressure:hPa                     0\n",
      "snow_density:kgm3                 2880\n",
      "snow_depth:cm                        0\n",
      "snow_drift:idx                       0\n",
      "snow_melt_10min:mm                   0\n",
      "snow_water:kgm2                      0\n",
      "sun_azimuth:d                        0\n",
      "sun_elevation:d                      0\n",
      "super_cooled_liquid_water:kgm2       0\n",
      "t_1000hPa:K                          0\n",
      "total_cloud_cover:p                  0\n",
      "visibility:m                         0\n",
      "wind_speed_10m:ms                    0\n",
      "wind_speed_u_10m:ms                  0\n",
      "wind_speed_v_10m:ms                  0\n",
      "wind_speed_w_1000hPa:ms              0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(f\"train_a: {train_a.isna().sum()}\")\n",
    "print(f\"train_b: {train_b.isna().sum()}\")\n",
    "print(f\"train_c: {train_c.isna().sum()}\")\n",
    "print(f\"X_train_estimated_a: {X_train_estimated_a.isna().sum()}\")\n",
    "print(f\"X_train_estimated_b: {X_train_estimated_b.isna().sum()}\")\n",
    "print(f\"X_train_estimated_c: {X_train_estimated_c.isna().sum()}\")\n",
    "print(f\"X_train_observed_a: {X_train_observed_a.isna().sum()}\")\n",
    "print(f\"X_train_observed_b: {X_train_observed_b.isna().sum()}\")\n",
    "print(f\"X_train_observed_c: {X_train_observed_c.isna().sum()}\")\n",
    "print(f\"X_test_estimated_a: {X_test_estimated_a.isna().sum()}\")\n",
    "print(f\"X_test_estimated_b: {X_test_estimated_b.isna().sum()}\")\n",
    "print(f\"X_test_estimated_c: {X_test_estimated_c.isna().sum()}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}