lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (2.13.0)\n",
      "Requirement already satisfied: tensorflow-intel==2.13.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow) (2.13.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.1.21 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (23.5.26)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (0.4.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (0.2.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (3.9.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (16.0.6)\n",
      "Requirement already satisfied: numpy<=1.24.3,>=1.22 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (1.23.5)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (4.24.3)\n",
      "Requirement already satisfied: setuptools in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (60.2.0)\n",
      "Requirement already satisfied: six>=1.12.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions<4.6.0,>=3.6.6 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (4.5.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (1.15.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (1.50.0)\n",
      "Requirement already satisfied: tensorboard<2.14,>=2.13 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (2.13.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.14,>=2.13.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (2.13.0)\n",
      "Requirement already satisfied: keras<2.14,>=2.13.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (2.13.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorflow-intel==2.13.0->tensorflow) (0.31.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from astunparse>=1.6.0->tensorflow-intel==2.13.0->tensorflow) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (2.23.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (2.28.2)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (4.9)\n",
      "Requirement already satisfied: urllib3<2.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (1.26.16)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from markdown>=2.6.8->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (6.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (2.1.3)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (3.17.0)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow-intel==2.13.0->tensorflow) (3.2.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\bruker\\onedrive\\ntnu semester 05\\tdt4173 maskinlæring\\ml_power_predictor\\venv\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from src.data.data_fetcher import get_raw_data\n",
    "from src.features.preprocess_data import fetch_preprocessed_uniform_data, get_preprocessed_test_data\n",
    "\n",
    "\n",
    "%pip install tensorflow\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_a, train_b, train_c, X_train_estimated_a, X_train_estimated_b, X_train_estimated_c, X_train_observed_a, X_train_observed_b, X_train_observed_c, X_test_estimated_a, X_test_estimated_b, X_test_estimated_c = get_raw_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Bruker\\OneDrive\\NTNU semester 05\\TDT4173 Maskinlæring\\ml_power_predictor\\src\\features\\feature_engineering.py:357: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  estimated_resampled = estimated.resample(\"1H\").mean()\n",
      "c:\\Users\\Bruker\\OneDrive\\NTNU semester 05\\TDT4173 Maskinlæring\\ml_power_predictor\\src\\features\\feature_engineering.py:357: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  estimated_resampled = estimated.resample(\"1H\").mean()\n",
      "c:\\Users\\Bruker\\OneDrive\\NTNU semester 05\\TDT4173 Maskinlæring\\ml_power_predictor\\src\\features\\feature_engineering.py:357: FutureWarning: The default value of numeric_only in DataFrameGroupBy.mean is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  estimated_resampled = estimated.resample(\"1H\").mean()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    time  pv_measurement       date_forecast  \\\n",
      "4355 2019-12-01 09:00:00            4.40 2019-12-01 09:00:00   \n",
      "4356 2019-12-01 10:00:00           66.00 2019-12-01 10:00:00   \n",
      "4357 2019-12-01 11:00:00           41.80 2019-12-01 11:00:00   \n",
      "4358 2019-12-01 12:00:00            4.62 2019-12-01 12:00:00   \n",
      "4359 2019-12-01 13:00:00            0.00 2019-12-01 13:00:00   \n",
      "\n",
      "      absolute_humidity_2m:gm3  air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
      "4355                     4.800              1.27200           27331.500000   \n",
      "4356                     4.825              1.27300           98679.046875   \n",
      "4357                     4.800              1.27350          142199.703125   \n",
      "4358                     4.750              1.27525          125991.281250   \n",
      "4359                     4.650              1.27700           61200.398438   \n",
      "\n",
      "      clear_sky_rad:W  cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  ...  \\\n",
      "4355        17.549999        456.250000              0.0      273.150024  ...   \n",
      "4356        35.825001        581.299988              0.0      273.174988  ...   \n",
      "4357        40.174999        704.500000              0.0      273.049988  ...   \n",
      "4358        27.475000        922.450012              0.0      272.950012  ...   \n",
      "4359         6.425000       1301.649902              0.0      272.600006  ...   \n",
      "\n",
      "      wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  location_a  location_b  \\\n",
      "4355               -1.950                      0.0           1           0   \n",
      "4356               -2.050                      0.0           1           0   \n",
      "4357               -2.075                      0.0           1           0   \n",
      "4358               -1.450                      0.0           1           0   \n",
      "4359               -0.700                      0.0           1           0   \n",
      "\n",
      "      location_c  month  season  year  day_of_year  day_segment  \n",
      "4355           0     12       1  2019          335           36  \n",
      "4356           0     12       1  2019          335           40  \n",
      "4357           0     12       1  2019          335           44  \n",
      "4358           0     12       1  2019          335           48  \n",
      "4359           0     12       1  2019          335           52  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "                     time  pv_measurement       date_forecast  \\\n",
      "30800 2022-12-15 03:00:00             0.0 2022-12-15 03:00:00   \n",
      "30801 2022-12-15 04:00:00             0.0 2022-12-15 04:00:00   \n",
      "30802 2022-12-15 05:00:00             0.0 2022-12-15 05:00:00   \n",
      "30803 2022-12-15 06:00:00             0.0 2022-12-15 06:00:00   \n",
      "30804 2022-12-15 07:00:00             0.0 2022-12-15 07:00:00   \n",
      "\n",
      "       absolute_humidity_2m:gm3  air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
      "30800                     2.700              1.30475                    0.0   \n",
      "30801                     2.850              1.30150                    0.0   \n",
      "30802                     3.050              1.29825                    0.0   \n",
      "30803                     3.200              1.29575                    0.0   \n",
      "30804                     3.325              1.29400                    0.0   \n",
      "\n",
      "       clear_sky_rad:W  cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  \\\n",
      "30800              0.0        360.349976              0.0      265.875000   \n",
      "30801              0.0        704.750000              0.0      266.524994   \n",
      "30802              0.0        839.400024              0.0      267.399994   \n",
      "30803              0.0        334.799988              0.0      267.950012   \n",
      "30804              0.0        478.525024              0.0      268.399994   \n",
      "\n",
      "       ...  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  location_a  \\\n",
      "30800  ...                1.875                      0.0           1   \n",
      "30801  ...                1.750                      0.0           1   \n",
      "30802  ...                2.025                      0.0           1   \n",
      "30803  ...                1.900                      0.0           1   \n",
      "30804  ...                1.875                      0.0           1   \n",
      "\n",
      "       location_b  location_c  month  season  year  day_of_year  day_segment  \n",
      "30800           0           0     12       1  2022          349           12  \n",
      "30801           0           0     12       1  2022          349           16  \n",
      "30802           0           0     12       1  2022          349           20  \n",
      "30803           0           0     12       1  2022          349           24  \n",
      "30804           0           0     12       1  2022          349           28  \n",
      "\n",
      "[5 rows x 54 columns]\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.4   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.3   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.3   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.3   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.3   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.286                    0.0              0.0   \n",
      "1                1.287                    0.0              0.0   \n",
      "2                1.287                    0.0              0.0   \n",
      "3                1.287                    0.0              0.0   \n",
      "4                1.287                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1041.199951              0.0      271.700012            0.0  ...   \n",
      "1       1054.800049              0.0      271.700012            0.0  ...   \n",
      "2       1068.300049              0.0      271.600006            0.0  ...   \n",
      "3       1081.900024              0.0      271.600006            0.0  ...   \n",
      "4       1095.400024              0.0      271.600006            0.0  ...   \n",
      "\n",
      "   wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  location_a  location_b  \\\n",
      "0                  3.4                     -0.0           1           0   \n",
      "1                  3.4                     -0.0           1           0   \n",
      "2                  3.3                     -0.0           1           0   \n",
      "3                  3.3                     -0.0           1           0   \n",
      "4                  3.3                     -0.0           1           0   \n",
      "\n",
      "   location_c  month  season  year  day_of_year  day_segment  \n",
      "0           0      5       2  2023          121            0  \n",
      "1           0      5       2  2023          121            1  \n",
      "2           0      5       2  2023          121            2  \n",
      "3           0      5       2  2023          121            3  \n",
      "4           0      5       2  2023          121            4  \n",
      "\n",
      "[5 rows x 53 columns]\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.3   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.3   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.3   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.3   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.3   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.283                    0.0              0.0   \n",
      "1                1.283                    0.0              0.0   \n",
      "2                1.283                    0.0              0.0   \n",
      "3                1.283                    0.0              0.0   \n",
      "4                1.283                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1039.900024              0.0      271.700012            0.0  ...   \n",
      "1       1053.099976              0.0      271.700012            0.0  ...   \n",
      "2       1066.400024              0.0      271.600006            0.0  ...   \n",
      "3       1079.599976              0.0      271.600006            0.0  ...   \n",
      "4       1092.900024              0.0      271.600006            0.0  ...   \n",
      "\n",
      "   wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  location_a  location_b  \\\n",
      "0                  3.4                     -0.0           0           1   \n",
      "1                  3.4                     -0.0           0           1   \n",
      "2                  3.4                     -0.0           0           1   \n",
      "3                  3.3                     -0.0           0           1   \n",
      "4                  3.3                     -0.0           0           1   \n",
      "\n",
      "   location_c  month  season  year  day_of_year  day_segment  \n",
      "0           0      5       2  2023          121            0  \n",
      "1           0      5       2  2023          121            1  \n",
      "2           0      5       2  2023          121            2  \n",
      "3           0      5       2  2023          121            3  \n",
      "4           0      5       2  2023          121            4  \n",
      "\n",
      "[5 rows x 53 columns]\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.2   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.2   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.1   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.1   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.1   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.286                    0.0              0.0   \n",
      "1                1.286                    0.0              0.0   \n",
      "2                1.286                    0.0              0.0   \n",
      "3                1.286                    0.0              0.0   \n",
      "4                1.286                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1178.000000              0.0      271.100006            0.0  ...   \n",
      "1       1217.000000              0.0      271.100006            0.0  ...   \n",
      "2       1256.000000              0.0      271.000000            0.0  ...   \n",
      "3       1295.000000              0.0      270.899994            0.0  ...   \n",
      "4       1334.099976              0.0      270.899994            0.0  ...   \n",
      "\n",
      "   wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  location_a  location_b  \\\n",
      "0                  3.1                     -0.0           0           0   \n",
      "1                  3.1                     -0.0           0           0   \n",
      "2                  3.1                     -0.0           0           0   \n",
      "3                  3.1                     -0.0           0           0   \n",
      "4                  3.1                     -0.0           0           0   \n",
      "\n",
      "   location_c  month  season  year  day_of_year  day_segment  \n",
      "0           1      5       2  2023          121            0  \n",
      "1           1      5       2  2023          121            1  \n",
      "2           1      5       2  2023          121            2  \n",
      "3           1      5       2  2023          121            3  \n",
      "4           1      5       2  2023          121            4  \n",
      "\n",
      "[5 rows x 53 columns]\n",
      "hei\n",
      "   absolute_humidity_2m:gm3  air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
      "0                       4.4                1.286                    0.0   \n",
      "1                       4.3                1.287                    0.0   \n",
      "2                       4.3                1.287                    0.0   \n",
      "3                       4.3                1.287                    0.0   \n",
      "4                       4.3                1.287                    0.0   \n",
      "\n",
      "   clear_sky_rad:W  cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  \\\n",
      "0              0.0       1041.199951              0.0      271.700012   \n",
      "1              0.0       1054.800049              0.0      271.700012   \n",
      "2              0.0       1068.300049              0.0      271.600006   \n",
      "3              0.0       1081.900024              0.0      271.600006   \n",
      "4              0.0       1095.400024              0.0      271.600006   \n",
      "\n",
      "   diffuse_rad:W  diffuse_rad_1h:J  direct_rad:W  ...  wind_speed_v_10m:ms  \\\n",
      "0            0.0               0.0           0.0  ...                  3.4   \n",
      "1            0.0               0.0           0.0  ...                  3.4   \n",
      "2            0.0               0.0           0.0  ...                  3.3   \n",
      "3            0.0               0.0           0.0  ...                  3.3   \n",
      "4            0.0               0.0           0.0  ...                  3.3   \n",
      "\n",
      "   wind_speed_w_1000hPa:ms  location_a  location_b  location_c  month  season  \\\n",
      "0                     -0.0           1           0           0      5       2   \n",
      "1                     -0.0           1           0           0      5       2   \n",
      "2                     -0.0           1           0           0      5       2   \n",
      "3                     -0.0           1           0           0      5       2   \n",
      "4                     -0.0           1           0           0      5       2   \n",
      "\n",
      "   year  day_of_year  day_segment  \n",
      "0  2023          121            0  \n",
      "1  2023          121            1  \n",
      "2  2023          121            2  \n",
      "3  2023          121            3  \n",
      "4  2023          121            4  \n",
      "\n",
      "[5 rows x 51 columns]\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train, X_val, y_val = fetch_preprocessed_uniform_data()\n",
    "X_test_estimated_a_processed, X_test_estimated_b_processed, X_test_estimated_c_processed = get_preprocessed_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2783, 51), (696, 51), (2783,), (696,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(X_train.shape, X_val.shape, \n",
    " y_train.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [[[1], [2], [3], [4], [5]]] [6]\n",
    "# [[[2], [3], [4], [5], [6]]] [7]\n",
    "# [[[3], [4], [5], [6], [7]]] [8]\n",
    "\n",
    "def df_to_X_y(df, window_size=5):\n",
    "    df_as_np = df.to_numpy()\n",
    "    X = []\n",
    "    y = []\n",
    "    for i in range(len(df_as_np) - window_size):\n",
    "        row = [r for r in df_as_np[i: i + window_size]]\n",
    "        X.append(row)\n",
    "        label = df_as_np[i+window_size]\n",
    "        y.append(label)\n",
    "    return np.array(X), np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train_obs_combined' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\Bruker\\OneDrive\\NTNU semester 05\\TDT4173 Maskinlæring\\ml_power_predictor\\lstm.ipynb Cell 5\u001b[0m line \u001b[0;36m5\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Bruker/OneDrive/NTNU%20semester%2005/TDT4173%20Maskinl%C3%A6ring/ml_power_predictor/lstm.ipynb#W4sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m scaler_X \u001b[39m=\u001b[39m MinMaxScaler(feature_range\u001b[39m=\u001b[39m(\u001b[39m0\u001b[39m, \u001b[39m1\u001b[39m))\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Bruker/OneDrive/NTNU%20semester%2005/TDT4173%20Maskinl%C3%A6ring/ml_power_predictor/lstm.ipynb#W4sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m scaler_Y \u001b[39m=\u001b[39m MinMaxScaler(feature_range\u001b[39m=\u001b[39m(\u001b[39m0\u001b[39m, \u001b[39m1\u001b[39m))\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Bruker/OneDrive/NTNU%20semester%2005/TDT4173%20Maskinl%C3%A6ring/ml_power_predictor/lstm.ipynb#W4sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m X_train_scaled \u001b[39m=\u001b[39m scaler_X\u001b[39m.\u001b[39mfit_transform(X_train_obs_combined)\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Bruker/OneDrive/NTNU%20semester%2005/TDT4173%20Maskinl%C3%A6ring/ml_power_predictor/lstm.ipynb#W4sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m y_train_scaled \u001b[39m=\u001b[39m scaler_Y\u001b[39m.\u001b[39mfit_transform(y_train_obs_combined\u001b[39m.\u001b[39mvalues\u001b[39m.\u001b[39mreshape(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39m1\u001b[39m))\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Bruker/OneDrive/NTNU%20semester%2005/TDT4173%20Maskinl%C3%A6ring/ml_power_predictor/lstm.ipynb#W4sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m X_val_scaled \u001b[39m=\u001b[39m scaler_X\u001b[39m.\u001b[39mtransform(X_train_obs_combined)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'X_train_obs_combined' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "# Scale the feature data\n",
    "scaler_X = MinMaxScaler(feature_range=(0, 1))\n",
    "scaler_Y = MinMaxScaler(feature_range=(0, 1))\n",
    "\n",
    "X_train_scaled = scaler_X.fit_transform(X_train)\n",
    "y_train_scaled = scaler_Y.fit_transform(y_train.values.reshape(-1, 1))\n",
    "X_val_scaled = scaler_X.transform(X_train)\n",
    "y_val_scaled = scaler_Y.transform(y_train.values.reshape(-1, 1))\n",
    "WINDOW_SIZE = 1\n",
    "# Reshape input to be [samples, time steps, features]\n",
    "X_train_reshaped = X_train_scaled.reshape((X_train_scaled.shape[0], WINDOW_SIZE, X_train_scaled.shape[1]))\n",
    "X_val_reshaped = X_val_scaled.reshape((X_val_scaled.shape[0], WINDOW_SIZE, X_val_scaled.shape[1]))\n",
    "\n",
    "# Construct the LSTM model\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.LSTM(50, input_shape=(X_train_reshaped.shape[1], X_train_reshaped.shape[2])),\n",
    "    tf.keras.layers.Dense(1)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam', loss='mean_absolute_error')\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(X_train_reshaped, y_train_scaled, epochs=50, batch_size=32, validation_data=(X_val_reshaped, y_val_scaled), verbose=2, shuffle=False)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74/74 [==============================] - 0s 1ms/step\n",
      "MSE on validation data: 15.05\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Make predictions on validation data\n",
    "y_val_pred_scaled = model.predict(X_val_reshaped)\n",
    "\n",
    "# Transform predictions back to original scale\n",
    "y_val_pred = scaler_Y.inverse_transform(y_val_pred_scaled).flatten()\n",
    "# Calculate Mean Absolute Error on validation data\n",
    "mae_val = mean_absolute_error(y_val_scaled, y_val_pred)\n",
    "print(f\"MSE on validation data: {mae_val:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2880, 53)\n",
      "(2343, 1, 53)\n",
      "90/90 [==============================] - 0s 1ms/step\n",
      "90/90 [==============================] - 0s 1ms/step\n",
      "90/90 [==============================] - 0s 1ms/step\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.4   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.3   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.3   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.3   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.3   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.286                    0.0              0.0   \n",
      "1                1.287                    0.0              0.0   \n",
      "2                1.287                    0.0              0.0   \n",
      "3                1.287                    0.0              0.0   \n",
      "4                1.287                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1041.199951              0.0      271.700012            0.0  ...   \n",
      "1       1054.800049              0.0      271.700012            0.0  ...   \n",
      "2       1068.300049              0.0      271.600006            0.0  ...   \n",
      "3       1081.900024              0.0      271.600006            0.0  ...   \n",
      "4       1095.400024              0.0      271.600006            0.0  ...   \n",
      "\n",
      "   wind_speed_u_10m:ms  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  \\\n",
      "0                  2.2                  3.4                     -0.0   \n",
      "1                  2.1                  3.4                     -0.0   \n",
      "2                  2.1                  3.3                     -0.0   \n",
      "3                  2.0                  3.3                     -0.0   \n",
      "4                  2.0                  3.3                     -0.0   \n",
      "\n",
      "   cloud_coverage_over_50%  no_clouds  month  season  year  day_of_year  \\\n",
      "0                        1          0      5       2  2023          121   \n",
      "1                        1          0      5       2  2023          121   \n",
      "2                        1          0      5       2  2023          121   \n",
      "3                        1          0      5       2  2023          121   \n",
      "4                        0          0      5       2  2023          121   \n",
      "\n",
      "   day_segment  \n",
      "0            0  \n",
      "1            1  \n",
      "2            2  \n",
      "3            3  \n",
      "4            4  \n",
      "\n",
      "[5 rows x 52 columns]\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.3   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.3   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.3   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.3   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.3   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.283                    0.0              0.0   \n",
      "1                1.283                    0.0              0.0   \n",
      "2                1.283                    0.0              0.0   \n",
      "3                1.283                    0.0              0.0   \n",
      "4                1.283                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1039.900024              0.0      271.700012            0.0  ...   \n",
      "1       1053.099976              0.0      271.700012            0.0  ...   \n",
      "2       1066.400024              0.0      271.600006            0.0  ...   \n",
      "3       1079.599976              0.0      271.600006            0.0  ...   \n",
      "4       1092.900024              0.0      271.600006            0.0  ...   \n",
      "\n",
      "   wind_speed_u_10m:ms  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  \\\n",
      "0                  2.2                  3.4                     -0.0   \n",
      "1                  2.1                  3.4                     -0.0   \n",
      "2                  2.1                  3.4                     -0.0   \n",
      "3                  2.0                  3.3                     -0.0   \n",
      "4                  2.0                  3.3                     -0.0   \n",
      "\n",
      "   cloud_coverage_over_50%  no_clouds  month  season  year  day_of_year  \\\n",
      "0                        1          0      5       2  2023          121   \n",
      "1                        1          0      5       2  2023          121   \n",
      "2                        1          0      5       2  2023          121   \n",
      "3                        1          0      5       2  2023          121   \n",
      "4                        0          0      5       2  2023          121   \n",
      "\n",
      "   day_segment  \n",
      "0            0  \n",
      "1            1  \n",
      "2            2  \n",
      "3            3  \n",
      "4            4  \n",
      "\n",
      "[5 rows x 52 columns]\n",
      "            date_calc       date_forecast  absolute_humidity_2m:gm3  \\\n",
      "0 2023-04-30 07:00:04 2023-05-01 00:00:00                       4.2   \n",
      "1 2023-04-30 07:00:04 2023-05-01 00:15:00                       4.2   \n",
      "2 2023-04-30 07:00:04 2023-05-01 00:30:00                       4.1   \n",
      "3 2023-04-30 07:00:04 2023-05-01 00:45:00                       4.1   \n",
      "4 2023-04-30 07:00:04 2023-05-01 01:00:00                       4.1   \n",
      "\n",
      "   air_density_2m:kgm3  clear_sky_energy_1h:J  clear_sky_rad:W  \\\n",
      "0                1.286                    0.0              0.0   \n",
      "1                1.286                    0.0              0.0   \n",
      "2                1.286                    0.0              0.0   \n",
      "3                1.286                    0.0              0.0   \n",
      "4                1.286                    0.0              0.0   \n",
      "\n",
      "   cloud_base_agl:m  dew_or_rime:idx  dew_point_2m:K  diffuse_rad:W  ...  \\\n",
      "0       1178.000000              0.0      271.100006            0.0  ...   \n",
      "1       1217.000000              0.0      271.100006            0.0  ...   \n",
      "2       1256.000000              0.0      271.000000            0.0  ...   \n",
      "3       1295.000000              0.0      270.899994            0.0  ...   \n",
      "4       1334.099976              0.0      270.899994            0.0  ...   \n",
      "\n",
      "   wind_speed_u_10m:ms  wind_speed_v_10m:ms  wind_speed_w_1000hPa:ms  \\\n",
      "0                  1.5                  3.1                     -0.0   \n",
      "1                  1.5                  3.1                     -0.0   \n",
      "2                  1.4                  3.1                     -0.0   \n",
      "3                  1.4                  3.1                     -0.0   \n",
      "4                  1.4                  3.1                     -0.0   \n",
      "\n",
      "   cloud_coverage_over_50%  no_clouds  month  season  year  day_of_year  \\\n",
      "0                        0          0      5       2  2023          121   \n",
      "1                        0          0      5       2  2023          121   \n",
      "2                        0          0      5       2  2023          121   \n",
      "3                        0          0      5       2  2023          121   \n",
      "4                        0          0      5       2  2023          121   \n",
      "\n",
      "   day_segment  \n",
      "0            0  \n",
      "1            1  \n",
      "2            2  \n",
      "3            3  \n",
      "4            4  \n",
      "\n",
      "[5 rows x 52 columns]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>time</th>\n",
       "      <th>location</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2023-05-01 00:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>31.798460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2023-05-01 01:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>33.788410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2023-05-01 02:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>30.757355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2023-05-01 03:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>33.042576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2023-05-01 04:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>43.879154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2023-05-01 05:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>74.732460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>2023-05-01 06:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>123.111153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>2023-05-01 07:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>200.208130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>2023-05-01 08:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>251.487213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>2023-05-01 09:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>247.227737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>2023-05-01 10:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>273.914185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>2023-05-01 11:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>282.054169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>2023-05-01 12:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>279.748077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>2023-05-01 13:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>253.002808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>2023-05-01 14:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>239.538315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>2023-05-01 15:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>206.678589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>2023-05-01 16:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>161.304489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>2023-05-01 17:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>135.516479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>2023-05-01 18:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>82.715096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>2023-05-01 19:00:00</td>\n",
       "      <td>A</td>\n",
       "      <td>38.315159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id                time location  prediction\n",
       "0    0 2023-05-01 00:00:00        A   31.798460\n",
       "1    1 2023-05-01 01:00:00        A   33.788410\n",
       "2    2 2023-05-01 02:00:00        A   30.757355\n",
       "3    3 2023-05-01 03:00:00        A   33.042576\n",
       "4    4 2023-05-01 04:00:00        A   43.879154\n",
       "5    5 2023-05-01 05:00:00        A   74.732460\n",
       "6    6 2023-05-01 06:00:00        A  123.111153\n",
       "7    7 2023-05-01 07:00:00        A  200.208130\n",
       "8    8 2023-05-01 08:00:00        A  251.487213\n",
       "9    9 2023-05-01 09:00:00        A  247.227737\n",
       "10  10 2023-05-01 10:00:00        A  273.914185\n",
       "11  11 2023-05-01 11:00:00        A  282.054169\n",
       "12  12 2023-05-01 12:00:00        A  279.748077\n",
       "13  13 2023-05-01 13:00:00        A  253.002808\n",
       "14  14 2023-05-01 14:00:00        A  239.538315\n",
       "15  15 2023-05-01 15:00:00        A  206.678589\n",
       "16  16 2023-05-01 16:00:00        A  161.304489\n",
       "17  17 2023-05-01 17:00:00        A  135.516479\n",
       "18  18 2023-05-01 18:00:00        A   82.715096\n",
       "19  19 2023-05-01 19:00:00        A   38.315159"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make predictions using the trained Linear Regression model on the processed test estimated data\n",
    "from src.features.preprocess_data import get_final_prediction\n",
    "\n",
    "print(X_test_estimated_a_processed.shape)\n",
    "\n",
    "print(X_val_reshaped.shape)\n",
    "scaler_X = MinMaxScaler(feature_range=(0, 1))\n",
    "\n",
    "X_test_estimated_a_processed = scaler_X.fit_transform(X_test_estimated_a_processed)\n",
    "X_test_estimated_b_processed = scaler_X.fit_transform(X_test_estimated_b_processed)\n",
    "X_test_estimated_c_processed = scaler_X.fit_transform(X_test_estimated_c_processed)\n",
    "\n",
    "X_test_estimated_a_reshaped = X_test_estimated_a_processed.reshape((X_test_estimated_a_processed.shape[0], WINDOW_SIZE, X_test_estimated_a_processed.shape[1]))\n",
    "X_test_estimated_b_reshaped = X_test_estimated_b_processed.reshape((X_test_estimated_b_processed.shape[0], WINDOW_SIZE, X_test_estimated_b_processed.shape[1]))\n",
    "X_test_estimated_c_reshaped = X_test_estimated_c_processed.reshape((X_test_estimated_c_processed.shape[0], WINDOW_SIZE, X_test_estimated_c_processed.shape[1]))\n",
    "\n",
    "y_pred_test_est_a = model.predict(X_test_estimated_a_reshaped)\n",
    "y_pred_test_est_b = model.predict(X_test_estimated_b_reshaped)\n",
    "y_pred_test_est_c = model.predict(X_test_estimated_c_reshaped)\n",
    "# Combine the predictions from the three locations into a single dataframe\n",
    "y_pred_test_est_a = scaler_Y.inverse_transform(y_pred_test_est_a).flatten()\n",
    "y_pred_test_est_b = scaler_Y.inverse_transform(y_pred_test_est_b).flatten()\n",
    "y_pred_test_est_c = scaler_Y.inverse_transform(y_pred_test_est_c).flatten()\n",
    "\n",
    "y_pred_test_est_combined = get_final_prediction(y_pred_test_est_a, y_pred_test_est_b, y_pred_test_est_c)\n",
    "y_pred_test_est_combined.head(20)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step : Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   id  prediction\n",
      "0   0   31.798460\n",
      "1   1   33.788410\n",
      "2   2   30.757355\n",
      "3   3   33.042576\n",
      "4   4   43.879154\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from src.models.saving import save_predictions\n",
    "\n",
    "\n",
    "save_predictions(y_pred_test_est_combined, 'lstm_only_obs_no_feature_engineering')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}