Data-Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b16a0b3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Preprocesses data for training.\n",
    "\n",
    "Preferrably, run using Jupyter Notebook.\n",
    "\"\"\"\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f13a530",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read raw data from the excel file.\n",
    "raw_data = pd.read_excel(\n",
    "    \"data/ouse93-96-raw-data.xlsx\",\n",
    "    index_col=0,\n",
    "    header=1,\n",
    "    usecols='A:I'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29ee7858",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data.loc[\n",
    "    [\n",
    "        '1993-02-13',\n",
    "        '1993-03-15',\n",
    "        '1993-03-16',\n",
    "        '1993-04-07',\n",
    "        '1993-04-27',\n",
    "        '1995-03-01',\n",
    "        '1996-02-09',\n",
    "        '1996-04-18'\n",
    "    ]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a27aab58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert columns to float values. Turn non-numeric values into np.nan.\n",
    "for column in raw_data.columns[1:]:\n",
    "    raw_data[column] = pd.to_numeric(raw_data[column], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a2ae057",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert negative values into np.nan.\n",
    "raw_data = raw_data.mask(raw_data < 0, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e77259f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print all rows with NAN values.\n",
    "raw_data[raw_data.isna().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcf7a024",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate mean and standard deviation for each numeric column.\n",
    "standard_deviations = raw_data.std()\n",
    "means = raw_data.mean()\n",
    "means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beecde18",
   "metadata": {},
   "outputs": [],
   "source": [
    "standard_deviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3350b568",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot river flow columns.\n",
    "raw_data[['Crakehill', 'Skip Bridge', 'Westwick', 'Skelton']].plot(xlabel=\"Date\", ylabel=\"Mean Daily Flow [Cumecs]\")\n",
    "plt.savefig('figures/river-flow.png') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "553c24ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the values that lies the furthest from the mean from Mean Daily Flow columns.\n",
    "skelton_max = raw_data['Skelton'].max()\n",
    "print(raw_data['Skelton'].sort_values(ascending=False).head())\n",
    "print((skelton_max - means['Skelton']) / standard_deviations['Skelton'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b8f948a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot river flow columns.\n",
    "raw_data[['Arkengarthdale', 'East Cowton', 'Malham Tarn', 'Snaizeholme']].plot(xlabel=\"Date\", ylabel=\"Daily Rainfall Total [mm]\")\n",
    "plt.savefig('figures/rainfall.png')\n",
    "\n",
    "print(raw_data['Arkengarthdale'].sort_values(ascending=False).head(1))\n",
    "print(raw_data['East Cowton'].sort_values(ascending=False).head(1))\n",
    "print(raw_data['Malham Tarn'].sort_values(ascending=False).head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "138d8fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interpolate outliers.\n",
    "for column in raw_data[['Arkengarthdale', 'East Cowton', 'Malham Tarn']].columns:\n",
    "    column_values = raw_data[column]\n",
    "    column_max_id = column_values.idxmax()\n",
    "    column_values.loc[column_max_id] = np.nan\n",
    "\n",
    "    \"\"\"\n",
    "    column_values = column_values.mask(\n",
    "        column_values > means[column] + 3 * standard_deviations[column],\n",
    "        np.nan\n",
    "    )\n",
    "    \n",
    "    # If this is a river flow column, check the lower bound as well.\n",
    "    if column in ['Crakehill', 'Skip Bridge', 'Westwick', 'Skelton']:\n",
    "        column_values = column_values.mask(\n",
    "            column_values < means[column] - 3 * standard_deviations[column],\n",
    "            np.nan\n",
    "        )\n",
    "    raw_data[column] = column_values.interpolate(method=\"linear\") \n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7e1ba23",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Imputate spurious data.\n",
    "raw_data = raw_data.interpolate(method=\"linear\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8aa3b8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data.loc[\n",
    "    [\n",
    "        '1993-02-13',\n",
    "        '1993-03-15',\n",
    "        '1993-03-16',\n",
    "        '1993-04-07',\n",
    "        '1993-04-27',\n",
    "        '1995-03-01',\n",
    "        '1996-02-09',\n",
    "        '1996-04-18'\n",
    "    ]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d12a1d41",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Print all rows with NAN values.\n",
    "raw_data[raw_data.isna().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0412654",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print values interpolated for outliers.\n",
    "print(raw_data.loc[\"1995-02-11\", \"Arkengarthdale\"])\n",
    "print(raw_data.loc[\"1995-02-28\", \"East Cowton\"])\n",
    "print(raw_data.loc[\"1996-01-10\", \"Malham Tarn\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fc45637",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Print all rows with NAN values.\n",
    "raw_data[raw_data.isna().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57284c4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the Skelton data\n",
    "raw_data['Skelton'].plot(kind='line', xlabel=\"Date\", ylabel=\"Mean Daily Flow [cumecs]\")\n",
    "plt.savefig('figures/skelton-data.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d0f247b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a column for Mean Daily Flow in Skelton one day before. \n",
    "raw_data.insert(0, 'Skelton T-1', raw_data['Skelton'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc8a3251",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictand column the rightmost column.\n",
    "columns = raw_data.columns\n",
    "skelton_column = raw_data.pop(\"Skelton\")\n",
    "raw_data.insert(len(columns) - 1, skelton_column.name, skelton_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09afff99",
   "metadata": {},
   "outputs": [],
   "source": [
    "correlation_data = raw_data.copy()\n",
    "correlation_data_flow = raw_data.copy()\n",
    "\n",
    "correlation_data.insert(0, \"Arkengarthdale MA 2\", correlation_data['Arkengarthdale'].rolling(2).mean())\n",
    "correlation_data.insert(0, \"East Cowton MA 2\", correlation_data['East Cowton'].rolling(2).mean())\n",
    "correlation_data.insert(0, \"Malham Tarn MA 2\", correlation_data['Malham Tarn'].rolling(2).mean())\n",
    "correlation_data.insert(0, \"Snaizeholme MA 2\", correlation_data['Snaizeholme'].rolling(2).mean())\n",
    "correlation_data.insert(0, \"Arkengarthdale MA 3\", correlation_data['Arkengarthdale'].rolling(3).mean())\n",
    "correlation_data.insert(0, \"East Cowton MA 3\", correlation_data['East Cowton'].rolling(3).mean())\n",
    "correlation_data.insert(0, \"Malham Tarn MA 3\", correlation_data['Malham Tarn'].rolling(3).mean())\n",
    "correlation_data.insert(0, \"Snaizeholme MA 3\", correlation_data['Snaizeholme'].rolling(3).mean())\n",
    "correlation_data.insert(0, \"Arkengarthdale MA 4\", correlation_data['Arkengarthdale'].rolling(4).mean())\n",
    "correlation_data.insert(0, \"East Cowton MA 4\", correlation_data['East Cowton'].rolling(4).mean())\n",
    "correlation_data.insert(0, \"Malham Tarn MA 4\", correlation_data['Malham Tarn'].rolling(4).mean())\n",
    "correlation_data.insert(0, \"Snaizeholme MA 4\", correlation_data['Snaizeholme'].rolling(4).mean())\n",
    "correlation_data['Arkengarthdale MA 2'] = correlation_data['Arkengarthdale MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['East Cowton MA 2'] = correlation_data['East Cowton MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Malham Tarn MA 2'] = correlation_data['Malham Tarn MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Snaizeholme MA 2'] = correlation_data['Snaizeholme MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Arkengarthdale MA 3'] = correlation_data['Arkengarthdale MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['East Cowton MA 3'] = correlation_data['East Cowton MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Malham Tarn MA 3'] = correlation_data['Malham Tarn MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Snaizeholme MA 3'] = correlation_data['Snaizeholme MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Arkengarthdale MA 4'] = correlation_data['Arkengarthdale MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['East Cowton MA 4'] = correlation_data['East Cowton MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Malham Tarn MA 4'] = correlation_data['Malham Tarn MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Snaizeholme MA 4'] = correlation_data['Snaizeholme MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Arkengarthdale'] = correlation_data['Arkengarthdale'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['East Cowton'] = correlation_data['East Cowton'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Malham Tarn'] = correlation_data['Malham Tarn'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Snaizeholme'] = correlation_data['Snaizeholme'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "correlation_data['Skelton T-1'] = correlation_data['Skelton T-1'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Crakehill'] = correlation_data['Crakehill'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Skip Bridge'] = correlation_data['Skip Bridge'].shift(periods=1, freq=\"D\")\n",
    "correlation_data['Westwick'] = correlation_data['Westwick'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "correlation_data_flow.insert(0, \"Skelton T-1 MA 2\", correlation_data_flow['Skelton T-1'].rolling(2).mean())\n",
    "correlation_data_flow.insert(0, \"Crakehill MA 2\", correlation_data_flow['Crakehill'].rolling(2).mean())\n",
    "correlation_data_flow.insert(0, \"Skip Bridge MA 2\", correlation_data_flow['Skip Bridge'].rolling(2).mean())\n",
    "correlation_data_flow.insert(0, \"Westwick MA 2\", correlation_data_flow['Westwick'].rolling(2).mean())\n",
    "correlation_data_flow.insert(0, \"Skelton T-1 MA 3\", correlation_data_flow['Skelton T-1'].rolling(3).mean())\n",
    "correlation_data_flow.insert(0, \"Crakehill MA 3\", correlation_data_flow['Crakehill'].rolling(3).mean())\n",
    "correlation_data_flow.insert(0, \"Skip Bridge MA 3\", correlation_data_flow['Skip Bridge'].rolling(3).mean())\n",
    "correlation_data_flow.insert(0, \"Westwick MA 3\", correlation_data_flow['Westwick'].rolling(3).mean())\n",
    "correlation_data_flow.insert(0, \"Skelton T-1 MA 4\", correlation_data_flow['Skelton T-1'].rolling(4).mean())\n",
    "correlation_data_flow.insert(0, \"Crakehill MA 4\", correlation_data_flow['Crakehill'].rolling(4).mean())\n",
    "correlation_data_flow.insert(0, \"Skip Bridge MA 4\", correlation_data_flow['Skip Bridge'].rolling(4).mean())\n",
    "correlation_data_flow.insert(0, \"Westwick MA 4\", correlation_data_flow['Westwick'].rolling(4).mean())\n",
    "\n",
    "correlation_data_flow['Skelton T-1 MA 2'] = correlation_data_flow['Skelton T-1 MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Crakehill MA 2'] = correlation_data_flow['Crakehill MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skip Bridge MA 2'] = correlation_data_flow['Skip Bridge MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Westwick MA 2'] = correlation_data_flow['Westwick MA 2'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skelton T-1 MA 3'] = correlation_data_flow['Skelton T-1 MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Crakehill MA 3'] = correlation_data_flow['Crakehill MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skip Bridge MA 3'] = correlation_data_flow['Skip Bridge MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Westwick MA 3'] = correlation_data_flow['Westwick MA 3'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skelton T-1 MA 4'] = correlation_data_flow['Skelton T-1 MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Crakehill MA 4'] = correlation_data_flow['Crakehill MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skip Bridge MA 4'] = correlation_data_flow['Skip Bridge MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Westwick MA 4'] = correlation_data_flow['Westwick MA 4'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skelton T-1'] = correlation_data_flow['Skelton T-1'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Crakehill'] = correlation_data_flow['Crakehill'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Skip Bridge'] = correlation_data_flow['Skip Bridge'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Westwick'] = correlation_data_flow['Westwick'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "correlation_data_flow['Arkengarthdale'] = correlation_data_flow['Arkengarthdale'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['East Cowton'] = correlation_data_flow['East Cowton'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Malham Tarn'] = correlation_data_flow['Malham Tarn'].shift(periods=1, freq=\"D\")\n",
    "correlation_data_flow['Snaizeholme'] = correlation_data_flow['Snaizeholme'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "\n",
    "\n",
    "# Plot correlations\n",
    "\n",
    "rainfall_correlations = correlation_data.corr()\n",
    "flow_correlations = correlation_data_flow.corr()\n",
    "\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15,6))\n",
    "\n",
    "plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
    "correlations_to_plot_flow = flow_correlations.drop(index=[\"Skelton\", \"Arkengarthdale\", \"East Cowton\", \"Malham Tarn\", \"Snaizeholme\"])[\"Skelton\"].sort_values(ascending=False).plot(xlabel=\"Potential Flow Predictors\", ylabel=\"Correlation with the predictand\", kind=\"bar\", ax=axes[0])\n",
    "correlations_to_plot = rainfall_correlations.drop(index=[\"Skelton\", \"Skelton T-1\", \"Crakehill\", \"Skip Bridge\", \"Westwick\"])[\"Skelton\"].sort_values(ascending=False).plot(xlabel=\"Potential Rainfall Predictors\", ylabel=\"Correlation with the predictand\", kind=\"bar\", ax=axes[1])\n",
    "plt.savefig('figures/flow-and-rainfall-correlations.png') \n",
    "#correlation_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "641f4533",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Explore moving average\n",
    "\n",
    "#weights = np.array([0.5, 0.25, 0.25])\n",
    "#sum_weights = np.sum(weights)\n",
    "\n",
    "raw_data.insert(0, \"Arkengarthdale MA\", raw_data['Arkengarthdale'].rolling(3).mean())\n",
    "raw_data.insert(0, \"East Cowton MA\", raw_data['East Cowton'].rolling(3).mean())\n",
    "raw_data.insert(0, \"Malham Tarn MA\", raw_data['Malham Tarn'].rolling(3).mean())\n",
    "raw_data.insert(0, \"Snaizeholme MA\", raw_data['Snaizeholme'].rolling(3).mean())\n",
    "raw_data[\"Snaizeholme MA\"][\"1993-01-01\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc023b9e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Explore the correlations between potential predictors and the predictand.\n",
    "\n",
    "print(raw_data.corr()[\"Skelton\"].sort_values(ascending=False))\n",
    "# Lag Mean Daily Flow potential predictor columns by one day.\n",
    "raw_data['Skelton T-1'] = raw_data['Skelton T-1'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Crakehill'] = raw_data['Crakehill'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Skip Bridge'] = raw_data['Skip Bridge'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Westwick'] = raw_data['Westwick'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "# Lag Rainfall columns by one day.\n",
    "raw_data['Arkengarthdale MA'] = raw_data['Arkengarthdale MA'].shift(periods=1, freq=\"D\")\n",
    "raw_data['East Cowton MA'] = raw_data['East Cowton MA'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Malham Tarn MA'] = raw_data['Malham Tarn MA'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Snaizeholme MA'] = raw_data['Snaizeholme MA'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Arkengarthdale'] = raw_data['Arkengarthdale'].shift(periods=1, freq=\"D\")\n",
    "raw_data['East Cowton'] = raw_data['East Cowton'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Malham Tarn'] = raw_data['Malham Tarn'].shift(periods=1, freq=\"D\")\n",
    "raw_data['Snaizeholme'] = raw_data['Snaizeholme'].shift(periods=1, freq=\"D\")\n",
    "\n",
    "correlations = raw_data.corr()\n",
    "correlations[\"Skelton\"].sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b94de867",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Drop rows containing missing values after lagging.\n",
    "raw_data = raw_data.dropna()\n",
    "raw_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "844fe46b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop non-moving-average rainfall columns.\n",
    "raw_data = raw_data.drop(columns=['Arkengarthdale', 'East Cowton', 'Malham Tarn', 'Snaizeholme'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9064647a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Plot correlation between lagged predictors and the predictand.\n",
    "\n",
    "fig, axes = plt.subplots(nrows=4, ncols=2, figsize=(15,15))\n",
    "\n",
    "plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
    "\n",
    "mean_daily_flow_columns = [\"Skelton T-1\", \"Crakehill\", \"Skip Bridge\", \"Westwick\"]\n",
    "daily_rainfall_total_columns = [\"Arkengarthdale MA\", \"East Cowton MA\", \"Malham Tarn MA\", \"Snaizeholme MA\"]\n",
    "\n",
    "\n",
    "for i in range(len(raw_data.loc[:, raw_data.columns != \"Skelton\"].columns)):\n",
    "    column_name = raw_data.iloc[:, i].name\n",
    "    x = raw_data.iloc[:, i].values.reshape(-1, 1)\n",
    "    y = raw_data['Skelton'].values.reshape(-1, 1)\n",
    "    linear_regressor = LinearRegression()\n",
    "    linear_regressor.fit(x, y)\n",
    "    y_pred = linear_regressor.predict(x)\n",
    "    axes[i // 2][i % 2].scatter(x, y, alpha=0.4)\n",
    "    axes[i // 2][i % 2].plot(x, y_pred, color='red')\n",
    "    # Decide on x-label.\n",
    "    if column_name in mean_daily_flow_columns:\n",
    "        axes[i // 2][ i % 2].set_xlabel(f\"Mean Daily Flow in {column_name} [cumecs]\")\n",
    "    elif column_name in daily_rainfall_total_columns:\n",
    "        axes[i // 2][ i % 2].set_xlabel(f\"Daily Rainfall Total in {column_name} [mm]\")\n",
    "    axes[i // 2][ i % 2].set_ylabel('Mean Daily Flow in Skelton [cumecs]')\n",
    "    axes[i // 2][ i % 2].set_title(f\"{column_name} to Mean Daily Flow in Skelton\")\n",
    "\n",
    "plt.savefig('figures/predictors-predictand.png')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "467712bc",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Split the data into into training-validation and test sets.\n",
    "train_and_validation, test = train_test_split(raw_data, test_size=0.2, random_state=5)\n",
    "\n",
    "# Calculate min and max values from training-validation set.\n",
    "max_values = train_and_validation.max(axis=0)\n",
    "min_values = train_and_validation.min(axis=0)\n",
    "print(max_values)\n",
    "print(min_values)\n",
    "\n",
    "# Split training-validation set into training and validation sets.\n",
    "train, validation = train_test_split(train_and_validation, test_size=0.25, random_state=6)\n",
    "validation['Skelton'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9ddd078",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def standardise(x: pd.Series, max_value: float, min_value: float):\n",
    "    \"\"\"Standardises data using minimum and maximum values.\n",
    "    \n",
    "    Args:\n",
    "    x: A pandas.Series instance.\n",
    "    max_value: A maximum value for the standardisation formula.\n",
    "    min_value: A minimum value for the standardisation formula.\n",
    "    \n",
    "    Returns:\n",
    "    pandas.Series.\n",
    "    \"\"\"\n",
    "    return 0.8 * ((x - min_value) / (max_value - min_value)) + 0.1\n",
    "    \n",
    "# Standardise the data.\n",
    "standardised_data_sets = []\n",
    "for data_set in [train, validation, test]:\n",
    "    standardised_columns = []\n",
    "    for column in data_set.columns:\n",
    "        standardised_column = data_set.loc[:, column].apply(standardise, args=(max_values[column], min_values[column]))\n",
    "        standardised_columns.append(standardised_column)\n",
    "    standardised_data_set = pd.concat(standardised_columns, axis=1)\n",
    "    standardised_data_sets.append(standardised_data_set)\n",
    "        \n",
    "train_standardised = standardised_data_sets[0]\n",
    "validation_standardised = standardised_data_sets[1]\n",
    "test_standardised = standardised_data_sets[2]\n",
    "train_standardised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cce47e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def destandardise(x: pd.Series, max_value: float, min_value: float):\n",
    "    \"\"\"Destandardises data using minimum and maximum values.\n",
    "    \n",
    "    Args:\n",
    "    x: A pandas.Series instance of standardised data.\n",
    "    max_value: A maximum value for the destandardisation formula.\n",
    "    min_value: A minimum value for the destandardisation formula.\n",
    "    \n",
    "    Returns:\n",
    "    pandas.Series.\n",
    "    \"\"\"\n",
    "    return ((x - 0.1) * (max_value - min_value)) / 0.8 + min_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c6ebc70",
   "metadata": {},
   "outputs": [],
   "source": [
    "destandardised_data_sets = []\n",
    "for data_set in standardised_data_sets:\n",
    "    destandardised_columns = []\n",
    "    for column in data_set.columns:\n",
    "        destandardised_column = data_set.loc[:, column].apply(destandardise, args=(max_values[column], min_values[column]))\n",
    "        destandardised_columns.append(destandardised_column)\n",
    "    destandardised_data_set = pd.concat(destandardised_columns, axis=1)\n",
    "    destandardised_data_sets.append(destandardised_data_set)\n",
    "#destandardised_data_sets[0]\n",
    "destandardised_data_sets[0].compare(train)\n",
    "print(train.dtypes)\n",
    "print(destandardised_data_sets[0].dtypes)\n",
    "#train.to_csv(\"hello.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cc7877f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save min and max values\n",
    "skelton_min_value = min_values['Skelton']\n",
    "skelton_max_value = max_values['Skelton']\n",
    "\n",
    "\n",
    "min_max_values = {\n",
    "    \"min\": skelton_min_value,\n",
    "    \"max\": skelton_max_value\n",
    "}\n",
    "\n",
    "with open(\"standardisation.json\", \"w\") as f:\n",
    "    json.dump(min_max_values, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93a84470",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_standardised.to_csv(\"data/training-set.csv\")\n",
    "validation_standardised.to_csv(\"data/validation-set.csv\")\n",
    "test_standardised.to_csv(\"data/test-set.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}