20220201_Application.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **Google Drive연결 및 Module 불러오기**"
      ],
      "metadata": {
        "id": "1rP4MWqKOLR1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#from google.colab import drive\n",
        "#drive.mount('/content/gdrive/')"
      ],
      "metadata": {
        "id": "C-4lBkK-NCAT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, BatchNormalization\n",
        "from keras.wrappers.scikit_learn import KerasRegressor\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import Pipeline\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "Kj6Kgf06Xtld"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QMXJIWRBUpP7",
        "outputId": "b65ff4b1-82ea-47d8-e30d-cd172c089b8e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Nov 21 15:06:31 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0    25W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **PM2.5 with Covariate 데이터 불러오기 및 전처리**"
      ],
      "metadata": {
        "id": "IvE21PO0OgGl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('PM25_with_covariate_220201_ver2.csv')\n",
        "df.head(6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "a6TxRfsEOf23",
        "outputId": "00e9d570-2db7-4c07-9ade-35496acac031"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4d150358-53fc-4cf0-a5f7-432f8650a52b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>code</th>\n",
              "      <th>x</th>\n",
              "      <th>y</th>\n",
              "      <th>t</th>\n",
              "      <th>z</th>\n",
              "      <th>tmp</th>\n",
              "      <th>uwind</th>\n",
              "      <th>vwind</th>\n",
              "      <th>wind</th>\n",
              "      <th>hum</th>\n",
              "      <th>prec</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>1</td>\n",
              "      <td>27</td>\n",
              "      <td>-0.127208</td>\n",
              "      <td>0.210263</td>\n",
              "      <td>-1.678297</td>\n",
              "      <td>0.608896</td>\n",
              "      <td>79.262265</td>\n",
              "      <td>0.042478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>2</td>\n",
              "      <td>19</td>\n",
              "      <td>-0.747990</td>\n",
              "      <td>0.257965</td>\n",
              "      <td>-1.256667</td>\n",
              "      <td>0.386535</td>\n",
              "      <td>72.087742</td>\n",
              "      <td>0.038822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>3</td>\n",
              "      <td>11</td>\n",
              "      <td>-0.793291</td>\n",
              "      <td>0.297994</td>\n",
              "      <td>-1.313214</td>\n",
              "      <td>0.493171</td>\n",
              "      <td>71.505221</td>\n",
              "      <td>0.020449</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>4</td>\n",
              "      <td>9</td>\n",
              "      <td>-1.753191</td>\n",
              "      <td>0.215887</td>\n",
              "      <td>-0.811954</td>\n",
              "      <td>0.243450</td>\n",
              "      <td>84.203073</td>\n",
              "      <td>0.051435</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>5</td>\n",
              "      <td>9</td>\n",
              "      <td>-2.032537</td>\n",
              "      <td>0.105124</td>\n",
              "      <td>-0.322796</td>\n",
              "      <td>-0.157497</td>\n",
              "      <td>87.085583</td>\n",
              "      <td>0.120312</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>126.9747</td>\n",
              "      <td>37.5643</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>-1.859001</td>\n",
              "      <td>0.054214</td>\n",
              "      <td>-0.760730</td>\n",
              "      <td>0.265878</td>\n",
              "      <td>88.018236</td>\n",
              "      <td>0.080176</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4d150358-53fc-4cf0-a5f7-432f8650a52b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4d150358-53fc-4cf0-a5f7-432f8650a52b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4d150358-53fc-4cf0-a5f7-432f8650a52b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   code         x        y  t  ...     vwind      wind        hum      prec\n",
              "0     1  126.9747  37.5643  1  ... -1.678297  0.608896  79.262265  0.042478\n",
              "1     1  126.9747  37.5643  2  ... -1.256667  0.386535  72.087742  0.038822\n",
              "2     1  126.9747  37.5643  3  ... -1.313214  0.493171  71.505221  0.020449\n",
              "3     1  126.9747  37.5643  4  ... -0.811954  0.243450  84.203073  0.051435\n",
              "4     1  126.9747  37.5643  5  ... -0.322796 -0.157497  87.085583  0.120312\n",
              "5     1  126.9747  37.5643  6  ... -0.760730  0.265878  88.018236  0.080176\n",
              "\n",
              "[6 rows x 11 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Target data processing\n",
        "df_pm25 = df['z']\n",
        "pm25 = df_pm25.values\n",
        "\n",
        "z = pm25[:,None]"
      ],
      "metadata": {
        "id": "Qk2Gn1z_NBb0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Lon and Lat data normalizing and processing\n",
        "lon = df.values[:,1]\n",
        "lat = df.values[:,2]\n",
        "\n",
        "normalized_lon = (lon-min(lon))/(max(lon)-min(lon))\n",
        "normalized_lat = (lat-min(lat))/(max(lat)-min(lat))\n",
        "\n",
        "\n",
        "# Input\n",
        "df[\"x\"] = normalized_lon\n",
        "df[\"y\"] = normalized_lat\n",
        "\n",
        "\n",
        "# Extraction\n",
        "aqs_lonlat=df.values[:,[1,2]]\n",
        "print(aqs_lonlat.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yK_NnCKtRBOa",
        "outputId": "e4d9f50e-b8b4-44ac-e884-509fac5ff0f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10704, 2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "N = lon.shape[0]\n",
        "print(N)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y2fRPCt0RBRL",
        "outputId": "1761b345-7fce-446d-ac74-0e832e6d4e20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "10704\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Make sptio-temporal basis function**"
      ],
      "metadata": {
        "id": "vUZVkb_0V0Ti"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# np.array 모두 보기\n",
        "import sys\n",
        "np.set_printoptions(threshold=sys.maxsize)"
      ],
      "metadata": {
        "id": "9K21RJp-WMsx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#########################################################\n",
        "# make sptio-temporal basis function\n",
        "num_basis = [10**2, 19**2, 37**2 ]\n",
        "\n",
        "knots_1dt = [ np.linspace( 0,25, int(i) ) for i in num_basis ] #1차원이므로 루트 없앰\n",
        "knots_1dx = [ np.linspace(0,1, int(np.sqrt(i)) ) for i in num_basis ] #error\n",
        "knots_1dy = [ np.linspace(0,1, int(np.sqrt(i)) ) for i in num_basis ] \n",
        "\n",
        "phi = np.zeros((N, sum(num_basis))) \n",
        "phi.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fCPPEZ76RBTu",
        "outputId": "e17c20e3-c463-44d7-92de-da59337c6b40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10704, 1830)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "T = np.linspace( 1, 24, 24) \n",
        "T = np.tile(T, int(N/24))\n",
        "\n",
        "#########################################################\n",
        "#################### Wendland kernel ####################\n",
        "basis_size = 0\n",
        "\n",
        "\n",
        "for res in range(len(num_basis)):\n",
        "    theta = 1 #/ np.sqrt(num_basis[res])\n",
        "    \n",
        "    knots_t = knots_1dt[res]\n",
        "    \n",
        "    knots_x, knots_y = np.meshgrid( knots_1dx[res] , knots_1dy[res] ) ###\n",
        "    knots = np.column_stack((knots_x.flatten(),knots_y.flatten()))\n",
        "        \n",
        "    if res == 0:\n",
        "        for i in range(num_basis[res]): # 100개 #가로\n",
        "            d = abs( T - np.linspace(knots_t[i], knots_t[i], N)  ) / theta # 1-Dimension #make\n",
        "        \n",
        "            for j in range(len(d)): # j = 1, \\cdots, 24 #세로 \n",
        "                if d[j] >= 0 and d[j] <= 1:\n",
        "                    phi[j,i + basis_size] = (1-d[j])**6 * (35 * d[j]**2 + 18 * d[j] + 3)/3\n",
        "                else:\n",
        "                    phi[j,i + basis_size] = 0\n",
        "                    \n",
        "        basis_size += num_basis[res]              \n",
        "            \n",
        "        \n",
        "    else: \n",
        "        \n",
        "        for i in range(num_basis[res]):\n",
        "            d = np.linalg.norm(np.vstack( (normalized_lon, normalized_lat) ).T - knots[i,:],axis=1)/theta\n",
        "            \n",
        "            for j in range(len(d)):\n",
        "                if d[j] >= 0 and d[j] <= 1:\n",
        "                    phi[j,i + basis_size] = (1-d[j])**6 * (35 * d[j]**2 + 18 * d[j] + 3)/3\n",
        "                else:\n",
        "                    phi[j,i + basis_size] = 0  \n",
        "                \n",
        "        basis_size = basis_size + num_basis[res]    \n",
        "\n",
        "\n",
        "    \n",
        "## Romove the all-zero columns\n",
        "idx_zero = np.array([], dtype=int)\n",
        "\n",
        "for i in range(phi.shape[1]):\n",
        "    if sum(phi[:,i]!=0)==0:\n",
        "        idx_zero = np.append(idx_zero,int(i))\n",
        "        \n",
        "phi_reduce = np.delete(phi,idx_zero,1)\n",
        "print(phi.shape)\n",
        "print(phi_reduce.shape) # zero reduced"
      ],
      "metadata": {
        "id": "DvEbRH77RBYY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "81688add-82d3-4fa9-f284-22e0c9ccc71a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10704, 1830)\n",
            "(10704, 1828)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Make Covariate**"
      ],
      "metadata": {
        "id": "CQ76l3kaXK65"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.iloc[[0,1],[5, 6, 7, 8, 9, 10]]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "UaF9vQtBXVZd",
        "outputId": "c4d642a8-e058-4b8d-c368-ee808306353f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-9c2bdadb-6547-4f52-8128-e388d71c4ae0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tmp</th>\n",
              "      <th>uwind</th>\n",
              "      <th>vwind</th>\n",
              "      <th>wind</th>\n",
              "      <th>hum</th>\n",
              "      <th>prec</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.127208</td>\n",
              "      <td>0.210263</td>\n",
              "      <td>-1.678297</td>\n",
              "      <td>0.608896</td>\n",
              "      <td>79.262265</td>\n",
              "      <td>0.042478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.747990</td>\n",
              "      <td>0.257965</td>\n",
              "      <td>-1.256667</td>\n",
              "      <td>0.386535</td>\n",
              "      <td>72.087742</td>\n",
              "      <td>0.038822</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9c2bdadb-6547-4f52-8128-e388d71c4ae0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9c2bdadb-6547-4f52-8128-e388d71c4ae0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9c2bdadb-6547-4f52-8128-e388d71c4ae0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "        tmp     uwind     vwind      wind        hum      prec\n",
              "0 -0.127208  0.210263 -1.678297  0.608896  79.262265  0.042478\n",
              "1 -0.747990  0.257965 -1.256667  0.386535  72.087742  0.038822"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "covariates = df.values[:,[1, 2, 5, 6, 7, 8, 9, 10]]\n",
        "covariates.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vctbxrtnRBfT",
        "outputId": "8b2bf198-4300-4a5c-e32d-18615e6f7b99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10704, 8)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "phi_obs = phi #v\n",
        " \n",
        "#~~\n",
        "\n",
        "s_obs = np.vstack((normalized_lon, normalized_lat)).T #v\n",
        "\n",
        "#~~\n",
        "\n",
        "X = covariates\n",
        "normalized_X = X #v\n",
        "\n",
        "for i in range(X.shape[1]):\n",
        "    normalized_X[:,i] = (X[:,i]-min(X[:,i]))/(max(X[:,i])-min(X[:,i]))\n",
        "\n",
        "N_obs = X.shape[0]\n",
        "\n",
        "print(N_obs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UTioHoeKXEst",
        "outputId": "5f0d4ba2-beff-4c56-ace5-c19dead39e82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "10704\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Make Ensemble DNN Model Function**"
      ],
      "metadata": {
        "id": "WDNJ5-cZXvkk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Function for Model Compile, Fit, Test, Prediction\n",
        "def deep_model(model, X_train, y_train, X_test, y_test):\n",
        "\n",
        "    model.compile(optimizer='adam', loss='mse', metrics=['mse','mae'])\n",
        "\n",
        "    model.fit(X_train, y_train, epochs = NB_START_EPOCHS, batch_size = BATCH_SIZE, verbose=0)\n",
        "    \n",
        "    results = model.evaluate(X_test, y_test, verbose=0)\n",
        "    \n",
        "    train_prediction = model.predict(X_train, verbose = 0)\n",
        "    test_prediction = model.predict(X_test, verbose = 0)\n",
        "\n",
        "    return train_prediction, test_prediction, results"
      ],
      "metadata": {
        "id": "2G-UWzgaXEvd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build Basis Version DNN Model\n",
        "def build_model1():\n",
        "  p1 = phi_reduce.shape[1]\n",
        "  p2 = s_obs.shape[1]\n",
        "  p = p1 + p2\n",
        "\n",
        "  n = p\n",
        "  m = 1\n",
        "\n",
        "\n",
        "  model = Sequential()\n",
        "  model.add(Dense(p+1+2, input_dim = p,  kernel_initializer='he_uniform', activation='relu'))\n",
        "  model.add(Dropout(rate=0.5))\n",
        "  model.add(BatchNormalization())\n",
        "\n",
        "  model.add(Dense(p+1+2, activation='relu'))\n",
        "  model.add(Dropout(rate=0.5))\n",
        "\n",
        "  model.add(Dense(p+1+2, activation='relu'))\n",
        "  model.add(BatchNormalization())\n",
        "  model.add(Dense(1, activation='linear'))\n",
        "  \n",
        "  return model\n",
        "\n",
        "# Build Covariate Version DNN Model => regression으로 대체하자\n",
        "def build_model2():\n",
        "  p1 = covariates.shape[1]\n",
        "  p2 = s_obs.shape[1]\n",
        "  p = p1 + p2\n",
        "\n",
        "  model = Sequential()\n",
        "  model.add(Dense(p+1+2, input_dim = p,  kernel_initializer='he_uniform', activation='relu'))\n",
        "  model.add(Dropout(rate=0.5))\n",
        "  model.add(BatchNormalization())\n",
        "\n",
        "  model.add(Dense(p+1+2, activation='relu'))\n",
        "  model.add(Dropout(rate=0.5))\n",
        "\n",
        "  model.add(Dense(p+1+2, activation='relu'))\n",
        "  model.add(BatchNormalization())\n",
        "  model.add(Dense(1, activation='linear'))\n",
        "\n",
        "  return model\n",
        "\n",
        "\n",
        "# Build Ensemble Version DNN Model\n",
        "def build_model3():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(10, input_dim = 2,  kernel_initializer='he_uniform', activation='relu'))\n",
        "  model.add(Dense(10, activation='relu'))\n",
        "  model.add(Dense(1, activation = 'linear'))\n",
        "  return model  "
      ],
      "metadata": {
        "id": "-UtE64upXEyc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import make_pipeline"
      ],
      "metadata": {
        "id": "WQqgq1iRiDHc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random"
      ],
      "metadata": {
        "id": "gtXkEyrJiJ24"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Make the inputs and targets\n",
        "inputs1 = np.hstack((phi_reduce , s_obs))\n",
        "inputs2 = np.hstack((s_obs, normalized_X))\n",
        "\n",
        "targets = z\n",
        "\n",
        "NB_START_EPOCHS = 200  # Number of epochs we usually start to train with\n",
        "BATCH_SIZE = 64  # Size of the batches used in the mini-batch gradient descent"
      ],
      "metadata": {
        "id": "UA4ADedwlKcX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **교차검증**"
      ],
      "metadata": {
        "id": "Pm-R609lU_U4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "date = [\"01:00\", \"02:00\", \"03:00\", \"04:00\", \"05:00\", \"06:00\", \"07:00\", \"08:00\",\n",
        "        \"09:00\", \"10:00\", \"11:00\", \"12:00\", \"13:00\", \"14:00\", \"15:00\", \"16:00\",\n",
        "        \"17:00\", \"18:00\", \"19:00\", \"20:00\", \"21:00\", \"22:00\", \"23:00\", \"00:00\"]\n",
        "\n",
        "loc1 = loc2 = loc3 = loc4 = loc5 = []\n",
        "time1 = time2 = time3 = time4 = time5 = []\n",
        "location = [ loc1, loc2, loc3, loc4, loc5 ]\n",
        "time = [time1, time2, time3, time4, time5]"
      ],
      "metadata": {
        "id": "kZKVsx8kgWo4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(5):\n",
        "  print(\"-----------------------------------------\")\n",
        "  j = i + 1\n",
        "  random.seed(j*10)\n",
        "  \n",
        "  time[i] = sorted(random.sample(range(24), 2))\n",
        "  location[i] = sorted(random.sample(range(446), 30))\n",
        "  \n",
        "  print(\"Test location\",i+1,\": \",location[i])\n",
        "  print(\"Test time\",i+1,\": \",time[i])\n",
        "  print(\"Test time\",i+1,\": \",date[ time[i][0] - 1 ], \"/\" , date[ time[i][1] - 1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OYZLkdZ7gZRR",
        "outputId": "5667573d-b6c1-4ea1-8d01-832f397a6833"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------------------------------------\n",
            "Test location 1 :  [7, 17, 22, 38, 71, 82, 105, 127, 142, 145, 167, 181, 184, 195, 215, 219, 236, 247, 250, 251, 266, 295, 308, 334, 381, 415, 416, 421, 423, 440]\n",
            "Test time 1 :  [1, 18]\n",
            "Test time 1 :  01:00 / 18:00\n",
            "-----------------------------------------\n",
            "Test location 2 :  [13, 38, 47, 51, 52, 64, 77, 86, 102, 106, 133, 162, 163, 167, 168, 171, 208, 210, 218, 230, 242, 293, 297, 320, 325, 345, 348, 392, 402, 434]\n",
            "Test time 2 :  [21, 23]\n",
            "Test time 2 :  21:00 / 23:00\n",
            "-----------------------------------------\n",
            "Test location 3 :  [3, 13, 15, 24, 34, 37, 41, 68, 81, 107, 124, 131, 178, 192, 203, 205, 236, 267, 271, 274, 306, 312, 318, 328, 334, 335, 417, 423, 425, 431]\n",
            "Test time 3 :  [9, 17]\n",
            "Test time 3 :  09:00 / 17:00\n",
            "-----------------------------------------\n",
            "Test location 4 :  [14, 16, 27, 30, 65, 66, 90, 103, 105, 125, 141, 144, 160, 178, 225, 235, 268, 270, 309, 317, 326, 328, 340, 362, 377, 380, 390, 405, 419, 435]\n",
            "Test time 4 :  [14, 18]\n",
            "Test time 4 :  14:00 / 18:00\n",
            "-----------------------------------------\n",
            "Test location 5 :  [32, 35, 43, 45, 50, 78, 96, 113, 114, 124, 162, 163, 168, 171, 177, 186, 219, 242, 275, 285, 309, 311, 325, 326, 346, 354, 392, 406, 422, 442]\n",
            "Test time 5 :  [8, 15]\n",
            "Test time 5 :  08:00 / 15:00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "i=1\n",
        "j=1\n",
        "k=1\n",
        "\n",
        "MSE_Basis = []\n",
        "MSE_Covariate = []\n",
        "MSE_Covariate_Reg = []\n",
        "MSE_DNN_DNN = []\n",
        "MSE_Reg_DNN = []\n",
        "MSE_Reg_Wgt = []\n",
        "MSE_DNN_Wgt = []\n",
        "\n",
        "for test_time in time:\n",
        "\n",
        "  for test_loc in location:\n",
        "\n",
        "      print('------------------------------------------------------------------------')\n",
        "      print('Test for time = ',i, ' location = ',j,'... (', k , '/', len(time)*len(location), ')' )\n",
        "\n",
        "      index1 = df[df['t'].isin( test_time )].index\n",
        "      index2 = df[df['code'].isin( test_loc )].index\n",
        "\n",
        "      train_idx_inv = list(set(index1) | set(index2)) \n",
        "      train_idx = list(set(df.index) - set(train_idx_inv))\n",
        "      test_idx = list(set(index1) & set(index2))\n",
        "\n",
        "\n",
        "\n",
        "      model1 = build_model1()\n",
        "      model2 = build_model2()\n",
        "      model3 = build_model3()\n",
        "\n",
        "      #------\n",
        "\n",
        "      train_pred1, test_pred1, result1 = deep_model(model1, inputs1[train_idx], targets[train_idx], inputs1[test_idx], targets[test_idx])\n",
        "\n",
        "      print(\"model1 (DNN with Basis) fit..\", end = \"\")\n",
        "      print(f'MSE = {result1[1]},,, MAE = {result1[2]}')\n",
        "      MSE_Basis.append(result1[1]) ##save\n",
        "\n",
        "      train_pred2, test_pred2, result2 = deep_model(model2, inputs2[train_idx], targets[train_idx],inputs2[test_idx], targets[test_idx])\n",
        "\n",
        "      print(\"model2 (DNN with Covariate) fit..\", end = \"\")\n",
        "      print(f'MSE = {result2[1]},,, MAE = {result2[2]}')\n",
        "      MSE_Covariate.append(result2[1])\n",
        "\n",
        "\n",
        "      mlr = LinearRegression()\n",
        "      mlr.fit(inputs2[train_idx], targets[train_idx]) \n",
        "      MSE_reg = sum((targets[test_idx] - mlr.predict(inputs2[test_idx]))**2)/ len(test_idx)\n",
        "      MAE_reg = sum( np.abs( targets[test_idx] - mlr.predict(inputs2[test_idx] ) ) )/ len(test_idx)\n",
        "      train_pred_reg = mlr.predict(inputs2[train_idx]) \n",
        "      test_pred_reg = mlr.predict(inputs2[test_idx])\n",
        "\n",
        "      print(\"model_reg (Regression with Covariate) fit..\", end = \"\")\n",
        "      print(f'MSE = {MSE_reg[0]},,, MAE = {MAE_reg[0]}')\n",
        "      MSE_Covariate_Reg.append(MSE_reg)\n",
        "\n",
        "      #------\n",
        "\n",
        "      X_train_new = np.hstack( ( train_pred1, train_pred2  )  )\n",
        "      X_test_new = np.hstack( ( test_pred1, test_pred2  )  )\n",
        "\n",
        "      X_train_new2 = np.hstack( ( train_pred1, train_pred_reg  )  )\n",
        "      X_test_new2 = np.hstack( ( test_pred1, test_pred_reg  )  )\n",
        "\n",
        "\n",
        "      train_pred3, test_pred3, result3 = deep_model(model3, X_train_new, targets[train_idx], X_test_new, targets[test_idx])\n",
        "\n",
        "      train_pred_reg, test_pred_reg, result_reg = deep_model(model3, X_train_new2, targets[train_idx], X_test_new2, targets[test_idx])\n",
        "\n",
        "\n",
        "      stacker = LinearRegression()\n",
        "      stacker.fit(X = X_train_new, y = targets[train_idx])\n",
        "      MSE_Wgt1 = np.sum((stacker.predict(X_test_new) - targets[test_idx])**2) / len(test_idx)\n",
        "      MAE_Wgt1 = np.sum( np.abs(stacker.predict(X_test_new) - targets[test_idx])) / len(test_idx)\n",
        "\n",
        "      stacker.fit(X = X_train_new2, y = targets[train_idx])\n",
        "      MSE_Wgt2 = np.sum((stacker.predict(X_test_new2) - targets[test_idx])**2) / len(test_idx)\n",
        "      MAE_Wgt2 = np.sum( np.abs(stacker.predict(X_test_new2) - targets[test_idx])) / len(test_idx)\n",
        "\n",
        "\n",
        "      print(f'-> The performance: MSE = {result3[1]}, MAE = {result3[2]} (DNN + DNN)')\n",
        "      print(f'-> The performance: MSE = {result_reg[1]}, MAE = {result_reg[2]} (Reg + DNN)')\n",
        "      MSE_DNN_DNN.append(result3[1])\n",
        "      MSE_Reg_DNN.append(result_reg[1])\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "      print(f'-> The performance: MSE = {MSE_Wgt1}, MAE = {MAE_Wgt1} (DNN + Wgt)')\n",
        "      print(f'-> The performance: MSE = {MSE_Wgt2}, MAE = {MAE_Wgt2} (Reg + Wgt)')\n",
        "      MSE_DNN_Wgt.append(MSE_Wgt1)\n",
        "      MSE_Reg_Wgt.append(MSE_Wgt2)\n",
        "\n",
        "\n",
        "\n",
        "      k = k + 1\n",
        "      j = j + 1\n",
        "    \n",
        "  i = i + 1\n",
        "  j = 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zQhPXfGggZVa",
        "outputId": "ae671a9c-768b-4bca-d030-71e45c6590b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------------------------------------------------------------------------\n",
            "Test for time =  1  location =  1 ... ( 1 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 47.361324310302734,,, MAE = 5.396949291229248\n",
            "model2 (DNN with Covariate) fit..MSE = 68.0084457397461,,, MAE = 6.49793815612793\n",
            "model_reg (Regression with Covariate) fit..MSE = 89.69480496436695,,, MAE = 7.283934983610507\n",
            "-> The performance: MSE = 38.534568786621094, MAE = 4.55187463760376 (DNN + DNN)\n",
            "-> The performance: MSE = 42.27595520019531, MAE = 4.875837326049805 (Reg + DNN)\n",
            "-> The performance: MSE = 37.98850121400107, MAE = 4.5239639600118 (DNN + Wgt)\n",
            "-> The performance: MSE = 38.20030614669864, MAE = 4.492674681735908 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  1  location =  2 ... ( 2 / 25 )\n",
            "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x7f76f21be680> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "model1 (DNN with Basis) fit..MSE = 41.314632415771484,,, MAE = 5.221771240234375\n",
            "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x7f76ce720560> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "model2 (DNN with Covariate) fit..MSE = 59.87761306762695,,, MAE = 6.629988193511963\n",
            "model_reg (Regression with Covariate) fit..MSE = 35.237038786464254,,, MAE = 4.885388035819318\n",
            "-> The performance: MSE = 32.15254211425781, MAE = 4.292019367218018 (DNN + DNN)\n",
            "-> The performance: MSE = 24.8686580657959, MAE = 3.9103195667266846 (Reg + DNN)\n",
            "-> The performance: MSE = 29.879249737587575, MAE = 4.139559364318847 (DNN + Wgt)\n",
            "-> The performance: MSE = 28.48570887552155, MAE = 4.080396224998571 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  1  location =  3 ... ( 3 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 68.30690002441406,,, MAE = 6.628565311431885\n",
            "model2 (DNN with Covariate) fit..MSE = 50.700138092041016,,, MAE = 5.413431644439697\n",
            "model_reg (Regression with Covariate) fit..MSE = 61.9877025934784,,, MAE = 6.249608848386318\n",
            "-> The performance: MSE = 48.60395431518555, MAE = 5.13988733291626 (DNN + DNN)\n",
            "-> The performance: MSE = 59.99981689453125, MAE = 5.800563335418701 (Reg + DNN)\n",
            "-> The performance: MSE = 45.513149609938836, MAE = 4.81594820022583 (DNN + Wgt)\n",
            "-> The performance: MSE = 52.91204775621518, MAE = 5.4808825504422 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  1  location =  4 ... ( 4 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 64.93734741210938,,, MAE = 5.967705249786377\n",
            "model2 (DNN with Covariate) fit..MSE = 85.48699188232422,,, MAE = 7.413478374481201\n",
            "model_reg (Regression with Covariate) fit..MSE = 77.03251775831363,,, MAE = 6.683320997785582\n",
            "-> The performance: MSE = 57.92545700073242, MAE = 5.228925704956055 (DNN + DNN)\n",
            "-> The performance: MSE = 55.9118766784668, MAE = 5.039112567901611 (Reg + DNN)\n",
            "-> The performance: MSE = 55.88251419371469, MAE = 5.097330474853516 (DNN + Wgt)\n",
            "-> The performance: MSE = 55.34420436621863, MAE = 5.058894273586353 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  1  location =  5 ... ( 5 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 39.91410827636719,,, MAE = 4.835289478302002\n",
            "model2 (DNN with Covariate) fit..MSE = 65.69345092773438,,, MAE = 6.531318187713623\n",
            "model_reg (Regression with Covariate) fit..MSE = 47.85780840071886,,, MAE = 5.503684759393534\n",
            "-> The performance: MSE = 39.26660919189453, MAE = 4.6416335105896 (DNN + DNN)\n",
            "-> The performance: MSE = 39.02820587158203, MAE = 4.656890869140625 (Reg + DNN)\n",
            "-> The performance: MSE = 38.90332866320338, MAE = 4.696145820617676 (DNN + Wgt)\n",
            "-> The performance: MSE = 38.903714386699825, MAE = 4.700714092226914 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  2  location =  1 ... ( 6 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 101.5326919555664,,, MAE = 8.976065635681152\n",
            "model2 (DNN with Covariate) fit..MSE = 63.537864685058594,,, MAE = 6.204771041870117\n",
            "model_reg (Regression with Covariate) fit..MSE = 58.627500083754555,,, MAE = 6.345969507045018\n",
            "-> The performance: MSE = 48.55113983154297, MAE = 5.769265651702881 (DNN + DNN)\n",
            "-> The performance: MSE = 55.36057662963867, MAE = 6.093358516693115 (Reg + DNN)\n",
            "-> The performance: MSE = 47.03014466091833, MAE = 5.685344123840332 (DNN + Wgt)\n",
            "-> The performance: MSE = 47.98683779276192, MAE = 5.7592592849433295 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  2  location =  2 ... ( 7 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 75.37444305419922,,, MAE = 7.006840229034424\n",
            "model2 (DNN with Covariate) fit..MSE = 52.36149597167969,,, MAE = 5.904928684234619\n",
            "model_reg (Regression with Covariate) fit..MSE = 71.66937062726632,,, MAE = 6.958977910823299\n",
            "-> The performance: MSE = 53.73301315307617, MAE = 5.559996128082275 (DNN + DNN)\n",
            "-> The performance: MSE = 66.53399658203125, MAE = 6.2346367835998535 (Reg + DNN)\n",
            "-> The performance: MSE = 52.99236429103839, MAE = 5.5391538619995115 (DNN + Wgt)\n",
            "-> The performance: MSE = 56.0611353706046, MAE = 5.697775885607007 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  2  location =  3 ... ( 8 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 128.46620178222656,,, MAE = 9.714258193969727\n",
            "model2 (DNN with Covariate) fit..MSE = 80.15924835205078,,, MAE = 7.264909744262695\n",
            "model_reg (Regression with Covariate) fit..MSE = 107.5067947845652,,, MAE = 8.72650791125583\n",
            "-> The performance: MSE = 92.44820404052734, MAE = 8.044989585876465 (DNN + DNN)\n",
            "-> The performance: MSE = 81.16930389404297, MAE = 7.547811031341553 (Reg + DNN)\n",
            "-> The performance: MSE = 80.07189454061638, MAE = 7.352380116780599 (DNN + Wgt)\n",
            "-> The performance: MSE = 84.43535902867183, MAE = 7.649670481683599 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  2  location =  4 ... ( 9 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 105.97840118408203,,, MAE = 8.85953140258789\n",
            "model2 (DNN with Covariate) fit..MSE = 122.76123046875,,, MAE = 9.234539985656738\n",
            "model_reg (Regression with Covariate) fit..MSE = 91.15291146963776,,, MAE = 7.804917579616301\n",
            "-> The performance: MSE = 80.25309753417969, MAE = 7.061639308929443 (DNN + DNN)\n",
            "-> The performance: MSE = 82.78193664550781, MAE = 7.30366325378418 (Reg + DNN)\n",
            "-> The performance: MSE = 79.47431777600323, MAE = 7.001428031921387 (DNN + Wgt)\n",
            "-> The performance: MSE = 80.6928201410111, MAE = 7.191407013331013 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  2  location =  5 ... ( 10 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 33.890323638916016,,, MAE = 4.646451950073242\n",
            "model2 (DNN with Covariate) fit..MSE = 81.38225555419922,,, MAE = 7.232239723205566\n",
            "model_reg (Regression with Covariate) fit..MSE = 67.1244321441721,,, MAE = 6.629694365571808\n",
            "-> The performance: MSE = 35.017578125, MAE = 4.815309524536133 (DNN + DNN)\n",
            "-> The performance: MSE = 35.80449295043945, MAE = 4.839445114135742 (Reg + DNN)\n",
            "-> The performance: MSE = 34.36162222796381, MAE = 4.755658054351807 (DNN + Wgt)\n",
            "-> The performance: MSE = 34.42434146236348, MAE = 4.7291734076308565 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  3  location =  1 ... ( 11 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 28.629383087158203,,, MAE = 4.347177505493164\n",
            "model2 (DNN with Covariate) fit..MSE = 46.087039947509766,,, MAE = 5.599017143249512\n",
            "model_reg (Regression with Covariate) fit..MSE = 56.11926429625044,,, MAE = 6.0857153840564155\n",
            "-> The performance: MSE = 25.53439712524414, MAE = 4.109285354614258 (DNN + DNN)\n",
            "-> The performance: MSE = 29.45334815979004, MAE = 4.396794319152832 (Reg + DNN)\n",
            "-> The performance: MSE = 24.915280208021493, MAE = 4.059902540842692 (DNN + Wgt)\n",
            "-> The performance: MSE = 26.89816748387414, MAE = 4.189186249705294 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  3  location =  2 ... ( 12 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 26.355066299438477,,, MAE = 4.4352288246154785\n",
            "model2 (DNN with Covariate) fit..MSE = 46.85441207885742,,, MAE = 5.667819023132324\n",
            "model_reg (Regression with Covariate) fit..MSE = 64.02591654716394,,, MAE = 6.520295508005414\n",
            "-> The performance: MSE = 18.18885612487793, MAE = 3.4540557861328125 (DNN + DNN)\n",
            "-> The performance: MSE = 21.72690773010254, MAE = 3.9052069187164307 (Reg + DNN)\n",
            "-> The performance: MSE = 17.5119977301864, MAE = 3.4201698621114094 (DNN + Wgt)\n",
            "-> The performance: MSE = 19.693735366177616, MAE = 3.541611503430672 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  3  location =  3 ... ( 13 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 51.14427185058594,,, MAE = 5.517996311187744\n",
            "model2 (DNN with Covariate) fit..MSE = 47.07538604736328,,, MAE = 5.468932151794434\n",
            "model_reg (Regression with Covariate) fit..MSE = 69.23345867189875,,, MAE = 6.829141901556197\n",
            "-> The performance: MSE = 46.60930252075195, MAE = 5.307685375213623 (DNN + DNN)\n",
            "-> The performance: MSE = 50.556575775146484, MAE = 5.474729061126709 (Reg + DNN)\n",
            "-> The performance: MSE = 46.978727193195546, MAE = 5.287035115559896 (DNN + Wgt)\n",
            "-> The performance: MSE = 49.99912526090248, MAE = 5.450246200837787 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  3  location =  4 ... ( 14 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 55.96818161010742,,, MAE = 6.016836166381836\n",
            "model2 (DNN with Covariate) fit..MSE = 70.79023742675781,,, MAE = 6.9938154220581055\n",
            "model_reg (Regression with Covariate) fit..MSE = 63.00274374608616,,, MAE = 6.557376853794059\n",
            "-> The performance: MSE = 35.96226501464844, MAE = 4.767612934112549 (DNN + DNN)\n",
            "-> The performance: MSE = 30.41466522216797, MAE = 4.425440788269043 (Reg + DNN)\n",
            "-> The performance: MSE = 36.8304486907311, MAE = 4.800974305470785 (DNN + Wgt)\n",
            "-> The performance: MSE = 34.41816266317115, MAE = 4.708024924361949 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  3  location =  5 ... ( 15 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 54.92424392700195,,, MAE = 5.674396991729736\n",
            "model2 (DNN with Covariate) fit..MSE = 74.96391296386719,,, MAE = 6.446969032287598\n",
            "model_reg (Regression with Covariate) fit..MSE = 74.95887783447061,,, MAE = 7.0280095810606165\n",
            "-> The performance: MSE = 43.59523010253906, MAE = 5.152462959289551 (DNN + DNN)\n",
            "-> The performance: MSE = 39.16468048095703, MAE = 4.948646068572998 (Reg + DNN)\n",
            "-> The performance: MSE = 47.471749314629896, MAE = 5.40271193186442 (DNN + Wgt)\n",
            "-> The performance: MSE = 49.5634744860246, MAE = 5.511572967644823 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  4  location =  1 ... ( 16 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 58.960899353027344,,, MAE = 6.153806686401367\n",
            "model2 (DNN with Covariate) fit..MSE = 119.51029968261719,,, MAE = 9.026639938354492\n",
            "model_reg (Regression with Covariate) fit..MSE = 53.713219770063844,,, MAE = 6.131381036660468\n",
            "-> The performance: MSE = 34.12858963012695, MAE = 4.740353107452393 (DNN + DNN)\n",
            "-> The performance: MSE = 36.20917892456055, MAE = 4.880011081695557 (Reg + DNN)\n",
            "-> The performance: MSE = 36.40301693170198, MAE = 4.85386110941569 (DNN + Wgt)\n",
            "-> The performance: MSE = 37.35246189213408, MAE = 4.944040631725473 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  4  location =  2 ... ( 17 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 16.133085250854492,,, MAE = 3.3199708461761475\n",
            "model2 (DNN with Covariate) fit..MSE = 64.2722396850586,,, MAE = 6.293751239776611\n",
            "model_reg (Regression with Covariate) fit..MSE = 38.31065475294171,,, MAE = 5.116750702138583\n",
            "-> The performance: MSE = 14.799116134643555, MAE = 3.2022690773010254 (DNN + DNN)\n",
            "-> The performance: MSE = 16.05940818786621, MAE = 3.3479247093200684 (Reg + DNN)\n",
            "-> The performance: MSE = 15.009115723205529, MAE = 3.21868995030721 (DNN + Wgt)\n",
            "-> The performance: MSE = 15.105372788518872, MAE = 3.222707344017556 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  4  location =  3 ... ( 18 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 67.15216827392578,,, MAE = 7.081632614135742\n",
            "model2 (DNN with Covariate) fit..MSE = 107.53901672363281,,, MAE = 8.880558967590332\n",
            "model_reg (Regression with Covariate) fit..MSE = 87.32659144235798,,, MAE = 7.922805749433893\n",
            "-> The performance: MSE = 40.124332427978516, MAE = 5.369822025299072 (DNN + DNN)\n",
            "-> The performance: MSE = 40.710121154785156, MAE = 5.40626859664917 (Reg + DNN)\n",
            "-> The performance: MSE = 42.388510287514265, MAE = 5.464337730407715 (DNN + Wgt)\n",
            "-> The performance: MSE = 42.76967258572653, MAE = 5.490343916242119 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  4  location =  4 ... ( 19 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 29.310144424438477,,, MAE = 4.344498634338379\n",
            "model2 (DNN with Covariate) fit..MSE = 171.30931091308594,,, MAE = 10.886877059936523\n",
            "model_reg (Regression with Covariate) fit..MSE = 85.52764837702753,,, MAE = 7.430034222422382\n",
            "-> The performance: MSE = 28.029273986816406, MAE = 4.222917079925537 (DNN + DNN)\n",
            "-> The performance: MSE = 28.389577865600586, MAE = 4.267334938049316 (Reg + DNN)\n",
            "-> The performance: MSE = 30.212051288872377, MAE = 4.248067315419515 (DNN + Wgt)\n",
            "-> The performance: MSE = 28.198920397232996, MAE = 4.197756799552339 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  4  location =  5 ... ( 20 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 26.84343910217285,,, MAE = 4.137299537658691\n",
            "model2 (DNN with Covariate) fit..MSE = 56.57352828979492,,, MAE = 6.256804943084717\n",
            "model_reg (Regression with Covariate) fit..MSE = 53.132940439840766,,, MAE = 5.951312376285846\n",
            "-> The performance: MSE = 17.26392936706543, MAE = 3.353224277496338 (DNN + DNN)\n",
            "-> The performance: MSE = 17.398771286010742, MAE = 3.351144552230835 (Reg + DNN)\n",
            "-> The performance: MSE = 18.09807110554211, MAE = 3.428986676534017 (DNN + Wgt)\n",
            "-> The performance: MSE = 17.909237043032885, MAE = 3.4186486354481302 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  5  location =  1 ... ( 21 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 49.180728912353516,,, MAE = 5.679454326629639\n",
            "model2 (DNN with Covariate) fit..MSE = 80.53467559814453,,, MAE = 7.154950141906738\n",
            "model_reg (Regression with Covariate) fit..MSE = 62.44720836112729,,, MAE = 6.259724202449138\n",
            "-> The performance: MSE = 32.39374542236328, MAE = 4.6827616691589355 (DNN + DNN)\n",
            "-> The performance: MSE = 33.7126579284668, MAE = 4.779542922973633 (Reg + DNN)\n",
            "-> The performance: MSE = 34.92046373513531, MAE = 4.87033478418986 (DNN + Wgt)\n",
            "-> The performance: MSE = 32.75360851820917, MAE = 4.66782935371726 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  5  location =  2 ... ( 22 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 26.534006118774414,,, MAE = 4.290737628936768\n",
            "model2 (DNN with Covariate) fit..MSE = 65.5051040649414,,, MAE = 6.8178534507751465\n",
            "model_reg (Regression with Covariate) fit..MSE = 78.95792606812034,,, MAE = 6.786921604567548\n",
            "-> The performance: MSE = 21.16136360168457, MAE = 3.9681382179260254 (DNN + DNN)\n",
            "-> The performance: MSE = 21.04265022277832, MAE = 3.9399993419647217 (Reg + DNN)\n",
            "-> The performance: MSE = 21.23291668018713, MAE = 3.972364393870036 (DNN + Wgt)\n",
            "-> The performance: MSE = 21.071965919516042, MAE = 3.9596174850465125 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  5  location =  3 ... ( 23 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 52.803199768066406,,, MAE = 5.764496803283691\n",
            "model2 (DNN with Covariate) fit..MSE = 81.19454956054688,,, MAE = 7.539787769317627\n",
            "model_reg (Regression with Covariate) fit..MSE = 94.11921892297671,,, MAE = 8.309930092889564\n",
            "-> The performance: MSE = 47.20465850830078, MAE = 5.557649612426758 (DNN + DNN)\n",
            "-> The performance: MSE = 44.209659576416016, MAE = 5.477892875671387 (Reg + DNN)\n",
            "-> The performance: MSE = 43.414455602918196, MAE = 5.313082059224446 (DNN + Wgt)\n",
            "-> The performance: MSE = 45.3918020266264, MAE = 5.525195560936373 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  5  location =  4 ... ( 24 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 46.52829360961914,,, MAE = 5.428498268127441\n",
            "model2 (DNN with Covariate) fit..MSE = 93.68230438232422,,, MAE = 7.660276412963867\n",
            "model_reg (Regression with Covariate) fit..MSE = 79.17181142953005,,, MAE = 6.976464703881106\n",
            "-> The performance: MSE = 37.806461334228516, MAE = 5.051970481872559 (DNN + DNN)\n",
            "-> The performance: MSE = 39.61344528198242, MAE = 5.156196117401123 (Reg + DNN)\n",
            "-> The performance: MSE = 35.877586254611074, MAE = 4.993135229746501 (DNN + Wgt)\n",
            "-> The performance: MSE = 36.63371544398165, MAE = 5.055936913449234 (Reg + Wgt)\n",
            "------------------------------------------------------------------------\n",
            "Test for time =  5  location =  5 ... ( 25 / 25 )\n",
            "model1 (DNN with Basis) fit..MSE = 34.058799743652344,,, MAE = 4.858368873596191\n",
            "model2 (DNN with Covariate) fit..MSE = 88.22698211669922,,, MAE = 7.151795864105225\n",
            "model_reg (Regression with Covariate) fit..MSE = 75.87292751514188,,, MAE = 6.81207924262165\n",
            "-> The performance: MSE = 25.329856872558594, MAE = 3.9843015670776367 (DNN + DNN)\n",
            "-> The performance: MSE = 24.588884353637695, MAE = 4.077737331390381 (Reg + DNN)\n",
            "-> The performance: MSE = 27.28894145743446, MAE = 4.124014027913412 (DNN + Wgt)\n",
            "-> The performance: MSE = 25.69762171397076, MAE = 4.05298444672511 (Reg + Wgt)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Results**"
      ],
      "metadata": {
        "id": "UBidMXHDk_te"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"MSE_Basis_DNN mean = \", np.mean(MSE_Basis), \"/ Std = \", np.std(MSE_Basis))\n",
        "print(\"MSE_Covariate_DNN mean = \", np.mean(MSE_Covariate), \"/ Std = \", np.std(MSE_Covariate))\n",
        "print(\"MSE_Covariate_Reg mean = \", np.mean(MSE_Covariate_Reg), \"/ Std = \", np.std(MSE_Covariate_Reg))\n",
        "print(\"--------------------------------------\")\n",
        "print(\"MSE_DNN_DNN mean = \", np.mean(MSE_DNN_DNN), \"/ Std = \", np.std(MSE_DNN_DNN))\n",
        "print(\"MSE_Reg_DNN mean = \", np.mean(MSE_Reg_DNN), \"/ Std = \", np.std(MSE_Reg_DNN))\n",
        "print(\"MSE_Reg_Wgt mean = \", np.mean(MSE_Reg_Wgt), \"/ Std = \", np.std(MSE_Reg_Wgt))\n",
        "print(\"MSE_DNN_Wgt mean = \", np.mean(MSE_DNN_Wgt), \"/ Std = \", np.std(MSE_DNN_Wgt))"
      ],
      "metadata": {
        "id": "fXKL95NJnGPb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2149ac5-30c9-495e-b047-3b1ad584f6db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MSE_Basis_DNN mean =  53.264091415405275 / Std =  26.61851993330698\n",
            "MSE_Covariate_DNN mean =  77.76350936889648 / Std =  28.10332543929333\n",
            "MSE_Covariate_Reg mean =  69.75245159150944 / Std =  17.2380682832753\n",
            "--------------------------------------\n",
            "MSE_DNN_DNN mean =  39.78470169067383 / Std =  17.76344630568472\n",
            "MSE_Reg_DNN mean =  40.6794140625 / Std =  17.772579876388473\n",
            "MSE_Reg_Wgt mean =  40.036140756634644 / Std =  17.02519447389566\n",
            "MSE_DNN_Wgt mean =  39.2260167647549 / Std =  16.030090635465918\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Sqrt_MSE_Basis_DNN mean = \", np.mean(np.sqrt(MSE_Basis))  , '/ Std = ', np.std(np.sqrt(MSE_Basis)) )  \n",
        "print(\"Sqrt_MSE_Covariate_DNN mean = \", np.mean(np.sqrt(MSE_Covariate))  , '/ Std = ', np.std(np.sqrt(MSE_Covariate))  )\n",
        "print(\"Sqrt_MSE_Covariate_Reg mean = \", np.mean(np.sqrt(MSE_Covariate_Reg))  , '/ Std = ', np.std(np.sqrt(MSE_Covariate_Reg) ) )\n",
        "print(\"--------------------------------------\")\n",
        "print(\"Sqrt_MSE_DNN_DNN mean = \", np.mean(np.sqrt(MSE_DNN_DNN))  , '/ Std = ', np.std(np.sqrt(MSE_DNN_DNN))  )\n",
        "print(\"Sqrt_MSE_Reg_DNN mean = \", np.mean(np.sqrt(MSE_Reg_DNN))  , '/ Std = ', np.std(np.sqrt(MSE_Reg_DNN))  )\n",
        "print(\"Sqrt_MSE_Reg_Wgt mean = \", np.mean(np.sqrt(MSE_Reg_Wgt))  , '/ Std = ', np.std(np.sqrt(MSE_Reg_Wgt))  )\n",
        "print(\"Sqrt_MSE_DNN_Wgt mean = \", np.mean(np.sqrt(MSE_DNN_Wgt))  , '/ Std = ', np.std(np.sqrt(MSE_DNN_Wgt))  )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XzkRUlL1gNs2",
        "outputId": "dcafc93c-dab2-434e-e298-afdccc9a2d87"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sqrt_MSE_Basis_DNN mean =  7.0903937798800225 / Std =  1.729279463141793\n",
            "Sqrt_MSE_Covariate_DNN mean =  8.694557188747206 / Std =  1.4724756909710848\n",
            "Sqrt_MSE_Covariate_Reg mean =  8.28455834978699 / Std =  1.0576126608943968\n",
            "--------------------------------------\n",
            "Sqrt_MSE_DNN_DNN mean =  6.16307683443463 / Std =  1.3420825695645757\n",
            "Sqrt_MSE_Reg_DNN mean =  6.232795037795374 / Std =  1.3533957585766847\n",
            "Sqrt_MSE_Reg_Wgt mean =  6.1910861457878426 / Std =  1.306366369770932\n",
            "Sqrt_MSE_DNN_Wgt mean =  6.137170178686738 / Std =  1.2494634698914124\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# R 결과\n",
        "\n",
        "**1. Without Covariate**\n",
        "\n",
        "```\n",
        "\n",
        "```\n",
        "\n",
        "---\n",
        "\n",
        "**2. With Covariate**\n",
        "\n",
        "```\n",
        "> acc\n",
        " [1]  67.73125  44.30942  61.77689  45.20541  49.44973  64.74137  43.25497  64.91315  40.22579\n",
        "[10]  29.79341  66.00340  60.69261  61.86075  87.26543  94.62138 104.20433  55.70525 102.67190\n",
        "[19]  88.77406  62.77618  59.88496  65.88396  71.57981  69.77092  72.30629\n",
        "> mean(acc)\n",
        "[1] 65.41611\n",
        "> sd(acc)\n",
        "[1] 18.88905\n",
        "> \n",
        "> \n",
        "> mean(sqrt(acc))\n",
        "[1] 8.006119\n",
        "> sd(sqrt(acc))\n",
        "[1] 1.171788\n",
        "```"
      ],
      "metadata": {
        "id": "N3DmuWzahJf0"
      }
    }
  ]
}