Time_Series_and_Entity_Embedding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "Time Series and Entity Embedding.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjnnsiWG3G_2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.regularizers import l2\n",
        "from tensorflow.keras.losses import MeanSquaredError\n",
        "from tensorflow.keras.layers import Input, Dense, Embedding, Concatenate, LSTM, Dropout"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tb6vmvz83G_7",
        "colab_type": "text"
      },
      "source": [
        "#### Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TSnwmnAa3G_8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess_categorical(x_train, x_test, categorical):\n",
        "    # Columns to be embedded: map to range [0, # values)\n",
        "    for cat in categorical:\n",
        "        raw_vals, val_map = np.unique(x_train[cat]), {}\n",
        "        for i in range(len(raw_vals)):\n",
        "            val_map[raw_vals[i]] = i  \n",
        "        x_train.loc[:,cat] = x_train.loc[:,cat].map(val_map)\n",
        "        x_test.loc[:,cat] = x_test.loc[:,cat].map(val_map) \n",
        "        \n",
        "    return x_train, x_test\n",
        "\n",
        "def prepare_data(x, y, categorical, numeric, steps):\n",
        "    _x, _y = [], []\n",
        "    columns = categorical + numeric\n",
        "    for col in columns:\n",
        "        inp = []\n",
        "        for i in range(len(x) - steps): \n",
        "            v = x.iloc[i:(i + steps)][col].values.astype(np.float32)\n",
        "            if col in numeric: \n",
        "                v = v.reshape(steps, 1)\n",
        "            inp.append(v)\n",
        "        inp = np.stack(inp, axis = 0)\n",
        "        _x.append(inp)   \n",
        "        \n",
        "    for i in range(len(x) - steps): \n",
        "      _y.append(y.iloc[i + steps].values)\n",
        "    _y = np.array(_y).astype(np.float32)\n",
        "    \n",
        "    return _x, _y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YisMOdo03G__",
        "colab_type": "text"
      },
      "source": [
        "#### Data Processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fjm4uhjJ3HAA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv(\"./California_SO2_Measures.csv\")\n",
        "\n",
        "# From date extract day, month, year for learning embeddings\n",
        "df[\"Date\"] = pd.to_datetime(df[\"Date\"], infer_datetime_format=True)\n",
        "df[\"Day\"] = df[\"Date\"].dt.day\n",
        "df[\"Month\"] = df[\"Date\"].dt.month\n",
        "df['lat/long'] = df['SITE_LATITUDE'].astype(\n",
        "    str) + \"-\" + df['SITE_LONGITUDE'].astype(str)\n",
        "df = df.drop(columns=['Source','Date', 'UNITS', 'AQS_PARAMETER_CODE', \n",
        "                      'SITE_LATITUDE', 'SITE_LONGITUDE'])\n",
        "\n",
        "df.loc[:,\"Site Name\"] = df.loc[:,\"Site Name\"].astype(\"category\").cat.codes\n",
        "df.loc[:,\"COUNTY\"] = df.loc[:,\"COUNTY\"].astype(\"category\").cat.codes\n",
        "df.loc[:,\"lat/long\"] = df.loc[:,\"lat/long\"].astype(\"category\").cat.codes\n",
        "\n",
        "# Categorize columns by type for pre-processing\n",
        "categorical =  ['Day','Month','Site ID','POC','Site Name', 'COUNTY', 'lat/long'] \n",
        "numeric = ['DAILY_AQI_VALUE', 'DAILY_OBS_COUNT', 'PERCENT_COMPLETE'] \n",
        "label = ['Daily Max 1-hour SO2 Concentration']\n",
        "\n",
        "# Train/Test split based on months\n",
        "train = df[df[\"Month\"].isin([1,2,3,4,5,6,7,8,9])].copy(deep=True)\n",
        "test = df[df[\"Month\"].isin([9])].copy(deep=True)\n",
        "\n",
        "# Separate features and label\n",
        "train_x = train[categorical + numeric]\n",
        "train_y = train[['Daily Max 1-hour SO2 Concentration']]\n",
        "\n",
        "test_x = test[categorical + numeric]\n",
        "test_y = test[['Daily Max 1-hour SO2 Concentration']]\n",
        "\n",
        "num_scalers = {}\n",
        "for num in numeric:\n",
        "    scaler = StandardScaler()\n",
        "    scaler.fit(train_x[num].values.reshape(-1, 1))\n",
        "    train_x.iloc[:][num] = scaler.transform(\n",
        "        train_x.iloc[:][num].values.reshape(-1, 1))\n",
        "    test_x.iloc[:][num] = scaler.transform(\n",
        "        test_x.iloc[:][num].values.reshape(-1, 1))\n",
        "    num_scalers[num] = scaler\n",
        "\n",
        "y_scaler = StandardScaler()\n",
        "y_scaler.fit(train_y.values.reshape(-1, 1))\n",
        "train_y.iloc[:][label] = y_scaler.transform(train_y.values.reshape(-1, 1))\n",
        "test_y.iloc[:][label] = y_scaler.transform(test_y.values.reshape(-1, 1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7u5dmh83HAD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "steps = 5 # hyper-parameter: window size\n",
        "train_x, test_x = preprocess_categorical(train_x, test_x, categorical)\n",
        "train_x, train_y = prepare_data(train_x, train_y, categorical, numeric, steps)\n",
        "test_x, test_y = prepare_data(test_x, test_y, categorical, numeric, steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FeILc03B3HAG",
        "colab_type": "text"
      },
      "source": [
        "#### Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OmK70Eb3HAM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs, embeddings = [], []\n",
        "\n",
        "for cat in categorical:\n",
        "    cat_input = Input(shape=(steps,), name=\"\".join([cat.replace(\" \", \"\"),\"_inp\"]))\n",
        "    unique_cat  = train[cat].nunique()\n",
        "    embedding_size = min(np.ceil((unique_cat)/2), 20)\n",
        "    embedding_size = int(embedding_size)\n",
        "    cat_dim = unique_cat + 1\n",
        "    inputs.append(cat_input)\n",
        "    embeddings.append(Embedding(cat_dim, embedding_size, input_length = steps,\n",
        "            name=\"\".join([cat.replace(\" \", \"\"),\"_emb\"]))(cat_input))\n",
        "for num in numeric:\n",
        "    num_input = Input(shape=(steps,1), \n",
        "          name=\"\".join([num.replace(\" \", \"\"),\"_inp\"]))\n",
        "    inputs.append(num_input)\n",
        "    embeddings.append(num_input)\n",
        "    \n",
        "x = Concatenate(name=\"concat\")(embeddings)\n",
        "x = LSTM(128, kernel_regularizer=l2(0.0001), \n",
        "        recurrent_regularizer=l2(0.0001),\n",
        "        return_sequences=False)(x)\n",
        "x = Dense(64, activation=\"relu\", kernel_regularizer=l2(0.0001))(x)\n",
        "out = Dense(1, activation='linear', name=\"output\")(x)\n",
        "model = Model(inputs, out, name = \"so2_model\")\n",
        "model.compile(optimizer = Adam(lr = 3e-4), loss=MeanSquaredError())\n",
        "model.fit(train_x, train_y, epochs = 30, batch_size = 24, verbose = 2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rF-n8aB13HAT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_y = model.predict(train_x)\n",
        "train_mse = mean_squared_error(y_scaler.inverse_transform(train_y), \n",
        "                y_scaler.inverse_transform(pred_y), squared = False)\n",
        "print(\"Train RMSE: \", train_mse)\n",
        "\n",
        "pred_y = model.predict(test_x)\n",
        "test_mse = mean_squared_error(y_scaler.inverse_transform(test_y), \n",
        "                y_scaler.inverse_transform(pred_y), squared = False)\n",
        "print(\"Test RMSE: \", test_mse)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "huqHTPFC3HAW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}