nbeats-test.sync.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "837bbb28",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:36.817976Z",
     "start_time": "2022-01-16T14:51:33.777017Z"
    }
   },
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "import os\n",
    "from pathlib import Path\n",
    "import warnings\n",
    "import keras_tuner as kt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from collections.abc import Iterable\n",
    "from hypernbeat import HyperNBeats\n",
    "from loss import LOSSES\n",
    "from data import M4Meta,read_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd9a899f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:36.886599Z",
     "start_time": "2022-01-16T14:51:36.821447Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def isnotebook():\n",
    "    try:\n",
    "        shell = get_ipython().__class__.__name__\n",
    "        if shell == 'ZMQInteractiveShell':\n",
    "            return True   # Jupyter notebook or qtconsole\n",
    "        elif shell == 'TerminalInteractiveShell':\n",
    "            return False  # Terminal running IPython\n",
    "        else:\n",
    "            return False  # Other type (?)\n",
    "    except NameError:\n",
    "        return False  \n",
    "if isnotebook():\n",
    "    from tqdm.notebook import tqdm\n",
    "    os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" \n",
    "else:\n",
    "    from tqdm import tqdm\n",
    "    os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7052ce05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:36.893910Z",
     "start_time": "2022-01-16T14:51:36.889321Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "tf.config.threading.set_inter_op_parallelism_threads(8)\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "267546c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:36.914760Z",
     "start_time": "2022-01-16T14:51:36.897763Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def last_insample_window(timeseries, insample_size):\n",
    "    \"\"\"\n",
    "    The last window of insample size of all timeseries.\n",
    "    This function does not support batching and does not reshuffle timeseries.\n",
    "    :return: Last insample window of all timeseries. Shape \"timeseries, insample size\"\n",
    "    \"\"\"\n",
    "    insample = np.zeros((len(timeseries), insample_size))\n",
    "    for i, ts in enumerate(timeseries):\n",
    "        ts_last_window = ts[-insample_size:]\n",
    "        insample[i, -len(ts):] = ts_last_window\n",
    "    return insample\n",
    "\n",
    "def predict_m4_timeseries(timeseries, targets, model, loss):\n",
    "    if loss==\"mase\":\n",
    "        insample_size=model.input_shape[0][1]\n",
    "        x_test=last_insample_window(timeseries,insample_size)\n",
    "        y_pred=model.predict((x_test,targets)).squeeze()\n",
    "    else:\n",
    "        insample_size=model.input_shape[1]\n",
    "        x_test=last_insample_window(timeseries,insample_size)\n",
    "        y_pred=model.predict(x_test).squeeze()\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d6f961ba",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:37.000734Z",
     "start_time": "2022-01-16T14:51:36.981287Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def ensemble_member(freq,lookback,loss,model_type=\"interpretable\",overwrite=False):\n",
    "\n",
    "    timeseries,targets=read_data(freq)\n",
    "\n",
    "    history_size=M4Meta.history_size[freq]\n",
    "    batch_size_power=10\n",
    "\n",
    "    if model_type==\"interpretable\":\n",
    "        project_name=f\"{freq}_{lookback}_{loss}\"\n",
    "    else:\n",
    "        project_name=f\"{model_type}_{freq}_{lookback}_{loss}\"\n",
    "         \n",
    "\n",
    "    print(project_name)\n",
    "\n",
    "    hp=kt.HyperParameters()\n",
    "    hp.Fixed(\"model_type\", model_type)\n",
    "    hp.Fixed(\"lookback\",lookback)\n",
    "    hp.Fixed(\"nb_blocks_per_stack\",3)\n",
    "    hp.Fixed(\"trend_layer_units_power\",8)\n",
    "    hp.Fixed(\"season_layer_units_power\",11)\n",
    "    hp.Fixed(\"degree_of_polynomial\",2)\n",
    "    hp.Fixed(\"loss_fn\",loss)\n",
    "    hp.Fixed(\"batch_size_power\",batch_size_power)\n",
    "    hp.Fixed(\"init_lr\",1e-3)\n",
    "    hp.Fixed(\"history_size\",history_size)\n",
    "    hp.Fixed(\"lr_decay_rate\", 0.5)\n",
    "    hp.Fixed(\"nb_harmonics\", 1)\n",
    "    hp.Fixed(\"epochs\", 100)\n",
    "    hp.Fixed(\"val_size\", 0.1)\n",
    "    hp.Fixed(\"stacks_num\", 15)\n",
    "\n",
    "    tuner = kt.RandomSearch(\n",
    "        HyperNBeats(freq),\n",
    "        hyperparameters=hp,\n",
    "        objective=kt.Objective(\"val_loss\", direction=\"min\"),\n",
    "        max_trials=15,\n",
    "        overwrite=overwrite,\n",
    "        directory=\"kt\",\n",
    "        project_name=project_name,\n",
    "        seed=64\n",
    "    )\n",
    "    es_callback=tf.keras.callbacks.EarlyStopping(\n",
    "        monitor='val_loss', patience=20,\n",
    "        restore_best_weights=True,\n",
    "        verbose=1\n",
    "    )\n",
    "\n",
    "    logdir = Path(\"logs\")/\"keras_tuner_log\"\n",
    "    tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)\n",
    "\n",
    "    tuner.search(timeseries, targets, \n",
    "                # epochs=100, \n",
    "                # steps_per_epoch=M4Meta.iterations[freq] // 100,\n",
    "                verbose=2,\n",
    "                callbacks=[es_callback,tensorboard_callback]\n",
    "                )\n",
    "\n",
    "    evaluate_path=Path('./nbeats_result')/project_name\n",
    "    evaluate_path.mkdir(exist_ok=True,parents=True)\n",
    "\n",
    "    print(\"Record pred\")\n",
    "    for i,model in tqdm(enumerate(tuner.get_best_models(num_models=15))):\n",
    "        y_pred=predict_m4_timeseries(timeseries, targets, model, loss) \n",
    "        np.save(evaluate_path/f\"{i}.npy\",y_pred)\n",
    "\n",
    "\n",
    "    return tuner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0ac1f02b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:37.009584Z",
     "start_time": "2022-01-16T14:51:37.002763Z"
    }
   },
   "outputs": [],
   "source": [
    "class Evaluater(object):\n",
    "\n",
    "    def __init__(self,freq):\n",
    "        self._freq = freq\n",
    "        self.timeseries,self.targets=read_data(freq)\n",
    "        self.eval_fn=LOSSES['smape']\n",
    "\n",
    "    def ensemble_results(self,results):\n",
    "        if not isinstance(results, Iterable):\n",
    "            results=[results]\n",
    "        preds=[np.load(fp) for fp in results]\n",
    "        print(f\"Ensembling from {len(preds)} models:\")\n",
    "        # mean_ensemble=np.mean(np.stack(preds),axis=0)\n",
    "        median_ensemble=np.median(np.stack(preds),axis=0)\n",
    "        return median_ensemble\n",
    "\n",
    "    def __call__(self,results):\n",
    "        y_pred=self.ensemble_results(results)\n",
    "        median_score=self.eval_fn(self.targets, y_pred).numpy()\n",
    "        print(f\"Median ensemble: {median_score}\")\n",
    "        return median_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e19e08a2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T14:51:37.088805Z",
     "start_time": "2022-01-16T14:51:37.012783Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def generic_model(freq,repeat=10):\n",
    "    results=[]\n",
    "    for lookback,loss in product([2,3,4,5,6,7],LOSSES.keys()):\n",
    "        project_name=f\"generic_{freq}_{lookback}_{loss}\"\n",
    "        member_results=list(Path('./nbeats_result').glob(f\"{project_name}/*.npy\"))\n",
    "        select_results=np.random.choice(member_results,size=repeat) # For accurate reporduction\n",
    "        results.extend(select_results)\n",
    "    return results\n",
    "\n",
    "def interpretable_model(freq,repeat=10):\n",
    "    results=[]\n",
    "    for lookback,loss in product([2,3,4,5,6,7],LOSSES.keys()):\n",
    "        project_name=f\"{freq}_{lookback}_{loss}\"\n",
    "        member_results=list(Path('./nbeats_result').glob(f\"{project_name}/*.npy\"))\n",
    "        select_results=np.random.choice(member_results,size=repeat) # For accurate reporduction\n",
    "        results.extend(select_results)\n",
    "    return results\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b3d089fe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-16T15:17:02.071973Z",
     "start_time": "2022-01-16T15:17:00.180828Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ensembling from 180 models:\n",
      "Median ensemble: 13.012983322143555\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "13.012983"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # freq=\"Yearly\"\n",
    "    # results=Path('./nbeats_result').glob(f\"{freq}_[234567]_*ma?e/[0-9].npy\")\n",
    "    # evaluate(results,freq)\n",
    "    for freq,lookback,loss in product([\"Yearly\"],[2,3,4,5,6,7],LOSSES.keys()):\n",
    "        # ensemble_member(freq,lookback,loss,model_type=\"generic\")\n",
    "        ensemble_member(freq,lookback,loss)\n",
    "    evaluater=Evaluater(freq)\n",
    "    results=interpretable_model(freq)\n",
    "\n",
    "    evaluater(results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "notify_time": "5",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}