SVR_submission.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "from sklearn.svm import SVR \n",
    "\n",
    "from modules.data_processing import make_Xy, post_process, RMSLE\n",
    "from modules.keras_processing import make_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data from csv\n",
    "X, y, Xsub, id_sub = make_Xy()\n",
    "\n",
    "# add cube_PCA to X and Xsub\n",
    "cube_PCA = np.load('data/cube_PCA.npy').transpose()\n",
    "cube_PCA = cube_PCA / np.max(cube_PCA)\n",
    "\n",
    "X = np.hstack((X, cube_PCA[:2400, ]))\n",
    "Xsub = np.hstack((Xsub, cube_PCA[-600:, ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trial 1  -  current min 0.0531\n",
      "trial 2  -  current min 0.0530\n",
      "trial 3  -  current min 0.0530\n",
      "trial 4  -  current min 0.0530\n",
      "trial 5  -  current min 0.0530\n",
      "trial 6  -  current min 0.0530\n",
      "trial 7  -  current min 0.0530\n",
      "trial 8  -  current min 0.0528\n",
      "trial 9  -  current min 0.0528\n",
      "trial 10  -  current min 0.0528\n",
      "trial 11  -  current min 0.0528\n",
      "trial 12  -  current min 0.0528\n",
      "trial 13  -  current min 0.0528\n",
      "trial 14  -  current min 0.0527\n",
      "trial 15  -  current min 0.0527\n",
      "trial 16  -  current min 0.0527\n",
      "trial 17  -  current min 0.0527\n",
      "trial 18  -  current min 0.0527\n",
      "trial 19  -  current min 0.0527\n",
      "trial 20  -  current min 0.0527\n",
      "trial 21  -  current min 0.0527\n",
      "trial 22  -  current min 0.0527\n",
      "trial 23  -  current min 0.0527\n",
      "trial 24  -  current min 0.0527\n",
      "trial 25  -  current min 0.0526\n",
      "trial 26  -  current min 0.0526\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-18139f99ebd3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mclf0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVR\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgamma\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdimX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rbf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m         \u001b[0mclf0\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0mclf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVR\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgamma\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdimX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rbf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/py36/lib/python3.6/site-packages/sklearn/svm/base.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m         \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'i'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msolver_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m         \u001b[0;31m# see comment on the other call to np.iinfo in this file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/py36/lib/python3.6/site-packages/sklearn/svm/base.py\u001b[0m in \u001b[0;36m_dense_fit\u001b[0;34m(self, X, y, sample_weight, solver_type, kernel, random_seed)\u001b[0m\n\u001b[1;32m    252\u001b[0m                 \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoef0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m                 \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gamma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 254\u001b[0;31m                 max_iter=self.max_iter, random_seed=random_seed)\n\u001b[0m\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_warn_from_fit_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dimX = X.shape[1]\n",
    "N = X.shape[0]\n",
    "\n",
    "C_list = [1.0 * 1.5**p for p in range(8,11)]\n",
    "epsilon_list = [0.1 * 1.5**p for p in range(-8,-3)]\n",
    "gamma_list = [1.0 * 1.5**p for p in range(0,5)]\n",
    "\n",
    "error0 = {}\n",
    "error1 = {}\n",
    "current_min  = np.inf\n",
    "\n",
    "while len(error0) < 100:\n",
    "    C = np.random.choice(C_list)\n",
    "    epsilon = np.random.choice(epsilon_list)\n",
    "    gamma = np.random.choice(gamma_list)\n",
    "    \n",
    "    t = (C, epsilon, gamma)\n",
    "    if t in error0:\n",
    "        continue\n",
    " \n",
    "    pred = np.tile(np.nan, (N,2))\n",
    "    for iCV in range(5):\n",
    "        ind_test = np.tile(False, N)\n",
    "        ind_test[iCV::5] = True\n",
    "\n",
    "        X_train, y_train = X[~ind_test, :], y[~ind_test, :]\n",
    "        X_test, y_test = X[ind_test, :], y[ind_test, :]\n",
    "\n",
    "        y_train = np.log(1+y_train)\n",
    "\n",
    "        clf0 = SVR(C=C, epsilon=epsilon, gamma=gamma/float(dimX), kernel='rbf')\n",
    "        clf0.fit(X_train, y_train[:, 0])\n",
    "\n",
    "        clf1 = SVR(C=C, epsilon=epsilon, gamma=gamma/float(dimX), kernel='rbf')\n",
    "        clf1.fit(X_train, y_train[:, 1])\n",
    "\n",
    "        pred[ind_test, 0] = clf0.predict(X_test)\n",
    "        pred[ind_test, 1] = clf1.predict(X_test)\n",
    "        \n",
    "    # post process\n",
    "    pred = post_process(pred)\n",
    "    \n",
    "    error0[t], error1[t] = RMSLE(y, pred)\n",
    "    current_min = 0.5*(min(error0.values()) + min(error1.values()))\n",
    "\n",
    "    print(\"trial\", len(error0), \" - \", \"current min\", '{0:.4f}'.format(current_min))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get best hyperparameters for each output\n",
    "(C0, epsilon0, gamma0) = min(error0, key=error0.get) \n",
    "(C1, epsilon1, gamma1) = min(error1, key=error1.get) \n",
    "\n",
    "# train SVR\n",
    "clf0 = SVR(C=C0, epsilon=epsilon0, gamma=gamma0/float(dimX), kernel='rbf')\n",
    "clf0.fit(X, np.log(1+y[:, 0]) )\n",
    "\n",
    "clf1 = SVR(C=C1, epsilon=epsilon1, gamma=gamma1/float(dimX), kernel='rbf')\n",
    "clf1.fit(X, np.log(1+y[:, 1]) )\n",
    "\n",
    "# predict submission samples\n",
    "ysub0 = clf0.predict(Xsub)\n",
    "ysub1 = clf1.predict(Xsub)\n",
    "\n",
    "# post process\n",
    "ysub0 = post_process(ysub0)\n",
    "ysub1 = post_process(ysub1)    \n",
    "\n",
    "# write submission\n",
    "submission = pd.concat([pd.DataFrame(id_sub), pd.DataFrame(ysub0), pd.DataFrame(ysub1)], axis=1)\n",
    "submission.columns = ['id','formation_energy_ev_natom', 'bandgap_energy_ev']\n",
    "submission.to_csv('output/submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}