training_baseline_yahoo_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: the binary library cannot be imported. You cannot train maps, but you can load and analyze ones that you have already saved.\n",
      "The problem occurs because either compilation failed when you installed Somoclu or a path is missing from the dependencies when you are trying to import it. Please refer to the documentation to see your options.\n"
     ]
    }
   ],
   "source": [
    "from PipelineResources import * # This is where we have all of the resources that will be used in the project. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "nasdaq = pd.read_pickle('nasdaq_qqq_yahoo.pckl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare training, validation and test sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First Test: Use a Random Forest Model to predict whether or not the next day will close above 1.5 %. \n",
    "The goal behind this exercise is to establish a baseline model and performance. In addition, it is also work out the way the data will be stored. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sample size is: 4780\n"
     ]
    }
   ],
   "source": [
    "number_of_samples = nasdaq.index.size\n",
    "print(\" Sample size is: {}\".format(number_of_samples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume',\n",
       "       'open_change', 'open_change_pct', 'next_day_open_change',\n",
       "       'next_day_open_change_pct', 'open_change_wrt_close',\n",
       "       'open_change_wrt_close_pct', 'open_change_wrt_high',\n",
       "       'open_change_wrt_low', 'open_change_wrt_volume',\n",
       "       'next_day_open_change_wrt_close', 'next_day_open_change_wrt_close_pct',\n",
       "       'next_day_open_change_wrt_high', 'next_day_open_change_wrt_low',\n",
       "       'next_day_open_change_wrt_volume', 'close_change_pct', 'close_change',\n",
       "       'high_change_pct', 'high_change', 'low_change_pct', 'low_change',\n",
       "       'volume_change_pct', 'volume_change', 'high_low_range',\n",
       "       'high_low_range_with_ref_open', 'high_low_range_with_ref_open_pct',\n",
       "       'high_low_range_with_ref_close', 'high_low_range_with_ref_close_pct',\n",
       "       'gt_1', 'gt_1.5', 'gt_2.5'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nasdaq.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of days per samples: 1\n",
      "(4776,)\n",
      "[0 1 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 0 0 1 1]\n",
      "(4776, 13)\n",
      "(3582, 13)\n"
     ]
    }
   ],
   "source": [
    "chosen_label = 'gt_2.5'\n",
    "random_state = 1\n",
    "x_train, x_val, x_test, y_train, y_val, y_test = get_input_sets(nasdaq, chosen_features, chosen_label, x_test=False,\n",
    "                                                               number_of_days_per_sample=1)\n",
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1130\n",
      "960\n",
      "(234,)\n",
      "(234, 13)\n",
      "3400\n",
      "2890\n",
      "(692,)\n",
      "(692, 13)\n"
     ]
    }
   ],
   "source": [
    "x_val, y_val = decrease_class_rows(x_val, y_val, class_number=0, percentage_cut=0.85)\n",
    "x_train, y_train = decrease_class_rows(x_train, y_train, class_number=0, percentage_cut=0.85)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking how the classes differ from one another"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply FFT on the training and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(692, 19)\n"
     ]
    }
   ],
   "source": [
    "z_train = []\n",
    "z_val = []\n",
    "for row_index in range(x_train.shape[0]):\n",
    "    z_train.append(get_fft(x_train[row_index], absolute_value=True))\n",
    "for row_index in range(x_val.shape[0]):\n",
    "    z_val.append(get_fft(x_val[row_index], absolute_value=True))\n",
    "z_train = np.array(z_train, dtype=np.float)\n",
    "z_val = np.array(z_val, dtype=np.float)\n",
    "x_train = np.concatenate((x_train, z_train), axis=1)\n",
    "x_val = np.concatenate((x_val, z_val), axis=1)\n",
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cv scores: [0.81294964 0.74100719 0.8115942  0.84782609 0.78985507]\n",
      "avg score: 0.800646439370243, std score: 0.03512957325194948, min: 0.7410071942446043, max: 0.8478260869565217\n",
      "Accuracy of random forest TRAINING: 0.80\n",
      "\n",
      "Classification report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.86      0.88      0.87       510\n",
      "          1       0.64      0.58      0.61       182\n",
      "\n",
      "avg / total       0.80      0.80      0.80       692\n",
      "\n",
      "Accuracy of random forest VALIDATION: 0.83\n",
      "\n",
      "Classification report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.91      0.89       170\n",
      "          1       0.73      0.62      0.67        64\n",
      "\n",
      "avg / total       0.83      0.83      0.83       234\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(15,0.5,'true labels')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAEWCAYAAACuU8gIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFQVJREFUeJzt3Xl4VPW9x/H3FwKyBcgCCFiKVpAr1mq1XjcUEa4botV6XUC8iFItaBEVaqEoKrWtXqsXN0BBEQWXq5WqVdSLG6IgLiCKSxUVEZWwJIBoEr73jzmhI0+WyQ8mZyb5vJ5nnpzlN2c+IcmHc86cmTF3R0QkRKO4A4hI9lKBiEgwFYiIBFOBiEgwFYiIBFOBiEgwFUgDYWbNzezvZrbBzB7age0MNLO5OzNbXMysl5m9H3eObGa6DiSzmNlZwCigB1ACvAVMdPeXd3C7ZwMXAYe6e9kOB81wZuZAN3f/KO4s9Zn2QDKImY0CbgL+CHQAugC3ASfthM3/GPigIZRHKswsJ+4M9YK765YBN6ANsBE4rZoxu5AomFXR7SZgl2hdb2AlcCnwNfAlMCRaNwH4HiiNHmMocBUwM2nbXQEHcqL5/wI+JrEX9AkwMGn5y0n3OxRYBGyIvh6atO554BpgfrSduUBhFd9bRf7RSflPBo4HPgDWAr9PGn8QsABYH429BWgarXsx+l42Rd/v6UnbHwOsBu6tWBbd5yfRY/w8mu8ErAF6x/27kcm32APoFv0g4FigrOIPuIoxVwOvAu2BdsArwDXRut7R/a8GmkR/eJuBvGj99oVRZYEALYFiYK9oXUegZzS9rUCAfGAdcHZ0vzOj+YJo/fPAP4HuQPNo/k9VfG8V+cdH+c8HvgHuB3KBnsAWYI9o/AHAwdHjdgXeA0Ymbc+BPSvZ/p9JFHHz5AKJxpwfbacF8DRwQ9y/F5l+0yFM5igA1nj1hxgDgavd/Wt3/4bEnsXZSetLo/Wl7v4kif999wrMsxXYx8yau/uX7r6skjEnAB+6+73uXubus4DlwIlJY6a7+wfu/i3wILBfNY9ZSuJ8TykwGygEbnb3kujxlwH7Arj7Ynd/NXrcFcBk4MgUvqcr3f27KM8PuPtU4EPgNRKlObaG7TV4KpDMUQQU1nBs3gn4NGn+02jZtm1sV0CbgVa1DeLum0js9l8AfGlmT5hZjxTyVGTqnDS/uhZ5ity9PJqu+AP/Kmn9txX3N7PuZva4ma02s2IS540Kq9k2wDfuvqWGMVOBfYBJ7v5dDWMbPBVI5lhAYhf95GrGrCJxMrRCl2hZiE0kdtUr7Jq80t2fdvd+JP4nXk7iD6umPBWZvgjMVBu3k8jVzd1bA78HrIb7VPuUo5m1InFe6S7gKjPL3xlB6zMVSIZw9w0kjv9vNbOTzayFmTUxs+PM7C/RsFnAODNrZ2aF0fiZgQ/5FnCEmXUxszbAFRUrzKyDmQ0ws5bAdyQOhcor2caTQHczO8vMcszsdGBv4PHATLWRS+I8zcZo7+jC7dZ/BexRy23eDCx29/OAJ4A7djhlPacCySDufiOJa0DGkTiB+DkwAvhbNORa4HVgCbAUeCNaFvJYzwAPRNtazA//6BuReDZnFYlnJo4EflPJNoqA/tHYIhLPoPR39zUhmWrpMuAsEs/uTCXxvSS7CrjHzNab2X/WtDEzO4nEiewLokWjgJ+b2cCdlrge0oVkIhJMeyAiEkwFIiLBVCAiEkwFIiLBMvYFRc33H6Gzu1lq3aJb4o4gO6BZTo3X02yjPRARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgKhARCaYCEZFgOXEHqM8aNTLm3zeaVV9v4NTf3sGUCYPodcCebNi4BYBh4+9lyQdf0L1rB6ZMGMR+PXbjqlse56Z7n4s5uSQ7rl8fWrRsSeNGjWic05hZDz7C+8uXc+3VV7J582Y6derMdX+5gVatWsUdtc6pQNJoxFlH8f4nX5Hbstm2Zb+/6W88+uxbPxi3bsMmLv3zQ5x41M/qOqKk6M7p95CXl79tfsL4sYy6fAwH/uIgHn3kYe6edicjLh4ZY8J4pO0Qxsx6mNkYM/sfM7s5mv63dD1epuncvi3HHt6T6Y++UuPYb9ZtZPG7n1FaVl4HyWRnWLHiEw448BcAHHLIYTz3zNyYE8UjLQViZmOA2YABC4FF0fQsM/tdOh4z01x/+amMvflvbN3qP1h+1fATWfjAFfzl0lNo2kQ7gFnB4ILzh3LGaafw8IMPALBnt+48Py9xqDn36adYvfrLOBPGJl2/wUOBnu5emrzQzG4ElgF/quxOZjYMGAaQs1tvcgp7pileeh3Xax++XlvCm+99Tq8Dum1bPn7SHFavKaZpkxxu/cOZXDqkL9dNeSrGpJKKe2bOon37DhQVFXHBeUPYfY89mHDNRP503UQm334bvY/qQ5MmTeOOGYt0FchWoBPw6XbLO0brKuXuU4ApAM33H+FVjct0h+y3B/2P/CnHHt6TXZo2oXXLZky7djDnjpsBwPelZcx47FVGDj465qSSivbtOwBQUFBAn779eGfpEs4ZMpTJU6cBicOZF194PsaE8UlXgYwEnjOzD4HPo2VdgD2BEWl6zIwxftIcxk+aA0CvA7oxcvDRnDtuBrsWtmb1mmIABhy1L+/+c1WcMSUFmzdvxn0rLVu2YvPmzSx4ZT6/vuA3FBUVUVBQwNatW5k6+XZOO/2MuKPGIi0F4u5PmVl34CCgM4nzHyuBRe7eYM8UTp94DoV5uZjBkvdXctHE2QB0KMhl/n2jyW3ZjK3ujBjYm/1PnUjJpi0xJ5a1RUVccvFwAMrKyzn+hP4c1usI7rv3HmbPuh+Ao/v24+RfnhpnzNiYe2YeKWTzIUxDt27RLXFHkB3QLAdLdayuRBWRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQlWY4GY2WFm1jKaHmRmN5rZj9MfTUQyXSp7ILcDm83sZ8BoEh+YPSOtqUQkK6RSIGWe+PzLk4Cb3f1mIDe9sUQkG6Ty4dolZnYFMAg4wswaA03SG0tEskEqeyCnA98BQ919NdAZuD6tqUQkK9S4BxKVxo1J85+hcyAiQjUFYmYlgFe2CnB3b522VCKSFaosEHfXiVIRqVZKF5KZ2eFmNiSaLjSz3dMbS0SyQSoXkl0JjAGuiBY1BWamM5SIZIdU9kB+CQwANgG4+yp0HYiIkFqBfB9dSOYAFZe1i4ikUiAPmtlkoK2ZnQ88C0xNbywRyQapXAdyg5n1A4qB7sB4d38m7clEJOOlcik7wFKgOYnDmKXpiyMi2SSVZ2HOAxYCpwC/Al41s3PTHUxEMl8qeyCXA/u7exGAmRUArwDT0hlMRDJfKidRVwIlSfMlwOfpiSMi2aS618KMiia/AF4zs8dInAM5icQhjYg0cNUdwlRcLPbP6FbhsfTFEZFsUt2L6SbUZRARyT41nkQ1s3Yk3gu1J9CsYrm790ljLhHJAqmcRL0PWA7sDkwAVgCL0phJRLJEKgVS4O53AaXu/oK7nwscnOZcIpIFUrkOpDT6+qWZnQCsAnZLXyQRyRaWeKFtNQPM+gMvAT8CJgGtgQnuPiedwdZtLq8+mGSsDd+W1jxIMlbXgmaW6tgaCyQuKpDspQLJbrUpkOouJJtE5W+qDIC7X1zLXCJSz1R3DuT1OkshIlmpugvJ7qnLICKSfVJ6V3YRkcqoQEQkmApERIKl8o5k3c3sOTN7J5rf18zGpT+aiGS6VPZAppL4UKlSAHdfApyRzlAikh1SKZAW7r79GwiVpSOMiGSXVApkjZn9hH99sNSvgC/TmkpEskIqL6YbDkwBepjZF8AnwKC0phKRrJDKB0t9DPSNPtKykbuX1HQfEWkYUnlHsvHbzQPg7lenKZOIZIlUDmE2JU03A/oD76Unjohkk1q/nN/MdgHmuPsx6YmUoJfzZy+9nD+71ebl/CFXorYA9gi4n4jUM6mcA1nKv94XpDHQDtD5DxFJ6RxI/6TpMuArd9eFZCJSfYGYWSPgCXffp47yiEgWqfYciLtvBd42sy51lEdEskgqhzAdgWVmtpCkp3TdfUDaUolIVkilQPQZuSJSqVQK5Hh3H5O8wMz+DLyQnkgiki1SuQ6kXyXLjtvZQUQk+1T3uTAXAr8B9jCzJUmrcoH56Q4mIpmvykvZzawNkAdcB/wuaVWJu69NdzBdyp69dCl7dtspn0zn7huADcCZOyOUiNQ/eld2EQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmmAhGRYCoQEQmWyifTyQ74dMUnjBszatv8F1+sZNiFF3HGwME8OGsmDz9wP40bN+bQXkdy0cjLYkwqFf574nhem/8ibfPymXLfIwAUF2/gj38YzVdfrqJDx06MveZ6clu3BuDtNxZxx83XU1ZWSps2edxw27Q449epKj8XJm718XNhysvLOfGY3tw1YzarvljJ9Dsnc+OkO2jatClr1xaRn18Qd8SdIts/F2bpm4tp1qIF1189dluB3HnrX8nNbc3pg4fywIy7KCkp5rzhl7CxpJhLfn0OE2+8jfa7dmT92iLaZvnPsTafC6NDmDr0+sJX6bxbFzp26swjD81m8JDzaNq0KUC9KY/64Kf7H7Bt76LCgpfm0ff4AQD0PX4AC16aB8C8uf/gsCOPpv2uHQGyvjxqSwVSh555+kn+49jjAfjs0xW8/eZizj37dC4cOph3ly2NOZ1UZ93atRQUtgOgoLAd69clPpxx5eefsrGkmMuHD2X4kDN45h9/jzNmnavzAjGzIdWsG2Zmr5vZ63dPm1qXsdKutPR7XnphHn36HQMkDmeKi4u5a8ZsRlxyGWNHjyJTDyelauXlZXz4/rtcc8Mk/vjX27l/+hRWfrYi7lh1Jo6TqBOA6ZWtcPcpwBSof+dAFrz8Env12JuCgkIA2nfYld5H98PM6LnPvjRq1Ij169aRl58fc1KpTF5+PkVrvqGgsB1Fa76hbV7i59SuXQfatMmjWfMWNGvegp/u93M+/ugDduvSNd7AdSQteyBmtqSK21KgQzoeM9PNfepfhy8AR/Tuw+KFrwGJw5nS0lLa5uXFFU9qcPDhvXn2yTkAPPvkHA7pdRQAhxxxFO+8/QblZWVs2fIty5ctpcuPd48zap1Ky7MwZvYVcAywbvtVwCvu3qmmbdSnPZAt337LgOP68Mjf59IqNxdIHNJce9U4Pnx/OTlNmnDxJZdz4EEHx5x058j2Z2GuGz+GJW++zob168nLz+fs8y7k0CP6MHHc5Xz91Wrad9iVsRNvoHXrNgA8dN/dzH3iMcyMYwecwimnD4r3G9hBtXkWJl0Fchcw3d1frmTd/e5+Vk3bqE8F0tBke4E0dLEXyM6gAsleKpDsputARKROqEBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCqUBEJJgKRESCmbvHnaFBMrNh7j4l7hwSRj+/BO2BxGdY3AFkh+jnhwpERHaACkREgqlA4tPgj5+znH5+6CSqiOwA7YGISDAViIgEU4HEwMyONbP3zewjM/td3HkkdWY2zcy+NrN34s6SCVQgdczMGgO3AscBewNnmtne8aaSWrgbODbuEJlCBVL3DgI+cveP3f17YDZwUsyZJEXu/iKwNu4cmUIFUvc6A58nza+MlolkHRVI3bNKlum5dMlKKpC6txL4UdL8bsCqmLKI7BAVSN1bBHQzs93NrClwBjAn5kwiQVQgdczdy4ARwNPAe8CD7r4s3lSSKjObBSwA9jKzlWY2NO5McdKl7CISTHsgIhJMBSIiwVQgIhJMBSIiwVQgIhJMBSLBzGxj9LWTmT1cw9iRZtYiaf5JM2ub7oySXnoaV37AzBq7e3mKYze6e6sUx64ADnT3NTuSTzKL9kAaEDPrambLzeweM1tiZg+bWQszW2Fm483sZeA0M/uJmT1lZovN7CUz6xHdf3czW2Bmi8zsmu22+0403djMbjCzpdFjXGRmFwOdgHlmNi8at8LMCqPpUWb2TnQbmbTN98xsqpktM7O5ZtY8Wnexmb0bbX92nf4jyg+5u24N5AZ0JfHCvcOi+WnAZcAKYHTSuOeAbtH0vwP/F03PAQZH08OBjUnbfSeavhD4XyAnms+Pvq4ACpMeYwVQCBwALAVaAq2AZcD+0TbLgP2i8Q8Cg6LpVcAu0XTbuP9dG/JNeyANz+fuPj+angkcHk0/AGBmrYBDgYfM7C1gMtAxGnMYMCuavreK7fcF7vDEJfu4e03vnXE48Ki7b3L3jcAjQK9o3Sfu/lY0vZhEqQAsAe4zs0EkSkZikhN3AKlz25/0qpjfFH1tBKx39/1SvP/2LIUx24+vyndJ0+VA82j6BOAIYADwBzPrWVFYUre0B9LwdDGzQ6LpM4GXk1e6ezHwiZmdBmAJP4tWzyfx6mGAgVVsfy5wgZnlRPfPj5aXALmVjH8RODk6F9MS+CXwUlXhzawR8CN3nweMBtqSOPSRGKhAGp73gHPMbAmQD9xeyZiBwFAze5vEOYmKt1z8LTDczBYBbarY/p3AZ8CS6P5nRcunAP+oOIlawd3fIPE+owuB14A73f3NavI3Bmaa2VLgTeCv7r6+mvGSRnoatwExs67A4+6+T8xRpJ7QHoiIBNMeiIgE0x6IiARTgYhIMBWIiARTgYhIMBWIiAT7f2G81M+HqA8YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAEWCAYAAACuU8gIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFAlJREFUeJzt3Xmc1XW9x/HXm6EUWUQYMsUQF9ArPizTFjcs0wo3zKspouZSprlxyUTUXNP0RpapVwPFFBXE9KapqWWm4ga4BBjkisnOyA5dncHP/eP8xo40zBy+w5nfHOb9fDzOY37L9/zOexzm7ff8zm/OUURgZpaiXd4BzKxyuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLpI2Q1EHS7yUtlXRPM44zWNJj6zNbXiTtI+nveeeoZPJ1IK2LpGOAocCOwHLgFeCKiJjQzOMeB5wJ7BkRdc0O2spJCqBPRLyRd5YNmWcgrYikocAvgSuBzYFewP8AA9fD4bcGXmsL5VEKSe3zzrBBiAjfWsEN2BRYARzZyJiNKBTMnOz2S2CjbN9XgFnAD4EFwFzgxGzfpcAHQG32GCcDlwB3FB27NxBA+2z9BOAtCrOgt4HBRdsnFN1vT2ASsDT7umfRvr8AlwPPZMd5DKhey/dWn//covyHAQcCrwGLgPOLxn8ReA5Yko29Hvhktu+p7HtZmX2/RxUdfxgwDxhTvy27z3bZY3w+W98SqAG+kve/jdZ8yz2Ab9kPAr4J1NX/Aq9lzGXA88CngB7As8Dl2b6vZPe/DPhE9ou3Ctgs279mYay1QICOwDJgh2zfFkC/bPmjAgG6AYuB47L7DcrWu2f7/wK8CfQFOmTrV63le6vPf1GW/3vAQuAuoDPQD/g/YNts/G7Al7PH7Q1MB4YUHS+A7Rs4/tUUirhDcYFkY76XHWcT4FFgRN7/Llr7zU9hWo/uQE00/hRjMHBZRCyIiIUUZhbHFe2vzfbXRsTDFP7vu0Ning+BnSV1iIi5EfFqA2MOAl6PiDERURcRY4EZwCFFY26NiNci4p/AeOBzjTxmLYXzPbXAOKAauDYilmeP/yqwC0BEvBgRz2ePOxP4NbBvCd/TxRHxfpbnYyJiFPA68AKF0rygieO1eS6Q1uM9oLqJ5+ZbAu8Urb+TbfvoGGsU0Cqg07oGiYiVFKb9pwJzJT0kaccS8tRn6lm0Pm8d8rwXEauz5fpf8PlF+/9Zf39JfSU9KGmepGUUzhtVN3JsgIUR8X9NjBkF7AxcFxHvNzG2zXOBtB7PUZiiH9bImDkUTobW65VtS7GSwlS93qeLd0bEoxFxAIX/E8+g8IvVVJ76TLMTM62LGynk6hMRXYDzATVxn0ZfcpTUicJ5pVuASyR1Wx9BN2QukFYiIpZSeP5/g6TDJG0i6ROSBkj672zYWOBCST0kVWfj70h8yFeA/pJ6SdoUGF6/Q9Lmkg6V1BF4n8JTodUNHONhoK+kYyS1l3QUsBPwYGKmddGZwnmaFdns6LQ19s8Htl3HY14LvBgR3wUeAm5qdsoNnAukFYmIayhcA3IhhROI7wJnAL/LhvwEmAxMAaYCL2XbUh7rj8Dd2bFe5OO/9O0ovJozh8IrE/sCP2jgGO8BB2dj36PwCsrBEVGTkmkdnQMcQ+HVnVEUvpdilwC3SVoi6dtNHUzSQAonsk/NNg0FPi9p8HpLvAHyhWRmlswzEDNL5gIxs2QuEDNL5gIxs2St9g+KOux6hs/uVqjFk67PO4I1w8btm7ye5iOegZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVzgZhZMheImSVrn3eADdVNFw9mQP+dWbhoObsfeSUAF3z/QE46fE8WLl4BwMXXP8CjE/5Gry268cp9F/LaOwsAmDh1JmddMS637PYvF104nKee/AvdunXnvvsfBODGG67j3t+Op9tm3QA4c8hQ9um/b54xc+MCKZMxv3+em+5+kpsvP/5j26+74wl+Oebxfxv/1qwavnz0VS0Vz0o08LDDGXTMsVwwfNjHth93/Al858STc0rVepStQCTtCAwEegIBzAEeiIjp5XrM1uSZl96k1xbd8o5hzbTb7l9g9uxZecdotcpyDkTSMGAcIGAiMClbHivpvHI8ZqU49ej+TLx7ODddPJiunTt8tL13z+48N3YYj918Nnvtul2OCa0U4+66kyO+dQgXXTicZUuX5h0nN+U6iXoy8IWIuCoi7shuVwFfzPY1SNIpkiZLmlxX82qZouVn1D1Ps9Mhl/Clo69iXs0yrhp6OADzapbRd8BF7DHoaob9/D5+c+UJdO64cc5pbW2+fdQgHnzkj4y/93569PgUI37Wdp96lqtAPgS2bGD7Ftm+BkXEyIjYPSJ2b1/dr0zR8rNg0XI+/DCICEbf9wy777w1AB/U1rFo6UoAXp7+Lm/NqqHP1p/KM6o1ont1NVVVVbRr147DjziSaVOn5h0pN+U6BzIEeFzS68C72bZewPbAGWV6zFbv09VdmFezDICB+32Wv705F4DqzTqxaOlKPvww6N2zO9v36sHbs2ryjGqNWLhwAT16FAr+z3/6E9v36ZNzovyUpUAi4hFJfSk8ZelJ4fzHLGBSRKwux2O2Nrf99AT22a0P1V078cYjl3P5TQ/Tf7c+7LLDVkQE78xdxJk/GQvA3p/fnh+fdhB1q1ezenVw5hXjWLxsVc7fgQEMO2cokydNZMmSxRywX39OO/1MJk+ayN9nzECCLbfsyY8vuSzvmLlRROSdoUEddj2jdQazJi2edH3eEawZNm6PSh3rK1HNLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySuUDMLJkLxMySNVkgkvaS1DFbPlbSNZK2Ln80M2vtSpmB3AiskvRZ4FzgHeD2sqYys4pQSoHUReHzLwcC10bEtUDn8sYys0pQyodrL5c0HDgW6C+pCvhEeWOZWSUoZQZyFPA+cHJEzAN6Aj8rayozqwhNzkCy0rimaP0f+ByImdFIgUhaDkRDu4CIiC5lS2VmFWGtBRIRPlFqZo0q6UIySXtLOjFbrpa0TXljmVklKOVCsouBYcDwbNMngTvKGcrMKkMpM5BvAYcCKwEiYg6+DsTMKK1APsguJAuA+svazcxKKZDxkn4NdJX0PeBPwKjyxjKzSlDKdSAjJB0ALAP6AhdFxB/LnszMWr1SLmUHmAp0oPA0Zmr54phZJSnlVZjvAhOBw4EjgOclnVTuYGbW+pUyA/kRsGtEvAcgqTvwLDC6nMHMrPUr5STqLGB50fpy4N3yxDGzStLY38IMzRZnAy9Iup/COZCBFJ7SmFkb19hTmPqLxd7MbvXuL18cM6skjf0x3aUtGcTMKk+TJ1El9aDwXqj9gI3rt0fEfmXMZWYVoJSTqHcCM4BtgEuBmcCkMmYyswpRSoF0j4hbgNqIeDIiTgK+XOZcZlYBSrkOpDb7OlfSQcAcYKvyRTKzSqHCH9o2MkA6GHga+AxwHdAFuDQiHihnsIUr6hoPZq3WgqXv5x3BmqFfz44qdWwpf0z3YLa4FPhqaigz2/A0diHZdTT8psoARMRZZUlkZhWjsRnI5BZLYWYVqbELyW5rySBmVnlKeld2M7OGuEDMLJkLxMySlfKOZH0lPS5pWra+i6QLyx/NzFq7UmYgoyh8qFQtQERMAY4uZygzqwylFMgmEbHmGwjVlSOMmVWWUgqkRtJ2/OuDpY4A5pY1lZlVhFL+mO50YCSwo6TZwNvAsWVNZWYVoZS/hXkL2D/7SMt2EbG8qfuYWdtQyjuSXbTGOgARcVmZMplZhSjlKczKouWNgYOB6eWJY2aVpJSnMD8vXpc0Aijre4GYWWVIuRJ1E2Db9R3EzCpPKedApvKv9wWpAnoAPv9hZiWdAzm4aLkOmB8RvpDMzBovEEntgIciYucWymNmFaTRcyAR8SHwV0m9WiiPmVWQUp7CbAG8KmkiRS/pRsShZUtlZhWhlALxZ+SaWYNKKZADI2JY8QZJVwNPlieSmVWKUq4DOaCBbQPWdxAzqzyNfS7MacAPgG0lTSna1Rl4ptzBzKz1a+wpzF3AH4CfAucVbV8eEYvKmsrMKkJjnwuzlMLHWQ5quThmVkn8ruxmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFYmbJXCBmlswFUmbz583lzFNOYPB/HsKxRx7K+LvGfGz/Xbffyt679WPJ4sU5JbRSrF69mh+eMogrzj8LgPlzZzPsB8dz+nEDGXHZMGpra3NOmA8XSJlVVbXnjP86lzvv/T0jfzOW++4Zy9tvvQEUymXyC8+y+ae3yDmlNeWh+8ayVa9tPlofM/JXHHLEYG4Ycz+dOnfh8Yd/l2O6/LhAyqy6Rw92+I+dANikY0d6b7MtNQsWAHDdNVdz2tk/RFKeEa0JNQvn8+LzT7P/gYcBEBFMfXkSe+z7NQC++vWDmfjME3lGzI0LpAXNnTOb12ZMZ6edd2HCk3+musfm9Om7Y96xrAmjbxjB8d8/G7Ur/LosX7aEjp06UVVV+Fy27j02572ahXlGzE2LF4ikExvZd4qkyZIm3z56VEvGKrtVq1ZywY+GcPY551FVVcVtt4zku6eekXcsa8Lk555i067d2K7vTh9ti/j3cW11FtnYZ+OWy6XArQ3tiIiRwEiAhSvqGvgxVaa62lou/NEQvj7gIPbd7wDefP015s6ZzQmDDgdg4YL5nDT4CEbdPo7u1T1yTmvFZkz7K5OefZKXXphA7QcfsGrVSkbfMIKVK1awenUdVVXteW/hfLp1r847ai4UDdVpcw8qTVnbLqBvRGzU1DE2lAKJCH5y8fl06dKFs88Z3uCYIw4+gJvHjKfrZpu1cLryWLD0/bwjlMW0VyZz//jbueDKX/GzS85lj/5fY+/9vsFNv7iCrbftw4CB38474nrRr2fHkqdT5ZqBbA58A1jztUkBz5bpMVulKa+8xKMPPcB22/f9aMbx/dOHsMfe/XNOZs1x3Clncc3lw7lr9A1ss/2O7D/gsLwj5aJcM5BbgFsjYkID++6KiGOaOsaGMgNpizbUGUhbsS4zkLIUyPrgAqlcLpDKti4F4pdxzSyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZC8TMkrlAzCyZIiLvDG2SpFMiYmTeOSyNf34FnoHk55S8A1iz+OeHC8TMmsEFYmbJXCD5afPPnyucf374JKqZNYNnIGaWzAViZslcIDmQ9E1Jf5f0hqTz8s5jpZM0WtICSdPyztIauEBamKQq4AZgALATMEjSTvmmsnXwG+CbeYdoLVwgLe+LwBsR8VZEfACMAwbmnMlKFBFPAYvyztFauEBaXk/g3aL1Wdk2s4rjAml5amCbX0u3iuQCaXmzgM8UrW8FzMkpi1mzuEBa3iSgj6RtJH0SOBp4IOdMZklcIC0sIuqAM4BHgenA+Ih4Nd9UVipJY4HngB0kzZJ0ct6Z8uRL2c0smWcgZpbMBWJmyVwgZpbMBWJmyVwgZpbMBWLJJK3Ivm4p6bdNjB0iaZOi9YcldS13Risvv4xrHyOpKiJWlzh2RUR0KnHsTGD3iKhpTj5rXTwDaUMk9ZY0Q9JtkqZI+q2kTSTNlHSRpAnAkZK2k/SIpBclPS1px+z+20h6TtIkSZevcdxp2XKVpBGSpmaPcaaks4AtgSckPZGNmympOlseKmladhtSdMzpkkZJelXSY5I6ZPvOkvS37PjjWvQ/on1cRPjWRm5Abwp/uLdXtj4aOAeYCZxbNO5xoE+2/CXgz9nyA8Dx2fLpwIqi407Llk8D7gXaZ+vdsq8zgeqix5gJVAO7AVOBjkAn4FVg1+yYdcDnsvHjgWOz5TnARtly17z/u7blm2cgbc+7EfFMtnwHsHe2fDeApE7AnsA9kl4Bfg1skY3ZCxibLY9Zy/H3B26KwiX7RERT752xN/C/EbEyIlYA9wH7ZPvejohXsuUXKZQKwBTgTknHUigZy0n7vANYi1vzpFf9+srsaztgSUR8rsT7r0kljFlz/Nq8X7S8GuiQLR8E9AcOBX4sqV99YVnL8gyk7eklaY9seRAwoXhnRCwD3pZ0JIAKPpvtfobCXw8DDF7L8R8DTpXUPrt/t2z7cqBzA+OfAg7LzsV0BL4FPL228JLaAZ+JiCeAc4GuFJ76WA5cIG3PdOA7kqYA3YAbGxgzGDhZ0l8pnJOof8vFs4HTJU0CNl3L8W8G/gFMye5/TLZ9JPCH+pOo9SLiJQrvMzoReAG4OSJebiR/FXCHpKnAy8AvImJJI+OtjPwybhsiqTfwYETsnHMU20B4BmJmyTwDMbNknoGYWTIXiJklc4GYWTIXiJklc4GYWbL/B1fQdDx/cKZvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = GaussianNB()\n",
    "\n",
    "scores = cross_validate(model, x_train, y_train, cv=5, n_jobs=10)\n",
    "print(\"cv scores: {}\".format(scores['test_score']))\n",
    "print(\"avg score: {}, std score: {}, min: {}, max: {}\".format(np.mean(scores['test_score']), np.std(scores['test_score']), \n",
    "                                                             np.min(scores['test_score']), np.max(scores['test_score'])))\n",
    "\n",
    "model.fit(x_train, y_train)\n",
    "\n",
    "pred_test = model.predict(x_train)  # Predict labels of test data using the trained classifier\n",
    "c_matrix = confusion_matrix(y_train, pred_test) \n",
    "\n",
    "rf_accuracy = accuracy_score(y_train, pred_test)\n",
    "print(\"Accuracy of random forest TRAINING: {:.2f}\".format(rf_accuracy))\n",
    "print(\"\\nClassification report:\")\n",
    "print(classification_report(y_train, pred_test))\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(4, 4))\n",
    "ax.set_title(\"Confusion matrix\")\n",
    "sns.heatmap(c_matrix, cmap='Blues', annot=True, fmt='g', cbar=False)\n",
    "ax.set_xlabel('predictions')\n",
    "ax.set_ylabel('true labels')\n",
    "\n",
    "pred_test = model.predict(x_val)  # Predict labels of test data using the trained classifier\n",
    "c_matrix = confusion_matrix(y_val, pred_test) \n",
    "rf_accuracy = accuracy_score(y_val, pred_test)\n",
    "print(\"Accuracy of random forest VALIDATION: {:.2f}\".format(rf_accuracy))\n",
    "print(\"\\nClassification report:\")\n",
    "print(classification_report(y_val, pred_test))\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(4, 4))\n",
    "ax.set_title(\"Confusion matrix\")\n",
    "sns.heatmap(c_matrix, cmap='Blues', annot=True, fmt='g', cbar=False)\n",
    "ax.set_xlabel('predictions')\n",
    "ax.set_ylabel('true labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_model = deepcopy(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = deepcopy(original_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.651341, 0.348659])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.class_prior_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'GaussianNB' object has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m-------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-b367482bb17c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mfeature\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimportance\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchosen_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"feature name: {}, importance : {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimportance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdpi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m80\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Plotting feature importance\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"importance level\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'GaussianNB' object has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "for feature, importance in zip(chosen_features, model.feature_importances_):\n",
    "    print(\"feature name: {}, importance : {}\".format(feature, importance))\n",
    "plt.figure( figsize=(30, 10), dpi=80)\n",
    "plt.title(\"Plotting feature importance\")\n",
    "plt.ylabel(\"importance level\")\n",
    "plt.xlabel(\"feature indexes\")\n",
    "#plt.bar(np.append(chosen_features, np.arange(0,6)), model.feature_importances_)\n",
    "plt.bar(np.arange(0, len(model.feature_importances_)), model.feature_importances_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Optimization for RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of random forest TRAINING: 0.84\n",
      "\n",
      "Classification report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.86      0.96      0.91      3156\n",
      "          1       0.74      0.40      0.52       852\n",
      "\n",
      "avg / total       0.83      0.84      0.82      4008\n",
      "\n",
      "Accuracy of random forest VALIDATION: 0.90\n",
      "\n",
      "Classification report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.97      0.94      1056\n",
      "          1       0.60      0.38      0.46       138\n",
      "\n",
      "avg / total       0.88      0.90      0.89      1194\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(15,0.5,'true labels')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAEWCAYAAACuU8gIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFsRJREFUeJzt3XmcleP/x/HXZ2YamlTalJCdKLLztSS+2cm+la0iRUj2LWTfd1IKXxHZQ35RJEuoSIuyJRVKaZ+2WT6/P849OWWaTleduec07+fjcR5zL9d93Z/TmXl339d9n3PM3RERCZEVdwEikrkUICISTAEiIsEUICISTAEiIsEUICISTAFSSZhZVTN7x8zmmdmra9FPGzP7YF3WFhczO8jMfoi7jkxmug+kYjGz1kBXoDGwABgN3OHun61lv2cDlwD7u3vhWhdawZmZA9u7+89x17I+0xFIBWJmXYGHgTuB+kAj4Eng+HXQ/ZbAj5UhPFJhZjlx17BecHc9KsADqAksBE4to80GJALmj+jxMLBBtK4FMA24AvgL+BNoG627FVgGFET7aA/cAvRN6nsrwIGcaP48YBKJo6BfgTZJyz9L2m5/YAQwL/q5f9K6ocBtwOdRPx8AdVfx3Erqvzqp/hOAo4EfgdnA9Unt9wGGA3Ojto8DudG6YdFzyY+e7+lJ/V8DTAdeKFkWbbNttI89ovmGwCygRdy/GxX5EXsBekQvBBwJFJb8Aa+iTXfgS2AToB7wBXBbtK5FtH13oEr0h7cIqBWtXzkwVhkgQDVgPrBjtG5ToEk0vTxAgNrAHODsaLszo/k60fqhwC/ADkDVaP7uVTy3kvq7RfVfAMwEXgKqA02AJcA2Ufs9gf2i/W4FTAC6JPXnwHal9H8PiSCumhwgUZsLon7ygEHA/XH/XlT0h05hKo46wCwv+xSjDdDd3f9y95kkjizOTlpfEK0vcPeBJP733TGwnmKgqZlVdfc/3X18KW2OAX5y9xfcvdDd+wETgeOS2jzr7j+6+2KgP7BbGfssIDHeUwC8DNQFHnH3BdH+xwO7Arj7KHf/MtrvZOBp4OAUntPN7r40qmcF7t4L+An4ikRo3rCa/io9BUjF8TdQdzXn5g2B35Lmf4uWLe9jpQBaBGy0poW4ez6Jw/6OwJ9m9p6ZNU6hnpKaNkuan74G9fzt7kXRdMkf+Iyk9YtLtjezHczsXTObbmbzSYwb1S2jb4CZ7r5kNW16AU2Bx9x96WraVnoKkIpjOIlD9BPKaPMHicHQEo2iZSHySRyql2iQvNLdB7n7YST+J55I4g9rdfWU1PR7YE1r4ikSdW3v7jWA6wFbzTZlXnI0s41IjCv1Bm4xs9rrotD1mQKkgnD3eSTO/58wsxPMLM/MqpjZUWZ2b9SsH3CjmdUzs7pR+76BuxwNNDezRmZWE7iuZIWZ1TezVmZWDVhK4lSoqJQ+BgI7mFlrM8sxs9OBnYF3A2taE9VJjNMsjI6OOq20fgawzRr2+Qgwyt3PB94Deqx1les5BUgF4u4PkrgH5EYSA4hTgc7AW1GT24GRwBhgLPBNtCxkXx8Cr0R9jWLFP/osEldz/iBxZeJg4KJS+vgbODZq+zeJKyjHuvuskJrW0JVAaxJXd3qReC7JbgGeN7O5Znba6jozs+NJDGR3jBZ1BfYwszbrrOL1kG4kE5FgOgIRkWAKEBEJpgARkWAKEBEJVmHfUFR1984a3c1Qc0Y8HncJshY2zFnt/TTL6QhERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkmAJERIIpQEQkWE7cBawvNsjNYXDvLuTm5pCTnc2bg7/l9h4D2bJhHV64uy21auYxesJU2t34PwoKizj/lAO58LTmFBUXk79oKRff3o+Jk6Yv72+LBrX45vUbuaPHQB5+YUiMz6zy6XbjdQz7ZCi1a9fhjbffBeDB++/hk6EfU6VKFTbfohHdb7+LGjVqMPyLz3nkoQcoKCigSpUqXH7FVey7339ifgblx9w97hpKVXX3zhWzsDJUq5pL/uJl5ORk8VGfrlx532tcetahvP3Rd7w6aBSP3nAGY3+cRq9XP6N6tQ1ZkL8EgGMO3oUOpx7E8Z2fXN5Xv/vPp7i4mBFjf8u4AJkz4vG4S1gro0aOIC8vjxuuu2Z5gHzx+Wfss+9+5OTk8NAD9wFw+RVXMWHC99SpU4dNNqnPTz/9SKcO7Rn88adxlr/WNszBUm2btlMYM2tsZteY2aNm9kg0vVO69lcR5C9eBkCVnGxycrJxdw7eewfeGPwtAC++8xXHtWgGsDw8IBE8zj95eVyLXfl12iy+/2U6Uv723GtvatSsucKy/Q84kJycxAH7rs12468Ziddmp512ZpNN6gOw3Xbbs2zpMpYtW1a+BccoLQFiZtcALwMGfA2MiKb7mdm16dhnRZCVZXz58rVMGXI3H305kUnTZjFvwWKKiooB+H3GHBpu8s8v5oWnNWf8gJu547ITuOLe1wDI2zCXK9oexh1PD4zlOcjqvfXG6xxwUPN/LR/8wSAa77QTubm5MVQVj3SNgbQHmrh7QfJCM3sQGA/cXdpGZtYB6ACQs3kLcuo2SVN56VFc7Ox3xt3U3Kgqrzx4AY23bvCvNslnjE/3H8bT/Ydx+pF7ce35R3JBtxe4qdMxPNb3o+VHM1Kx9Hr6KbJzsjnm2FYrLP/55594+KH76dGzT0yVxSNdAVIMNAR+W2n5ptG6Url7T6AnZOYYSIl5CxczbORP7LPLVtSsXpXs7CyKiorZrH4t/pw571/t+w8axSPXnw7A3k235MSWu3FHlxOoWb0qxcXOkmUF9HhlWHk/DVnJgLfeZNgnQ+nZ+znM/hkmmDF9Opdf2pnb77yHLRo1irHC8peuAOkCDDGzn4Cp0bJGwHZA5zTtM1Z1a21EQUER8xYuZsMNqnDovjvywHODGTbyR05quTuvDhpFm+P25d2hYwDYtlE9fpkyE4CjDmrCz1MT0y3bP7y8zxsuPJr8RUsVHhXA558O49nevej9fF+qVq26fPn8+fPp3KkDl3Xpyu577BljhfFIS4C4+/+Z2Q7APsBmJMY/pgEj3L0oHfuMW4O6NejV/Wyys7LIyjJe//Ab3v90HBMm/ckLd7fl5ouO5bsfpvLcW8MB6HR6cw7ZtzEFhUXMnb+IC276X8zPQEpcc2VXRo74mrlz53DYoc3pdPEl9OnVk2UFy+h4flsAdmnWjJtu7s7LL/VlytQp9OzxJD17JK6iPdWrD3Xq1InzKZQbXcaVdS7TL+NWdhXiMq6IrP8UICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISbLUBYmYHmFm1aPosM3vQzLZMf2kiUtGlcgTyFLDIzJoBV5P4wmx9D6OIpBQghZ74/svjgUfc/RGgenrLEpFMkMqXay8ws+uAs4DmZpYNVElvWSKSCVI5AjkdWAq0d/fpwGbAfWmtSkQywmqPQKLQeDBpfgoaAxERyggQM1sAeGmrAHf3GmmrSkQywioDxN01UCoiZUrpRjIzO9DM2kbTdc1s6/SWJSKZIJUbyW4GrgGuixblAn3TWZSIZIZUjkBOBFoB+QDu/ge6D0RESC1AlkU3kjlAyW3tIiKpBEh/M3sa2NjMLgAGA73SW5aIZIJU7gO538wOA+YDOwDd3P3DtFcmIhVeKreyA4wFqpI4jRmbvnJEJJOkchXmfOBr4CTgFOBLM2uX7sJEpOJL5QjkKmB3d/8bwMzqAF8AfdJZmIhUfKkMok4DFiTNLwCmpqccEckkZb0Xpms0+TvwlZm9TWIM5HgSpzQiUsmVdQpTcrPYL9GjxNvpK0dEMklZb6a7tTwLEZHMs9pBVDOrR+KzUJsAG5Ysd/dD01iXiGSAVAZRXwQmAlsDtwKTgRFprElEMkQqAVLH3XsDBe7+ibu3A/ZLc10ikgFSuQ+kIPr5p5kdA/wBbJ6+kkQkU1jijbZlNDA7FvgU2AJ4DKgB3OruA9JZ2KSZS8ouTCqsahtkx12CrIX6NapYqm1XGyBxUYBkLgVIZluTACnrRrLHKP1DlQFw90vXsC4RWc+UNQYystyqEJGMVNaNZM+XZyEiknlS+lR2EZHSKEBEJJgCRESCpfKJZDuY2RAzGxfN72pmN6a/NBGp6FI5AulF4kulCgDcfQxwRjqLEpHMkEqA5Ln7yh8gVJiOYkQks6QSILPMbFv++WKpU4A/01qViGSEVN5MdzHQE2hsZr8DvwJnpbUqEckIqXyx1CSgZfSVllnuvmB124hI5ZDKJ5J1W2keAHfvnqaaRCRDpHIKk580vSFwLDAhPeWISCZZ47fzm9kGwAB3PyI9JSXo7fyZS2/nz2xr8nb+kDtR84BtArYTkfVMKmMgY/nnc0GygXqAxj9EJKUxkGOTpguBGe6uG8lEpOwAMbMs4D13b1pO9YhIBilzDMTdi4HvzKxROdUjIhkklVOYTYHxZvY1SZd03b1V2qoSkYyQSoDoO3JFpFSpBMjR7n5N8gIzuwf4JD0liUimSOU+kMNKWXbUui5ERDJPWd8L0wm4CNjGzMYkraoOfJ7uwkSk4ivrFOYl4H3gLuDapOUL3H12WqsSkYxQ1vfCzAPmAWeWXzkikkn0qewiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEkwBIiLBFCAiEiyVb6aTAOeechR5eXlkZWWTnZ3No737sWD+PO7qdjUzpv9B/QYNua77fVSvUYP8hQu4t/v1zJwxnaKiQk4+81wOP+aEuJ9CpbV06VIu6XAuBQXLKCososV/D6PdhZ2Xr3/4vjt5/503GTRsxArbDR3yAd2u7UrP51+m8c5Ny7vsWChA0ujuR5+h5sa1ls/379uH3fbch9PObk//F3rTv29v2l90Oe+88QqNttqGW+99jLlzZnNB6+M55PBjqFKlSozVV165ubk8/FQf8vLyKCws4OLzz2Hf/Q+iyS7NmPj9OBYumP+vbRbl5/PaKy+yc9NdY6g4PjqFKUfDP/2Ylke1AqDlUa0Y/unHAJgZixctwt1ZsngR1WvUJDs7O85SKzUzIy8vD4DCwkIKCwsxM4qKinjq0QfoeOkV/9rmmR6P0frstuTm5pZ3ubFSgKSJGdzQtSOXtDuDgW+/BsDcObOpXbceALXr1mPenMQX/B138hlM/W0SbU5oSadzT6HjZVeTlaWXJk5FRUW0a30yxx/enL32/Q87N92VN/q/xAHND6Fu9BqW+PGHCfw1Yzr7H9QinmJjVO6/pWbWtox1HcxspJmN7Pe/3uVZ1jr3wFPP83ifV7jtgSd4941XGDt61CrbjvrqC7bZvjEvvjWYJ57tz5MP3UV+/sJyrFZWlp2dTZ+XXue194YwcfxYRn8zkqFDPuCk01qv0K64uJjHH7yHi7tcFVOl8Yrjv7lbV7XC3Xu6+17uvteZ57Qvz5rWuTp1NwFg41p12L/5ofzw/Tg2rlWb2bNmAjB71kxq1qoNwIcD3+aAg/+LmdFw80Y02HQzpv32a2y1yz+qV6/Bbnvuzbejvub3qVNofdLRnNbqcJYsWcKZJx7FokX5/PrLz1zWsS2ntTqc78eN4borLmHi9+PiLr1cpGUQ1czGrGoVUD8d+6xIlixeRLE7eXnVWLJ4Ed+MGE7r8y5kvwNbMPj9AZx2dnsGvz+A/xx0CAD16jdg9MivaNpsD+bM/ptpUybToOHmMT+LymvunNlk5+RQvXoNli5Zwqivv6T1Oe14a9Any9sc0Xxv+r35PgDvDP5s+fJLLzyPiy67Uldh1lJ94AhgzkrLDfgiTfusMObMns1t118OQFFRIS0OO5q99juAHXZqwp3drmLQe29Rr34DbrjtfgBan9eBB+64iU7nnIy7065TlxWu3kj5+nvWTO685QaKiovwYueQlkdUyvGNVJi7r/tOzXoDz7r7Z6Wse8ndW5ey2QomzVyy7guTclFtA11BymT1a1SxVNumJUDWBQVI5lKAZLY1CRBdKxSRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYObucddQKZlZB3fvGXcdEkavX4KOQOLTIe4CZK3o9UMBIiJrQQEiIsEUIPGp9OfPGU6vHxpEFZG1oCMQEQmmABGRYAqQGJjZkWb2g5n9bGbXxl2PpM7M+pjZX2Y2Lu5aKgIFSDkzs2zgCeAoYGfgTDPbOd6qZA08BxwZdxEVhQKk/O0D/Ozuk9x9GfAycHzMNUmK3H0YMDvuOioKBUj52wyYmjQ/LVomknEUIOXPSlmma+mSkRQg5W8asEXS/ObAHzHVIrJWFCDlbwSwvZltbWa5wBnAgJhrEgmiACln7l4IdAYGAROA/u4+Pt6qJFVm1g8YDuxoZtPMrH3cNcVJt7KLSDAdgYhIMAWIiARTgIhIMAWIiARTgIhIMAWIBDOzhdHPhmb22mradjGzvKT5gWa2cbprlPTSZVxZgZllu3tRim0XuvtGKbadDOzl7rPWpj6pWHQEUomY2VZmNtHMnjezMWb2mpnlmdlkM+tmZp8Bp5rZtmb2f2Y2ysw+NbPG0fZbm9lwMxthZret1O+4aDrbzO43s7HRPi4xs0uBhsDHZvZx1G6ymdWNprua2bjo0SWpzwlm1svMxpvZB2ZWNVp3qZl9H/X/crn+I8qK3F2PSvIAtiLxxr0Dovk+wJXAZODqpHZDgO2j6X2Bj6LpAcA50fTFwMKkfsdF052A14GcaL529HMyUDdpH5OBusCewFigGrARMB7YPeqzENgtat8fOCua/gPYIJreOO5/18r80BFI5TPV3T+PpvsCB0bTrwCY2UbA/sCrZjYaeBrYNGpzANAvmn5hFf23BHp44pZ93H11n51xIPCmu+e7+0LgDeCgaN2v7j46mh5FIlQAxgAvmtlZJEJGYpITdwFS7lYe9CqZz49+ZgFz3X23FLdfmaXQZuX2q7I0aboIqBpNHwM0B1oBN5lZk5LAkvKlI5DKp5GZ/SeaPhP4LHmlu88HfjWzUwEsoVm0+nMS7x4GaLOK/j8AOppZTrR97Wj5AqB6Ke2HASdEYzHVgBOBT1dVvJllAVu4+8fA1cDGJE59JAYKkMpnAnCumY0BagNPldKmDdDezL4jMSZR8pGLlwEXm9kIoOYq+n8GmAKMibZvHS3vCbxfMohawt2/IfE5o18DXwHPuPu3ZdSfDfQ1s7HAt8BD7j63jPaSRrqMW4mY2VbAu+7eNOZSZD2hIxARCaYjEBEJpiMQEQmmABGRYAoQEQmmABGRYAoQEQn2/wfYb2TIeJLAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAEWCAYAAACuU8gIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFOhJREFUeJzt3XmYFNW5x/HvOzMguyyDKAbEBSTRuCTEBXcNriDqEzQCJiLqdSEuxIsb4ha3xGskalBADIoKKCiiRlwxgCBLogPENREF2RmEYWeG9/7RNaQls/Qc6Knumd/nefqZqjqnT79NM7+pOl3dZe6OiEiInLgLEJHspQARkWAKEBEJpgARkWAKEBEJpgARkWAKkFrCzOqb2UQzW2NmL+zEOL3M7M1dWVtczOw4M/ss7jqymek8kMxiZj2B/kBHoAj4CLjH3afu5LgXAb8BOrt78U4XmuHMzIH27v5l3LXUZNoDySBm1h94GLgXaAW0Bf4MdN8Fw+8DfF4bwiMVZpYXdw01grvrlgE3YHdgHdCjgj67kQiYxdHtYWC3qO1EYBHwW2A5sAToE7XdCWwBtkaP0Re4AxiVNHY7wIG8aP1i4N8k9oK+AnolbZ+adL/OwCxgTfSzc1LbZOBuYFo0zptAfjnPrbT+AUn1nwOcCXwOFAK3JPU/ApgOfBf1fRSoG7X9LXou66Pne0HS+DcCS4FnSrdF99k/eoyfROutgZXAiXH/38jkW+wF6Ba9EHA6UFz6C1xOn7uAGcAeQEvgA+DuqO3E6P53AXWiX7wNQLOofcfAKDdAgIbAWuDAqG0v4KBoeXuAAM2B1cBF0f0ujNZbRO2TgX8BHYD60fr95Ty30voHRfVfBqwAngMaAwcBm4D9ov4/BY6KHrcd8AlwXdJ4DhxQxvgPkAji+skBEvW5LBqnATAJeDDu/xeZftMhTOZoAaz0ig8xegF3uftyd19BYs/ioqT2rVH7Vnd/ncRf3wMD69kGHGxm9d19ibvPL6PPWcAX7v6Muxe7+/PAp0C3pD5Pufvn7r4RGAscVsFjbiUx37MVGA3kA4PdvSh6/PnAIQDuPsfdZ0SPuwB4Ajghhed0u7tvjur5HncfBnwBfEgiNG+tZLxaTwGSOVYB+ZUcm7cGvk5a/zratn2MHQJoA9CoqoW4+3oSu/1XAEvM7DUz65hCPaU17Z20vrQK9axy95JoufQXfFlS+8bS+5tZBzN71cyWmtlaEvNG+RWMDbDC3TdV0mcYcDDwiLtvrqRvracAyRzTSeyin1NBn8UkJkNLtY22hVhPYle91J7Jje4+yd27kPhL/CmJX6zK6imt6dvAmqpiCIm62rt7E+AWwCq5T4VvOZpZIxLzSk8Cd5hZ811RaE2mAMkQ7r6GxPH/Y2Z2jpk1MLM6ZnaGmf0+6vY8MNDMWppZftR/VOBDfgQcb2ZtzWx34ObSBjNrZWZnm1lDYDOJQ6GSMsZ4HehgZj3NLM/MLgB+BLwaWFNVNCYxT7Mu2ju6cof2ZcB+VRxzMDDH3S8FXgMe3+kqazgFSAZx94dInAMykMQE4kKgH/By1OV3wGygAJgL/D3aFvJYbwFjorHm8P1f+hwS7+YsJvHOxAnAVWWMsQroGvVdReIdlK7uvjKkpiq6AehJ4t2dYSSeS7I7gJFm9p2ZnV/ZYGbWncRE9hXRpv7AT8ys1y6ruAbSiWQiEkx7ICISTAEiIsEUICISTAEiIsEy9gNF9Q/vp9ndLLV61qNxlyA7oV5epefTbKc9EBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWAKEBEJpgARkWB5cRdQUzx+ey/OOP5gVhQW0anHvQA0a9KAZx64hH1aN+frxYX0HvAk3xVt5JdndKL/xV0AWL9xM9fcO4a5n39b7jgSn82bN9PnV73YumULxSUldDn1NK7qdw233XITs2fPpHGjxgDcdc/9dPzhD2OutvppD2QXeWbiDLpf/dj3tt3QpwuTZ37Gj7vfxeSZn3FDn1MBWLB4Fade+jBHXHAf9w17g8cGXljhOBKfunXrMnzESF546RXGjnuZaVOnUPDxRwD0/+0Axo6fwNjxE2pleEAaA8TMOprZjWb2JzMbHC3X2H/laX//F4VrNnxvW9cTD2HUxA8BGDXxQ7qddAgAMz7+iu+KNgIws+Ar9m7VtMJxJD5mRoOGDQEoLi6muLgYzGKuKnOkJUDM7EZgNGDATGBWtPy8md2UjsfMRHu0aMzSlWsBWLpyLS2bN/6vPhef05lJ0/5Z3aVJFZSUlHD+ed056bjOHHV0Zw455FAAHvnTH/nFud34w/33smXLlpirjEe69kD6Aj9z9/vdfVR0ux84Imork5ldbmazzWx28cr5aSotcxzfqT2/PudoBg6eEHcpUoHc3FzGjp/Am+++z7y5BXzxxedcc31/Jrz6Bs+NGceaNWsYMXxo3GXGIl0Bsg1oXcb2vaK2Mrn7UHfv5O6d8vIPSlNp1Wf5qiL2zG8CwJ75TVhRWLS97eD2rRkyqCc9rh9K4Zr1cZUoVdCkSRN+dsSRfDB1Ci1b7oGZUbduXbqfex7z5s2Nu7xYpCtArgPeMbO/mtnQ6PYG8A5wbZoeM+O89v5cenc7EoDe3Y7k1ckFALTZsxmjH7yMvrc9zZffLI+zRKlEYWEha9cmDkM3bdrEjOkf0G7f/VixIvG6uTvvvfM2BxzQPs4yY2Punp6BzXJIHLLsTWL+YxEwy91LUrl//cP7paewNBl538Uc99P25DdtxPLCtdz9+OtMfK+AUQ9cQpu9mrFwyWp6DXiS1Ws38OdBPTnnlMP4ZkkhAMUl2zi21+/LHWfky9PjfGpVtnrWo3GXsMt8/tmnDLzlJrZtK2HbNufU007niqv6cWmfX7F69WrcnQM7duS2QXdun2zNdvXySHmWOG0BsrOyLUDkP2pSgNRGVQkQnQciIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISTAEiIsEUICISrNIAMbNjzKxhtNzbzB4ys33SX5qIZLpU9kCGABvM7FBgAPA18HRaqxKRrJBKgBR74vqX3YHB7j4YaJzeskQkG+Sl0KfIzG4GegPHm1kuUCe9ZYlINkhlD+QCYDPQ192XAnsDf0hrVSKSFSrdA4lC46Gk9W/QHIiIUEGAmFkR4GU1Ae7uTdJWlYhkhXIDxN01USoiFUrpRDIzO9bM+kTL+Wa2b3rLEpFskMqJZLcDNwI3R5vqAqPSWZSIZIdU9kDOBc4G1gO4+2J0HoiIkFqAbIlOJHOA0tPaRURSCZCxZvYE0NTMLgPeBoaltywRyQapnAfyoJl1AdYCHYBB7v5W2isTkYyXyqnsAHOB+iQOY+amrxwRySapvAtzKTATOA/4BTDDzC5Jd2EikvlS2QP5X+Bwd18FYGYtgA+AEeksTEQyXyqTqIuAoqT1ImBhesoRkWxS0Wdh+keL3wIfmtkEEnMg3Ukc0ohILVfRIUzpyWL/im6lJqSvHBHJJhV9mO7O6ixERLJPpZOoZtaSxHehHgTUK93u7iensS4RyQKpTKI+C3wK7AvcCSwAZqWxJhHJEqkESAt3fxLY6u7vu/slwFFprktEskAq54FsjX4uMbOzgMXAD9JXkohkC0t80LaCDmZdgSlAG+ARoAlwp7u/ks7CCteXVFyYZKzd6uiCh9msYV2zVPtWGiBxUYBkLwVIdqtKgFR0ItkjlP2lygC4+zVVrEtEapiK5kBmV1sVIpKVdAgju5wOYbJbVQ5h9EqLSDAFiIgEU4CISLBUvpGsg5m9Y2bzovVDzGxg+ksTkUyXyh7IMBIXldoK4O4FwC/TWZSIZIdUAqSBu+/4BULF6ShGRLJLKgGy0sz25z8XlvoFsCStVYlIVkjlw3RXA0OBjmb2LfAV0DutVYlIVkj5RLLokpY57l5UaeddQCeSZS+dSJbddslnYUqZ2aAd1gFw97uqXJmI1CipHMKsT1quB3QFPklPOSKSTar8WRgz2w14xd1PS09JCTqEyV46hMlu6f4sTANgv4D7iUgNk8ocyFz+870guUBLQPMfIpLSHEjXpOViYJm760QyEak4QMwsB3jN3Q+upnpEJItUOAfi7tuAj82sbTXVIyJZJJVDmL2A+WY2k6S3dN397LRVJSJZIZUA0TVyRaRMqQTIme5+Y/IGM3sAeD89JYlItkjlPJAuZWw7Y1cXIiLZp6LrwlwJXAXsZ2YFSU2NgWnpLkxEMl+5p7Kb2e5AM+A+4KakpiJ3L0x3YTqVPXvpVPbspktbSqwUINlN14URkWqhABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmmABGRYAoQEQmWypXpZCc9P2okE19+ETNj/wM6cOsd91C3bl2eeGww7749iZycXM7rcQHnX3hR3KVKGc467WQaNmhITm4uubm5PDtmHH/8v98zZfJ75NWpQ5s2bbnj7ntp3KRJ3KVWOwVImi1fvowXRo/iuRcnUq9ePW698XrenvQ67s6yZUsZPf41cnJyKCxcFXepUoEnRjxNs2bNtq8fdXRnfnNtf/Ly8hj80IOMGD6Ua/vfEGOF8dAhTDUoKSlh8+ZNFBcXs2njJvJb7sH4F8dwyWVXkpOTeAmaN28Rc5VSFUd3Ppa8vMTf3x8feijLly2NuaJ4KEDSbI89WtHzoj6ce+YpdDv1BBo1bsSRRx/Dt4u+4Z03/0qfXj24vt/lLPxmQdylSjnMjKv/py89zz+PcS+M+a/2CS+No/Oxx8dQWfyqPUDMrE8FbZeb2Wwzmz1yxLDqLCtt1q5dw5TJ7zLu1beYOGkymzZu5I3XXmHrli3U3W03nnr2Bbqf24N77hgYd6lSjqeefo7nxo7n0SHDGDv6OebMnrW9bfjQx8nLzePMrt1irDA+ceyB3Fleg7sPdfdO7t7p15dcVp01pc2sD6ez195706xZc/Lq1OGEk7swt+AjWrbak5NOORWAE07+OV9++XnMlUp5Wu7RCoDmLVpw0ik/Z/68xLXmJ054iSnvv8fv7v8DlvrVIGuUtASImRWUc5sLtErHY2aqPffci/lzP2bTxo24O7NnzqDdvvtxwomnMHvmDAD+MWcWbdu2i7dQKdPGDRtYv37d9uUZH0xj/wM6MG3qFP4yYjgPPzKE+vXrx1xlfNJycW0zWwacBqzesQn4wN1bVzZGTbq49rAhj/D2W2+Ql5tLhwN/yM2D7mbz5k3ccesAli5dQoP6DRhw6+2079Ax7lJ3iZp0ce1FCxfy2+v6AYnJ8NPP7Mqll1/B2WeeytYtW9i9aVMAfnzIodw6qNyd66xSlYtrpytAngSecvepZbQ95+49KxujJgVIbVOTAqQ2ij1AdgUFSPZSgGS3qgSIXmkRCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCaYAEZFgChARCWbuHncNtZKZXe7uQ+OuQ8Lo9UvQHkh8Lo+7ANkpev1QgIjITlCAiEgwBUh8av3xc5bT64cmUUVkJ2gPRESCKUBEJJgCJAZmdrqZfWZmX5rZTXHXI6kzsxFmttzM5sVdSyZQgFQzM8sFHgPOAH4EXGhmP4q3KqmCvwCnx11EplCAVL8jgC/d/d/uvgUYDXSPuSZJkbv/DSiMu45MoQCpfnsDC5PWF0XbRLKOAqT6WRnb9F66ZCUFSPVbBLRJWv8BsDimWkR2igKk+s0C2pvZvmZWF/gl8ErMNYkEUYBUM3cvBvoBk4BPgLHuPj/eqiRVZvY8MB040MwWmVnfuGuKk05lF5Fg2gMRkWAKEBEJpgARkWAKEBEJpgARkWAKEAlmZuuin63N7MVK+l5nZg2S1l83s6bprlHSS2/jyveYWa67l6TYd527N0qx7wKgk7uv3Jn6JLNoD6QWMbN2ZvapmY00swIze9HMGpjZAjMbZGZTgR5mtr+ZvWFmc8xsipl1jO6/r5lNN7NZZnb3DuPOi5ZzzexBM5sbPcZvzOwaoDXwnpm9F/VbYGb50XJ/M5sX3a5LGvMTMxtmZvPN7E0zqx+1XWNm/4zGH12t/4jyfe6uWy25Ae1IfHDvmGh9BHADsAAYkNTvHaB9tHwk8G60/Arwq2j5amBd0rjzouUrgXFAXrTePPq5AMhPeowFQD7wU2Au0BBoBMwHDo/GLAYOi/qPBXpHy4uB3aLlpnH/u9bmm/ZAap+F7j4tWh4FHBstjwEws0ZAZ+AFM/sIeALYK+pzDPB8tPxMOeP/HHjcE6fs4+6VfXfGscBL7r7e3dcB44Hjorav3P2jaHkOiVABKACeNbPeJEJGYpIXdwFS7Xac9CpdXx/9zAG+c/fDUrz/jiyFPjv2L8/mpOUSoH60fBZwPHA2cJuZHVQaWFK9tAdS+7Q1s6Oj5QuBqcmN7r4W+MrMegBYwqFR8zQSnx4G6FXO+G8CV5hZXnT/5tH2IqBxGf3/BpwTzcU0BM4FppRXvJnlAG3c/T1gANCUxKGPxEABUvt8AvzazAqA5sCQMvr0Avqa2cck5iRKv3LxWuBqM5sF7F7O+MOBb4CC6P49o+1Dgb+WTqKWcve/k/ie0ZnAh8Bwd/9HBfXnAqPMbC7wD+CP7v5dBf0ljfQ2bi1iZu2AV9394JhLkRpCeyAiEkx7ICISTHsgIhJMASIiwRQgIhJMASIiwRQgIhLs/wFe7PJ9GZIiKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This is where we attempt to find optimum parameters for the classfier by producing a random search through a grid\n",
    "# In order to limi computation time, we only visit 10 options. \n",
    "M = 50\n",
    "max_depth = np.array(np.random.uniform(1, 200, M), dtype=int)\n",
    "min_samples_split = np.array(np.random.uniform(2, 72, M), dtype=int)\n",
    "a = int(x_train.shape[0] / 10)\n",
    "b = int(x_train.shape[0] / 0.1)\n",
    "n_estimators = np.array(np.random.uniform(a, b, M), dtype=int)\n",
    "min_weight_fraction_leaf= np.random.uniform(0.0, 0.5, M)\n",
    "min_samples_leaf = np.array(np.random.uniform(2, 50, M), dtype=int)\n",
    "parameters = {'n_estimators': n_estimators,\n",
    "             #'max_depth': max_depth, \n",
    "             #'min_samples_split': min_samples_split,\n",
    "             #'min_weight_fraction_leaf': min_weight_fraction_leaf, \n",
    "             #'min_samples_leaf': min_samples_leaf\n",
    "             }\n",
    "\n",
    "model_test = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
    "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=19, min_samples_split=64,\n",
    "            min_weight_fraction_leaf=0.02550492908746793,\n",
    "            n_estimators=13421, n_jobs=10,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "#clf = GridSearchCV(model_test, parameters, n_jobs=40)\n",
    "clf = RandomizedSearchCV(model_test, parameters, refit=True, \n",
    "                        n_iter=20, n_jobs=40,\n",
    "                       cv=3, scoring='f1')\n",
    "clf.fit(x_train, y_train)\n",
    "model = clf.best_estimator_\n",
    "pred_test = model.predict(x_train)  # Predict labels of test data using the trained classifier\n",
    "c_matrix = confusion_matrix(y_train, pred_test) \n",
    "\n",
    "rf_accuracy = accuracy_score(y_train, pred_test)\n",
    "print(\"Accuracy of random forest TRAINING: {:.2f}\".format(rf_accuracy))\n",
    "print(\"\\nClassification report:\")\n",
    "print(classification_report(y_train, pred_test))\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(4, 4))\n",
    "ax.set_title(\"Confusion matrix\")\n",
    "sns.heatmap(c_matrix, cmap='Blues', annot=True, fmt='g', cbar=False)\n",
    "ax.set_xlabel('predictions')\n",
    "ax.set_ylabel('true labels')\n",
    "\n",
    "pred_test = model.predict(x_val)  # Predict labels of test data using the trained classifier\n",
    "c_matrix = confusion_matrix(y_val, pred_test) \n",
    "rf_accuracy = accuracy_score(y_val, pred_test)\n",
    "print(\"Accuracy of random forest VALIDATION: {:.2f}\".format(rf_accuracy))\n",
    "print(\"\\nClassification report:\")\n",
    "print(classification_report(y_val, pred_test))\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(4, 4))\n",
    "ax.set_title(\"Confusion matrix\")\n",
    "sns.heatmap(c_matrix, cmap='Blues', annot=True, fmt='g', cbar=False)\n",
    "ax.set_xlabel('predictions')\n",
    "ax.set_ylabel('true labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method BaseEstimator.get_params of RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=19, min_samples_split=64,\n",
       "            min_weight_fraction_leaf=0.02550492908746793,\n",
       "            n_estimators=13421, n_jobs=10, oob_score=False,\n",
       "            random_state=None, verbose=0, warm_start=False)>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dir_path = 'models'\n",
    "model_filename = \"{:s}/{}_random_forest_model_acc-{:2.3f}_{:s}.model\".format(\n",
    "    model_dir_path, 'yahoo_data', rf_accuracy, strftime(\"%Y_%m_%d_%H_%M_%S\", localtime())\n",
    ")\n",
    "pickle.dump(model, open(model_filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}