tradML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.svm import SVR\n",
    "from sklearn import linear_model\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "seed = 42\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "\n",
    "pca_t=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import dataset\n",
    "df = pd.read_csv('dataset-ml-25m/dataset.csv')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Prepocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train set:  9946\n",
      "Number of validation set:  1106\n",
      "Numebr of test set:  2764\n"
     ]
    }
   ],
   "source": [
    "#split data and labels \n",
    "X = df.drop(['rating'], axis=1)\n",
    "y = df['rating']\n",
    "\n",
    "X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.20, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.1, random_state=42)\n",
    "\n",
    "#count the numebr of x_train x_val and x_test\n",
    "print(\"Number of train set: \", X_train.shape[0])\n",
    "print(\"Number of validation set: \", X_val.shape[0])\n",
    "print(\"Numebr of test set: \", X_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA is not applied\n"
     ]
    }
   ],
   "source": [
    "if pca_t == True:\n",
    "    pca = PCA(n_components=0.95)\n",
    "    pca.fit(X_train)\n",
    "    X_train = pca.transform(X_train)\n",
    "    X_val = pca.transform(X_val)\n",
    "    X_test = pca.transform(X_test)\n",
    "else:\n",
    "    print (\"PCA is not applied\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regressor\n",
    "Con PCA:\n",
    "- Mean Square Error: 0.00643\n",
    "- R-square: 0.97096\n",
    "\n",
    "Senza PCA:\n",
    "- mse: 0.00544\n",
    "- R2-square: 0.97543"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Square Error: 0.005445406919048585\n",
      "R2-square: 0.9754341910082424\n"
     ]
    }
   ],
   "source": [
    "log_name = f\"linear_regression\"\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/LinearRegression/{log_name}\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/LinearRegression/{log_name}\")\n",
    "\n",
    "lin_regr = linear_model.LinearRegression()\n",
    "lin_regr.fit(X_train, y_train)\n",
    "y_pred = lin_regr.predict(X_test)\n",
    "\n",
    "# Compute the RSS\n",
    "mse = mean_squared_error(y_test, y_pred) \n",
    "print('Mean Square Error:', mse)\n",
    "writer.add_scalar('Loss', mse)\n",
    "writer.flush()\n",
    "\n",
    "# Compute the R-square index\n",
    "rsquare = r2_score(y_test, y_pred) \n",
    "print('R2-square:', rsquare)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Regressor\n",
    "Con PCA:\n",
    "- Mean Squared Error:  0.03735\n",
    "- R2 Score:  0.83150\n",
    "\n",
    "Senza PCA:\n",
    "- mse:  0.01255\n",
    "- R2 Score:  0.94334"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyper:  {'criterion': 'friedman_mse', 'n_estimators': 30}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                \"n_estimators\": [10, 15, 20, 25, 30],\n",
    "                \"criterion\": [\"squared_error\", \"friedman_mse\"]\n",
    "              }\n",
    "rf = RandomForestRegressor()\n",
    "grid_search = GridSearchCV(rf, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_train, y_train)\n",
    "best_hyper= grid_search.best_params_\n",
    "print(\"Best hyper: \", best_hyper)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "#display(grid_search.cv_results_)\n",
    "## con pca\n",
    "print(pca_t)\n",
    "if pca_t == True:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/pca/RandomForestRegressor.csv\")\n",
    "else:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/no_pca/RandomForestRegressor.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only neg_mean_squared_error and convert to positive\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "results = results[['param_n_estimators', 'param_criterion', 'mean_test_score']]\n",
    "results['mean_test_score'] = results['mean_test_score'] * -1\n",
    "\n",
    "#save mse to tensorboard\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/RandomForest/\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/RandomForest/\")\n",
    "for index, row in results.iterrows():\n",
    "    mse = row['mean_test_score']\n",
    "    n_estimators = row['param_n_estimators']\n",
    "    criterion = row['param_criterion']\n",
    "    writer.add_hparams({\"n_estimators\": n_estimators, \"criterion\": criterion},{\"hparam/mse\": mse})\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['value'])\n",
    "rf = rf.T\n",
    "if pca_t == True:\n",
    "    rf.to_csv('tradML/best_params/pca/random_forest.csv', index=False)\n",
    "else:\n",
    "    rf.to_csv('tradML/best_params/no_pca/random_forest.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error:  0.012738678622681958\n",
      "R2 Score:  0.9425321283598643\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestRegressor(criterion= 'friedman_mse', n_estimators=30)\n",
    "rf= rf.fit(X_train,y_train)\n",
    "y_pred= rf.predict(X_test)\n",
    "\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_test, y_pred))\n",
    "print(\"R2 Score: \", r2_score(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAG0CAYAAAAW6KJPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxUklEQVR4nO3deZhkZXn38e+PAURZJRAXdg1iUAR0xDVqFBOMCr6KEVQUo+ISlcQlEl9fF0xi3HeNqCiKBkUNQUWBuMVdhkV2IuACBAUB2VQQvN8/zmmmaHpmuuv09OlT/f1cV1/T51T11F0z3dW/ep7n3E+qCkmSJI1nnb4LkCRJGjLDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJK0qCV5VZIP912HJK1K7DMlTa4kPwXuBNw8cvoeVfW/Hf/O51TVf3WrbniSvA74k6p6et+1SFo8HJmSJt/jq2qjkY+xg9R8SLJun48/rqHWLWntM0xJS1CSTZN8JMmlSS5J8k9JlrW33T3J15JckeRXST6ZZLP2tk8A2wJfSHJdkn9I8ogkF0/7+3+aZM/289cl+WySI5NcAxy4usefodbXJTmy/Xz7JJXkWUkuSnJVkucnuX+S05P8Osl7R772wCTfSfLeJFcnOTfJo0Zuv2uSY5NcmeT8JM+d9rijdT8feBXwlPa5/6i937OSnJPk2iQXJnneyN/xiCQXJ3lZksva5/uskdtvn+RtSX7W1vftJLdvb3tgku+2z+lHSR4x7Xld2D7mT5I8bU7fAJLmle+0pKXpY8BlwJ8AGwJfBC4CPggEeCPw38AmwOeA1wF/V1UHJPkzRqb5Rn/Jr8Y+wJOBZwC3Az61msefjQcAOwIPA44FvgLsCawHnJrk6Kr65sh9PwtsATwR+HySHarqSuAo4EzgrsA9gROTXFBVX1tF3Vtw22m+y4DHARe29Xw5yUlVdUp7+52BTYGtgEcDn01yTFVdBbwVuBfwYOAXba1/SLIV8CXggPa5PQr4XJJ7Ar8B3g3cv6rOS3IXYPNZ/rtJWgscmZIm3zHt6MavkxyT5E7AX9GEo+ur6jLgHcB+AFV1flWdWFU3VNXlwNuBh3es4XtVdUxV/YEmoK3y8WfpDVX1u6o6Abge+PequqyqLgG+Bew+ct/LgHdW1e+r6tPAecBjk2wDPAR4Zft3nQZ8mCY43abuqvrtTIVU1Zeq6oJqfBM4Afizkbv8Hji0ffzjgOuAnZKsA/wNcHBVXVJVN1fVd6vqBuDpwHFVdVz72CcCK9p/N4A/APdOcvuqurSqzprDv52keebIlDT5njC6WDzJHjQjOJcmmTq9Ds3IEG3YehdNINi4ve2qjjVcNPL5dqt7/Fn65cjnv53heKOR40vq1lfa/IxmJOquwJVVde2025avou4ZJXkM8FrgHjTP4w7AGSN3uaKqbho5/k1b3xbABsAFM/y12wFPTvL4kXPrAV+vquuTPAV4OfCRJN8BXlZV566pVklrhyNT0tJzEXADsEVVbdZ+bFJV92pv/xeggF2qahOaUZKMfP30S4CvpwkQALRrn7acdp/Rr1nT48+3rTKS2mjWfP1v+7F5ko2n3XbJKuq+zXGS29FMg74VuFNVbQYcx63/vVblV8DvgLvPcNtFwCdG/n02q6oNq+pfAarq+Kp6NHAX4FzgQ7N4PElriWFKWmKq6lKaqai3JdkkyTrtovOpqbyNaaairm7X7rxi2l/xS+BuI8f/A2yQ5LFJ1gNeTbO+aNzHn29/DLwkyXpJngz8Kc0U2kXAd4E3JtkgyX2AZwNHrubv+iWwfTtFB7A+zXO9HLipHaX6i9kU1U55Hg68vV0IvyzJg9qAdiTw+CR/2Z7foF3MvnWSOyXZJ8mGNKH0OpppP0k9MUxJS9MzaILA2TRTeJ+lGeUAeD1wX+BqmkXQn5/2tW8EXt2uwXp5VV0NvJBmvdElNCNVF7N6q3v8+fYDmsXqvwL+Gdi3qq5ob9sf2J5mlOo/gNeuoX/W0e2fVyQ5pZ0ifAnwGZrn8VSaBfGz9XKaKcGTgCuBNwHrtEFvH5qrBy+nGal6Bc1r9jrAS9uar6RZz/aCOTympHlm005JEyvJgTRXHj6071okTS5HpiRJkjowTEmSJHXgNJ8kSVIHjkxJkiR1YJiSJEnqoLcO6FtssUVtv/32fT28JEnSrJ188sm/qqrpDYmBHsPU9ttvz4oVK/p6eEmSpFlL8rNV3eY0nyRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHUwqzCVZK8k5yU5P8khM9x+YJLLk5zWfjxn/kuVJElafNbYGiHJMuB9wKOBi4GTkhxbVWdPu+unq+pFa6FGSZKkRWs2I1N7AOdX1YVVdSNwFLDP2i1LkiRpGGYTprYCLho5vrg9N92Tkpye5LNJtpnpL0pyUJIVSVZcfvnlY5QrSZK0uMzXAvQvANtX1X2AE4EjZrpTVR1WVcuravmWW87YkV2SJGlQZhOmLgFGR5q2bs/doqquqKob2sMPA/ebn/IkSZIWt9nszXcSsGOSHWhC1H7AU0fvkOQuVXVpe7g3cM68VjnP/vfXN/ZdwmrddbP1+y5BkiTN0hrDVFXdlORFwPHAMuDwqjoryaHAiqo6FnhJkr2Bm4ArgQPXYs2SJEmLRqqqlwdevnx5rVixopfHdmRKkiTNRZKTq2r5TLfZAV2SJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA5mFaaS7JXkvCTnJzlkNfd7UpJKsnz+SpQkSVq81himkiwD3gc8BtgZ2D/JzjPcb2PgYOAH812kJEnSYjWbkak9gPOr6sKquhE4Cthnhvu9AXgT8Lt5rE+SJGlRm02Y2gq4aOT44vbcLZLcF9imqr40j7VJkiQtep0XoCdZB3g78LJZ3PegJCuSrLj88su7PrQkSVLvZhOmLgG2GTneuj03ZWPg3sA3kvwUeCBw7EyL0KvqsKpaXlXLt9xyy/GrliRJWiRmE6ZOAnZMskOS9YH9gGOnbqyqq6tqi6ravqq2B74P7F1VK9ZKxZIkSYvIGsNUVd0EvAg4HjgH+ExVnZXk0CR7r+0CJUmSFrN1Z3OnqjoOOG7audes4r6P6F6WJEnSMNgBXZIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDmYVppLsleS8JOcnOWSG25+f5IwkpyX5dpKd579USZKkxWeNYSrJMuB9wGOAnYH9ZwhLn6qqXapqN+DNwNvnu1BJkqTFaDYjU3sA51fVhVV1I3AUsM/oHarqmpHDDYGavxIlSZIWr3VncZ+tgItGji8GHjD9Tkn+FngpsD7wyHmpTpIkaZGbtwXoVfW+qro78Erg1TPdJ8lBSVYkWXH55ZfP10NLkiT1ZjZh6hJgm5Hjrdtzq3IU8ISZbqiqw6pqeVUt33LLLWddpCRJ0mI1mzB1ErBjkh2SrA/sBxw7eockO44cPhb48fyVKEmStHitcc1UVd2U5EXA8cAy4PCqOivJocCKqjoWeFGSPYHfA1cBz1ybRUuSJC0Ws1mATlUdBxw37dxrRj4/eJ7rkiRJGgQ7oEuSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqYNZhakkeyU5L8n5SQ6Z4faXJjk7yelJvppku/kvVZIkafFZY5hKsgx4H/AYYGdg/yQ7T7vbqcDyqroP8FngzfNdqCRJ0mI0m5GpPYDzq+rCqroROArYZ/QOVfX1qvpNe/h9YOv5LVOSJGlxmk2Y2gq4aOT44vbcqjwb+PJMNyQ5KMmKJCsuv/zy2VcpSZK0SM3rAvQkTweWA2+Z6faqOqyqllfV8i233HI+H1qSJKkX687iPpcA24wcb92eu5UkewL/F3h4Vd0wP+VJkiQtbrMZmToJ2DHJDknWB/YDjh29Q5LdgQ8Ce1fVZfNfpiRJ0uK0xjBVVTcBLwKOB84BPlNVZyU5NMne7d3eAmwEHJ3ktCTHruKvkyRJmiizmeajqo4Djpt27jUjn+85z3VJkiQNgh3QJUmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdrNt3ARrfmZdc13cJa3TvrTbquwRJktYqR6YkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjpYt+8CJIBvnHdl3yWs0SN22rzvEiRJi5AjU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQOZhWmkuyV5Lwk5yc5ZIbbH5bklCQ3Jdl3/suUJElanNYYppIsA94HPAbYGdg/yc7T7vZz4EDgU/NdoCRJ0mI2m+1k9gDOr6oLAZIcBewDnD11h6r6aXvbH9ZCjZIkSYvWbKb5tgIuGjm+uD03Z0kOSrIiyYrLL798nL9CkiRpUVnQBehVdVhVLa+q5VtuueVCPrQkSdJaMZswdQmwzcjx1u05SZKkJW82YeokYMckOyRZH9gPOHbtliVJkjQMawxTVXUT8CLgeOAc4DNVdVaSQ5PsDZDk/kkuBp4MfDDJWWuzaEmSpMViNlfzUVXHAcdNO/eakc9Popn+kyRJWlLsgC5JktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6MExJkiR1YJiSJEnqwDAlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHRimJEmSOjBMSZIkdWCYkiRJ6sAwJUmS1IFhSpIkqQPDlCRJUgeGKUmSpA4MU5IkSR0YpiRJkjowTEmSJHVgmJIkSerAMCVJktTBun0XIE2aj6+4qO8SVusZy7fpuwRJmiiOTEmSJHVgmJIkSerAMCVJktSBYUqSJKkDw5QkSVIHhilJkqQODFOSJEkdGKYkSZI6sGmnpBm95vgf913CGh36lzv2XYIkOTIlSZLUhSNTkibeAZ/8Ud8lrNEnnrZr3yVIGpNhSpIG5BHv/G7fJazWN/7uwX2XIC04p/kkSZI6MExJkiR1YJiSJEnqwDAlSZLUgQvQJUm92OmVx/ddwmqd96a/7LsEDcSsRqaS7JXkvCTnJzlkhttvl+TT7e0/SLL9vFcqSZK0CK1xZCrJMuB9wKOBi4GTkhxbVWeP3O3ZwFVV9SdJ9gPeBDxlbRQsSdJissWBR/Vdwhr96mP7zep+t3/MO9ZyJd399st/33cJtzGbkak9gPOr6sKquhE4Cthn2n32AY5oP/8s8Kgkmb8yJUmSFqdU1ervkOwL7FVVz2mPDwAeUFUvGrnPme19Lm6PL2jv86tpf9dBwEHt4U7AefP1RHq2BfCrNd5rGCbluUzK8wCfy2I0Kc8DfC6L1aQ8l0l5HgDbVdWWM92woAvQq+ow4LCFfMyFkGRFVS3vu475MCnPZVKeB/hcFqNJeR7gc1msJuW5TMrzWJPZTPNdAmwzcrx1e27G+yRZF9gUuGI+CpQkSVrMZhOmTgJ2TLJDkvWB/YBjp93nWOCZ7ef7Al+rNc0fSpIkTYA1TvNV1U1JXgQcDywDDq+qs5IcCqyoqmOBjwCfSHI+cCVN4FpKJmnqclKey6Q8D/C5LEaT8jzA57JYTcpzmZTnsVprXIAuSZKkVXM7GUmSpA4MU5IkSR0YpuYoyTpJHtx3HZI0V0mWJVl87aPH1O7QIfXOMDVHVfUHmu11tIgkedNszmlhJFk3yfOSfCXJ6e3Hl5M8P8l6fdc3V5PyJqqqbgb277uOefTjJG9JsnPfhXSVZPMZPob4s3JEks1Gju+Y5PAeS1oQLkAfQ5K3At8DPm8LiMUhySlVdd9p506vqvv0VVMXSTaf4fS1VfX7BS9mDEn+Hfg1zTZTF7ent6ZpobJ5VQ1u784kp1bV7n3X0VWSdwDrAZ8Grp86X1Wn9FbUmJJsTHP1+LNoBgcOB46qqmt6LWwMSX5K06/xKiDAZsAvgF8Cz62qk3srbg5m+jmZlJ+d1TFMjSHJtcCGwM3Ab2m+8auqNum1sDlI8gVglf/5VbX3ApYztiQvAF4I3A24YOSmjYHvVNXTeymso6G/sCb5n6q6x1xvW8wm5U1Ukq/PcLqq6pELXsw8SvJw4FM0PyufBd5QVef3WtQcJPkQ8NmqOr49/gvgScBHgXdV1QP6rG+2kvwIeERVXdUebw58s6p26beytcswtUS1LzwATwTuDBzZHu8P/LKqBrGuIsmmwB2BNwKHjNx0bVVd2U9V3Q39hTXJ94G3AZ9rp8ZJsg7wZOCli73+mUzCm6hJ066ZeizNyNT2wCeATwJ/BvzLkEJ7kjOmB46p0fUkp1XVbj2VNidJngG8Cjia5mdkX+Cfq+oTvRa2lhmmxpAkwNOAHarqDUm2Ae5SVT/subQ5m2nfpKHupZRkV5oXUYBvVdWP+qyni6G/sCbZHngT8EhWjq5tCnwdOKSqftJfdUtTkqdX1ZFJXjrT7VX19oWuqaskF9J8T32kqr477bZ3V9VL+qls7pKcAHwVOKo99RTg0cBewEnTlzEsZu0atqmRzq9V1dl91rMQFnSj4wnyfuAPNN8sbwCuo1mUfv8+ixrThknuVlUXAiTZgebd96AkeQlwEPD59tSRSQ6rqvf0WFYXlyZ5Jbd+Yf1l+078D/2VNTtV9VOamknyR+25Qe/XOQFvoqZ+rjfutYr5dZ+qum6mG4YUpFpPBV4LHNMef6c9twz4655qmrUkm1TVNe203i9oplynbtt8yDMFs+HI1BimFjuPLqpL8qOq2rXv2uYqyV407f4vpBk92A543tT00lAkOR14UFVd3x5vCHxvwAvQt6B5YX1oe+o7wOuBq4Fth7QWBG4J6bsDZ1fVuX3XM44kH6B9E1VVf5rkjsAJVTXEN1ETIcmWwHNppvhuGRyoqr/pq6alKskXq+pxSX7CrdfjTk2H362n0haEI1Pj+X07QlBwyw/0oh8tmElVfSXJjsA921PnVtUNfdY0ptCsZZlyc3tukKrqV8CLV3Hzog9SSY6pqie0n+8DvBP4BvCvSf6lqj7WW3Hje8DUmyiAqrqq3fx9UJJsDbwHeEh76lvAwVV18aq/atH6T5r6/4tb//wPTpJ7AC/ntsFwEBcGVNXj2j936LuWPhimxvNu4D+AP07yzzQL7F7db0njSXIH4KXAdlX13CQ7Jtmpqr7Yd21z9FHgB0n+oz1+As0G3IM09BdWmhHOKa+kGc35STvi9lXgY71U1c2kvIn6KM0UzJPb46e35x7dW0Xju0NVvbLvIubJ0cC/AR9mwMEwyUOA06rq+iRPB+4LvLOqft5zaWuV03xjSnJP4FE0ox9frapzei5pLEk+DZwMPKOq7t2Gq+8u9gXOM0lyX1ZOi32rqk7ts54u2suL/43m/+aWF9bF3hJhymjfryQ/rKo9Rm4bZM+ZJE+jWQd2X5r+WfsCr66qo3stbI5muoBhCBc1zCTJP9G8Xh3Xdy1dJTm5qu7Xdx1dtUsudgXuQ/Om6cPAX1fVw1f3dUPnyNQYkrybpjHcJHRCv3tVPSXJ/gBV9Zt2oe0gTGtu+dP245bbBrzo8aaq+kDfRXSwa5JraN5s3C7JXarq0nZabJBbgFTVJ5OczMo3UU8Y6JuoK9oRg39vj/cHBnVxQNumomj+H16V5EZgqqHtUNtVfCHJC2lmPW5ZajHA17Cbqqra6f33VtVHkjy776LWNsPUeE4GXp1kJ5pv/KOqakXPNY3rxiS3Z+XUxd0Z+UEegJNZ+aI6XdE08xyiQb+wVtWqAtMdgOctZC1dTQvsl7EyhAw1sP8NzZqpd9D8jHyXpk/TYFTVJF2ROOWZ7Z+vGDk3xNewa5P8I8308cPa/nKD2xZnrpzm66B9kX0SzXYG21bVjj2XNGdtM8j/C+wMnECzKPVZVTVTl2QtkPaKmOkm/oqYxWjk6qTRwD517P9Jz5I8kWZ6v2im94/pt6KlLcmdaVo6nFRV30qyLU1H9I/3XNpaZZjqIMkeNGso9gHOqarH91zSWNo+QA+k+eXw/fZKssHxRXXxaHswvQXYCvgy8JZq9xUcvdJPC2+S2gkkeT/wJ6wcLXwKcEFV/W1/Vc1NkkdW1dfa16/bqKrPz3Rei4vTfGNI8mbg/9DsBXcUzR5Qv+61qDEl+WpVPQr40gznBmOGF9XnJ3n0kF5UYaJeWA8HPgd8H3g28M0kj28bd2632q9cZNoLG1aphrdB8MS0E6BpnPyn1Y4KJDkCOKvfkubs4cDXgJnejBcrGxEPwsh6NoD1aab4rquqTfurau0zTI3nApoGkYMcwQFIsgHN+pUt2uaDU1MYm9CMJgzNJLyowuS8sG5ZVf/Wfv7idsHzfyfZm9VssL1IvW01txUrt80YiklqJ3A+sC3ws/Z4GwbQh21UVb22/fTQ6dsstc1uB2V0PVt7MdM+NDMfE81pvjG0C+qeCtytqg5t54TvPKBtJUhyMPB3wF2BS1gZpq4BPlRV7+2ptLEk+SLwt1X1s/Z4O5orSQY39dp+f+1bVZ/pu5ZxJTkLuF9V/W7k3J407R42rKq79FbcEjdh7QS+SbON1w9pgu0ewAqanQKoqr37q25uRtuJjJyblHYJg2yHMheGqTFM0rYSSV5cw92/7haT9KIKw91sekqSvwdOqapvTju/O/DmqhpMg8gJmnoFbpmG2ZDmKtHfs3Ih/eDaCSRZbe+i6d9/i1Hbs/BewJu59ZV8mwCvqKp79VLYmKb9nKwDLAceXlUP6qmkBeE033gmYlsJgKp6T5J701zNt8HI+aFdefGavguYZ/+V5OXAp4Hrp04O5TL8qnrHKs6fyvA6bU/K1CswWW0FhhCWZmEn4HHAZtz6e+xamgsFhmb0OdxE0/tvn35KWTiOTI0hyQ+AB9Nc+nnf9uqYE4Y4jJnktcAjaMLUccBjgG9X1b591rXU2RphcZmEqddJlOSBND2z/pRmsfMy4PqBjrI9qKq+13cdGo8jU+OZmL35aGrfFTi1qp6V5E7AkT3XtOTVEt0sdLGqqj8k+QfAMLW4vJemz9/RNNNJzwDu0WtF43t+knOmrgxvl4+8bWgtK9pF8y/mtq03BrXUYq4MU2OYoG0lAH7b/qK4KckmNB2et+m7qKUuyXrAC4CHtae+AXxwqleTejHoqddJVVXnJ1lWVTcDH22XX/xj33WN4T6jLXba5SODm+0AjqHZZP4LDHMj8LEYpuZgAreVAFiRZDPgQzRbs1wHONTcvw/Q9Gd5f3t8QHvuOb1VNIa2IezraDrrF/BtmkvAB7UXXOsp7Z+jvcuGuN3H1KjHNtx65GBo/bIAftOuVz2t7f93Kc2i5yFaJ8kdq+oquOX3zRB/R/+uqt7ddxELzTVTczDDthJT/3iD3Fai7QGydVVd1B5vD2xSVaf3WtgcJDmD1fQtqqr7LGA58ybJj6pq1zWdW+ySnAj8Nyunjp9Gs7XEnv1VtbQleQNwIE2/vKmfnaqqofXLmmqB8kua9VJ/D2wKvL+qBtVrCiDJM4BX0UxZhmYJxj9X1Sd6LWyOkjwV2JFme7LRfUWHGNZnzTC1xCU5o6p26buOcbUvprBytGDqhedpAFV1yIIXNQ+SnAI8uaouaI/vBnx2eh+axS7JmVV172nnBvs9l+TB3HYtyKCufE1yHrBLVd3Ydy1dJFkGfLyqntZ3LfMlyb2AP28Pv1ZVZ/dZzziSvJFmJP0CVk7zDTKsz8UQhxB7l+T/0HyjX90eb0bzbvuYPusa0ylJ7l9VJ/VdyDhGmnQ+etrVlIe0gWSQYYqm38zXk1zYHm8PPKu/csZ2QpL9WLlwe1/g+B7rGVuSTwB3B05j5TYsBQwqTAFn0lyGf1nPdXRSVTcn2S7J+kMPhlOq6qwkl9O2qUmybVX9vOey5urJNA2tJ+L/ZLYcmRpDktOqardp5wbZ4TXJuTR72v2MZlHt1JTloKbHkpxG0wH9O+3xg2mG+3frs65xtdv9vIzmIodfAycB7xjtKD4EIw0ip96hrsPKxduDahSZ5Bxg5xr4i2aS5TT7853JradhBne1VZKP07RFOJZbXxTw9t6KGlO71dLbaHaluIxmD8tzBti08xjgoKoadFifK0emxjPTAsfB/Vu2a6YOYuW+VkP2bODwJJvSBMKrgEFdUjzNx2m29nlDe/xUminMJ/dW0RgmqUEkTfi4M80i5yE7AngTcAbDv9rqgvZjHWDo32tvoNnD7r+qavckfw48veeaxrEZcG6Skxh4WJ8LR6bGkORwmtGC97Wn/hbYvKoO7KumcQ15/cpM2jDF1BTsUCU5u6p2XtO5IWjfcd/S4qGqvthnPXOV5As003kbA7vRbFk02F8SSU4a4tZXk25qC6kkPwJ2b1vWDPGikxm3+JmQbvWrNLjRlEXixcD/o+k3U8CJ3Ppy6SEZ9JqpUUkeS7PH1QbNoBtU1aG9FjW+U5I8sKq+D5DkATR7DQ5Kkn+l2TPxk+2pg5M8pKqG1AforTSjnW8CnjByfurc0HyrXSR8LAO/2mok6I66muZn5YMDmxb/dZKNaK5+/WSSyxiZuhyKSQ9Nq+LI1BLXrpnakWb/pCGvmfo34A40V8J8mGah8w+r6tm9FjZHI60e1qPZs+vn7fF2wLlDG5lKcjqwW1X9oT1eRtNtf1DfX9BcYTn9asokpw/tuST5+gynB3m1VZJ3AVuysuffU2imx4umzcsBfdU2V0k2BH5LM2X5NJo2D58caE+2JccwtcSNtBa4lamr5IZi6pfayJ8bAV+uqj/ru7a5WNX/x5Qh/r/QXOl6ZXu8Oc1U32ACSJIXAC+kac55wchNGwPfqaohrmuZCDNNWU6dS3LW0BZva7ic5lviqupnSR4K7FhVH02zafNGfdc1ht+2f/4myV2BK4C79FjPWIYWlmbhjcCp7WhIaNZODWmKD+BTwJdpnstoq41rh7TrQZKXru72IV4BB2w02j4gybasfP1aUpfmq1+GqTEk+aNJGXpN8lqaDUJ3Aj5KM710JM32H0Pyxbbf11uAU2iG+T/Ua0Wiqv49yTdo1k0BvLKqftFjSXPWXsxwNbB/37V0NPSr3WbyMuDbSS6gCes7AC9sp8yO6LWyJWZSd6OYLaf5xpDkxzSN+z5KM5U02H/Etj/T7sApU32yhrgOZFSS2wEbDP2KvkmQ5KtV9ag1nZPG1f6837M9PG9gi84nxqTuRjFbjkyN5x7AnjR9jN6d5DPAx6rqf/otayw3VlUlKbhlEeTgtE0uXwg8lHZD3SQf8IW1H+3/xx2ALdJsqju1n+UmwFa9FbaEJfmHqnpzkvcwwwhCVb2kh7I6SbIe8DxGWm8k+WBV/b7HsuZkUkZ0Jng3ilkxTI2hHYk6ETixbax2JM3Q8o+AQ6rqe70WODefSfJBYLMkz6UJiEOcHvs4cC3wnvZ4kE0uJ8jzgL+j6eZ8MivD1DXAe3uqaamb2udtcC02VuMDNEsT3t8eH9Cee05vFc3d49o/ZxzRGaC07U9Gd6OYqdH1RHGabwxJ/oimM+0BNDuWf4SmZ8tuwNFVtUN/1c1dkkcDf0HzC+/4qjqx55LmbJKaXE6SJC+uqves+Z5a25J8oqoOSHJwVb2r73rmw0xNLYfY6BJm3pJspnYci12S+wGH07R2uGU3iiH2MZsLR6bG8z2adw9PqKqLR86vaPsdDUZ7hc+nhxigppmIJpeTxiC1qNyvvdL1b9o97TJ645CuTBxxc5K7V9UFAEnuxspNqIdmIkZ0qupkYNdJ2Y1itgxT49lpVYvOq2poHZE3Bk5IciVNR/ejq+qXPdc0a9OaXH43ya2aXPZZm7TI/BvwVZp+WaNTr9D8zNytj6I6egXw9SQX0jyf7RjunpwTsb9oW/9radexJfkmcOikhyqn+eZgFVsX3GJoe3SNSnIfmu7BTwIurqo9ey5pViatyaW0trUXZryg7zrmQ3slHzStXQDOA6iqG2b+isVv6CM6ST5Hsyn4VGuKA4Bdq+qJ/VW19hmm5mBVGzhOGfKeREnuTLNYez9g46FcQaLFL8kTGbnKsqr+o+eSNCFWscXP4NYZwW1HdIBBjugkOa2qdlvTuUnjNN8cDDksrUqSFwJ/TbO/1dHAc6vq7NV/lTQ7Sd4P/Akr9057XpI9q2qoG4NrEWjf/G0F3D7J7ty69cYdeiusm8NpRnT+uj0+gKaX4dBGdH6b5KFV9W2AJA9h5Q4VE8swNYZV9AWZ2qn8nwbWHX0b4GCad0NTa4+k+fJI4E+n1hgmOQI4q9+SNAH+EjgQ2BoY3QbnWuBVfRQ0D+5eVU8aOX5921R5aF4AHDGy9utK4Jn9lrT2GabG82WaK0Y+1R7vR/Nu6BfAx4DH91PWWH5B0yfr8zTf+EcmOcyrsDRPzge2BabWrm3TnpPGVlVH0PzCflJVfa7veubJRIzoVNVpNFfzbdIeX9NvRQvDNVNjWN08fZIzqmqXvmqbqySnAw+qquvb4w2B77lmSl2MXKyxKc2+fD9sjx8A/LCqHtFfdZokSR4L3AvYYOpcVR3aX0XjSbIbzaLtW43oVNXpfdY1V5Oy9muuHJkaz7Ike1TVDwGS3B9Y1t52U39ljSXcui/LzUzrPyON4a19F6DJ1/b1uwPw58CHgX1pgvvgTNCIzqSs/ZoTR6bG0Ianw4GNaILHNTTbF5wFPLaqPtNjeXPSNu18JjB1hdUTaPYZfGdfNUnSbExtyj7y50Y0m8//Wd+1zdWkjOh4NZ9mrapOAnZZRT+QwQQpgKp6e5Jv0Fy6DvCsqjq1x5I0AZJcy+p7sm2ygOVock2tKfpN2939CuAuPdbTxaSM6EzE2q+5MkyNoW0U9yRge2DdpJkVG+I8PUC7Z9JE75ukhVVVGwMkeQNwKc32S6HZvHWov+y0+HwxyWbAW2hew4pmum+IJvlqvgN7rWgBOM03hiRfoWmFcDIj642q6m29FSUtQpO0Ea0Wt/ZN7gZDmxabkuR7wCumjei8taoe1G9l45mAtV9z4sjUeLauqr36LkIagOuTPA04imbUYH/g+n5L0qRIsgx4LO0sQXuOqnr76r5ukZqIEZ12He7oMbSDD+0i+4lkmBrPd5PsUlVn9F2ItMg9FXhX+1HAd9pz0nz4AvA74AzgDz3X0skEXc23vP34Qnv8OOB04PlJjq6qN/dW2VrkNN8YkpxNs0XGT4AbaN5FlL2ZJGnhTF3F13cd82H6iE5rcCM6Sf4b+Kuquq493gj4ErAXzXPZuc/61hZHpsbzmL4LkIYgyQ7AixmZhgGoqr37qkkT5ctJ/qKqTui7kHkwKSM6f0wzyDDl98Cdquq3SW5YxdcMnmFqDpJs0g69Xtt3LdJAHAN8hOYXxKCnYbQofR/4jyTr0PzSnpolGGLrja2B+46M6LyWZkTnYTQXOw0lTH0S+EGS/2yPHw98qt1d4+z+ylq7nOabgyRfrKrHJfkJzfqP0U7hVVV366k0aVFK8oOqekDfdWgyta/F+wBn1MB/mSU5F9ilqn7fHt8O+FFV3TPJqVW1e78Vzl6S5cBD2sPvVNWKPutZCI5MzUFVPa79c4e+a5EG4l3tO+wTGBn6b3ubSV1dBJw59CDVmpgRnTY8TXyAGuXI1Bja/h+nVdX1SZ4O3Bd4Z1X9vOfSpEUlyRtpOjlfwMppvqqqR/ZXlSZFko8BdwO+zK3D+hBbIyzJEZ1JYZgaQ5LTgV2B+wAfo+m4+9dV9fA+65IWmyTnAztX1Y1916LJ04563kZVvX6ha9HS5jTfeG6qqkqyD/DeqvpIkmf3XZS0CJ0JbAZc1nMdmkBToSnJHarqN33Xo6XLMDWea5P8I/B04GHtlSTr9VyTtBhtBpyb5CRuPQ1jawR1luRBNFeLbgRsm2RX4HlV9cJ+K9NS4zTfGJLcmaaL80lV9a0k2wKPqKqP91yatKgkmXHqu6q+udC1aPIk+QGwL3Ds1NVuSc6sqnv3W5mWGsOUJGmQplpvjLYOcCNt9cFpvjlI8u2qemiSa2n6TN1yE8NtFCfNO39WtEAuSvJgoJKsBxwMnNNzTVqCHJmSJA1Ski1oNtHekyaonwAcXFVX9FqYlhzD1BwlWQacVVX37LsWSZLUP6f55qiqbk5yXpJtbdIpSf1JsiXwXG67kfbf9FWTlibD1HjuCJyV5IfA9VMnvdxbkhbUfwLfAv4LuLnnWrSEOc03Bi/3lqT+JTmtqnbruw5pnb4LGKI2NJ0HbApsApxnkJKkBffFJH/VdxGSI1NjSPIc4DXA12iuIHk4cGhVHd5rYZK0hLStNzak6a7/e2y9oZ4YpsaQ5DzgwVOX3yb5I+C7VbVTv5VJkqSF5gL08VwBXDtyfG17TpK0liW5Z1Wdm+S+M91eVacsdE1a2hyZGkOSjwO70FxJUsA+wOntB1X19v6qk6TJluSwqjooyddnuLmq6pELXpSWNMPUGJK8dnW3V9XrF6oWSZLUL8OUJGmQkvwt8Mmq+nV7fEdg/6p6f6+FackxTI2hHVq+zT+cQ8uStHBm6jOV5NSq2r2nkrREuQB9PC8f+XwD4EnATT3VIklL1bIkqXZUoN07df2ea9ISZJgaQ1WdPO3Ud9qtZSRJC+crwKeTfLA9fl57TlpQTvONIcnmI4frAPcD3m2fKUlaOEnWAQ4C9mxPnQh8uKrcp08LyjA1hiQ/oVkzFZrpvZ/QdED/dq+FSZKkBWeYkiRJ6sCNjseQ5MlJNm4/f3WSz6+qE68kSZpshqnx/L+qujbJQ2nm6j8CfKDnmiRJUg+8mm88U4sbHwscVlVfSvJPfRYkSUtNki9w255/VwMrgA9W1e8WviotRY5MjeeS9lLcpwDHJbkd/ltK0kK7ELgO+FD7cQ3NxvP3aI+lBeEC9DEkuQOwF3BGVf04yV2AXarqhJ5Lk6QlI8lJVXX/mc4lOauq7tVXbVpanOYbQ1X9Bvj8yPGlwKX9VSRJS9JGSbatqp8DJNkW2Ki97cb+ytJSY5iSJA3Vy4BvJ7mApu/fDsALk2wIHNFrZVpSnOaTJA1Wu2b1nu3heS46Vx8MU5KkwUryYGB7RmZaqurjvRWkJclpPknSICX5BHB34DRWtqwpwDClBeXIlCRpkJKcA+xc/iJTz+yNJEkaqjOBO/ddhOQ0nyRpqLYAzk7yQ+CGqZNVtXd/JWkpMkxJkobqdX0XIIFrpiRJkjpxZEqSNChJvl1VD01yLbfe6DhAVdUmPZWmJcqRKUmSpA68mk+SNEhJPpJkt2nnXtdPNVrKDFOSpKH6S+CIJM8cOeeVfFpwhilJ0lBdBjwM2DfJ+5KsS7NuSlpQhilJ0lClqq6uqscDlwPfADbttyQtRYYpSdJQHTv1SVW9DngT8NO+itHS5dV8kiRJHTgyJUkapCQPTHJSkuuS3Jjk5iRX912Xlh7DlCRpqN4L7A/8GLg98Bzg/b1WpCXJMCVJGqyqOh9YVlU3V9VHgb36rklLj9vJSJKG6jdJ1gdOS/Jm4FIcJFAP/KaTJA3VAcAy4EXA9cA2wJN6rUhLklfzSZIkdeDIlCRpkJI8LsmpSa5Mck2Sa5Nc03ddWnocmZIkDVKS84EnAmeUv8zUI0emJElDdRFwpkFKfXNkSpI0SEnuD7wB+CZww9T5qnp7b0VpSbI1giRpqP4ZuA7YAFi/51q0hBmmJElDddequnffRUiumZIkDdVxSf6i7yIk10xJkgYpybXAhjTrpX4PBKiq2qTXwrTkGKYkSZI6cM2UJGmwktwR2JFmEToAVfXf/VWkpcgwJUkapCTPAQ4GtgZOAx4IfA94ZI9laQlyAbokaagOBu4P/Kyq/hzYHfh1rxVpSTJMSZKG6ndV9TuAJLerqnOBnXquSUuQ03ySpKG6OMlmwDHAiUmuAn7Wa0VakryaT5I0eEkeDmwKfKWqbuy7Hi0thilJkqQOXDMlSZLUgWFKkiSpA8OUJElSB4YpSZKkDgxTkiRJHfx/SasE2V1RFaQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#features the first 10 most important with random forest with no PCA\n",
    "if pca_t == False:\n",
    "    importances = rf.feature_importances_\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "    features = X.columns\n",
    "\n",
    "    # Crea il grafico\n",
    "    my_palette = sns.color_palette(\"Blues\", n_colors=10)\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.title(\"Feature importances\")\n",
    "    plt.bar(range(10), importances[indices[0:10]], color=my_palette, align=\"center\")\n",
    "    plt.xticks(range(10), features[indices[0:10]], rotation=90)\n",
    "    plt.xlim([-1, 10])\n",
    "    plt.show()\n",
    "else:\n",
    "    print(\"PCA is applied, so no feature importance graph is shown\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge Regressor\n",
    "Con PCA:\n",
    "- Mean Squared Error:  0.00639\n",
    "- R2 Score:  0.97113\n",
    "\n",
    "Senza PCA:\n",
    "- mse: 0.00532\n",
    "- R2 Score:  0.97598"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyper:  {'alpha': 5}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                \"alpha\":[0.0001, 0.001, 0.1, 0.5, 1, 5, 10, 20]\n",
    "              }\n",
    "ridge = Ridge()\n",
    "grid_search = GridSearchCV(ridge, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_train,y_train)\n",
    "best_hyper= grid_search.best_params_\n",
    "print(\"Best hyper: \", best_hyper)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display(grid_search.cv_results_)\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"tradML/cv_results/pca/Ridge/\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"tradML/cv_results/no_pca/Ridge/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only neg_mean_squared_error and convert to positive\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "results = results[['param_alpha','mean_test_score']]\n",
    "results['mean_test_score'] = results['mean_test_score'] * -1\n",
    "\n",
    "#save mse to tensorboard\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/Ridge\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/Ridge\")\n",
    "for index, row in results.iterrows():\n",
    "    mse = row['mean_test_score']\n",
    "    alpha = row['param_alpha']\n",
    "    writer.add_hparams({\"alpha\": alpha},{\"/hparam/mse\": mse})\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge = pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['value'])\n",
    "ridge = ridge.T\n",
    "if pca_t == True:\n",
    "    ridge.to_csv('tradML/best_params/pca/ridge.csv', index=False)\n",
    "else:\n",
    "    ridge.to_csv('tradML/best_params/no_pca/ridge.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error:  0.005323110704793611\n",
      "R2 Score:  0.9759859046789496\n"
     ]
    }
   ],
   "source": [
    "ridge = Ridge(**best_hyper)\n",
    "ridge.fit(X_train, y_train)\n",
    "y_pred= ridge.predict(X_test)\n",
    "\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_test, y_pred))\n",
    "print(\"R2 Score: \", r2_score(y_test, y_pred))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN Regressor\n",
    "Con PCA:\n",
    "- Mean Squared Error:  0.04028\n",
    "- R2 Score:  0.81828\n",
    "\n",
    "Senza PCA:\n",
    "- mse: 0.04050\n",
    "- R2 Score:  0.81727"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyper:  {'n_neighbors': 10, 'weights': 'distance'}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                'n_neighbors': [7, 8, 9, 10, 15, 20],\n",
    "                'weights': ['uniform', 'distance']\n",
    "              }\n",
    "knn = KNeighborsRegressor()\n",
    "grid_search = GridSearchCV(knn, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_train,y_train)\n",
    "best_hyper= grid_search.best_params_\n",
    "print(\"Best hyper: \", best_hyper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display(grid_search.cv_results_)\n",
    "if pca_t == True:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/pca/KNN.csv\")\n",
    "else:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/no_pca/KNN.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only neg_mean_squared_error and convert to positive\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "results = results[['param_n_neighbors', 'param_weights', 'mean_test_score']]\n",
    "results['mean_test_score'] = results['mean_test_score'] * -1\n",
    "\n",
    "#save mse to tensorboard\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/KNN/\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/KNN/\")\n",
    "writer = SummaryWriter(f\"results/tradML/pca/KNN/\")\n",
    "for index, row in results.iterrows():\n",
    "    mse = row['mean_test_score']\n",
    "    n_neighbors = row['param_n_neighbors']\n",
    "    weights = row['param_weights']\n",
    "    writer.add_hparams({\"n_neighbors\": n_neighbors, \"weights\":weights},{\"hparam/mse\": mse})\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['value'])\n",
    "knn = knn.T\n",
    "if pca_t == True:\n",
    "    knn.to_csv('tradML/best_params/pca/knn.csv', index=False)\n",
    "else:\n",
    "    knn.to_csv('tradML/best_params/no_pca/knn.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error:  0.040503488522491256\n",
      "R2 Score:  0.8172770231251678\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsRegressor(**best_hyper)\n",
    "knn.fit(X_train, y_train)\n",
    "y_pred= knn.predict(X_test)\n",
    "\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_test, y_pred))\n",
    "print(\"R2 Score: \", r2_score(y_test, y_pred))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lasso Regressor\n",
    "Con PCA:\n",
    "- Mean Squared Error:  0.00641\n",
    "- R2 Score:  0.97106\n",
    "\n",
    "Senza PCA:\n",
    "- 0.00542\n",
    "- R2 Score:  0.97553\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 5.527e+00, tolerance: 1.880e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 6.996e+00, tolerance: 1.910e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 6.742e+00, tolerance: 1.920e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 5.814e+00, tolerance: 1.897e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 6.194e+00, tolerance: 1.885e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.590e+01, tolerance: 1.880e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.585e+01, tolerance: 1.910e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.563e+01, tolerance: 1.920e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.560e+01, tolerance: 1.897e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.543e+01, tolerance: 1.885e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyper:  {'alpha': 1e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 2.046e+01, tolerance: 2.373e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                'alpha':[1e-1, 1e-2, 1e-3, 1e-4, 1e-5]\n",
    "              }\n",
    "lasso = Lasso()\n",
    "grid_search = GridSearchCV(lasso, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_train,y_train)\n",
    "best_hyper= grid_search.best_params_\n",
    "print(\"Best hyper: \", best_hyper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display(grid_search.cv_results_)\n",
    "if pca_t == True:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/pca/Lasso.csv\")\n",
    "else:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/no_pca/Lasso.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only neg_mean_squared_error and convert to positive\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "results = results[['param_alpha','mean_test_score']]\n",
    "results['mean_test_score'] = results['mean_test_score'] * -1\n",
    "\n",
    "#save mse to tensorboard\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/Lasso/\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/Lasso/\")\n",
    "for index, row in results.iterrows():\n",
    "    mse = row['mean_test_score']\n",
    "    alpha = row['param_alpha']\n",
    "    writer.add_hparams({\"alpha\": alpha},{\"hparam/mse\": mse})\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso = pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['value'])\n",
    "lasso= lasso.T\n",
    "if pca_t == True:\n",
    "    lasso.to_csv('tradML/best_params/pca/lasso.csv', index=False)\n",
    "else:\n",
    "    lasso.to_csv('tradML/best_params/no_pca/lasso.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error:  0.005423553353396875\n",
      "R2 Score:  0.9755327787772686\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:647: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 2.046e+01, tolerance: 2.373e-01\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    }
   ],
   "source": [
    "lasso = Lasso(**best_hyper)\n",
    "lasso.fit(X_train, y_train)\n",
    "y_pred= lasso.predict(X_test)\n",
    "\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_test, y_pred))\n",
    "print(\"R2 Score: \", r2_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features that are zero:  146\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "007           -0.097706\n",
       "1920s          0.379278\n",
       "3d            -0.153728\n",
       "aardman        0.060788\n",
       "afterlife     -0.079177\n",
       "alcoholism     0.206545\n",
       "almodovar      0.083976\n",
       "amnesia        0.062696\n",
       "animation      0.024383\n",
       "arms dealer   -0.073337\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if pca_t == False:\n",
    "    #print null coefficients for lasso and name of features without pca\n",
    "    lasso_coef = pd.DataFrame(lasso.coef_, index=X.columns, columns=['value'])\n",
    "    lasso_coef = lasso_coef[lasso_coef['value'] == 0]\n",
    "    print(\"Number of features that are zero: \", len(lasso_coef))\n",
    "\n",
    "    lasso_coef = pd.DataFrame(lasso.coef_, index=X.columns, columns=['value'])\n",
    "    lasso_coef = lasso_coef[lasso_coef['value'] == 0]\n",
    "    X_zero_coef = X[lasso_coef.index]\n",
    "    # corr between features set to 0 and y\n",
    "    corr = X_zero_coef.corrwith(y)\n",
    "    display(corr.head(10))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVR\n",
    "Con PCA:\n",
    "- Mean Squared Error: 0.00645\n",
    "- R2 Score: 0.97086\n",
    "\n",
    "Senza PCA:\n",
    "- mse: 0.00544\n",
    "- R2 Score: 0.97543"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyper  {'epsilon': 0.001, 'kernel': 'rbf'}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                'kernel': ['linear', 'poly', 'rbf'],\n",
    "                'epsilon': [0.001, 0.01, 0.1, 1]\n",
    "              }\n",
    "svr = SVR()\n",
    "grid_search = GridSearchCV(svr, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_train,y_train)\n",
    "best_hyper = grid_search.best_params_\n",
    "print(\"Best hyper \", best_hyper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display(grid_search.cv_results_)\n",
    "if pca_t == True:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/pca/SVR.csv\")\n",
    "else:\n",
    "    pd.DataFrame(grid_search.cv_results_).to_csv(\"tradML/cv_results/no_pca/SVR.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only neg_mean_squared_error and convert to positive\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "results = results[['param_kernel','param_epsilon' ,'mean_test_score']]\n",
    "results['mean_test_score'] = results['mean_test_score'] * -1\n",
    "\n",
    "\n",
    "#save mse to tensorboard\n",
    "if pca_t == True:\n",
    "    writer = SummaryWriter(f\"results/tradML/pca/SVR/\")\n",
    "else:\n",
    "    writer = SummaryWriter(f\"results/tradML/no_pca/SVR/\")\n",
    "for index, row in results.iterrows():\n",
    "    mse = row['mean_test_score']\n",
    "    kernel = row['param_kernel']\n",
    "    epsilon = row['param_epsilon']\n",
    "    writer.add_hparams({\"kernel\": kernel, \"epsilon\":epsilon},{\"hparam/mse\": mse})\n",
    "    writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "svr = pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['value'])\n",
    "svr = svr.T\n",
    "if pca_t == True:   \n",
    "    svr.to_csv('tradML/best_params/pca/svr.csv', index=False)\n",
    "else:\n",
    "    svr.to_csv('tradML/best_params/no_pca/svr.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error:  0.005214488613478723\n",
      "R2 Score:  0.9764759304175574\n"
     ]
    }
   ],
   "source": [
    "svr = SVR(**best_hyper)\n",
    "svr.fit(X_train, y_train)\n",
    "y_pred= svr.predict(X_test)\n",
    "\n",
    "print(\"Mean Squared Error: \", mean_squared_error(y_test, y_pred))\n",
    "print(\"R2 Score: \", r2_score(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}