05_ML_model_eval_best_param.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fe0855d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import numpy as np\n",
    "import time\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import joblib\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.pipeline import Pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4698dbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "def train_random_forest_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create a random forest regressor object\n",
    "    rf = RandomForestRegressor()\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(estimator=rf, param_grid=param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "\n",
    "    # Use the best hyperparameters to create a random forest model\n",
    "    best_rf = RandomForestRegressor(**grid_search.best_params_)\n",
    "    best_rf.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = best_rf.predict(X_val)\n",
    "\n",
    "    # Compute the evaluation metrics on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(val_mse)\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = best_rf.predict(X_test)\n",
    "\n",
    "    # Compute the evaluation metrics on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "\n",
    "    # Calculate MSE for validation and test sets\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    return best_rf, grid_search.best_params_, val_mae, val_rmse, val_mse, val_r2, test_mae, test_rmse, test_mse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e96c8486",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_linear_regression_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create a linear regression object\n",
    "    lr = LinearRegression()\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(estimator=lr, param_grid=param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "\n",
    "    # Use the best hyperparameters to create a linear regression model\n",
    "    best_lr = LinearRegression(**grid_search.best_params_)\n",
    "    best_lr.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = best_lr.predict(X_val)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(val_mse)\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = best_lr.predict(X_test)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "\n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "\n",
    "    return best_lr, grid_search.best_params_, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea050eed",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "def train_svr_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    \n",
    "    # Print the shape of the datasets\n",
    "   # print(\"Shape of X: \", X.shape)\n",
    "    #print(\"Shape of X_train: \", X_train.shape)\n",
    "    #print(\"Shape of X_test: \", X_test.shape)\n",
    "    #print(\"Shape of X_val: \", X_val.shape)\n",
    "    \n",
    "    \n",
    "    # Create an SVR object\n",
    "    svr = SVR()\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(estimator=svr, param_grid=param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "\n",
    "    # Print the best hyperparameters and corresponding mean cross-validated score on validation set\n",
    "    #print(f'Best hyperparameters: {grid_search.best_params_}')\n",
    "    #print(f'Best cross-validated score on validation set: {grid_search.best_score_}')\n",
    "\n",
    "    # Use the best hyperparameters to create an SVR model\n",
    "    best_svr = SVR(**grid_search.best_params_)\n",
    "    best_svr.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = best_svr.predict(X_val)\n",
    "\n",
    "    # Compute the RMSE on the validation set\n",
    "    val_rmse = np.sqrt(mean_squared_error(y_val, y_pred_val))\n",
    "    \n",
    "    # Compute the MAE on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    \n",
    "    # Compute the MSE on the validation set\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    \n",
    "    # Compute the R2 on the validation set\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = best_svr.predict(X_test)\n",
    "\n",
    "    # Compute the RMSE on the test set\n",
    "    test_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "    \n",
    "    # Compute the MAE on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    \n",
    "    # Compute the MSE on the test set\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    \n",
    "    # Compute the R2 on the test set\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    return best_svr, grid_search.best_params_, val_rmse, val_mae, val_mse, val_r2, test_rmse, test_mae, test_mse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "65b556a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "def train_dt_model(df, target_col, param_grid, test_size=0.15, random_state=42):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=random_state)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create the base estimator\n",
    "    base_estimator = DecisionTreeRegressor()\n",
    "\n",
    "    # Create the grid search object\n",
    "    grid_search = GridSearchCV(estimator=base_estimator, param_grid=param_grid, cv=5)\n",
    "\n",
    "    # Train the grid search object\n",
    "    grid_search.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Get the best model and its parameters\n",
    "    best_model = grid_search.best_estimator_\n",
    "    best_params = grid_search.best_params_\n",
    "\n",
    "    # Use the best model to predict the target variable on the validation set\n",
    "    y_pred_val = best_model.predict(X_val)\n",
    "\n",
    "    # Compute the evaluation metrics on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(mean_squared_error(y_val, y_pred_val))\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the best model to predict the target variable on the test set\n",
    "    y_pred_test = best_model.predict(X_test)\n",
    "\n",
    "    # Compute the evaluation metrics on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "\n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "\n",
    "    return best_model, best_params, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "befeee72",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import VotingRegressor\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "\n",
    "def train_dt_vot_model(df, target_col, param_grid, cv=5):\n",
    "    start_time = time.time()\n",
    "\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create the base estimator\n",
    "    base_estimator = DecisionTreeRegressor()\n",
    "\n",
    "    # Create the voting regressor model\n",
    "    model = VotingRegressor(estimators=[\n",
    "        ('estimator_1', base_estimator), \n",
    "        ('estimator_2', DecisionTreeRegressor(max_depth=3)), \n",
    "        ('estimator_3', DecisionTreeRegressor(max_depth=5))\n",
    "    ])\n",
    "\n",
    "    # Train the model\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=cv)\n",
    "    grid_search.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = grid_search.predict(X_val)\n",
    "\n",
    "    # Compute the evaluation metrics on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(mean_squared_error(y_val, y_pred_val))\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = grid_search.predict(X_test)\n",
    "\n",
    "    # Compute the evaluation metrics on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "\n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "\n",
    "    return grid_search.best_estimator_, grid_search.best_params_, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3433db8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "def train_dt_bag_model(df, target_col, param_grid=None):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "    \n",
    "    # Create the base estimator\n",
    "    base_estimator = DecisionTreeRegressor()\n",
    "    \n",
    "    # Create the ensemble model\n",
    "    if param_grid:\n",
    "        model = GridSearchCV(estimator=base_estimator, param_grid=param_grid, cv=5)\n",
    "    else:\n",
    "        model = BaggingRegressor(base_estimator=base_estimator, n_estimators=10, random_state=42)\n",
    "    \n",
    "    # Train the model\n",
    "    model.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = model.predict(X_val)\n",
    "\n",
    "    # Compute the evaluation metrics on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(mean_squared_error(y_val, y_pred_val))\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = model.predict(X_test)\n",
    "\n",
    "    # Compute the evaluation metrics on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    # Get the best parameters (if applicable)\n",
    "    best_params = model.best_params_ if param_grid else None\n",
    "    \n",
    "    return model, best_params, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f0cddd4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_knn_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create a KNN regressor object\n",
    "    knn = KNeighborsRegressor()\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(estimator=knn, param_grid=param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "\n",
    "    # Use the best hyperparameters to create a KNNRegressor model\n",
    "    best_knn = KNeighborsRegressor(**grid_search.best_params_)\n",
    "    best_knn.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = best_knn.predict(X_val)\n",
    "\n",
    "    # Calculate MAE on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "\n",
    "    # Calculate MSE on the validation set\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "\n",
    "    # Calculate RMSE on the validation set\n",
    "    val_rmse = np.sqrt(val_mse)\n",
    "\n",
    "    # Calculate R2 on the validation set\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = best_knn.predict(X_test)\n",
    "\n",
    "    # Calculate MAE on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "\n",
    "    # Calculate MSE on the test set\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "\n",
    "    # Calculate RMSE on the test set\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "\n",
    "    # Calculate R2 on the test set\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    return best_knn, grid_search.best_params_, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2121789c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "def train_lasso_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create a Lasso object\n",
    "    lasso = Lasso()\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(estimator=lasso, param_grid=param_grid, cv=5)\n",
    "    grid_search.fit(X_train, y_train)\n",
    "\n",
    "    # Use the best hyperparameters to create a Lasso model\n",
    "    best_lasso = Lasso(**grid_search.best_params_)\n",
    "    best_lasso.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the validation set\n",
    "    y_pred_val = best_lasso.predict(X_val)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the validation set\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(val_mse)\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the trained model to predict the target variable on the test set\n",
    "    y_pred_test = best_lasso.predict(X_test)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the test set\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    return best_lasso, grid_search.best_params_, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4b99184c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_pol_model(df, target_col, param_grid):\n",
    "    start_time = time.time()\n",
    "    # Split the data into training, validation, and testing sets\n",
    "    X = df.drop(target_col, axis=1)\n",
    "    y = df[target_col]\n",
    "    X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.1765, random_state=42)  # 0.1765 = 15/85\n",
    "\n",
    "    # Create a Pipeline object that includes PolynomialFeatures and LinearRegression\n",
    "    pol_regression = Pipeline([('poly', PolynomialFeatures()),\n",
    "                               ('linear', LinearRegression())])\n",
    "\n",
    "    # Set up the GridSearchCV object with the given param_grid and the pol_regression pipeline\n",
    "    grid_search = GridSearchCV(pol_regression, param_grid, cv=5, return_train_score=True)\n",
    "\n",
    "    # Fit the model on the training data\n",
    "    grid_search.fit(X_trainval, y_trainval)\n",
    "\n",
    "    # Use the best estimator from the grid search to predict the target variable on the validation set\n",
    "    y_pred_val = grid_search.best_estimator_.predict(X_val)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the validation set using the best estimator\n",
    "    val_mae = mean_absolute_error(y_val, y_pred_val)\n",
    "    val_mse = mean_squared_error(y_val, y_pred_val)\n",
    "    val_rmse = np.sqrt(val_mse)\n",
    "    val_r2 = r2_score(y_val, y_pred_val)\n",
    "\n",
    "    # Use the best estimator from the grid search to predict the target variable on the test set\n",
    "    y_pred_test = grid_search.best_estimator_.predict(X_test)\n",
    "\n",
    "    # Compute the MAE, MSE, RMSE, and R2 on the test set using the best estimator\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "    test_r2 = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    elapsed_time = end_time - start_time\n",
    "    \n",
    "    return grid_search.best_estimator_, grid_search.best_params_, val_mae, val_mse, val_rmse, val_r2, test_mae, test_mse, test_rmse, test_r2, elapsed_time\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c4ff365",
   "metadata": {},
   "source": [
    "### Load dataset and set target_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d2e3b315",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csvdf = pd.read_csv('./data/movies_only_numeric.csv')\n",
    "target_col = \"Rating\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6955eff3",
   "metadata": {},
   "source": [
    "#### RF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9084e6a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for RF\n",
    "rf_param_grid = {\n",
    "    'n_estimators': [50, 100],\n",
    "    'max_depth': [5, 10, 15],\n",
    "    'min_samples_split': [2, 5],\n",
    "    'min_samples_leaf': [1, 2],\n",
    "}\n",
    "# model, best_params, val_rmse, test_rmse, time\n",
    "model_rf = train_random_forest_model(df, target_col, rf_param_grid)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf5901fc",
   "metadata": {},
   "source": [
    "#### Lin Reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4463ecef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for the function\n",
    "lin_param_grid = {\n",
    "    'fit_intercept': [True, False],\n",
    "    'normalize': [True, False],\n",
    "}\n",
    "# Call the function\n",
    "# model, best_params, val_rmse, test_rmse, time\n",
    "model_lin = train_linear_regression_model(df, target_col, lin_param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dc38632",
   "metadata": {},
   "source": [
    "#### SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fb6a9795",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the parameter grid\n",
    "svr_param_grid = {\n",
    "    'C': [1.0],\n",
    "    'kernel': ['rbf'],\n",
    "    'gamma': ['scale'],\n",
    "    'epsilon': [0.1]\n",
    "}\n",
    "\n",
    "# Train the SVR model\n",
    "#model_svr = train_svr_model(df, target_col, svr_param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b748f0a",
   "metadata": {},
   "source": [
    "#### Descision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9b8dc4fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the hyperparameter grid for DecisionTreeRegressor\n",
    "dt_param_grid = {\n",
    "    'max_depth': [2, 4, 6],\n",
    "    'min_samples_split': [2, 5],\n",
    "    'min_samples_leaf': [1, 3],\n",
    "    'max_features': ['auto', 'sqrt'],\n",
    "    'criterion': ['mse', 'mae']\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "# Call the train_dt_model function with default hyperparameters\n",
    "model_dt = train_dt_model(df, target_col, dt_param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0b8cabc",
   "metadata": {},
   "source": [
    "#### Descision Tree - Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "439db187",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_bag_param_grid = {\n",
    "    'max_depth': [3, 5, 7],\n",
    "    'min_samples_split': [2, 4, 6]\n",
    "}\n",
    "\n",
    "model_dt_bag = train_dt_model(df, target_col, dt_bag_param_grid)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "545e5169",
   "metadata": {},
   "source": [
    "#### Descision Tree - Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a80ef60b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define parameter grid\n",
    "dt_vot_param_grid = {\n",
    "    'weights': [[1,2,1], [1,3,1], [1,4,1], [1,5,1]],\n",
    "    'n_jobs': [-1],\n",
    "    'verbose': [1]\n",
    "}\n",
    "\n",
    "# Train decision tree model with the best parameters\n",
    "model_dt_vot = train_dt_vot_model(df, target_col, dt_vot_param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "085ec47e",
   "metadata": {},
   "source": [
    "#### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e606553a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load your data into a pandas DataFrame called 'df'\n",
    "# Define your target column name as a string called 'target_col'\n",
    "\n",
    "# Define the hyperparameter grid\n",
    "knn_param_grid = {\n",
    "    'n_neighbors': [5],\n",
    "    'weights': ['uniform'],\n",
    "    'p': [2],\n",
    "}\n",
    "\n",
    "# Train the KNN model with default hyperparameters\n",
    "#model_knn = train_knn_model(df, target_col, param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c96ca74",
   "metadata": {},
   "source": [
    "#### Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1f97f9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load your data into a pandas DataFrame called 'df'\n",
    "# Define your target column name as a string called 'target_col'\n",
    "\n",
    "# Define the hyperparameter grid\n",
    "lasso_param_grid = {\n",
    "    'alpha': [1.0],\n",
    "    'fit_intercept': [True],\n",
    "    'normalize': [False],\n",
    "    'precompute': [False],\n",
    "    'max_iter': [1000],\n",
    "    'tol': [0.0001],\n",
    "    'warm_start': [False],\n",
    "    'positive': [False],\n",
    "    'random_state': [None],\n",
    "    'selection': ['cyclic']\n",
    "}\n",
    "\n",
    "# Train the Lasso model with default hyperparameters\n",
    "#model_lasso = train_lasso_model(df, target_col, param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e545f5eb",
   "metadata": {},
   "source": [
    "#### Polynomal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9148e563",
   "metadata": {},
   "outputs": [],
   "source": [
    "pol_param_grid = {\n",
    "    'poly__degree': [2, 3],\n",
    "    'linear__fit_intercept': [True, False],\n",
    "    'linear__normalize': [True, False]\n",
    "}\n",
    "\n",
    "\n",
    "# Train the model with default hyperparameters\n",
    "model_pol = train_pol_model(df, target_col, pol_param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef3f8f81",
   "metadata": {},
   "source": [
    "#### Overview of models with tuned hyper params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "123f70a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Val_Mae</th>\n",
       "      <th>Val_Mse</th>\n",
       "      <th>Val_Rmse</th>\n",
       "      <th>Val_R2</th>\n",
       "      <th>Test_Mae</th>\n",
       "      <th>Test_Mse</th>\n",
       "      <th>Test_Rmse</th>\n",
       "      <th>Test_R2</th>\n",
       "      <th>Best_Params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RF</td>\n",
       "      <td>0.283971</td>\n",
       "      <td>0.375343</td>\n",
       "      <td>0.140883</td>\n",
       "      <td>0.774153</td>\n",
       "      <td>0.563977</td>\n",
       "      <td>0.738370</td>\n",
       "      <td>0.545190</td>\n",
       "      <td>0.328288</td>\n",
       "      <td>{'max_depth': 10, 'min_samples_leaf': 2, 'min_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lin</td>\n",
       "      <td>0.490564</td>\n",
       "      <td>0.420064</td>\n",
       "      <td>0.648123</td>\n",
       "      <td>0.326602</td>\n",
       "      <td>0.573687</td>\n",
       "      <td>0.567870</td>\n",
       "      <td>0.753572</td>\n",
       "      <td>0.300345</td>\n",
       "      <td>{'fit_intercept': True, 'normalize': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pol</td>\n",
       "      <td>0.546667</td>\n",
       "      <td>0.499145</td>\n",
       "      <td>0.706502</td>\n",
       "      <td>0.199828</td>\n",
       "      <td>0.682752</td>\n",
       "      <td>1.010921</td>\n",
       "      <td>1.005445</td>\n",
       "      <td>-0.245524</td>\n",
       "      <td>{'linear__fit_intercept': False, 'linear__norm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DT</td>\n",
       "      <td>0.521600</td>\n",
       "      <td>0.464177</td>\n",
       "      <td>0.681305</td>\n",
       "      <td>0.255885</td>\n",
       "      <td>0.607513</td>\n",
       "      <td>0.630077</td>\n",
       "      <td>0.793774</td>\n",
       "      <td>0.223702</td>\n",
       "      <td>{'criterion': 'mse', 'max_depth': 4, 'max_feat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DT Vot</td>\n",
       "      <td>0.426909</td>\n",
       "      <td>0.315032</td>\n",
       "      <td>0.561277</td>\n",
       "      <td>0.494976</td>\n",
       "      <td>0.603543</td>\n",
       "      <td>0.629452</td>\n",
       "      <td>0.793380</td>\n",
       "      <td>0.224472</td>\n",
       "      <td>{'n_jobs': -1, 'verbose': 1, 'weights': [1, 4,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DT Bag</td>\n",
       "      <td>0.503116</td>\n",
       "      <td>0.450275</td>\n",
       "      <td>0.671026</td>\n",
       "      <td>0.278170</td>\n",
       "      <td>0.627766</td>\n",
       "      <td>0.707227</td>\n",
       "      <td>0.840968</td>\n",
       "      <td>0.128648</td>\n",
       "      <td>{'max_depth': 5, 'min_samples_split': 4}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Model   Val_Mae   Val_Mse  Val_Rmse    Val_R2  Test_Mae  Test_Mse  \\\n",
       "0      RF  0.283971  0.375343  0.140883  0.774153  0.563977  0.738370   \n",
       "1     Lin  0.490564  0.420064  0.648123  0.326602  0.573687  0.567870   \n",
       "2     Pol  0.546667  0.499145  0.706502  0.199828  0.682752  1.010921   \n",
       "3      DT  0.521600  0.464177  0.681305  0.255885  0.607513  0.630077   \n",
       "4  DT Vot  0.426909  0.315032  0.561277  0.494976  0.603543  0.629452   \n",
       "5  DT Bag  0.503116  0.450275  0.671026  0.278170  0.627766  0.707227   \n",
       "\n",
       "   Test_Rmse   Test_R2                                        Best_Params  \n",
       "0   0.545190  0.328288  {'max_depth': 10, 'min_samples_leaf': 2, 'min_...  \n",
       "1   0.753572  0.300345        {'fit_intercept': True, 'normalize': False}  \n",
       "2   1.005445 -0.245524  {'linear__fit_intercept': False, 'linear__norm...  \n",
       "3   0.793774  0.223702  {'criterion': 'mse', 'max_depth': 4, 'max_feat...  \n",
       "4   0.793380  0.224472  {'n_jobs': -1, 'verbose': 1, 'weights': [1, 4,...  \n",
       "5   0.840968  0.128648           {'max_depth': 5, 'min_samples_split': 4}  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "models_overview = { 'Model': ['RF', 'Lin', 'Pol', 'DT', 'DT Vot', 'DT Bag'],\n",
    "                'Val_Mae': [model_rf[2], model_lin[2], model_pol[2], model_dt[2], model_dt_vot[2], model_dt_bag[2]],\n",
    "                'Val_Mse': [model_rf[3], model_lin[3], model_pol[3], model_dt[3], model_dt_vot[3], model_dt_bag[3]],\n",
    "                'Val_Rmse': [model_rf[4], model_lin[4], model_pol[4], model_dt[4], model_dt_vot[4], model_dt_bag[4]],\n",
    "                'Val_R2': [model_rf[5], model_lin[5], model_pol[5], model_dt[5], model_dt_vot[5], model_dt_bag[5]],\n",
    "                'Test_Mae': [model_rf[6], model_lin[6], model_pol[6], model_dt[6], model_dt_vot[6], model_dt_bag[6]],\n",
    "                'Test_Mse': [model_rf[7], model_lin[7], model_pol[7], model_dt[7], model_dt_vot[7], model_dt_bag[7]],\n",
    "                'Test_Rmse': [model_rf[8], model_lin[8], model_pol[8], model_dt[8], model_dt_vot[8], model_dt_bag[8]], \n",
    "                'Test_R2': [model_rf[9], model_lin[9], model_pol[9], model_dt[9], model_dt_vot[9], model_dt_bag[9]],\n",
    "                'Best_Params': [model_rf[1], model_lin[1], model_pol[1], model_dt[1], model_dt_vot[1], model_dt_bag[1]]}\n",
    "df = pd.DataFrame(models_overview)\n",
    "df.head(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7f9fcc5",
   "metadata": {},
   "source": [
    "#### Save the models with joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a0093828",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_models():\n",
    "    models = [model_rf[0], model_lin[0], model_pol[0], model_dt[0], model_dt_vot[0], model_dt_bag[0]]\n",
    "    file_name = [\"rf\", \"lin\", \"pol\", \"dt\", \"dt_vot\", \"dt_bag\"]\n",
    "    for i, model in enumerate(models):\n",
    "        filename = f'./models/model_{file_name[i]}.joblib'\n",
    "        joblib.dump(model, filename)\n",
    "\n",
    "#save_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5e708ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5d22ddc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab8f052f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}