DATA_441_Final_Project_RandomForest.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyPSQdieC7MCYJoHgue99mo6"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","from sklearn.preprocessing import StandardScaler\n","from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, RandomForestClassifier, GradientBoostingClassifier\n","from sklearn.pipeline import Pipeline\n","from sklearn.model_selection import train_test_split as tts, KFold, GridSearchCV\n","from sklearn.metrics import mean_squared_error as mse\n","import os"],"metadata":{"id":"sTHYjPvS1dVA","executionInfo":{"status":"ok","timestamp":1684030422566,"user_tz":240,"elapsed":1749,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":1,"outputs":[]},{"cell_type":"code","source":["scaler = StandardScaler()"],"metadata":{"id":"3Fkgj-w3W8xY","executionInfo":{"status":"ok","timestamp":1684030425470,"user_tz":240,"elapsed":375,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":2,"outputs":[]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Pq6HH-_51_u6","executionInfo":{"status":"ok","timestamp":1684030448895,"user_tz":240,"elapsed":23136,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"8a2a8bcf-3505-4333-bcfb-c525e4d4af0e"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["data = pd.read_csv('drive/MyDrive/Colab Notebooks/data/telecom_churn.csv')"],"metadata":{"id":"zilsPACO1wdS","executionInfo":{"status":"ok","timestamp":1684030450496,"user_tz":240,"elapsed":1613,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["X = data.loc[:, 'AccountWeeks':].values\n","y = data['Churn'].values"],"metadata":{"id":"Z7U-2MfHZvCO","executionInfo":{"status":"ok","timestamp":1684030450498,"user_tz":240,"elapsed":33,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["# here is our pipeline for grid search\n","rfr_pipe = Pipeline([('zscores', StandardScaler()),\n","                     ('rfr', RandomForestRegressor())])"],"metadata":{"id":"HzFtT_HbWUJl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# now we specify which parameters we want to consider\n","params = [{'rfr__n_estimators': [65,75,85],\n","         'rfr__max_depth': [5,6,7],\n","         'rfr__min_samples_leaf': [2],\n","         'rfr__min_samples_split': [2]}]"],"metadata":{"id":"Rt-ozU9GYKM5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# finally we perform the grid search\n","gs_rfr = GridSearchCV(rfr_pipe,\n","                      param_grid=params,\n","                      scoring='neg_mean_squared_error',\n","                      cv=5)\n","gs_rfr.fit(X, y)\n","gs_rfr.best_params_"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Hcm5f7gTaKm4","executionInfo":{"status":"ok","timestamp":1682913624855,"user_tz":240,"elapsed":27446,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"b53cd0f6-1656-447a-b6d7-bbb9c286adf9"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'rfr__max_depth': 6,\n"," 'rfr__min_samples_leaf': 2,\n"," 'rfr__min_samples_split': 2,\n"," 'rfr__n_estimators': 65}"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["# and we look at our score\n","gs_rfr.score(X,y)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HHQr9euiaN2r","executionInfo":{"status":"ok","timestamp":1682913659996,"user_tz":240,"elapsed":225,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"57dd61cf-3f31-4381-cb58-774a6378d285"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["-0.04114885296727684"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","source":["# here is our pipeline for grid search\n","gbr_pipe = Pipeline([('zscores', StandardScaler()),\n","                     ('rfr', GradientBoostingRegressor())])"],"metadata":{"id":"LtOwOR-dQqDa","executionInfo":{"status":"ok","timestamp":1684000765042,"user_tz":240,"elapsed":573,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["# now we specify which parameters we want to consider\n","params = [{'rfr__n_estimators': [55],\n","         'rfr__max_depth': [4],\n","         'rfr__min_samples_leaf': [3],\n","         'rfr__min_samples_split': [3]}]"],"metadata":{"id":"6umqQhQcQoT4","executionInfo":{"status":"ok","timestamp":1684000992904,"user_tz":240,"elapsed":592,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["# finally we perform the grid search\n","gs_gbr = GridSearchCV(gbr_pipe,\n","                      param_grid=params,\n","                      scoring='neg_mean_squared_error',\n","                      cv=5)\n","gs_gbr.fit(X, y)\n","gs_gbr.best_params_"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7X_6v8EHQrnH","executionInfo":{"status":"ok","timestamp":1684000998186,"user_tz":240,"elapsed":4398,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"2e616031-6429-4cc4-f3e5-1dcf687c5059"},"execution_count":28,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'rfr__max_depth': 4,\n"," 'rfr__min_samples_leaf': 3,\n"," 'rfr__min_samples_split': 3,\n"," 'rfr__n_estimators': 55}"]},"metadata":{},"execution_count":28}]},{"cell_type":"code","source":["# here is our pipeline for grid search\n","rfc_pipe = Pipeline([('zscores', StandardScaler()),\n","                     ('rfr', RandomForestClassifier())])"],"metadata":{"id":"opqxQ9DcCzCb","executionInfo":{"status":"ok","timestamp":1684030603882,"user_tz":240,"elapsed":438,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# now we specify which parameters we want to consider\n","params = [{'rfr__n_estimators': [90,95,100],\n","         'rfr__max_depth': [7],\n","         'rfr__min_samples_leaf': [2],\n","         'rfr__min_samples_split': [2]}]"],"metadata":{"id":"8Z9jTkR4CzCc","executionInfo":{"status":"ok","timestamp":1684030766231,"user_tz":240,"elapsed":278,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["# finally we perform the grid search\n","gs_rfc = GridSearchCV(rfc_pipe,\n","                      param_grid=params,\n","                      scoring='neg_mean_squared_error',\n","                      cv=5)\n","gs_rfc.fit(X, y)\n","gs_rfc.best_params_"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1684030782852,"user_tz":240,"elapsed":15458,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"d7e0a35e-6278-4f16-e50d-6f6df57e83a1","id":"iS2zh3vHCzCc"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py:378: FitFailedWarning: \n","30 fits failed out of a total of 60.\n","The score on these train-test partitions for these parameters will be set to nan.\n","If these failures are not expected, you can try to debug them by setting error_score='raise'.\n","\n","Below are more details about the failures:\n","--------------------------------------------------------------------------------\n","30 fits failed with the following error:\n","Traceback (most recent call last):\n","  File \"/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n","    estimator.fit(X_train, y_train, **fit_params)\n","  File \"/usr/local/lib/python3.10/dist-packages/sklearn/pipeline.py\", line 405, in fit\n","    self._final_estimator.fit(Xt, y, **fit_params_last_step)\n","  File \"/usr/local/lib/python3.10/dist-packages/sklearn/ensemble/_forest.py\", line 340, in fit\n","    self._validate_params()\n","  File \"/usr/local/lib/python3.10/dist-packages/sklearn/base.py\", line 600, in _validate_params\n","    validate_parameter_constraints(\n","  File \"/usr/local/lib/python3.10/dist-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n","    raise InvalidParameterError(\n","sklearn.utils._param_validation.InvalidParameterError: The 'min_samples_split' parameter of RandomForestClassifier must be an int in the range [2, inf) or a float in the range (0.0, 1.0]. Got 1 instead.\n","\n","  warnings.warn(some_fits_failed_message, FitFailedWarning)\n","/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_search.py:952: UserWarning: One or more of the test scores are non-finite: [        nan         nan         nan -0.06360378 -0.06450423 -0.06270423\n","         nan         nan         nan -0.06390318 -0.06240348 -0.06330438]\n","  warnings.warn(\n"]},{"output_type":"execute_result","data":{"text/plain":["{'rfr__max_depth': 7,\n"," 'rfr__min_samples_leaf': 2,\n"," 'rfr__min_samples_split': 2,\n"," 'rfr__n_estimators': 95}"]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["# here is our pipeline for grid search\n","gbc_pipe = Pipeline([('zscores', StandardScaler()),\n","                     ('rfr', GradientBoostingClassifier())])"],"metadata":{"executionInfo":{"status":"ok","timestamp":1684030990670,"user_tz":240,"elapsed":396,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"id":"B631l0J5EBT_"},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["# now we specify which parameters we want to consider\n","params = [{'rfr__n_estimators': [70],\n","         'rfr__max_depth':[3],\n","         'rfr__min_samples_leaf': [5,6],\n","         'rfr__min_samples_split': [2]}]"],"metadata":{"id":"0mZmxzWGD5wk","executionInfo":{"status":"ok","timestamp":1684031359445,"user_tz":240,"elapsed":316,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["# finally we perform the grid search\n","gs_gbc = GridSearchCV(gbc_pipe,\n","                      param_grid=params,\n","                      scoring='neg_mean_squared_error',\n","                      cv=5)\n","gs_gbc.fit(X, y)\n","gs_gbc.best_params_"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1684031366840,"user_tz":240,"elapsed":4978,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"26d38d50-46f1-4d63-b9b3-8a20e536aac9","id":"d7ParBrLD5wn"},"execution_count":28,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'rfr__max_depth': 3,\n"," 'rfr__min_samples_leaf': 5,\n"," 'rfr__min_samples_split': 2,\n"," 'rfr__n_estimators': 70}"]},"metadata":{},"execution_count":28}]},{"cell_type":"code","source":["mses = []\n","acc = []\n","kf = KFold(n_splits=10,shuffle=True,random_state=1234)\n","\n","model = RandomForestRegressor(max_depth=6, min_samples_leaf=2, min_samples_split=2, n_estimators=65)\n","\n","for idxtrain, idxtest in kf.split(X):\n","  xtrain = X[idxtrain]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  xtest = X[idxtest]\n","  xtrain = scaler.fit_transform(xtrain)\n","  xtest = scaler.transform(xtest)\n","\n","  model.fit(xtrain,ytrain)\n","  yhat = model.predict(xtest)\n","\n","  mses.append(mse(ytest,yhat))\n","  acc.append(100*(1 - (np.sum(np.abs(np.around(yhat)-ytest))/len(yhat))))\n","\n","print(\"The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : \"+str(np.mean(mses)))\n","print(\"Average accuracy:\" + str(np.mean(acc)) + '%')\n","for i in acc:\n","  print(str(i) + '%')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"CRawUkl4946c","executionInfo":{"status":"ok","timestamp":1684000552810,"user_tz":240,"elapsed":7698,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"4c0f9012-6e9b-45f4-ae58-280d633289e7"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : 0.07052201902501304\n","Average accuracy:92.94779809749869%\n","95.80838323353294%\n","93.41317365269461%\n","94.61077844311377%\n","92.1921921921922%\n","93.69369369369369%\n","90.69069069069069%\n","93.3933933933934%\n","93.69369369369369%\n","92.1921921921922%\n","89.7897897897898%\n"]}]},{"cell_type":"code","source":["mses = []\n","acc = []\n","kf = KFold(n_splits=10,shuffle=True,random_state=1234)\n","\n","model = GradientBoostingRegressor(max_depth=4, min_samples_leaf=3, min_samples_split=3, n_estimators=55)\n","\n","for idxtrain, idxtest in kf.split(X):\n","  xtrain = X[idxtrain]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  xtest = X[idxtest]\n","  xtrain = scaler.fit_transform(xtrain)\n","  xtest = scaler.transform(xtest)\n","\n","  model.fit(xtrain,ytrain)\n","  yhat = model.predict(xtest)\n","\n","  mses.append(mse(ytest,yhat))\n","  acc.append(100*(1 - (np.sum(np.abs(np.around(yhat)-ytest))/len(yhat))))\n","\n","print(\"The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : \"+str(np.mean(mses)))\n","print(\"Average accuracy:\" + str(np.mean(acc)) + '%')\n","for i in acc:\n","  print(str(i) + '%')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bAPHqc1UQ-OX","executionInfo":{"status":"ok","timestamp":1684001032781,"user_tz":240,"elapsed":4189,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"bd85f6e9-80a5-45fa-f1db-438160c3fc8e"},"execution_count":29,"outputs":[{"output_type":"stream","name":"stdout","text":["The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : 0.05377227528040629\n","Average accuracy:93.36758914603226%\n","96.10778443113772%\n","94.311377245509%\n","95.50898203592814%\n","92.7927927927928%\n","94.29429429429429%\n","91.89189189189189%\n","92.49249249249249%\n","93.09309309309309%\n","93.09309309309309%\n","90.09009009009009%\n"]}]},{"cell_type":"code","source":["def boosted(x, y, xnew):\n","  model1 = RandomForestClassifier(max_depth=7, min_samples_leaf=2, min_samples_split=2, n_estimators=95)\n","  model1.fit(x,y)\n","  residuals1 = y - model1.predict(x)\n","  model2 = GradientBoostingClassifier(max_depth=3, min_samples_leaf=5, min_samples_split=2, n_estimators=70)\n","  model2.fit(x,residuals1)\n","  output = model1.predict(xnew) + model2.predict(xnew)\n","  return output "],"metadata":{"id":"v4-QirsNXHhh","executionInfo":{"status":"ok","timestamp":1684031699146,"user_tz":240,"elapsed":966,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}}},"execution_count":30,"outputs":[]},{"cell_type":"code","source":["mses = []\n","acc = []\n","kf = KFold(n_splits=10,shuffle=True)\n","\n","for idxtrain, idxtest in kf.split(X):\n","  xtrain = X[idxtrain]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  xtest = X[idxtest]\n","  xtrain = scaler.fit_transform(xtrain)\n","  xtest = scaler.transform(xtest)\n","\n","  yhat = boosted(xtrain, ytrain, xtest)\n","\n","  mses.append(mse(ytest,yhat))\n","  acc.append(100*(1 - (np.sum(np.abs(np.around(yhat)-ytest))/len(yhat))))\n","\n","print(\"The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : \"+str(np.mean(mses)))\n","print(\"Average accuracy:\" + str(np.mean(acc)) + '%')\n","for i in acc:\n","  print(str(i) + '%')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ASUHm8KcMKEo","executionInfo":{"status":"ok","timestamp":1684031724858,"user_tz":240,"elapsed":22253,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"a67db283-2847-44fb-f1f5-8d6e2566c8ee"},"execution_count":31,"outputs":[{"output_type":"stream","name":"stdout","text":["The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : 0.06151031270791749\n","Average accuracy:93.84896872920825%\n","92.21556886227545%\n","94.91017964071857%\n","95.80838323353294%\n","94.5945945945946%\n","93.3933933933934%\n","92.1921921921922%\n","93.09309309309309%\n","93.09309309309309%\n","96.09609609609609%\n","93.09309309309309%\n"]}]},{"cell_type":"code","source":["mses = []\n","acc = []\n","kf = KFold(n_splits=10,shuffle=True,random_state=1234)\n","\n","model = RandomForestClassifier(max_depth=7, min_samples_leaf=2, min_samples_split=2, n_estimators=95)\n","\n","for idxtrain, idxtest in kf.split(X):\n","  xtrain = X[idxtrain]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  xtest = X[idxtest]\n","  xtrain = scaler.fit_transform(xtrain)\n","  xtest = scaler.transform(xtest)\n","\n","  model.fit(xtrain,ytrain)\n","  yhat = model.predict(xtest)\n","\n","  mses.append(mse(ytest,yhat))\n","  acc.append(100*(1 - (np.sum(np.abs(np.around(yhat)-ytest))/len(yhat))))\n","\n","print(\"The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : \"+str(np.mean(mses)))\n","print(\"Average accuracy:\" + str(np.mean(acc)) + '%')\n","for i in acc:\n","  print(str(i) + '%')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HV864JS3NXSW","executionInfo":{"status":"ok","timestamp":1684030838000,"user_tz":240,"elapsed":5376,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"065b5b0e-a668-4451-d51c-29fb707942ab"},"execution_count":15,"outputs":[{"output_type":"stream","name":"stdout","text":["The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : 0.062120803438168705\n","Average accuracy:93.78791965618313%\n","95.80838323353294%\n","94.91017964071857%\n","95.50898203592814%\n","93.3933933933934%\n","94.5945945945946%\n","91.89189189189189%\n","93.09309309309309%\n","94.8948948948949%\n","93.3933933933934%\n","90.39039039039038%\n"]}]},{"cell_type":"code","source":["mses = []\n","acc = []\n","kf = KFold(n_splits=10,shuffle=True,random_state=1234)\n","\n","model = GradientBoostingClassifier(max_depth=3, min_samples_leaf=5, min_samples_split=2, n_estimators=70)\n","\n","for idxtrain, idxtest in kf.split(X):\n","  xtrain = X[idxtrain]\n","  ytrain = y[idxtrain]\n","  ytest = y[idxtest]\n","  xtest = X[idxtest]\n","  xtrain = scaler.fit_transform(xtrain)\n","  xtest = scaler.transform(xtest)\n","\n","  model.fit(xtrain,ytrain)\n","  yhat = model.predict(xtest)\n","\n","  mses.append(mse(ytest,yhat))\n","  acc.append(100*(1 - (np.sum(np.abs(np.around(yhat)-ytest))/len(yhat))))\n","\n","print(\"The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : \"+str(np.mean(mses)))\n","print(\"Average accuracy:\" + str(np.mean(acc)) + '%')\n","for i in acc:\n","  print(str(i) + '%')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"C-8bpcToCdtx","executionInfo":{"status":"ok","timestamp":1684031576041,"user_tz":240,"elapsed":5105,"user":{"displayName":"Matthew Cheng","userId":"00814230256052145695"}},"outputId":"026f6b70-403a-45f7-9272-58275116eb26"},"execution_count":29,"outputs":[{"output_type":"stream","name":"stdout","text":["The new Cross-validated Mean Squared Error for Gramfort's Locally Weighted Regression is : 0.06152020283756811\n","Average accuracy:93.84797971624317%\n","96.7065868263473%\n","94.311377245509%\n","95.20958083832335%\n","93.993993993994%\n","94.29429429429429%\n","92.7927927927928%\n","93.09309309309309%\n","93.993993993994%\n","93.3933933933934%\n","90.69069069069069%\n"]}]},{"cell_type":"code","source":[],"metadata":{"id":"Oc92IkCxCrGc"},"execution_count":null,"outputs":[]}]}