Linear Regression/Linear Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load package\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# set path\n",
    "import os\n",
    "default_path = \"/Users/mayritaspring/Desktop/Github/Data-Science/Example_LasVegasTrip/\"\n",
    "os.chdir(default_path)\n",
    "\n",
    "# skip interation warning\n",
    "import warnings; \n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   User country  Nr. reviews  Nr. hotel reviews  Helpful votes  Score  \\\n",
      "ID                                                                      \n",
      "1           USA           11                  4             13      5   \n",
      "2           USA          119                 21             75      3   \n",
      "3           USA           36                  9             25      5   \n",
      "4            UK           14                  7             14      4   \n",
      "5        Canada            5                  5              2      4   \n",
      "\n",
      "   Period of stay Traveler type Pool  Gym Tennis court Spa Casino  \\\n",
      "ID                                                                  \n",
      "1         Dec-Feb       Friends   NO  YES           NO  NO    YES   \n",
      "2         Dec-Feb      Business   NO  YES           NO  NO    YES   \n",
      "3         Mar-May      Families   NO  YES           NO  NO    YES   \n",
      "4         Mar-May       Friends   NO  YES           NO  NO    YES   \n",
      "5         Mar-May          Solo   NO  YES           NO  NO    YES   \n",
      "\n",
      "   Free internet                              Hotel name  Hotel stars  \\\n",
      "ID                                                                      \n",
      "1            YES  Circus Circus Hotel & Casino Las Vegas            3   \n",
      "2            YES  Circus Circus Hotel & Casino Las Vegas            3   \n",
      "3            YES  Circus Circus Hotel & Casino Las Vegas            3   \n",
      "4            YES  Circus Circus Hotel & Casino Las Vegas            3   \n",
      "5            YES  Circus Circus Hotel & Casino Las Vegas            3   \n",
      "\n",
      "    Nr. rooms User continent  Member years Review month Review weekday  \n",
      "ID                                                                      \n",
      "1      3773.0  North America           9.0      January       Thursday  \n",
      "2      3773.0  North America           3.0      January         Friday  \n",
      "3      3773.0  North America           2.0     February       Saturday  \n",
      "4      3773.0         Europe           6.0     February         Friday  \n",
      "5      3773.0  North America           7.0        March        Tuesday  \n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 504 entries, 1 to 504\n",
      "Data columns (total 20 columns):\n",
      "User country         504 non-null object\n",
      "Nr. reviews          504 non-null int64\n",
      "Nr. hotel reviews    504 non-null int64\n",
      "Helpful votes        504 non-null int64\n",
      "Score                504 non-null int64\n",
      "Period of stay       504 non-null object\n",
      "Traveler type        504 non-null object\n",
      "Pool                 504 non-null object\n",
      "Gym                  504 non-null object\n",
      "Tennis court         504 non-null object\n",
      "Spa                  504 non-null object\n",
      "Casino               504 non-null object\n",
      "Free internet        504 non-null object\n",
      "Hotel name           504 non-null object\n",
      "Hotel stars          504 non-null int64\n",
      "Nr. rooms            408 non-null float64\n",
      "User continent       408 non-null object\n",
      "Member years         408 non-null float64\n",
      "Review month         408 non-null object\n",
      "Review weekday       408 non-null object\n",
      "dtypes: float64(2), int64(5), object(13)\n",
      "memory usage: 82.7+ KB\n"
     ]
    }
   ],
   "source": [
    "# Data Prepararion\n",
    "# read data\n",
    "review_data = pd.read_csv(\"LasVegasTripAdvisorReviews.csv\",index_col=\"ID\")\n",
    "print(review_data.head())\n",
    "\n",
    "#data description\n",
    "review_data.head\n",
    "review_data.values\n",
    "review_data.shape\n",
    "review_data.columns\n",
    "review_data.index\n",
    "review_data.info()\n",
    "review_data.dtypes\n",
    "review_data.describe()\n",
    "\n",
    "#one hot encoding\n",
    "category_var = ['User country', 'Period of stay', 'Pool', 'Gym', 'Tennis court','Spa','Casino','Free internet', 'Traveler type','Hotel name','User continent','Review month','Review weekday']\n",
    "for col in category_var: \n",
    "    review_data[col] = review_data[col].astype('category')\n",
    "    dummies = pd.get_dummies(review_data.loc[:, col], prefix=col ) \n",
    "    review_data = pd.concat( [review_data, dummies], axis = 1)\n",
    "\n",
    "# drop original variable\n",
    "fields_to_drop =  category_var\n",
    "review_data = review_data.drop(fields_to_drop, axis = 1 )\n",
    "\n",
    "# replace missing value with zero\n",
    "review_data = review_data.fillna(review_data.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split to Training and Testing\n",
    "from sklearn import cross_validation\n",
    "seed = 7\n",
    "test_size = 0.3\n",
    "X = review_data.loc[:, review_data.columns != 'Score']\n",
    "y = review_data[['Score']]\n",
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=test_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.23665633470138747\n",
      "Coefficients: \n",
      " [[ 2.64652589e-04  1.60420539e-06  1.79533303e-04  2.33841550e-01\n",
      "  -1.46983707e-04 -1.09650791e-03  2.81607029e-01 -8.63709753e-01\n",
      "   1.57877061e+00  1.12318057e-01 -2.12923536e+00  2.46707331e-01\n",
      "  -3.15563730e-01  1.86795024e-14  2.56351761e-01 -2.82779079e-01\n",
      "  -3.21675960e-01  5.53904393e-01  4.11722530e-01 -9.54401478e-01\n",
      "   2.31491902e-03 -1.99285033e-14  4.31824295e-01 -7.16769637e-02\n",
      "  -1.19388197e-01 -1.19087449e+00  4.31574689e-01 -4.75324254e-01\n",
      "   2.17650017e-02 -7.60994741e-01 -5.32907052e-15  5.03329834e-01\n",
      "  -6.79120237e-01  3.23356609e-02 -1.77276555e-01  1.15837561e+00\n",
      "   1.97374821e-01  1.19983169e-02  1.00348586e+00  5.55111512e-16\n",
      "   1.99798318e+00 -4.55191440e-15  1.84197616e+00 -4.17365208e-01\n",
      "  -4.85462680e-01 -1.05901280e+00 -1.11022302e-15  2.07311472e-01\n",
      "  -9.70840496e-01  5.49315236e-01 -1.42752185e-02  1.61149710e-01\n",
      "   1.83712419e-01 -8.88231695e-01  2.70931593e-01  3.53932504e-01\n",
      "  -2.71031338e-01 -3.53832759e-01 -2.42080418e-01  2.42080418e-01\n",
      "  -1.49202397e-01  1.49202397e-01 -2.67389291e-02  2.67389291e-02\n",
      "   2.89063788e-02 -2.89063788e-02  4.34821119e-02 -4.34821119e-02\n",
      "  -2.73060616e-01  2.73060616e-01 -2.23461028e-01  1.14291630e-01\n",
      "  -1.00956009e-01  1.46385265e-01  6.37401422e-02  1.63472973e-01\n",
      "  -3.67004691e-01 -2.42080418e-01  4.20226790e-01  2.34630082e-02\n",
      "  -2.07196451e-01  2.68111426e-01  3.76707082e-01 -2.73060616e-01\n",
      "  -1.04293006e-01 -3.01352703e-01 -1.49202397e-01  1.29151752e-01\n",
      "   2.16677163e-01 -2.12174268e-01 -4.32725273e-01 -1.38860164e-01\n",
      "   2.31815427e-01  6.85538772e-02  2.50678563e-01  2.79091926e-01\n",
      "  -2.64911925e-01  4.96952907e-01  1.12841523e-01  7.55558853e-02\n",
      "   3.59889862e-02 -1.20272205e+00  4.43516296e-01 -2.21786695e-01\n",
      "  -1.25488942e-01 -7.12594464e-01 -2.18715396e-01 -4.85562142e-01\n",
      "  -7.74255047e-01  5.16118673e-01 -1.17813031e-02  4.53765886e-01\n",
      "   2.02563602e-01  1.87924859e-01  1.43091776e-01 -4.27899258e-01\n",
      "   2.13663606e-01 -1.90313651e-01  1.76792065e-01 -3.13587638e-01\n",
      "  -3.48041572e-01]]\n",
      "Mean squared error: 1.23\n",
      "Variance score: -0.24\n"
     ]
    }
   ],
   "source": [
    "# Linear Regression (OLS)\n",
    "# load package\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "# Create linear regression object\n",
    "regr = linear_model.LinearRegression()\n",
    "\n",
    "# Train the model using the training sets\n",
    "regr.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using the testing set\n",
    "y_pred = regr.predict(X_test)\n",
    "print(regr.score(X_test, y_test, sample_weight=None))\n",
    "\n",
    "# The coefficients\n",
    "print('Coefficients: \\n', regr.coef_)\n",
    "# The mean squared error\n",
    "print(\"Mean squared error: %.2f\" % mean_squared_error(y_test, y_pred))\n",
    "regr_mse = mean_squared_error(y_test, y_pred)\n",
    "\n",
    "# Explained variance score: 1 is perfect prediction\n",
    "print('Variance score: %.2f' % r2_score(y_test, y_pred))\n",
    "regr_score = r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.19494904157746062\n",
      "Coefficients: \n",
      " [[ 3.46648074e-04 -3.84793279e-05  1.17344868e-04  2.29855402e-01\n",
      "  -1.48308844e-04 -1.05373817e-03  1.21029410e-01 -7.39672594e-01\n",
      "   7.49275914e-01 -7.31868270e-02 -1.27933791e+00  6.37647695e-02\n",
      "  -2.98099729e-01  0.00000000e+00  1.22180559e-01 -3.07604184e-01\n",
      "  -3.49432087e-01  3.39290690e-01  2.50284757e-01 -7.64783032e-01\n",
      "  -1.73873286e-01  0.00000000e+00  2.41475236e-01  2.63406954e-01\n",
      "   2.30621399e-01 -5.32336932e-01  3.04082746e-01 -5.78339473e-02\n",
      "  -1.11175606e-01 -6.94709945e-01  0.00000000e+00  3.42412779e-01\n",
      "  -1.95334668e-01 -1.19901730e-01  1.76739421e-01  8.44773854e-01\n",
      "   6.94990289e-02 -1.24841418e-01  7.40209979e-01  0.00000000e+00\n",
      "   1.31213137e+00  0.00000000e+00  1.32058026e+00 -5.06979195e-03\n",
      "  -3.84360306e-01 -1.01433370e+00  0.00000000e+00  5.65735345e-02\n",
      "  -4.13106430e-01  3.46068648e-01  8.90574533e-02  4.23251183e-02\n",
      "   1.08090354e-02 -3.97598791e-01  2.76881716e-01  2.79982536e-01\n",
      "  -2.31493654e-01 -3.25370598e-01 -2.44542462e-01  2.44542462e-01\n",
      "  -1.36642984e-01  1.36642984e-01 -2.15116645e-02  2.15116645e-02\n",
      "   3.05421664e-02 -3.05421664e-02  3.75298968e-02 -3.75298968e-02\n",
      "  -2.80320999e-01  2.80320999e-01 -2.46822465e-01  1.19497021e-01\n",
      "  -8.87175547e-02  1.50825267e-01  6.52177325e-02  1.51660656e-01\n",
      "  -3.42934977e-01 -2.44542462e-01  4.12389176e-01  6.93462754e-03\n",
      "  -1.90806613e-01  2.89865638e-01  3.74197716e-01 -2.80320999e-01\n",
      "  -1.14531435e-01 -2.81076950e-01 -1.36642984e-01  1.21130597e-01\n",
      "   2.31654666e-01 -2.20843537e-01 -4.22262638e-01 -1.32578967e-01\n",
      "   2.02355217e-01  7.46771635e-02  2.28336510e-01  2.73339596e-01\n",
      "  -3.49551711e-01 -6.27912220e-02  8.45211246e-02  1.11595519e-01\n",
      "   4.10659842e-02 -5.80596575e-01  3.99949020e-01 -1.84661463e-01\n",
      "  -1.38535162e-01 -6.73272109e-01 -2.29827996e-01 -4.10846055e-01\n",
      "  -6.92473220e-01  4.62056396e-01 -4.83033122e-02  4.33152507e-01\n",
      "   1.73198622e-01  1.53805891e-01  1.44387101e-01 -4.11341652e-01\n",
      "   2.02944140e-01 -2.07594394e-01  1.73281664e-01 -3.11697471e-01\n",
      "  -3.45736269e-01]]\n",
      "Mean squared error: 1.19\n",
      "Variance score: -0.19\n"
     ]
    }
   ],
   "source": [
    "# Ridge regression\n",
    "# set parameter\n",
    "alphas = np.logspace(-4, -0.5, 30) # default base = 10\n",
    "tuned_parameters = [{'alpha': alphas}]\n",
    "n_folds = 3\n",
    "\n",
    "# load package\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# model\n",
    "model = Ridge()\n",
    "ridge = GridSearchCV(estimator=model, param_grid = tuned_parameters, cv=n_folds, refit=False)\n",
    "ridge.fit(X_train, y_train)\n",
    "\n",
    "# Final Model\n",
    "ridge_final = Ridge(alpha = ridge.best_params_['alpha'])\n",
    "ridge_final.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using the testing set\n",
    "y_pred = ridge_final.predict(X_test)\n",
    "# ridge_score = ridge_final.score(X_test, y_test, sample_weight=None)\n",
    "print(ridge_final.score(X_test, y_test, sample_weight=None))\n",
    "\n",
    "# The coefficients\n",
    "print('Coefficients: \\n', ridge_final.coef_)\n",
    "\n",
    "# The mean squared error\n",
    "print(\"Mean squared error: %.2f\" % mean_squared_error(y_test, y_pred))\n",
    "ridge_mse = mean_squared_error(y_test, y_pred)\n",
    "\n",
    "# Explained variance score: 1 is perfect prediction\n",
    "print('Variance score: %.2f' % r2_score(y_test, y_pred))\n",
    "ridge_score = r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.07871713952384263\n",
      "Coefficients: \n",
      " [ 6.70515176e-04  1.98742761e-04 -6.40541926e-04  2.39246916e-01\n",
      " -1.49171723e-04 -8.26865436e-04  0.00000000e+00 -0.00000000e+00\n",
      "  0.00000000e+00 -1.10789484e-01 -0.00000000e+00  0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00  0.00000000e+00 -0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00  0.00000000e+00 -0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00  0.00000000e+00  1.93437320e-02\n",
      "  0.00000000e+00 -0.00000000e+00  1.24343096e-01 -0.00000000e+00\n",
      " -0.00000000e+00 -0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      " -0.00000000e+00 -0.00000000e+00  0.00000000e+00  4.29185868e-01\n",
      "  0.00000000e+00 -0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "  0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      " -0.00000000e+00 -2.20126890e-01  0.00000000e+00  0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00 -0.00000000e+00  0.00000000e+00\n",
      "  0.00000000e+00 -0.00000000e+00  1.05088523e-01  0.00000000e+00\n",
      " -0.00000000e+00 -1.32160431e-01 -4.87286996e-01  3.82954122e-16\n",
      " -2.60602105e-01  3.74941930e-15 -1.40907979e-02  0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00  0.00000000e+00 -0.00000000e+00\n",
      " -7.64167721e-01  0.00000000e+00 -2.73445332e-01  1.23049924e-01\n",
      " -4.46478815e-02  8.56746912e-02  0.00000000e+00  0.00000000e+00\n",
      " -1.95708863e-01 -7.55022023e-02  3.20526027e-01 -0.00000000e+00\n",
      " -0.00000000e+00  5.63960805e-02  2.87096198e-01 -2.54282354e-02\n",
      " -3.15353936e-02 -1.06428348e-01 -3.80670322e-02  8.43569754e-03\n",
      "  1.70530407e-01 -1.70857677e-01 -2.37797792e-01 -8.69697316e-03\n",
      "  0.00000000e+00  1.39408182e-02  1.04436351e-01  1.63543445e-01\n",
      " -3.79733192e-01 -1.26366849e-01  0.00000000e+00  0.00000000e+00\n",
      " -0.00000000e+00  0.00000000e+00  7.37222115e-02  0.00000000e+00\n",
      "  0.00000000e+00 -3.01427992e-01 -0.00000000e+00 -6.99391882e-02\n",
      " -3.53946676e-01  1.46755433e-01 -2.19577258e-01  1.09159475e-01\n",
      " -0.00000000e+00 -0.00000000e+00  0.00000000e+00 -3.94818076e-01\n",
      "  5.95681410e-02 -2.86723379e-01  2.91111526e-02 -3.91220282e-01\n",
      " -4.25030392e-01]\n",
      "Mean squared error: 1.07\n",
      "Variance score: -0.08\n"
     ]
    }
   ],
   "source": [
    "# LASSO\n",
    "# load packages\n",
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "# model\n",
    "lasso = Lasso(random_state=0, normalize = True)\n",
    "clf = GridSearchCV(lasso,  param_grid= tuned_parameters, cv=n_folds, refit=False)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Final Model\n",
    "clf_final = Lasso(alpha = clf.best_params_['alpha'])\n",
    "clf_final.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using the testing set\n",
    "y_pred = clf_final.predict(X_test)\n",
    "print(clf_final.score(X_test, y_test, sample_weight=None))\n",
    "\n",
    "# The coefficients\n",
    "print('Coefficients: \\n', clf_final.coef_)\n",
    "\n",
    "# The mean squared error\n",
    "print(\"Mean squared error: %.2f\" % mean_squared_error(y_test, y_pred))\n",
    "clf_mse = mean_squared_error(y_test, y_pred)\n",
    "\n",
    "# Explained variance score: 1 is perfect prediction\n",
    "print('Variance score: %.2f' % r2_score(y_test, y_pred))\n",
    "clf_score = r2_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------\n",
      "Compare Score: \n",
      " -0.2367 -0.1949 -0.0787\n",
      "Compare MSE: \n",
      " 1.2315 1.19 1.0742\n"
     ]
    }
   ],
   "source": [
    "# compare\n",
    "print('------------------------------------------------')\n",
    "print('Compare Score: \\n', round(regr_score,4), round(ridge_score,4), round(clf_score,4))\n",
    "print('Compare MSE: \\n', round(regr_mse,4), round(ridge_mse,4), round(clf_mse,4))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}