UnderSampled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.svm import SVR\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "\n",
    "\n",
    "from sklearn.metrics import median_absolute_error\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn import linear_model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfFinal = pd.read_csv('News_Final.csv')\n",
    "dfFb_Ec = pd.read_csv('Facebook_Economy.csv')\n",
    "dfFb_Ms = pd.read_csv('Facebook_Microsoft.csv')\n",
    "dfFb_Ob = pd.read_csv('Facebook_Obama.csv')\n",
    "dfFb_Pa = pd.read_csv('Facebook_Palestine.csv')\n",
    "dfGo_Pa = pd.read_csv('GooglePlus_Palestine.csv')\n",
    "dfGo_Ob = pd.read_csv('GooglePlus_Obama.csv')\n",
    "dfGo_Ms = pd.read_csv('GooglePlus_Microsoft.csv')\n",
    "dfGo_Ec = pd.read_csv('GooglePlus_Economy.csv')\n",
    "dfLd_Pa = pd.read_csv('LinkedIn_Palestine.csv')\n",
    "dfLd_Ob = pd.read_csv('LinkedIn_Obama.csv')\n",
    "dfLd_Ms = pd.read_csv('LinkedIn_Microsoft.csv')\n",
    "dfLd_Ec = pd.read_csv('LinkedIn_Economy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Edit original Data frame to assign group rank\n",
    "def assignRank(row,rankDictionary):\n",
    "    if row.IDLink in rankDictionary:\n",
    "        return rankDictionary[row.IDLink]\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rank_model_preprocess(df):\n",
    "    #removing time from publish date\n",
    "    df['PublishDate'] = df['PublishDate'].astype('datetime64[ns]')\n",
    "    df['DateOnly'] = df['PublishDate'].dt.date\n",
    "\n",
    "    #count number of days since Jan 1 1970. These columns would be used to form groups of 3 days\n",
    "    df['DaysSince1'] = (pd.to_datetime(df['DateOnly']) - pd.datetime(1970,1,1)).dt.days\n",
    "    df['DaysSince1'] = df['DaysSince1'].subtract(df['DaysSince1'].min()).add(1)\n",
    "    df['DaysSince2'] = df['DaysSince1'] + 1\n",
    "    df['DaysSince3'] = df['DaysSince1'] + 2\n",
    "\n",
    "    print \"Calculating rank...\"\n",
    "    # Calculate rank of the news item within the 3 day window group\n",
    "\n",
    "    loop_start = df['DaysSince1'].min()\n",
    "    loop_end = df['DaysSince1'].max()\n",
    "    print loop_start\n",
    "    print loop_end\n",
    "    rankDictionary = {}\n",
    "    for x in range(loop_start, loop_end):\n",
    "        df_temp = df[(df.DaysSince1 == x) | (df.DaysSince2 == x) | (df.DaysSince3 == x)]\n",
    "        df_temp.sort_values(by='Facebook')# Made it ascending so that we rank them in reverse order #, ascending=False)\n",
    "        df_temp['GroupRanking'] = df_temp['Facebook'].rank(ascending=False)\n",
    "        for index, row in df_temp.iterrows():\n",
    "            if row['DaysSince1'] == x:\n",
    "                rankDictionary[row['IDLink']] = row['GroupRanking']\n",
    "    df['groupRank'] = df.apply (lambda row: assignRank (row,rankDictionary),axis=1)\n",
    "    #df['Facebook'] = df['Facebook'].apply(lambda x: np.log(x + 1))\n",
    "    #Split DataFrame into X and Y\n",
    "    #df_X_Without_Rank = pd.get_dummies(df[['Source', 'Topic', 'SentimentTitle', 'SentimentHeadline']])\n",
    "    df_X = pd.get_dummies(df[['Source', 'Topic', 'SentimentTitle', 'SentimentHeadline', 'groupRank']])\n",
    "    df_Y = df[['Facebook']]\n",
    "    #xtrain, xtest, ytrain, ytest = train_test_split(df_X, df_Y, test_size=0.25)\n",
    "    return df_X,df_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.combine import SMOTEENN, SMOTETomek\n",
    "\n",
    "\n",
    "def rank_model_With_SMOTE(dfFinal):\n",
    "    df_model = dfFinal[(dfFinal.Facebook != -1) & (dfFinal.GooglePlus != -1) & (dfFinal.LinkedIn != -1)]\n",
    "    df_X,df_Y = rank_model_preprocess(df_model)\n",
    "    print \"using SMOTE...\"\n",
    "    sampler = SMOTETomek(random_state=0)\n",
    "    \n",
    "    X_res, y_res = sampler.fit_sample(df_X, df_Y)\n",
    "    \n",
    "    xtrain, xtest, ytrain, ytest = train_test_split(X_res, y_res, test_size=0.25)\n",
    "    return df_model, xtrain, ytrain,xtest, ytest\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating rank...\n",
      "1\n",
      "243\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "df_model = dfFinal[(dfFinal.Facebook != -1) & (dfFinal.GooglePlus != -1) & (dfFinal.LinkedIn != -1)]\n",
    "df_X,df_Y = rank_model_preprocess(df_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd = TruncatedSVD(n_components=50, n_iter=7, random_state=42)\n",
    "trunacted_X= svd.fit_transform(df_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using SMOTE...\n"
     ]
    }
   ],
   "source": [
    "print \"using SMOTE...\"\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "rus = RandomUnderSampler(ratio= 'majority',return_indices=True)\n",
    "X_resampled, y_resampled, idx_resampled = rus.fit_sample(trunacted_X, df_Y)\n",
    "\n",
    "y_resampled_log =[ np.log(x+1) for x in y_resampled ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def under_sampled(X_resampled,y_resampled_log):\n",
    "     \n",
    "    xtrain, xtest, ytrain, ytest = train_test_split(X_resampled, y_resampled_log, test_size=0.25)\n",
    "    return df_model, xtrain, ytrain,xtest, ytest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "models= [under_sampled(X_resampled,y_resampled_log)]\n",
    "model_names = [\"Under Sampling\" ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "ModelEvaluationResults = {}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "results = {}\n",
    "\n",
    "for (m,models_) in enumerate(models):\n",
    "    df,train_X,train_Y,test_X,test_Y = models_   \n",
    "    print \"Evalutaion of Model:  \"+str(model_names[m])\n",
    "    \n",
    "    clfs = [ \n",
    "        \n",
    "    SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.2, gamma='auto',kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
    "    ,linear_model.BayesianRidge(),\n",
    "    linear_model.LassoLars()]\n",
    "    #KernelRidge(alpha=1.0, coef0=1, degree=3, gamma=None, kernel='linear',kernel_params=None)]\n",
    "\n",
    "    #clf_names = ['Logistic Regression','KNeighborsRegressor','SVR','KernelRidge']\n",
    "    clf_names = ['SVR','BayesianRidge','LassoLars'\n",
    "                ]\n",
    "\n",
    "    \n",
    "    ClassifierEvaluationResults = {}\n",
    "    \n",
    "    for (i, clf_) in enumerate(clfs):\n",
    "        print \"********************************************************************\"\n",
    "        print \"Training validations evaluations for : \"+str(clf_names[i])\n",
    "        scores = cross_val_score(clf_, train_X, train_Y, cv=5)\n",
    "        predicted_ratings = cross_val_predict(clf_, test_X,test_Y, cv=5)\n",
    "        print scores\n",
    "        train_mean_score=scores.mean()\n",
    "        print \"Testing validations evaluations for : \"+str(clf_names[i])\n",
    "\n",
    "\n",
    "        r2 = r2_score(test_Y, predicted_ratings, multioutput='uniform_average')\n",
    "        mae = median_absolute_error(test_Y, predicted_ratings)\n",
    "        msle =mean_squared_log_error(test_Y, predicted_ratings) \n",
    "        mse = mean_squared_error(test_Y, predicted_ratings)\n",
    "        mae = mean_absolute_error(test_Y, predicted_ratings)\n",
    "        evs = explained_variance_score(test_Y, predicted_ratings)  \n",
    "        \n",
    "        data ={'training score':train_mean_score,\n",
    "            'r2 score':r2,\n",
    "            'median absolute error':mae,\n",
    "            'mean squared log error':msle,\n",
    "            'mean sqaured error':mse,\n",
    "            'mean absolute error':mae,\n",
    "            'explained variance error':evs}\n",
    "   \n",
    "        \n",
    "        ClassifierEvaluationResults[clf_names[i]] = data\n",
    "        print \"********************************************************************\"\n",
    "        print data\n",
    "        print \"********************************************************************\"\n",
    "        \n",
    "    print \"*******************Evaluation Done*************************************************\"\n",
    "    ModelEvaluationResults[model_names[m]]=ClassifierEvaluationResults\n",
    "    print ClassifierEvaluationResults\n",
    "    print \"********************************************************************\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}