ML_supervised.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Procedure\n",
    "1. Prepare\n",
    "2. N Fold Split\n",
    "3. Rebalance (Up-sampling)\n",
    "4. Feature Selection\n",
    "5. Training\n",
    "6. Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'calculate': {'analysis': True,\n",
      "               'friends': False,\n",
      "               'network': True,\n",
      "               'uniquetweets': True,\n",
      "               'uniqueusers': True},\n",
      " 'data': {'dates': ['2018-03-11', '2018-03-12', '2018-03-13'],\n",
      "          'eventname': \"Givenchy's Death\",\n",
      "          'phrases': ['givenchy%20death', 'givenchy%20passed%20away'],\n",
      "          'starttime': 'Mar 12 08:20:00 -0500 2018'},\n",
      " 'path': {'cwd': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy',\n",
      "          'ml': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle',\n",
      "          'networkx': {'all': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/networkx_all.dat',\n",
      "                       'friends': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/networkx_friends.dat',\n",
      "                       'potential': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/networkx_potential.dat'},\n",
      "          'newcrawl': '/Users/lzhou/git/github/uclresearchanalysis/other/newcrawl.dat',\n",
      "          'pickle': {'friends': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/friends.dat',\n",
      "                     'needcrawl': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/needcrawl.dat',\n",
      "                     'network': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/network.dat',\n",
      "                     'tweets': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/tweets.dat',\n",
      "                     'users': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/users.dat'},\n",
      "          'result': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/result',\n",
      "          'twitter': '/Users/lzhou/git/github/uclresearchanalysis/data/givenchy/twitter'},\n",
      " 'save_to_file': 'False',\n",
      " 'timeframe': '1440'}\n",
      "Loading data file from path /Users/lzhou/git/github/uclresearchanalysis/data/givenchy/pickle/30_data.dat\n",
      "'Loaded 226984 entires'\n",
      "[Original] data counts, with uninfected (0): 45869, infected (1): 181115\n",
      "Start cross validation\n",
      "===Processing fold 0===\n",
      "tCurrent       0.506844\n",
      "t0             0.506769\n",
      "tAverage       0.487076\n",
      "t-1            0.411991\n",
      "kOut-1         0.175329\n",
      "kOutMin        0.169642\n",
      "kOutAverage    0.148361\n",
      "nNodes         0.058107\n",
      "kOut0          0.025893\n",
      "kOutMax        0.011677\n",
      "kIn            0.004062\n",
      "Name: label, dtype: float64\n",
      "[[10049 12886]\n",
      " [ 4281 86277]]\n",
      "===Processing fold 1===\n",
      "t0             0.511856\n",
      "tCurrent       0.511413\n",
      "tAverage       0.492286\n",
      "t-1            0.417729\n",
      "kOut-1         0.167234\n",
      "kOutMin        0.161051\n",
      "kOutAverage    0.136642\n",
      "nNodes         0.052326\n",
      "kOut0          0.017155\n",
      "kIn            0.003877\n",
      "kOutMax        0.000222\n",
      "Name: label, dtype: float64\n",
      "[[11132 11802]\n",
      " [ 4394 86163]]\n",
      "Finished cross validation\n",
      "0    0.146984\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Import settings\n",
    "import config\n",
    "from config import load_ml_data\n",
    "# Imports\n",
    "import pandas as pd\n",
    "from pandas import *\n",
    "import numpy as np\n",
    "# Import scikit-learn\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import *\n",
    "from sklearn.metrics import *\n",
    "import random\n",
    "import pickle\n",
    "\n",
    "# Parameters\n",
    "n_folds = 2\n",
    "interval = 30\n",
    "\n",
    "# 1. Prepare\n",
    "def prepare_data(interval):\n",
    "    df = load_ml_data(interval)\n",
    "    # Converting all type to float, to prepare for feature selection\n",
    "    df = df.astype('float')\n",
    "    # Reset index, with drop equals to true to avoid setting old index as a new column\n",
    "    df = df.reset_index(drop=True)\n",
    "    # Visualize distribution\n",
    "    print('[Original] data counts, with uninfected (0): {}, infected (1): {}'.format(\n",
    "        df['label'].value_counts()[0],\n",
    "        df['label'].value_counts()[1]\n",
    "    ))\n",
    "    df.groupby(['tCurrent','label']).size().unstack(fill_value=0).plot.bar(title='Original Data Distribution')\n",
    "    \n",
    "    columns = list(df.columns)\n",
    "    columns.remove('label')\n",
    "    \n",
    "    X = df[columns]\n",
    "    y = df[['label']]\n",
    "    return df, X, y\n",
    "\n",
    "\n",
    "# 1. Prepare\n",
    "df, X, y = prepare_data(interval)\n",
    "\n",
    "# 2. N Fold Split\n",
    "\n",
    "# Stratified K-Folds cross-validator\n",
    "# Provides train/test indices to split data in train/test sets. \n",
    "# This cross-validation object is a variation of KFold that returns stratified folds. \n",
    "# The folds are made by preserving the percentage of samples for each class.\n",
    "skf = StratifiedKFold(n_splits=n_folds, shuffle=True)\n",
    "\n",
    "mse = []\n",
    "fold_count = 0\n",
    "\n",
    "print(\"Start cross validation\")\n",
    "for train, test in skf.split(X, y):\n",
    "    print(\"===Processing fold %s===\" % fold_count)\n",
    "    train_fold = df.loc[train]\n",
    "    test_fold = df.loc[test]\n",
    "    \n",
    "    # 3. Rebalance (Up-sampling)\n",
    "    \n",
    "    # 4. Feature Selection\n",
    "    corr = train_fold.corr()['label'][train_fold.corr()['label'] < 1].abs()\n",
    "    corr = corr.sort_values(ascending=False)\n",
    "    print(corr)\n",
    "    features = corr.index[range(10)].values\n",
    "    \n",
    "    # 5. Training\n",
    "    # Get training examples\n",
    "    train_fold_input  = train_fold[features].values\n",
    "    train_fold_output = train_fold['label']\n",
    "    # Fit logistic regression\n",
    "    logreg = LogisticRegression()\n",
    "    logreg.fit(train_fold_input, train_fold_output)\n",
    "    \n",
    "    pickle_out = open(\"rubbish_predictor.pickle\",\"wb\")\n",
    "    pickle.dump(logreg, pickle_out)\n",
    "    pickle_out.close()\n",
    "    \n",
    "    # 6. Testing\n",
    "    # Check MSE on test set\n",
    "    pred = logreg.predict(test_fold[features])\n",
    "    mse.append(mean_squared_error(test_fold.label, pred))\n",
    "    \n",
    "    cm = confusion_matrix(test_fold.label, pred)\n",
    "    print(cm)\n",
    "    \n",
    "    # Done with the fold\n",
    "    fold_count += 1\n",
    "print(\"Finished cross validation\")\n",
    "print(DataFrame(mse).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reference\n",
    "- http://www.alfredo.motta.name/cross-validation-done-wrong/\n",
    "- https://www.marcoaltini.com/blog/dealing-with-imbalanced-data-undersampling-oversampling-and-proper-cross-validation"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}