skl_to_vw.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[['Age']].fillna(train.Age.mean(), inplace=True)\n",
    "train[['Pclass']].fillna(train.Pclass.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop(\"Cabin\",inplace=True,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sex = pd.get_dummies(train[\"Sex\"],drop_first=True)\n",
    "embarked = pd.get_dummies(train[\"Embarked\"],drop_first=True)\n",
    "pclass = pd.get_dummies(train[\"Pclass\"],drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train,pclass,sex,embarked],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop([\"PassengerId\",\"Pclass\",\"Name\",\"Sex\",\"Ticket\",\"Embarked\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.drop(\"Survived\",axis=1)\n",
    "y = train[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 101)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "logmodel = LogisticRegression()\n",
    "logmodel.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.84      0.82       128\n",
      "           1       0.74      0.70      0.72        86\n",
      "\n",
      "   micro avg       0.78      0.78      0.78       214\n",
      "   macro avg       0.77      0.77      0.77       214\n",
      "weighted avg       0.78      0.78      0.78       214\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = logmodel.predict(X_test)\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[107,  21],\n",
       "       [ 26,  60]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VW version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vowpalwabbit.sklearn_vw import VWClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vw requires {+1,-1} for input data!\n",
    "train['Survived'] = train['Survived'].map({1: 1, 0: -1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.drop(\"Survived\",axis=1)\n",
    "y = train[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 101)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'quiet': True, 'link': 'logistic', 'loss_function': 'logistic'}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = VWClassifier(loss_function='logistic', link='logistic', convert_to_vw=True)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "??VWClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        -1.0       0.00      0.00      0.00      1015\n",
      "         1.0       0.49      1.00      0.66       985\n",
      "\n",
      "   micro avg       0.49      0.49      0.49      2000\n",
      "   macro avg       0.25      0.50      0.33      2000\n",
      "weighted avg       0.24      0.49      0.33      2000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(X_test)\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.12423094, 0.25162295, 0.64815784, 0.14973623, 0.12362408,\n",
       "       0.12415335, 0.44522566, 0.44779125, 0.68772942, 0.14583515,\n",
       "       0.17864318, 0.50195384, 0.44544691, 0.57823026, 0.74179959,\n",
       "       0.12390354, 0.2243094 , 0.12320259, 0.12388254, 0.44680718,\n",
       "       0.26820895, 0.33443668, 0.14063697, 0.62826377, 0.22965927,\n",
       "       0.50346828, 0.48044318, 0.14962368, 0.44849795, 0.26814157,\n",
       "       0.65122521, 0.77636963, 0.12334119, 0.50165135, 0.26297376,\n",
       "       0.2267397 , 0.65316325, 0.33865812, 0.28503555, 0.69031906,\n",
       "       0.72231436, 0.24867713, 0.1106369 , 0.1348743 , 0.12436007,\n",
       "       0.26804549, 0.12111377, 0.62469631, 0.22568081, 0.14611416,\n",
       "       0.12499657, 0.67572641, 0.32461175, 0.12429279, 0.44320431,\n",
       "       0.12410071, 0.75641769, 0.12405992, 0.65592659, 0.12347862,\n",
       "       0.12490128, 0.13860182, 0.44582352, 0.12403497, 0.69659501,\n",
       "       0.1242573 , 0.62412882, 0.62881762, 0.12311032, 0.14160031,\n",
       "       0.70086938, 0.14519574, 0.25341967, 0.28541306, 0.32320389,\n",
       "       0.25196487, 0.12416648, 0.12718584, 0.22557501, 0.12371588,\n",
       "       0.22610389, 0.14226344, 0.22515257, 0.47796717, 0.12356904,\n",
       "       0.12403497, 0.22504702, 0.22388861, 0.50112802, 0.12443913,\n",
       "       0.19396327, 0.66835737, 0.71500933, 0.44652909, 0.22428828,\n",
       "       0.12577409, 0.3024289 , 0.68331552, 0.71773672, 0.25224495,\n",
       "       0.72777218, 0.22844149, 0.84102839, 0.46916535, 0.11023755,\n",
       "       0.2499226 , 0.7397548 , 0.63346517, 0.14669481, 0.59456277,\n",
       "       0.44722599, 0.72280234, 0.1247347 , 0.12438381, 0.62285072,\n",
       "       0.54096532, 0.51767021, 0.62469631, 0.15474701, 0.71985573,\n",
       "       0.12371588, 0.12347208, 0.12403233, 0.22515266, 0.13538928,\n",
       "       0.44955215, 0.12406915, 0.12365029, 0.25053099, 0.43388572,\n",
       "       0.653602  , 0.72010761, 0.2267397 , 0.28916386, 0.82612592,\n",
       "       0.62445801, 0.69781637, 0.12614711, 0.68751609, 0.12253121,\n",
       "       0.79964441, 0.12382478, 0.74238098, 0.62313485, 0.50119758,\n",
       "       0.49503273, 0.27602491, 0.75950897, 0.12379458, 0.65565324,\n",
       "       0.24171858, 0.12548448, 0.62639689, 0.47038031, 0.44616127,\n",
       "       0.14534445, 0.12367949, 0.1454197 , 0.12310867, 0.22441468,\n",
       "       0.25801855, 0.44580555, 0.12434292, 0.67549843, 0.23148338,\n",
       "       0.75636876, 0.12264056, 0.34310687, 0.26569065, 0.12590477,\n",
       "       0.2943024 , 0.2339025 , 0.63248128, 0.6895287 , 0.12382083,\n",
       "       0.12547509, 0.26455289, 0.26115209, 0.14764555, 0.23379408,\n",
       "       0.12482592, 0.12342756, 0.51279819, 0.50640529, 0.12412044,\n",
       "       0.12601382, 0.32707056, 0.12409415, 0.14670241, 0.45327872,\n",
       "       0.12376571, 0.73101777, 0.63111317, 0.1384602 , 0.67830056,\n",
       "       0.12305906, 0.58568567, 0.12352974, 0.1252058 , 0.24757326,\n",
       "       0.23391633, 0.63223898, 0.49918783, 0.65851581, 0.62142843,\n",
       "       0.25935516, 0.23559852, 0.14115967, 0.25750056, 0.25060368,\n",
       "       0.2267397 , 0.47164378, 0.24901639, 0.76459944])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'quiet': True, 'link': 'logistic', 'loss_function': 'logistic'}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from vowpalwabbit.sklearn_vw import VWClassifier\n",
    "\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# get some data\n",
    "X, y = datasets.make_hastie_10_2(n_samples=10000, random_state=1)\n",
    "X = X.astype(np.float32)\n",
    "\n",
    "# split train and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=256)\n",
    "\n",
    "# build vowpal wabbit model\n",
    "model = VWClassifier(loss_function='logistic', link='logistic')\n",
    "model.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X_test).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'quiet': True, 'loss_function': 'logistic'}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from vowpalwabbit.sklearn_vw import VWClassifier\n",
    "\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# get some data\n",
    "X, y = datasets.make_hastie_10_2(n_samples=10000, random_state=1)\n",
    "X = X.astype(np.float32)\n",
    "\n",
    "# split train and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=256)\n",
    "\n",
    "# build vowpal wabbit model\n",
    "model = VWClassifier()\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  1., ...,  1., -1.,  1.])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/coreylynch/pyFM\n",
      "  Cloning https://github.com/coreylynch/pyFM to /tmp/pip-req-build-zq1xj3dj\n",
      "  Running command git clone -q https://github.com/coreylynch/pyFM /tmp/pip-req-build-zq1xj3dj\n",
      "Building wheels for collected packages: pyfm\n",
      "  Building wheel for pyfm (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyfm: filename=pyfm-0.0.0-cp36-cp36m-linux_x86_64.whl size=232327 sha256=013b9499ece74325c261fa9c2a55f84a95edbd63d4d1bb5bf06bff404f4c0457\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-139pphx1/wheels/3b/d9/ef/1b148c527d39344632833679e79b3db1798a40b0f64f917b13\n",
      "Successfully built pyfm\n",
      "Installing collected packages: pyfm\n",
      "Successfully installed pyfm-0.0.0\n"
     ]
    }
   ],
   "source": [
    "!pip install --user git+https://github.com/coreylynch/pyFM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating validation dataset of 0.01 of training for adaptive regularization\n",
      "-- Epoch 1\n",
      "Training log loss: 2.04571\n",
      "-- Epoch 2\n",
      "Training log loss: 1.77167\n",
      "-- Epoch 3\n",
      "Training log loss: 1.53345\n",
      "-- Epoch 4\n",
      "Training log loss: 1.32768\n",
      "-- Epoch 5\n",
      "Training log loss: 1.15034\n",
      "-- Epoch 6\n",
      "Training log loss: 0.99773\n",
      "-- Epoch 7\n",
      "Training log loss: 0.86666\n",
      "-- Epoch 8\n",
      "Training log loss: 0.75433\n",
      "-- Epoch 9\n",
      "Training log loss: 0.65825\n",
      "-- Epoch 10\n",
      "Training log loss: 0.57632\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pyfm import pylibfm\n",
    "\n",
    "from sklearn.datasets import make_classification\n",
    "\n",
    "X, y = make_classification(n_samples=1000,n_features=100, n_clusters_per_class=1)\n",
    "data = [ {v: k for k, v in dict(zip(i, range(len(i)))).items()}  for i in X]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.1, random_state=42)\n",
    "\n",
    "v = DictVectorizer()\n",
    "X_train = v.fit_transform(X_train)\n",
    "X_test = v.transform(X_test)\n",
    "\n",
    "fm = pylibfm.FM(num_factors=50, num_iter=10, verbose=True, task=\"classification\", initial_learning_rate=0.0001, learning_rate_schedule=\"optimal\")\n",
    "\n",
    "fm.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}