note/02.NLP_scikit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/user_detail.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.review = df.review.apply(lambda x: x.replace('\\r', ' ')) \n",
    "df.review = df.review.apply(lambda x: x.replace('\\n', ' '))\n",
    "df.review = df.review.apply(lambda x: x.replace(\"\\\\\" , \"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I have played Lithium: Inmate 39 and it seems a very interesting game, it really has a good atmosphere something between a Tim Burton film with a more gory result. The game makes you remember old horror games like Nightmare Creatures, Silent Hill, Resident Evil and their challenge.'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.review.loc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEGCAYAAAB1iW6ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xlc1Ne9//HXMOyLgIKggDsexRUxLlETkyY32pul7W2zNEmbrUvatOl6b9Nfb5vb3La5vVu9bdrbNEmbLjdma9OkNTGLicZEjaC4exQQFURWRTZlm98fAxYFYYABhi/v5+PhQ+Y7Z77z+cLwnsOZ8z1fl8fjQUREhr+goS5ARET8Q4EuIuIQCnQREYdQoIuIOIQCXUTEIYKH6olzcnI0vUZEpA+ysrJcXW0fskAHyMrKGsqnv6ScnJyAra0vnHQ8OpbA5KRjgcA+npycnEvepyEXERGHUKCLiDiEAl1ExCEU6CIiDqFAFxFxCAW6iIhDKNBFRByix3noxpingOuBMmvt7C7uvx34p7abtcD91tpdfq1SRER65EsP/TfAqm7uPwJcaa2dCzwCPO6HukREpJd67KFbazcZYyZ1c//7HW5uBVL9UJeIyKB4bUthp22JoYNehl/4+9T/e4FXfW3c3SmsQy2Qa+sLJx2PjiUwDddjOXqsttO2xGnRw/J4/Bboxpir8Ab6cl8fE8hrJQRqbX3hpOPRsQSm4Xws5Y2FXWytDNjj6e6Nxi+BboyZCzwBrLbWVvpjnyIi0jv9nrZojJkA/BG401p7qP8liYhIX/gybfEZYCWQYIwpAr4HhABYa/8X+C4wBvi5MQag2Vq7cKAKFhGRrvkyy+W2Hu6/D7jPbxWJiEif6ExRERGHUKCLiDiEAl1ExCEU6CIiDqFAFxFxCAW6iIhDKNBFRBxCgS4i4hAKdBERh1Cgi4g4hAJdRMQhFOgiIg6hQBcRcQgFuoiIQyjQRUQcQoEuIuIQCnQREYdQoIuIOIQCXUTEIRToIiIOoUAXEXEIBbqIiEMo0EVEHEKBLiLiEAp0ERGHCO6pgTHmKeB6oMxaO7uL+13AGuDDQD1wl7V2h78LFRGR7vnSQ/8NsKqb+1cD6W3/Pgv8ov9liYhIb/UY6NbaTUBVN01uAn5rrfVYa7cCccaYcf4qUEREfNPjkIsPUoDjHW4XtW0r6emBOTk5fnj6gRHItfWFk45HxxKYhuuxHD1W22lb4rToYXk8/gh0VxfbPL48MCsryw9P7385OTkBW1tfOOl4dCyBaTgfS3ljYRdbKwP2eLp7o/HHLJciIK3D7VTghB/2KyIiveCPHvrLwAPGmLXAYqDaWtvjcIuIiPiXL9MWnwFWAgnGmCLge0AIgLX2f4F1eKcs5uGdtnj3QBUrIiKX1mOgW2tv6+F+D/BFv1UkIiJ9ojNFRUQcQoEuIuIQCnQREYdQoIuIOIQCXUTEIRToIiIOoUAXEXEIBbqIiEMo0EVEHEKBLiLiEAp0ERGHUKCLiDiEAl1ExCEU6CIiDqFAFxFxCAW6iIhDKNBFRBxCgS4i4hAKdBERh+jxmqIiIoHitS2FnbatWjppsMsIWOqhi4g4hAJdRMQhFOgiIg6hQBcRcQgFuoiIQ/g0y8UYswpYA7iBJ6y1j150/wTgaSCurc23rLXr/FyriIh0o8ceujHGDTwGrAYygNuMMRkXNfsO8Jy1NhO4Ffi5vwsVEZHu+TLksgjIs9YWWGsbgbXATRe18QCj2r6OBU74r0QREfGFL0MuKcDxDreLgMUXtXkYeN0Y8yUgCrjGlyfPycnxpdmQCOTa+sJJx6NjCUyDcSxHj9V2ft7QSr/vM3Fa9LD82fgS6K4utnkuun0b8Btr7X8aY5YCvzPGzLbWtna346ysLB/LHFw5OTkBW1tfOOl4dCyBabCOpbyxsNO2rKxJft8nVAbsz6a7NxpfAr0ISOtwO5XOQyr3AqsArLVbjDHhQAJQ1qtKRWRYys6r7TIYdVr+4PJlDH07kG6MmWyMCcX7oefLF7U5BnwIwBgzEwgHyv1ZqIiIdK/HQLfWNgMPAOuBA3hns+wzxnzfGHNjW7OvA58xxuwCngHustZePCwjIiIDyKd56G1zytddtO27Hb7eDyzzb2kiItIbOlNURMQhFOgiIg6hQBcRcQgFuoiIQyjQRUQcQoEuIuIQCnQREYdQoIuIOIQCXUTEIRToIiIOoUAXEXEIBbqIiEMo0EVEHEKBLiLiEAp0ERGHUKCLiDiEAl1ExCEU6CIiDqFAFxFxCAW6iIhDKNBFRBxCgS4i4hAKdBERh1Cgi4g4hAJdRMQhFOgiIg4R7EsjY8wqYA3gBp6w1j7aRZubgYcBD7DLWvtJP9YpIiI96LGHboxxA48Bq4EM4DZjTMZFbdKBh4Bl1tpZwFcGoFYREemGL0Mui4A8a22BtbYRWAvcdFGbzwCPWWtPAVhry/xbpoiI9MSXIZcU4HiH20XA4ovaTAcwxryHd1jmYWvtaz3tOCcnx8cyB18g19YXTjoeHUtgOnrsaKdtOaGVfn6OWr8/R1f7TJwWPSx/Nr4EuquLbZ4u9pMOrARSgXeNMbOttae723FWVpYvNQ66nJycgK2tL5x0PDqWwJSdt5GJEyZ22p6VNcmvz1PeWOj35+hqn1AZsD+b7t5ofBlyKQLSOtxOBU500ebP1toma+0RwOINeBERGSS+BPp2IN0YM9kYEwrcCrx8UZuXgKsAjDEJeIdgCvxZqIiIdK/HQLfWNgMPAOuBA8Bz1tp9xpjvG2NubGu2Hqg0xuwH3ga+aa317+CZiIh0y6d56NbadcC6i7Z9t8PXHuBrbf9ERGQI6ExRERGHUKCLiDiEAl1ExCEU6CIiDqFAFxFxCAW6iIhDKNBFRBxCgS4i4hAKdBERh/DpTFERGb5e21LYaduqpZMGuwwZBOqhi4g4hAJdRMQhFOgiIg6hQBcRcQgFuoiIQyjQRUQcQoEuIuIQCnQREYdQoIuIOIQCXUTEIRToIiIOoUAXEXEIBbqIiEMo0EVEHEKBLiLiED6th26MWQWsAdzAE9baRy/R7uPA88Bl1tpsv1UpIiI96rGHboxxA48Bq4EM4DZjTEYX7WKALwPb/F2kiIj0zJce+iIgz1pbAGCMWQvcBOy/qN0jwI+Bb/i1wiGQnVdLeWNhp+26youIBDJfAj0FON7hdhGwuGMDY0wmkGat/YsxxudAz8nJ8bXpoDt67GinbTmhlUNQiX8E8ve6t3QsvXP0WG3n5x2A1/Jg/M4MxLF0tc/EadHD8nXmS6C7utjmaf/CGBME/DdwV2+fPCsrq7cPGRTZeRuZOGFip+1ZWZMGvxg/yMnJCdjvdW/pWHqvq782/f1aHqzfmYE4lq72CZUB+zrr7o3Gl1kuRUBah9upwIkOt2OA2cA7xphCYAnwsjFmYW8LFRGRvvOlh74dSDfGTAaKgVuBT7bfaa2tBhLabxtj3gG+oVkuIiKDq8ceurW2GXgAWA8cAJ6z1u4zxnzfGHPjQBcoIiK+8WkeurV2HbDuom3fvUTblf0vS0REektnioqIOIQCXUTEIRToIiIOoUAXEXEIBbqIiEP4NMtFRPyvqzWDtF6Q9Id66CIiDqFAFxFxCAW6iIhDaAxdRKRNWVU9G3cWkz7OzYIFHlyurhabDVzqoYvIiOfxeNhhy3jx7cOUnarnvf01rH3j0FCX1WsKdBEZ8XbnVbBlTwnhYcFcu2gCcVFu/m/9QV7dUjjUpfWKAl1ERrSWVg+5h8oJdgdx8zXTmT4hnjuvSiA2OpT//eNuyk7VD3WJPlOgi8iIll90mtqGJjImjyYqPASAMaNCuHN1Bq2tHt7OOd7DHgKHAl1ERiyPx8Ouw+UAzJ2WcMF9y+eNJzQ4iLezj+PxeLp6eMBRoIvIiLX/SBVlpxqYkhJLbHTYBfdFRYSwZPY4isvrsMdODVGFvaNpiyJdeG1LYadtOi3fef68KR+AeekJXd5/9WVpbMotZkP2cWZMHD2YpfWJeugiMiLV1Dfywb6TjIkNZ9yYqC7bzE9PJD4mjHd3FtPU3DLIFfaeAl1ERqSte0poafUwPS3+kicQud1BrMxKo7ahiQ/2lw5yhb2nQBeREWnz7hMATE2N7bbdFZkpAHyw7+SA19RfCnQRGXFq6hvZdaicaamdPwy92JTxscRGh5J7qCzgZ7so0EVkxGkfblk2L6XHtkFBLualJ1J15hzHSmsGobq+U6CLyIizeZd3uGX5vPE+tc+cnghA7qHyAavJHxToIjKinKlrZNdh73BL8iVmt1xs/vSxAOy0ZQNZWr8p0EUk4FVWN/Cnd/LILz5N/dmmfu1r617vcMtyH4Zb2iXERZCWFM3egsqAnr7o04lFxphVwBrADTxhrX30ovu/BtwHNAPlwD3W2qN+rlVERqCjJWd4+FdbqKg+e35bxuTRrFyQ2qf1yt9rG25Z5uNwS7v508fyyrsFHCw8xZxpXZ+INNR67KEbY9zAY8BqIAO4zRiTcVGzncBCa+1c4AXgx/4uVERGnr35FfzTz96lovosn/hQOotnJRM/Koz9R6rYaXs/nn2mrpHcw+VMS4vzebilXfs4+s5DgTvs4suQyyIgz1pbYK1tBNYCN3VsYK1921rbvsbkViDVv2WKSCCqP9tE9oFSCkvPkXuonLONzX7bd21DEz96ejvnmlr4+icX8KkPZ7BwZhI3XTGV6IgQtuwtobDkTK/2uXVvCa2tHpbP7V3vHGD21ASC3S52BvAHo74MuaQAHdePLAIWd9P+XuBVX548JyfHl2ZD4uixziNGOaGVQ1CJfwTy97q3BuNYjh6r7fy8A/Dzv/h1NjDPMTDHUne2hSdeL+NUbfuYcgO7Dp1k8YxoIkKD+v08r+Wc5kxdI9fMH0UMZeTklJ0/lvmTw3j/QBPrtx7BJJ5lVKTbp32u2+QN41h31fnXUVffn8Rp0V2+zsaPDiG/6DSbt2w/f4yBxJdA72qQqsvZ9caYO4CFwJW+PHlWVpYvzQZddt5GJk6Y2Gl7VtakwS/GD3JycgL2e91bv3y2889mIBbNKm8s7LTN3z//rl5nA/EaG4hjOdfUwnd+8R6nalv40GVpVFZW0uKKYE9+JdsONXDTiqnExYT1+XmOl9awfe3bjBsTxRduu4KQYG9gtx/LRMAdXsnGHUXsOObm67f3/Po+U9fIkbWvMS0tjmuu/FuftKvvD1R2+TtzsPwga9+whMSkkjV7XB+OrP+669D48hZTBKR1uJ0KnLi4kTHmGuD/ATdaa8/1skYRGSY8Hg8/eWYHB4+e4srMVB68JZPUhFBWzE9h8axkauub+Mt7BTS3tPb5OZ56ZR8trR7uvmHW+TC/WMbk0STGR/DOjiIOHKnqcZ9b9niHW1b08sPQjtrXTN+dX9HnfQwkXwJ9O5BujJlsjAkFbgVe7tjAGJMJ/BJvmAfuJwYi0m+bc0+wedcJZk0Zw4O3zj8/08TlcrFwZhJzpyVQXdvIjj5Gwa7D5WQfKGVeegJLZidfsl2Qy8WKtqmHj7+0m9bWS5+W7/F4WL+1EIDL+zB+3s5MjCckOIg9ecM00K21zcADwHrgAPCctXafMeb7xpgb25r9OxANPG+MyTXGvHyJ3YnIMNbc0srvXj2AO8jFg7dkdtl7XjwrmajwYHIOlnGivPP4dHc8Hg/PvG4BuOvvZ/U4LXFcQhQrF6SSV1TN69suPVN6py3n8PHTLJ0zrtezWzoKDXEzc9Jojpw4w5m6xj7vZ6D4NA/dWrsOWHfRtu92+PoaP9clIgFo/dajlFTWcf2yyYxL6DoYQ0PcrJifwmtbj/LzF3fxyOcu93m++J78CvYVVLJwZhLT0uJ8esxd12fwwf6T/Pov+8iakURifMQF93s8Hta+4X2TuOWa6T7tsztzpyWwO6+CvfkV/ertD4TA+5hWRAJSw7lm1r5hCQ91c/O13QfjlJRYJibHsOtwBe/vLvH5Odp757f9nfH5MWNiI7j3xtnUn23mZy/kdloRcU9+BQcKq1iUkczUVN/eJLrTflLR7gAcdlGgi4hPXn43n9M15/jIldOIjwnvtq3L5WL5/BSC3S6eemUv55p6Pl1+T34Fe/MryZoxlukT4ntV27WLJrDAjGXHwTLe/ODY+e3NLa383/q23nkPb0K+Sk+LJyzUrUAXkeGp/mwTL72TT0xkCB9dOdWnx8RFh3HjiqmUnWrgpY153bZtbfXwm7/sA3rXO2/ncrl44BPziQgL5mfP5/LLP+3m8PFTfPOn754fwuntm8SlhAQHkTFpNMdLazhVc7bnBwwiBbr4xWtbCjv9E+f463tHqG1o4qYrpxIZHuLz4265djpx0WE8/9ZhKqsbLtnujQ+OcujYaa7ITMH08WLMifERfO++JYxLiOIvm4/wtZ9sIu/4aa5emMY37/DveRhz073LAOw6HFi9dAX6EFIIynDQcK6ZlzbmExURwvXLpvTqsZHhIdyxeibnGlv4n2dzaeliamF17Tme/ut+IsKCueeGWf2qddaUMfz0G1dxx6oZTEyO4Zt3ZPHV2xb06k3IF1kzvMvp7jgYWNcZ9WmWi4gMLy0trew7UsmW3SXkFZ1mamocaUkxuIN6vzrhq+8Xcqaukdv+zhAV0ftgvHbRBLbuLSH7QCm/f/UAn/77v63t5/F4ePqv+6mpb+LeG2czJjaimz35JiTYzS3XGm65tvdDN76aNG4Uo0eFscOW0drqIagP39eBoEAXcZiC4moeeWobFaf/NsRx8OgpIsKCWbkglSkp3V8UuaO6hib++M5hIsKCuXFF73rn7YKCXHz99iy+/pONvLDhMCmJ0Vy9MI3G5hZ+8eJuNmQfZ9K4UdywfHKf9j8UXC4XC0wSb24/Rn7xadLT/DM+318K9BEgO6+203oVA7H+iQy9g4VVPPyrLdSfa+a6JRNZPm88uYe8J9XsP1LFq1sKWTZ3PNctmejT3PA/rD9IdW0jd6yeQXRkaJ/rio4I4Tv3LObrazax5tmd3iGW8GBKKuqYPiGOb31qEW738BoBXjBjLG9uP8aOg2UBE+jD6zsoIpe0r6CSf/7l+zQ0tvC1T2bxwCfmM3/6WJLHRLFifgofWzmVyPBg3tt9gl/9eW+PV7A/cqKav24uYHxCFB9bOa3f9aUlxfDD+5dx7aIJAJRU1HHdkok8+sXlnU4GGg4ypycS5IKcg4Gz2ol66CIOUFndwKNPb6epuZWHPn0ZS7pYCTAxPpKPX53OXzYf4ZV3C2hubuXzH5vb5fhva6uHX7y4m1YPfO6jcy+5QFZvTUuL48u3ZNLa6qHubBMx/ej1D7XoyFDMxNHYo1XU1DcGxLGoh97Byco61r5h2X6oluwDpZysrOuxFyMy1JpbWvm332ZzuvYc99w4q8swbxcTGcpHrpzKlPGxvLqlkJ89n9tpVcT2DyoPFFZx+dxxLGib0eFPQUGugAjA/sqaMZZWD+QGyEUvFOh4T5r416e28ZkfvskfXjtI6elmtu07yYtv5/Gnjfn9viityEBqD9/l88Zzw/KeP7iMCAvmX++/nKmpsbzxwTG+9pON5BedBryzY9Y8u5M/vpNHSmIUn/3InIEuf1hrf7PLPhAY0xdH/JDLqZqzPPyrrRQUVzNjYjyrlk5i94F8QiLisUdPUVhyhufeOsxqfYgoAei9XSd4aWM+qWOj+dLN831eBCsmMpQf3r+MJ1/ex+vbjvK1NZuIiw6l4VwzDedaSE+L43v3LSE2OmyAj2B4m5oSR2J8BFv2lHD/PzQTHjq0kTqiA72yuoFvPbaZk5X1XLdkIvd/bC5udxB5BUeYmBrH1JRYdtpytuwt4aWNeSyencyMPp7FJuLxeDh8/DSbd53gg30nqa6pJ/5oPgmxEcxNTyS6l3O8i8pqWPPsTsJD3Tz06ct6ffJMZHgIX7p5Psvnjefpdfupb2hmVFQY01Lj+OxH5xARNqLjwSdBQS5WLkjl+bcO88G+k1yRObSXUx6xP7HGphZ+8OsPOFlZz83XTOeOVTM69W5cLhcLZowlLiaM17YU8siT2/j3L69gfEL00BQtw1bVmbP89Lnc83+aR4S5aW72cLy0luOltezKqyBj8miWzhnnU6+44VwzP3p6Ow3nmvnG7VlMSB7V59oyzVgyjf/HyUeKq7LSeP6tw7ydUzTkgT4ix9A9Hg+PvbCLw23rPHQV5h1NSYnlygWpnKlr5OHHt1Jdqyvsie+27DnBA//+NtkHSpk7LYHv3L2I3//Laq7LiuUzH5nNVVmpREeEsDe/ki/8eAPv7izu9sP42oYmvvf4Fo6drOH6ZZO5csHQhshIl5YUw7TUWHbYMk7XDG02jMge+ivvFrAh+zjTJ8TxxY/P82nccdaUMYwdHclzbx7i+09u5Qf3Lxvy8TLxn+OlNbydc5zcQ+U0Nbdypq6RMbHhTE2JZUJyTJ+m7bW2ei+s8MzrltAQN5//6BxWXz75gmmCocFuMiaPYcbE0ezJr+CD/aX8+PfZbMhJ4u7rMzr1vE/VnOXhx7dScKKaKzNTufem2f0+dum/q7LSyCvay6bcIm5c4dtqlANhxCXSwcIqnnplH3ExYXz7rkWEhvj+i3rHqhlUnG5gQ/Zx/uP3OTx016I+rY0hgePoyTM8/qc959e2Dna7CAtx09TcStWZsxw+fprQkCCyTBJXLUwjzMfXy9lzzax5diebd50gaXQk/3zPYiaOu/SwSFCQi3npidz54Zk89vwusg+UsuNgKVdkpjJjYjxj4iLYvr+UjTuLONfYwqqlk7j/EnPIZfCtyEzhyVf28XaOAn3QVNee499+ux2Px8M378jq9UJA7WsuV1WfZdu+k/zixV184R/m6ZdqGDrX1MIz6w/y0sZ8Wlo9zE9P5NrFE1g8exxhIW5eff8IFacbyCuq9i5ytbeEz//oTW5fNYOrFk7o9o28oLiaH/8um+LyWmZNGcNDn77M59ki4xOi+dfPX872A6U8/df9vLOjiHd2FJ2/f2x8BDetnsoNK6b4PKNFBl58TDgLzFiyD5Ry4EgVMycPzeSJERPoLa0e/vuZHVRUn+WO1TOYOy2xT/sJCQ7iW5++jId+vpn1W4/S1NzKl2+eP+zWoRguyqrq2byrmPziaqrOnOVYSQ0JxYWMjgljfGI0KWN7/wH1waNV/OSZHRSX1zE2PoLPfWwuizIuvLq8y+UiMT6SxPhI75VwbCl78ytZ82wuL23M55ZrDYtmJV/QY6+sbuDV9wt58e08mlta+ciVU/nUhzMICe7da8PlcrEoI5msGUkUFJ+muLyO0qo6pqbEkWnG6q/CAHXzh6aTfaCU3766nx/ev2xI3nBHTKA//df95BwsY8GMsXzi6v5diioqIoQf3L+Mh3+1hQ3Zx2k418xXbs30+5rLI1Vrq4ete0t4aWM+BwqrLrgvKAjOFFdTAHCwjMjwYIrKarkyM4XpE+K7/SWqrj3H2jcs6947gge48Yop3Ll6Zo+fhYSFulk6ZzxfujmTP7x2kA3Zx/jx77IJD3UzLz2RYHcQdWeb2JNXQUurh9joUL5y6wIWzkzq1/fBHeQiPS0+YBZ+ku7NnDyayzKS2L6/lJ2HylkwBDOHRkSgr996lD+9k0dKYjTfvD3LL0MkMZGhPPK5y/nBrz9gy54SDhRWcff1GaxckNbl/j0eD6dqznG8tIbyUw20tLayt6CS6PAQ4keFERMVStAw+hO6srqB7ftLOV5WQ2llPcXltYSHuokICyZpdCTj+jC1s6WllXdzi3nurcMcL63B5YL56YmsyExhfnoi8aPCefKFTSSOTaGiuoEjJ86QV3SaV94t4JV3C0geE8kVmalckZnCxLYPE1tbPRScqGbr3hJe3lRAw7lmxiVE8eWb5zN7akKv6kuIi+DBWzP52FXTeDvnOO/mFrNt38nz909IjuH65VNYuSBVc7hHqDtXz2T7/lJ+t24/mdMTB72X7vhXXfaBUn7x4i5iIkP53n1L+rUE6MUiw0N4+DNL+OPbeTz31mH++5mdPPnyPmZOGk1KYjSNTS3UnW3iRHkdRWU11J1tvuS+wkLcTEiOISoihKwZY/vc26843cDBo1UUl9dSUlFHc7OHvGN1HKsqJn5UOGNGhTO2jyvbVVY38P7uEt7bfYL9Ryq51My6PfmVAGzIPsbSOeNZMjuZaWnxlxwqqKlvZHNuMX96J5+SyjqCglxcvTCNj1+dTlpSzAVtXS4XUREhREWEMDF5FCvmj2dsfCSbdhazdW8Jz715iOfePERMZAhhIW7ONbVQU+9dumFUVCh3rp7DqqWTej0M0lFaUgyf+nAGd66eyamacwS5XIQEBxEZHqxx7RFu8vhYrpifwqbcYt7OKeLqhWmD+vyODvRNO4v4r//bgTvIxf+7exHjEqL8/hztV0e5KiuNtW9Ycg+XX9BrA+/MiXEJ0cxNjyYtKYak0ZGEBAexO6+CmrpGTtWc40RFLYePn+bHv8sm2B3E3PQElsxKZtGs5G4/vK1v+1M/93A5uYfKKSqr7bJdceXfrn0Y7Haxbd9J5k9PZG56IlPGx17yr5auQtzlgozJY1g2dzxmYjxJoyPZnFvM2aYWauubOFlZR3F5LScq6nhhw2Fe2HCY6IgQ5kxLIC0phjGx4biAslMNHDlRTe6hclpaPQS7g1i9dBIfu2oayWN8+1m5g4K4LCOZyzKSOdvYzPZ9pWzKLaK4vI6m5haCg733LzBjuSwjya/DYi6Xi9Gjwv22P3GG21fPYPuBkzz2fC6pY6P9dnFqXzgy0FtbPbz8bgFPvbKXiLBg/vmexcyaMmZAn3Ps6Ei+fEsm4O0lV5xuIKxtCCIhLoLgLj40PdfYcv5rj8dDRfVZ3EEutu09yY6DZew4WMbPX9zNlJRYJiTFkDQmEndQEI1NLVRUN1BQXE1RaQ3tl2kMD3WzcGYSc6aOYULyKMYnRBEW6uZ3f95K/Jgkqs6cpfx0A8Vltew8VM7OthXiosKDmZA8itSx0YSFuvF4vGc2FhRXU1pVD/wtxJfPG8/lc8d3CjK3O4godxBR4SEkjY5kXnoiK7NS2WnLyT5QSu6hMrbsKWHLnpJO34cpKd5ezcqs1H5dgix4tBYDAAAH0ElEQVQ8NJgVmSmsyEzp8z5E+mt8QjT/eOdlPPLkVh55ahv/+eAVjI2PHJTndlygHy+t4bEXdrGvoJK46DD+5bNLe3XJLX9IiIsgIa73UyIT4yJYtXQSn7xuBmVV9Wzbd5Jt+0rYV1BJQXF1p8dEhLmZOXkMs6eOYX56Imbi6C6HEiLCgkgaHUnS6Ehmtm1bPCuZXXkV7DpUzoHCKuyxU50+gIyNDj3fs+0qxHsSHhrM0jnjWDpnHB6Ph7JTDZRV1VNZ3YAHGBsfSfKYSL9cR1IkkCycmcS9N83mVy/t5ds/f48Hb81kTi8/s+kLnwLdGLMKWAO4gSestY9edH8Y8FsgC6gEbrHWFvq31EtraWll1+EKXv/gKNv2ltDc4mHpnHF87qNzhm1YjB0dyQ0rpnDDiim0tHooP1VPaWU9HjyEhQQTGx1K8pioPn/AGz8qnJULUlnZdtp4U3MrJyvraG5pJcjlIjoyhNGjwv02Juxyuc6/qYiMBDcsn0JtfRPPvmH59s/f47olE1m9dBJTUmIH7LOWHgPdGOMGHgOuBYqA7caYl621+zs0uxc4Za2dZoy5Ffg34JaBKPhMXSPr3j/CmbpG6s82UVRay5ET1TQ2exfpn5gcw+2rZrJ0zqUX+R9u3EEuksdE+Tyu3BchwUGdPoAUkb5zuVx88roZLJyZxJpnd7J+61HWbz1KYnwEX/iHef2e1toVX3roi4A8a20BgDFmLXAT0DHQbwIebvv6BeBnxhiXtdbvl/vZacv4w2sHz992B7mYmDyKGZPi+dBlE0hPi9NMAxEJGNMnxPOTr65k274Stu09eX7ywkAEuqunS6wZYz4OrLLW3td2+05gsbX2gQ5t9ra1KWq7nd/WpqKrfQLk5OTo2m4iIn2QlZXVZa/Vlx56Vw+8OIx9aeNTQSIi0je+nF1RBHScHZ8KnLhUG2NMMBALVCEiIoPGlx76diDdGDMZKAZuBT55UZuXgU8DW4CPAxsGYvxcREQurcceurW2GXgAWA8cAJ6z1u4zxnzfGHNjW7MngTHGmDzga8C3BqpgERHpWo8fioqIyPCgRbxFRBxCgS4i4hCOW8ulv3pa5mC4MMak4V2OIRloBR631q4Z2qr6p+2s5Wyg2Fp7/VDX0x/GmDjgCWA23im+91hrtwxtVX1jjPkqcB/e49gD3G2tPTu0VfnOGPMUcD1QZq2d3bZtNPAsMAkoBG621p4aqhp9pR56Bx2WOVgNZAC3GWMyhraqPmsGvm6tnQksAb44jI+l3YN4P5h3gjXAa9baGcA8hulxGWNSgC8DC9vC0I13Jtxw8htg1UXbvgW8Za1NB95imEz0UKBf6PwyB9baRqB9mYNhx1pbYq3d0fZ1Dd7AGLbryhpjUoG/x9urHdaMMaOAK/DODsNa22itPT20VfVLMBDRdg5KJJ3PUwlo1tpNdD5v5ibg6bavnwY+MqhF9ZEC/UIpwPEOt4sYxiHYzhgzCcgEtg1xKf3xE+Af8Q4fDXdTgHLg18aYncaYJ4wxA7fy2gCy1hYD/wEcA0qAamvt60NblV8kWWtLwNs5Agb/AqF9oEC/UK+XMAh0xpho4EXgK9baM0NdT18YY9rHN3OGuhY/CQYWAL+w1mYCdQyTP+kvZoyJx9ubnQyMB6KMMXcMbVUjlwL9Qr4sczBsGGNC8Ib5H6y1fxzqevphGXCjMaYQ7zDY1caY3w9pRf1TBBRZa9v/YnoBb8APR9cAR6y15dbaJuCPwOVDXJM/lBpjxgG0/V82xPX4RIF+ofPLHBhjQvF+uPPyENfUJ8YYF94x2gPW2v8a6nr6w1r7kLU21Vo7Ce/PZIO1dtj2Aq21J4HjxhjTtulDXLgc9XByDFhijIlse819iGH6Ae9F2pczoe3/Pw9hLT7TtMUOrLXNxpj2ZQ7cwFPW2n1DXFZfLQPuBPYYY3Lbtn3bWrtuCGuSv/kS8Ie2jkMBcPcQ19Mn1tptxpgXgB14Z1btBB4f2qp6xxjzDLASSDDGFAHfAx4FnjPG3Iv3TesTQ1eh73Tqv4iIQ2jIRUTEIRToIiIOoUAXEXEIBbqIiEMo0EVEHEKBLiLiEAp0kV5qW5VTJOBoHro4ljEmEu9KebOAJsBaa282xtyDdylegEbgemttqTHmU8A38a7fkw98zlpbZoy5C+8ZquV4l1W+FygFfgpMACKAZ6y1Pxy0gxPpgnro4mTXAfHW2gxr7Tzgc8aYlcC3gevatl0FVBtjZuM9O/DvrLVzgb14A7vdcuBha22WtTYX78VD/sdauwjIAlYbY64dtCMT6YJO/Rcn2wXMMMY8BrwD/BXvmuq/bVtPBWttLYAx5ipgXfuSqcAv2x7fbrO1Nr+tbRTeU8UT/7YcCzHATOCNATwekW4p0MWxrLUFxpiZeBeMWg38kEsvsuSi81LJHW/Xdvg6qO2+y9pWGBQJCBpyEcdqu8pRi7X2JeCrQCLwCvApY0xSW5toY0wY3suMfdgYk9z28M8Ab3a137YrQL1LhzXMjTFpHR4rMiQU6OJkc4AtxphdwAfAj6y1G4EfAW+2bd8AxLWtqvkQ8IYxZjfe63w+eIn9AtwOZBhj9hhj9uC9oHDcAB6LSI80y0VExCHUQxcRcQgFuoiIQyjQRUQcQoEuIuIQCnQREYdQoIuIOIQCXUTEIf4/be5mbqJr/jcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06cc0600f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "sns.distplot(df.score)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.score = df.score.apply(lambda x: 0 if x< 6 else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75830, 25277, 75830, 25277)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df.review, df.score)\n",
    "\n",
    "len(X_train), len(X_test), len(y_train), len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "  ...ue,\n",
       "        vocabulary=None)), ('clf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "clf = Pipeline([\n",
    "    ('vect', TfidfVectorizer(ngram_range=(1,2), stop_words='english'\n",
    "                            ,tokenizer =nltk.word_tokenize)), \n",
    "    ('clf', MultinomialNB(alpha=0.01))])\n",
    "\n",
    "model = clf.fit(X_train, y_train)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.8642243937176088\n",
      "[[ 3610  2892]\n",
      " [  540 18235]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.56      0.68      6502\n",
      "          1       0.86      0.97      0.91     18775\n",
      "\n",
      "avg / total       0.86      0.86      0.85     25277\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import *\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "print(\"acc:\", accuracy_score(y_test, y_pred))\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = X_test.reset_index(drop=True)\n",
    "X_test[0][:100], X_test[1][:100], X_test[2][:100]\n",
    "\n",
    "result = model.predict([X_test[0], X_test[1], X_test[2]])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good A decent game , the story was quiet boring althoug\n",
      "good The Devil May Cry collection is just as great on t\n",
      "good Its a solid 7 like the critics say..... If it was \n"
     ]
    }
   ],
   "source": [
    "classification_dict = {\n",
    "    0: \"bad\", 1: \"good\",\n",
    "}\n",
    "\n",
    "for idx, category in enumerate(result):\n",
    "    print(classification_dict[category], X_test[idx][:50])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[['review', 'score']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.to_csv('test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>101102</th>\n",
       "      <td>Had fun with it after playing through on easy ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101103</th>\n",
       "      <td>I get the distinct impression that Rare stumbl...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101104</th>\n",
       "      <td>What you have here is a very well rounded, inc...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101105</th>\n",
       "      <td>A real step up from the previous game, but spo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101106</th>\n",
       "      <td>Konami Cancelled Zone of the Enders 3. A small...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   review  score\n",
       "101102  Had fun with it after playing through on easy ...      1\n",
       "101103  I get the distinct impression that Rare stumbl...      1\n",
       "101104  What you have here is a very well rounded, inc...      1\n",
       "101105  A real step up from the previous game, but spo...      0\n",
       "101106  Konami Cancelled Zone of the Enders 3. A small...      1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stream_docs(path):\n",
    "    with open(path, 'r') as csv:\n",
    "        next(csv) #skip header\n",
    "        for line in csv:\n",
    "            text, label = line[1:-4], int(line[-2])\n",
    "            yield text, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"Kazuma Kiryu's last game (no spoilers)  I managed to get Yakuza 6 a little earlier and my opinion on it is highly positive. The game is really immersive with a cast made out of very wholesome, comedic and plain evil characters. The story takes you to both Onomichi and Kamurocho in yet another tale of our beloved Dragon.  The story itself is really good. Yakuza 6 has a really interesting plot and definitely goes for that less grim Yakuza 5 atmosphere in comparison to the one that we had in 0. The substories this time around are fully voiced. The substories can be sad, funny and have a whole lot of effort put in them. The substory-voice actors really did put that extra weight on their characters and it really helps out because not every character has that much screen time.  The mini games including the new Clan Creator, Baseball manager and underwater fishing game are done really well. One moment you're fighting a squid monster that fires little squids as projectiles and other moments you'll find yourself waging war in the streets of Hiroshima and Tokyo. And don't worry about previous minigames, as far as I can tell they have all been improved and streamlined.  The game does have a couple of minor issues however. The combat for one is shallow compared to previous installments. With only one style and a limited amount of heat moves the combat does feels repetitive at times. At some point in the game I had been using the same combo string for 10 fights in a row! BUT repetitive doesn't mean bad, keep that in mind. Kamurocho is also missing part of the upper map which made me a little upset. And finally it's time to address the engine. While the engine is drop dead gorgeous it does have some issues: minimal street NPC's, fade-aways, pop ups and some physics based issues.  All in all Yakuza 6 is a great game but I can't easily ignore some of the issues it has. However i'm happy to say that Yakuza 6's positive aspects easily overshadow the negatives aspects. The game is fun, well written, unique, dynamic, addictive and worth your time. I rate Yakuza 6 a 9 out of 10.\",\n",
       " 1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(stream_docs(path='./test.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_minibatch(doc_stream, size):\n",
    "    docs, y = [], []\n",
    "    try:\n",
    "         for _ in range(size):\n",
    "            text, label = next(doc_stream)\n",
    "            docs.append(text)\n",
    "            y.append(label)\n",
    "    except StopIteration:\n",
    "        return None, None\n",
    "    return docs, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "\n",
    "vect = HashingVectorizer(\n",
    "#                         ngram_range=(1,2), \n",
    "                        stop_words='english',\n",
    "                        decode_error='ignore',\n",
    "                        n_features=2**21,\n",
    "                        preprocessor=None,\n",
    "#                         non_negative=True,\n",
    "                        tokenizer =nltk.word_tokenize,\n",
    "                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(loss='log', random_state=1, max_iter=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [##############################] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:06:22\n"
     ]
    }
   ],
   "source": [
    "import pyprind\n",
    "import pickle\n",
    "\n",
    "pbar = pyprind.ProgBar(31)\n",
    "classes = np.array([0, 1])\n",
    "for _ in range(31):\n",
    "    X_train, y_train = get_minibatch(stream_docs(path='./test.csv'), size = 5000)\n",
    "    if not X_train:\n",
    "        break\n",
    "    X_train = vect.transform(X_train)\n",
    "    clf.partial_fit(X_train, y_train, classes=classes)\n",
    "    pbar.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, y_test = get_minibatch(stream_docs(path='./test.csv'), size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc: 0.865\n",
      "[[ 884 1221]\n",
      " [ 129 7766]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.42      0.57      2105\n",
      "          1       0.86      0.98      0.92      7895\n",
      "\n",
      "avg / total       0.87      0.86      0.85     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "print(\"acc:\", accuracy_score(y_test, y_pred))\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='log', max_iter=1, n_iter=None,\n",
       "       n_jobs=1, penalty='l2', power_t=0.5, random_state=1, shuffle=True,\n",
       "       tol=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.partial_fit(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "update"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}