prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from src.utils import encode_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data, Encode Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>all</th>\n",
       "      <th>pct_punctuation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.107143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.025751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.040193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.074627</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
       "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  all  pct_punctuation  \n",
       "0             0        0       0       0              0    0         0.041667  \n",
       "1             0        0       0       0              0    0         0.107143  \n",
       "2             0        0       0       0              0    0         0.025751  \n",
       "3             0        0       0       0              0    0         0.040193  \n",
       "4             0        0       0       0              0    0         0.074627  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_df = pd.read_csv('data/raw_combined.csv')\n",
    "full_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab, encoded_text = encode_text(\n",
    "    full_df, \n",
    "    use_cache=False,\n",
    "    max_length=500,\n",
    "    vocab_length=10000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(223549, 501)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# should be (n comments, max_length + 1)\n",
    "encoded_text.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_columns = ['toxic', 'severe_toxic', 'obscene', 'threat',\n",
    "                 'insult', 'identity_hate']\n",
    "labels = full_df[label_columns].values\n",
    "\n",
    "full_data = np.concatenate([labels, encoded_text], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train / Val / Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'processed/'\n",
    "# save vocab to each so it's available\n",
    "for subdir in ['train', 'val', 'test']:\n",
    "    directory = os.path.join(data_dir, subdir)\n",
    "    os.makedirs(directory, exist_ok=True)\n",
    "    with open(os.path.join(directory, 'vocab.joblib'), 'wb') as f:\n",
    "        joblib.dump(vocab, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([    21384,      1962,     12140,       689,     11304,      2117,\n",
       "         7744492, 213261217, 223964859, 216712792, 213177422, 205209873,\n",
       "       201224153, 191998815, 185584467, 178825053, 170897287, 164096046,\n",
       "       158443177, 150738627, 145606809, 138269683, 133714111, 127896100,\n",
       "       122244228, 119127843, 113033385, 110180465, 105880091, 102308451,\n",
       "        98500116,  95175741,  90900924,  88345281,  85230909,  82705616,\n",
       "        79894987,  78157806,  74878281,  71368929,  70190487,  67232490,\n",
       "        66367736,  64637938,  62539568,  59918456,  58483573,  56881980,\n",
       "        56152450,  54191132,  52627351,  50914997,  50218546,  48968241,\n",
       "        47225603,  46110606,  44647032,  43271547,  42622661,  42044759,\n",
       "        41495751,  40631647,  39290787,  37936104,  37804002,  36650875,\n",
       "        36425726,  35242451,  35147855,  33520883,  33115700,  31984577,\n",
       "        31687274,  31255952,  31005558,  29999140,  29854503,  28476173,\n",
       "        28412655,  28079409,  27442453,  26639072,  26446693,  25912961,\n",
       "        24991313,  24778697,  24797337,  24120547,  24024928,  22777174,\n",
       "        23147538,  22797274,  22036567,  22000179,  21043823,  20952940,\n",
       "        20167740,  20162810,  20325671,  19493774,  19003992,  18301988,\n",
       "        18566555,  18364571,  17588951,  17404525,  17431124,  16786601,\n",
       "        16632961,  16115609,  16057936,  15980189,  15892639,  15657636,\n",
       "        15793977,  15311442,  14811688,  14884852,  14713406,  14518380,\n",
       "        14575816,  14311672,  14043885,  13594609,  13504425,  13467236,\n",
       "        13176054,  13185391,  12248463,  12913704,  12373324,  12478675,\n",
       "        12281941,  12216283,  12168450,  11855739,  11653081,  11410781,\n",
       "        11725970,  11399438,  11138143,  10885238,  10780593,  10620100,\n",
       "        10804038,  10514337,  10353720,  10511564,  10465937,   9867704,\n",
       "         9987790,   9899833,   9847894,   9536322,   9777806,   9571193,\n",
       "         9460259,   9345272,   9333141,   9159155,   9096596,   8970255,\n",
       "         8591801,   8941229,   8654599,   8665392,   8679863,   8360266,\n",
       "         8607067,   8424341,   8304159,   8291445,   8163293,   8168736,\n",
       "         8115961,   8061484,   7904619,   8127714,   8120750,   7666299,\n",
       "         7701700,   7786195,   7494145,   7663768,   7346850,   7564116,\n",
       "         7493737,   7370051,   7101447,   7083816,   6987858,   7218974,\n",
       "         6562798,   7135808,   6906106,   6716776,   6770582,   6539478,\n",
       "         6786219,   6620417,   6845044,   6429468,   6482709,   6676041,\n",
       "         6111344,   6393201,   6037209,   6058235,   6144421,   5697833,\n",
       "         5935586,   6052037,   6043239,   6016853,   5840550,   5914117,\n",
       "         5875530,   5535878,   5499950,   5541957,   5538770,   5798077,\n",
       "         5481191,   5621882,   5506512,   5577617,   5520763,   5582532,\n",
       "         5422918,   5661402,   5288993,   5260198,   5243254,   5104354,\n",
       "         5055810,   5367943,   4957399,   5236398,   5169875,   4909320,\n",
       "         4980716,   4876134,   5096641,   5043667,   5011735,   4821486,\n",
       "         4974974,   4856757,   4710760,   4581837,   4610503,   4706122,\n",
       "         4494126,   4680244,   4486554,   4335474,   4883726,   4478557,\n",
       "         4553710,   4612296,   4300526,   4142987,   4504819,   4230197,\n",
       "         4328220,   4285626,   4224280,   4058865,   4336677,   4475743,\n",
       "         4188023,   4154911,   4297393,   4135272,   4289237,   4137794,\n",
       "         4002836,   3950447,   3898168,   3751407,   3911748,   3769105,\n",
       "         3834558,   4080056,   3860569,   3763215,   3921999,   3836919,\n",
       "         3664043,   3707049,   3637002,   3624416,   3749689,   3612268,\n",
       "         3677436,   3654856,   3515836,   3647590,   3523439,   3365275,\n",
       "         3367042,   3354266,   3381340,   3497047,   3392353,   3479698,\n",
       "         3285686,   3236200,   3415358,   3027124,   3362188,   3396417,\n",
       "         3284992,   3131287,   3197172,   3133161,   3354637,   3128902,\n",
       "         3109166,   2933315,   3061136,   3107360,   3064300,   2944190,\n",
       "         2911702,   2848214,   2678241,   2819666,   2843210,   2844090,\n",
       "         2837213,   2712362,   2863102,   2872785,   2897508,   2858715,\n",
       "         2781462,   2785433,   2680018,   2643128,   2602717,   2634318,\n",
       "         2744642,   2566485,   2506995,   2485328,   2492446,   2490120,\n",
       "         2639139,   2637868,   2500406,   2488364,   2579027,   2585292,\n",
       "         2396494,   2473687,   2357854,   2259048,   2455938,   2294757,\n",
       "         2232198,   2314976,   2138921,   2257272,   2197617,   2203962,\n",
       "         2290823,   2269434,   2062348,   2109032,   2293484,   2178803,\n",
       "         2072937,   1990070,   2025168,   2016470,   2059780,   2048314,\n",
       "         1974744,   2181655,   2166559,   1950475,   2023099,   1811089,\n",
       "         1893677,   1825554,   1813523,   1832054,   2023199,   1683337,\n",
       "         1965520,   1799053,   1826869,   1817443,   1787128,   1724743,\n",
       "         1678769,   1729607,   1795386,   1618803,   1554809,   1530534,\n",
       "         1543161,   1701598,   1597507,   1642011,   1515915,   1564313,\n",
       "         1564593,   1446814,   1485925,   1434038,   1483070,   1442816,\n",
       "         1400690,   1494930,   1420147,   1354712,   1306561,   1325702,\n",
       "         1148432,   1316034,   1357033,   1192151,   1343691,   1206664,\n",
       "         1144899,   1143409,   1217711,   1231056,   1076005,   1060962,\n",
       "         1131331,   1109089,   1144585,   1099848,   1114239,   1011956,\n",
       "         1022520,   1042975,    980439,    986614,    954727,    974518,\n",
       "          976659,    977048,    876150,    973934,    869051,    836765,\n",
       "          823594,    872305,    883625,    844206,    913790,    819413,\n",
       "          897439,    871032,    797858,    830672,    762163,    753845,\n",
       "          811716,    857826,    753481,    694212,    716149,    746699,\n",
       "          726082,    732363,    693409,    665295,    783283,    698169,\n",
       "          700339,    711483,    655691,    696145,    676209,    631027,\n",
       "          652530,    643338,    614765,    693527,    677116,    629620,\n",
       "          567862,    683715,    687774,    587531,    625376,    684447,\n",
       "          634784,    636195,    635087,    623265,    575485,    589008,\n",
       "          586241,    580211,    597296,    589101,    555707,    583762,\n",
       "          642471,    627086,    531700])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_data.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(160955, 507) (40239, 507) (22355, 507)\n"
     ]
    }
   ],
   "source": [
    "# split out test first\n",
    "train, test = train_test_split(\n",
    "    full_data, \n",
    "    test_size=0.1, \n",
    "    random_state=1,\n",
    "    # make sure we have some of each class in \n",
    "    # each set. this is kind of a hack, looking\n",
    "    # for the max class label for each, but it should\n",
    "    # be good enough\n",
    "    stratify=full_data[:, :6].argmax(axis=1)\n",
    ")\n",
    "\n",
    "with open(os.path.join(data_dir, 'test/data.joblib'), 'wb') as f:\n",
    "    joblib.dump(test, f)\n",
    "\n",
    "train, val = train_test_split(\n",
    "    train,\n",
    "    test_size=0.2,\n",
    "    random_state=1,\n",
    "    stratify=train[:, :6].argmax(axis=1)\n",
    ")\n",
    "\n",
    "\n",
    "with open(os.path.join(data_dir, 'train/data.joblib'), 'wb') as f:\n",
    "    joblib.dump(train, f)\n",
    "with open(os.path.join(data_dir, 'val/data.joblib'), 'wb') as f:\n",
    "    joblib.dump(val, f)\n",
    "\n",
    "print(train.shape, val.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([15326,  1405,  8738,   493,  8105,  1524])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.sum(axis=0)[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1000652356248641"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "has_positive = train[:, :6].max(axis=1)\n",
    "has_positive.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(160955,)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " train[:, :6].max(axis=1).shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}