MeLi_BaseGen/MeLi_BaseGen_V2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20.5 s, sys: 1.45 s, total: 21.9 s\n",
      "Wall time: 24.6 s\n"
     ]
    }
   ],
   "source": [
    "%time data = pd.read_csv('../train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('../test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "samplesub = pd.read_csv('../sample_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.sample(frac = 1) # keep index!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(curr):\n",
    "    # remove accent\n",
    "    curr = curr.str.normalize('NFKD').str.encode('ascii', errors='ignore').str.decode('utf-8')\n",
    "    # to lower case\n",
    "    curr = curr.str.lower()\n",
    "    # remove not alphanumerics or . ,\n",
    "    curr = curr.str.replace('[^a-zA-Z0-9.,]', ' ')\n",
    "    \n",
    "    # let , and . be the same char\n",
    "    curr = curr.str.replace('[.]', ',')\n",
    "    \n",
    "    # remove . , not between numbers\n",
    "    curr = curr.str.replace('(?<=[0-9])[,]+(?=[0-9])', '.')\n",
    "    curr = curr.str.replace('[,]', ' ')\n",
    "    \n",
    "    # set all digits to 0\n",
    "    curr = curr.str.replace('[0-9]', '0')\n",
    "    \n",
    "    # separate ' <digits><letters ' like in 22g or 12ms\n",
    "    # curr = curr.str.replace('(^| )([0-9]+)([a-zA-Z]+)($| )', r'\\1\\2 \\3\\4')\n",
    "    \n",
    "    # remove some Pt plurals\n",
    "    curr = curr.str.replace('([a-zA-Z]+[aeiou])(s)', r'\\1')\n",
    "    \n",
    "    # Other ideas: \n",
    "    \n",
    "    return curr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data = data.title\n",
    "test_data = test.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_full = pd.concat([X_data, test_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 26s, sys: 2.25 s, total: 3min 29s\n",
      "Wall time: 3min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "X_full = normalize(X_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp2 = len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min, sys: 71.8 ms, total: 1min\n",
      "Wall time: 1min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# remove tokens with numbers\n",
    "X_full = X_full.str.replace('[a-z0-9_]*[0-9][a-z0-9_]*', ' ')\n",
    "X_norm = X_full[:sp2]\n",
    "test_norm = X_full[sp2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 33.2 s, sys: 12 ms, total: 33.2 s\n",
      "Wall time: 33.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# train_words = X_norm.str.split(expand=True).stack().value_counts().to_dict()\n",
    "train_words = dict()\n",
    "for t in X_norm:\n",
    "    for w in t.split():\n",
    "        train_words[w] = train_words.get(w, 0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 387 ms, sys: 35 µs, total: 387 ms\n",
      "Wall time: 387 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# test_words = test_norm.str.split(expand=True).stack().value_counts().to_dict()\n",
    "test_words = dict()\n",
    "for t in test_norm:\n",
    "    for w in t.split():\n",
    "        test_words[w] = test_words.get(w, 0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65398, 686965)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_words), len(train_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "testwseries = pd.Series(test_words).index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_test = testwseries.isin(pd.Series(train_words).index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9534389430869445"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['quianyu', 'renauklt', 'graaan', 'whiteni', 'boguilla', 'batalh',\n",
       "       'zaragaton', 'platozote', 'cracion', 'abcdefghijlmnopqrstuvxz',\n",
       "       'avgb', 'turimovr', 'fzlante', 'almohadapileta', 'cabf', 'fltdlc',\n",
       "       'civemaq', 'autoignicao', 'teleprint', 'supeemercado', 'alyo',\n",
       "       'estactora', 'megabits', 'gmcb', 'litewind', 'ecolaminator',\n",
       "       'sainthealth', 'profesionane', 'treab', 'bowo', 'nanoanillo',\n",
       "       'vitamaca', 'fotolopolimerizador', 'francobolli', 'paei',\n",
       "       'progyro', 'vehen', 'unicaerv', 'daguer', 'tornazo',\n",
       "       'portaacrilico', 'barrafix', 'omgear', 'maxjuli', 'lacrex',\n",
       "       'dynamicut', 'drtagon', 'noboru', 'capacipativa', 'cardiocom',\n",
       "       'protrectore', 'duplivox', 'microdin', 'plotteable', 'aeroburne',\n",
       "       'megafome', 'pecorelli', 'abdefg', 'armatazo', 'rockernapper',\n",
       "       'cormans', 'swereco', 'hirvonen', 'propicepci', 'pitoporium',\n",
       "       'lidschatten', 'tocadidco', 'conns', 'danning', 'caeprotetora',\n",
       "       'compostero', 'premiumkw', 'dabin', 'littly', 'souplatss',\n",
       "       'giracar', 'adpitadop', 'radifon', 'antibalitica', 'dslnikon',\n",
       "       'servicegy', 'megaxone', 'sellegra', 'natuel', 'neckknife',\n",
       "       'stillesitema', 'zoadiaco', 'canixhito', 'oktoberfe', 'dallacot',\n",
       "       'atraum', 'compea', 'rindete', 'thiriet', 'fornotedeco',\n",
       "       'gathermoval', 'microconvexa', 'ssggg', 'megadone',\n",
       "       'mayoritaozanamx', 'zapemdia', 'melloding', 'bototnera',\n",
       "       'completaaaaaaa', 'xpn', 'retroviol', 'globalfiltro', 'taekeondo',\n",
       "       'protozoologia', 'ganchso', 'contempraneo', 'canelera', 'thyemy',\n",
       "       'scancutt', 'fuyk', 'revitaliing', 'homologafo', 'submersz',\n",
       "       'globalma', 'ushui', 'whearing', 'buceadorecuchillo', 'ribermed',\n",
       "       'tumdo', 'cerramient', 'roaar', 'espesificacione', 'camcrane',\n",
       "       'bertedor', 'yoadhoad', 'cheguered', 'cesonello', 'hecer',\n",
       "       'ubertuner', 'viideo', 'inatillab', 'simbu', 'reultads',\n",
       "       'woodensound', 'brutematic', 'inalambricomultidispoitivo',\n",
       "       'pargue', 'bkdown', 'occipit', 'restaunte', 'magohany',\n",
       "       'refractar', 'valyne', 'cartinteria', 'strixt', 'inoxidavem',\n",
       "       'looj', 'coppoia', 'silverwings', 'ambocruzado', 'escotv', 'simci',\n",
       "       'audioconexion', 'presumiado', 'superficir', 'acuaticahello',\n",
       "       'corteim', 'abstratomarinho', 'joybeauti', 'calsamento',\n",
       "       'autorotacion', 'matricce', 'stelpro', 'caixavigo', 'matercoool',\n",
       "       'crispolinihop', 'damaen', 'diatonics', 'miraaaaaaaaaaaaaaa',\n",
       "       'tdupl', 'jarrie', 'vanityd', 'autonronzeador', 'shuaibei',\n",
       "       'autoadehi', 'loder', 'stakol', 'zauza', 'neutralbeige',\n",
       "       'requeimado', 'xrxl', 'casetttera', 'rsbx', 'paaikivi', 'laukka',\n",
       "       'kullerv', 'sxrl', 'lenzatic', 'vidrasaria', 'deencravad',\n",
       "       'shabbychi', 'redondatoma', 'jacone', 'rafiel', 'esquive'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testwseries[~temp_test].values[-200:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 42s, sys: 35.9 ms, total: 1min 42s\n",
      "Wall time: 1min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## how many instances are necessary (assuming X_norm is shuffled already)\n",
    "test_counter = test_words.copy()\n",
    "priorities = np.full(len(X_norm), 999_999)\n",
    "\n",
    "for i, t in enumerate(X_norm):\n",
    "    for w in t.split():\n",
    "        if w in test_counter:\n",
    "            priorities[i] = min(priorities[i], test_words[w] - test_counter[w])\n",
    "            test_counter[w] -= 1 # 1 for equal ammount, .5 for twice in train, ... # actually any positive val works\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['priorities'] = priorities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.sort_values(['category', 'label_quality', 'priorities'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.16 s, sys: 280 ms, total: 1.44 s\n",
      "Wall time: 1.44 s\n"
     ]
    }
   ],
   "source": [
    "%time reduced = data.groupby(['category'], as_index=False).head(800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17 s, sys: 16 ms, total: 17 s\n",
      "Wall time: 17 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "red_norm = normalize(reduced.title)\n",
    "red_norm = red_norm.str.replace('[a-z0-9_]*[0-9][a-z0-9_]*', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.93 s, sys: 448 ms, total: 9.38 s\n",
      "Wall time: 8.35 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "red_words = red_norm.str.split(expand=True).stack().value_counts().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp2_test = testwseries.isin(pd.Series(red_words).index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9371387504205022"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2_test.mean() # 800 ->937, 1000 -> 94"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.93 s, sys: 596 ms, total: 9.53 s\n",
      "Wall time: 6.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "left = data.loc[~data.index.isin(reduced.index)].query('priorities <= 5') # 0 -> 1650, 1,0 -> 32k, 1-5 -> 97k, 1-10 -> 188k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "188739"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced = pd.concat([reduced, left]) # next time do a shuffle after this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31.6 s, sys: 356 ms, total: 32 s\n",
      "Wall time: 30.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "red_norm = normalize(reduced.title)\n",
    "red_norm = red_norm.str.replace('[a-z0-9_]*[0-9][a-z0-9_]*', ' ')\n",
    "red_words = red_norm.str.split(expand=True).stack().value_counts().to_dict()\n",
    "temp2_test = testwseries.isin(pd.Series(red_words).index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9534389430869445"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2_test.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Simplification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.fit(data.category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3D_GLASSES', '3D_PENS', '3D_PRINTERS', ..., 'YARNS', 'YOGA_MATS',\n",
       "       'YOGURT_MAKERS'], dtype=object)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# must save this for later\n",
    "\n",
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change target to numeric\n",
    "\n",
    "reduced['category'] = le.transform(reduced.category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change label_quality to binary integer\n",
    "\n",
    "reduced['label_quality'] =  reduced.label_quality.map({'reliable' : 0, 'unreliable' : 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change language to binary integer\n",
    "\n",
    "reduced['language'] =  reduced.language.map({'spanish' : 0, 'portuguese' : 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>label_quality</th>\n",
       "      <th>language</th>\n",
       "      <th>category</th>\n",
       "      <th>priorities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6737325</th>\n",
       "      <td>Anteojo Lg 3d Pasivo</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13643907</th>\n",
       "      <td>Samsung 55 3d-un55f8000. Ultima Semana D Publi...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301995</th>\n",
       "      <td>Samsung Lentes 3d Activo Ssg-3300gr  Caja Cerr...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18914336</th>\n",
       "      <td>Anteojos Activos 3d (x1 Lente) Noblex-jvc-pion...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13966351</th>\n",
       "      <td>Sony Lentes 3d Recargables Tdg-br250 !! Oferta...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>573</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      title  label_quality  \\\n",
       "6737325                                Anteojo Lg 3d Pasivo              0   \n",
       "13643907  Samsung 55 3d-un55f8000. Ultima Semana D Publi...              0   \n",
       "301995    Samsung Lentes 3d Activo Ssg-3300gr  Caja Cerr...              0   \n",
       "18914336  Anteojos Activos 3d (x1 Lente) Noblex-jvc-pion...              0   \n",
       "13966351  Sony Lentes 3d Recargables Tdg-br250 !! Oferta...              0   \n",
       "\n",
       "          language  category  priorities  \n",
       "6737325          0         0          79  \n",
       "13643907         0         0         120  \n",
       "301995           0         0         230  \n",
       "18914336         0         0         350  \n",
       "13966351         0         0         573  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduced.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1444624, 5)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced = reduced.sample(frac = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced.to_csv('../data-reduced-800-v2-shuffled.csv', index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}