Building a French LM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOS = 'xbos'  # beginning-of-sentence tag\n",
    "FLD = 'xfld'  # data field tag\n",
    "\n",
    "PATH=Path('../../data/fr-wiki/french')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standarize format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_FILENAMES = [str(f) for f in PATH.rglob(\"*/*\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3906,\n",
       " ['..\\\\..\\\\data\\\\fr-wiki\\\\french\\\\AA\\\\wiki_00',\n",
       "  '..\\\\..\\\\data\\\\fr-wiki\\\\french\\\\AA\\\\wiki_01',\n",
       "  '..\\\\..\\\\data\\\\fr-wiki\\\\french\\\\AA\\\\wiki_02',\n",
       "  '..\\\\..\\\\data\\\\fr-wiki\\\\french\\\\AA\\\\wiki_03',\n",
       "  '..\\\\..\\\\data\\\\fr-wiki\\\\french\\\\AA\\\\wiki_04'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(LANG_FILENAMES), LANG_FILENAMES[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 3906/3906 [01:59<00:00, 32.67it/s]\n"
     ]
    }
   ],
   "source": [
    "LANG_TEXT = []\n",
    "for fn in tqdm(LANG_FILENAMES):\n",
    "    for line in open(fn, encoding='utf8'):\n",
    "        LANG_TEXT.append(json.loads(line))\n",
    "        \n",
    "LANG_TEXT = pd.DataFrame(LANG_TEXT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Antoine Meillet\\n\\nPaul Jules Antoine Meillet,...</td>\n",
       "      <td>Antoine Meillet</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>Algèbre linéaire\\n\\nL’algèbre linéaire est la ...</td>\n",
       "      <td>Algèbre linéaire</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>Algèbre générale\\n\\nL'algèbre générale, ou alg...</td>\n",
       "      <td>Algèbre générale</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>Algorithmique\\n\\nLalgorithmique est l'étude et...</td>\n",
       "      <td>Algorithmique</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>Politique en Argentine\\n\\nL'Argentine est une ...</td>\n",
       "      <td>Politique en Argentine</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  \\\n",
       "0   3  Antoine Meillet\\n\\nPaul Jules Antoine Meillet,...   \n",
       "1   7  Algèbre linéaire\\n\\nL’algèbre linéaire est la ...   \n",
       "2   9  Algèbre générale\\n\\nL'algèbre générale, ou alg...   \n",
       "3  10  Algorithmique\\n\\nLalgorithmique est l'étude et...   \n",
       "4  11  Politique en Argentine\\n\\nL'Argentine est une ...   \n",
       "\n",
       "                    title                                     url  \n",
       "0         Antoine Meillet   https://fr.wikipedia.org/wiki?curid=3  \n",
       "1        Algèbre linéaire   https://fr.wikipedia.org/wiki?curid=7  \n",
       "2        Algèbre générale   https://fr.wikipedia.org/wiki?curid=9  \n",
       "3           Algorithmique  https://fr.wikipedia.org/wiki?curid=10  \n",
       "4  Politique en Argentine  https://fr.wikipedia.org/wiki?curid=11  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LANG_TEXT.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT.to_csv(PATH/'wiki_fr.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting rid of the title name in the text field\n",
    "def split_title_from_text(text):\n",
    "    words = text.split(\"\\n\\n\")\n",
    "    if len(words) >= 2:\n",
    "        return ''.join(words[1:])\n",
    "    else:\n",
    "        return ''.join(words)\n",
    "    \n",
    "LANG_TEXT['text'] = LANG_TEXT['text'].apply(lambda x: split_title_from_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Paul Jules Antoine Meillet, né le à Moulins (A...</td>\n",
       "      <td>Antoine Meillet</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>L’algèbre linéaire est la branche des mathémat...</td>\n",
       "      <td>Algèbre linéaire</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>L'algèbre générale, ou algèbre abstraite, est ...</td>\n",
       "      <td>Algèbre générale</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>Lalgorithmique est l'étude et la production de...</td>\n",
       "      <td>Algorithmique</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>L'Argentine est une république présidentielle ...</td>\n",
       "      <td>Politique en Argentine</td>\n",
       "      <td>https://fr.wikipedia.org/wiki?curid=11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  \\\n",
       "0   3  Paul Jules Antoine Meillet, né le à Moulins (A...   \n",
       "1   7  L’algèbre linéaire est la branche des mathémat...   \n",
       "2   9  L'algèbre générale, ou algèbre abstraite, est ...   \n",
       "3  10  Lalgorithmique est l'étude et la production de...   \n",
       "4  11  L'Argentine est une république présidentielle ...   \n",
       "\n",
       "                    title                                     url  \n",
       "0         Antoine Meillet   https://fr.wikipedia.org/wiki?curid=3  \n",
       "1        Algèbre linéaire   https://fr.wikipedia.org/wiki?curid=7  \n",
       "2        Algèbre générale   https://fr.wikipedia.org/wiki?curid=9  \n",
       "3           Algorithmique  https://fr.wikipedia.org/wiki?curid=10  \n",
       "4  Politique en Argentine  https://fr.wikipedia.org/wiki?curid=11  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LANG_TEXT.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT.to_csv(PATH/'wiki_fr1.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sorting the articles by length and keeping the first million."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = pd.read_csv(PATH/'wiki_fr1.csv', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = LANG_TEXT.assign(length = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT.columns = ['id', 'text', 'title', 'url', 'length']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = LANG_TEXT.assign(labels = 0).pipe(lambda x: x[['labels', 'text', 'length']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Paul Jules Antoine Meillet, né le à Moulins (A...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>L’algèbre linéaire est la branche des mathémat...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>L'algèbre générale, ou algèbre abstraite, est ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Lalgorithmique est l'étude et la production de...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>L'Argentine est une république présidentielle ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text  length\n",
       "0       0  Paul Jules Antoine Meillet, né le à Moulins (A...       0\n",
       "1       0  L’algèbre linéaire est la branche des mathémat...       0\n",
       "2       0  L'algèbre générale, ou algèbre abstraite, est ...       0\n",
       "3       0  Lalgorithmique est l'étude et la production de...       0\n",
       "4       0  L'Argentine est une république présidentielle ...       0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LANG_TEXT.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT['length'] = LANG_TEXT['text'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = LANG_TEXT.sort_values(by=['length'], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT.to_csv(PATH/'wiki_fr1.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = LANG_TEXT[LANG_TEXT['length'] > 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "LANG_TEXT = LANG_TEXT.iloc[0:1000000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Splitting 10% for validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts,val_texts = sklearn.model_selection.train_test_split(LANG_TEXT.pipe(lambda x: x[['labels', 'text']]), test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts.to_csv(PATH/'train.csv', header=False, index=False)\n",
    "val_texts.to_csv(PATH/'valid.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunksize = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "re1 = re.compile(r'  +')\n",
    "\n",
    "def fixup(x):\n",
    "    x = x.replace('#39;', \"'\").replace('amp;', '&').replace('#146;', \"'\").replace(\n",
    "        'nbsp;', ' ').replace('#36;', '$').replace('\\\\n', \"\\n\").replace('quot;', \"'\").replace(\n",
    "        '<br />', \"\\n\").replace('\\\\\"', '\"').replace('<unk>','u_n').replace(' @.@ ','.').replace(\n",
    "        ' @-@ ','-').replace('\\\\', ' \\\\ ')\n",
    "    return re1.sub(' ', html.unescape(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.read_csv(PATH/'train.csv', header=None, chunksize=chunksize)\n",
    "df_val = pd.read_csv(PATH/'valid.csv', header=None, chunksize=chunksize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_texts(df, n_lbls=1):\n",
    "    labels = df.iloc[:,range(n_lbls)].values.astype(np.int64)\n",
    "    texts = f'\\n{BOS} {FLD} 1 ' + df[n_lbls].astype(str)\n",
    "    for i in range(n_lbls+1, len(df.columns)): texts += f' {FLD} {i-n_lbls} ' + df[i].astype(str)\n",
    "    texts = texts.apply(fixup).values.astype(str)\n",
    "    tok = Tokenizer.proc_all_mp(partition_by_cores(texts), lang='fr')\n",
    "    return tok, list(labels)\n",
    "\n",
    "def get_all(df, name, n_lbls=1):\n",
    "    for i, r in enumerate(df):\n",
    "        print(i)\n",
    "        tok_, labels_ = get_texts(r, n_lbls)\n",
    "        #save the partial tokens instead of regrouping them in one big array.\n",
    "        np.save(PATH/f'{name}_tok{i}.npy', tok_)\n",
    "    return tok, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_all(df_trn,'trn',1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_all(df_val,'val',1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numericalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the Counter object from all the splitted files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_them_all(names):\n",
    "    cnt = Counter()\n",
    "    for name in names:\n",
    "        for file in PATH.glob(f'tmp/{name}_tok*'):\n",
    "            tok = np.load(file)\n",
    "            cnt_tok = Counter(word for sent in tok for word in sent)\n",
    "            cnt += cnt_tok\n",
    "    return cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = count_them_all(['trn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(',', 404771),\n",
       " ('de', 403453),\n",
       " ('.', 240390),\n",
       " ('la', 229440),\n",
       " ('le', 174666),\n",
       " ('et', 165450),\n",
       " ('à', 143193),\n",
       " ('en', 131044),\n",
       " (\"l'\", 120337),\n",
       " ('les', 110875),\n",
       " ('\"', 110053),\n",
       " ('des', 105044),\n",
       " ('du', 95649),\n",
       " ('est', 91542),\n",
       " (\"d'\", 84666),\n",
       " ('un', 75756),\n",
       " ('une', 68106),\n",
       " ('il', 65473),\n",
       " ('\\r\\r\\n', 64888),\n",
       " ('(', 59492),\n",
       " ('dans', 59185),\n",
       " (')', 54383),\n",
       " ('par', 53859),\n",
       " ('au', 51115),\n",
       " ('-', 49281),\n",
       " ('pour', 46147),\n",
       " ('qui', 39651),\n",
       " ('a', 34453),\n",
       " ('sur', 33718),\n",
       " ('que', 33183),\n",
       " ('son', 30532),\n",
       " ('t_up', 30484),\n",
       " ('avec', 29857),\n",
       " ('se', 25568),\n",
       " ('plus', 25213),\n",
       " (':', 24598),\n",
       " ('l’', 22258),\n",
       " ('sont', 22242),\n",
       " ('elle', 19998),\n",
       " ('sa', 19070),\n",
       " ('ce', 18273),\n",
       " ('ou', 18171),\n",
       " ('«', 17461),\n",
       " ('»', 17358),\n",
       " (\"s'\", 16638),\n",
       " ('aux', 16534),\n",
       " ('été', 15906),\n",
       " ('ses', 15597),\n",
       " (\"qu'\", 15529),\n",
       " ('comme', 15440),\n",
       " ('1', 15148),\n",
       " ('d’', 14818),\n",
       " ('deux', 14337),\n",
       " ('cette', 14224),\n",
       " ('mais', 13323),\n",
       " ('pas', 13032),\n",
       " ('ne', 12358),\n",
       " ('\\n', 12050),\n",
       " ('xbos', 12000),\n",
       " ('xfld', 12000),\n",
       " ('\\xa0', 11408),\n",
       " ('après', 10851),\n",
       " ('entre', 10787),\n",
       " ('fait', 10383),\n",
       " ('ont', 10330),\n",
       " ('on', 9098),\n",
       " (\"n'\", 8996),\n",
       " ('était', 8904),\n",
       " ('leur', 8853),\n",
       " ('ainsi', 8269),\n",
       " ('fut', 8266),\n",
       " ('être', 8189),\n",
       " ('alors', 8174),\n",
       " ('dont', 8125),\n",
       " ('lui', 8063),\n",
       " ('où', 8003),\n",
       " ('même', 7969),\n",
       " ('sous', 7730),\n",
       " ('aussi', 7678),\n",
       " ('également', 7674),\n",
       " ('y', 7551),\n",
       " ('ces', 7482),\n",
       " (';', 7358),\n",
       " ('lors', 7220),\n",
       " ('autres', 7208),\n",
       " ('ils', 7147),\n",
       " ('premier', 6980),\n",
       " ('première', 6862),\n",
       " (\"'\", 6727),\n",
       " ('france', 6695),\n",
       " ('puis', 6326),\n",
       " ('depuis', 6251),\n",
       " ('nom', 6084),\n",
       " ('ville', 6005),\n",
       " ('ans', 5932),\n",
       " ('%', 5925),\n",
       " ('partie', 5844),\n",
       " ('années', 5825),\n",
       " ('plusieurs', 5676),\n",
       " ('trois', 5659),\n",
       " ('avant', 5395),\n",
       " ('contre', 5386),\n",
       " ('très', 5338),\n",
       " ('guerre', 5192),\n",
       " ('peut', 5154),\n",
       " ('groupe', 5144),\n",
       " ('tout', 5010),\n",
       " (\"c'\", 4900),\n",
       " ('/', 4721),\n",
       " ('fin', 4698),\n",
       " (\"jusqu'\", 4697),\n",
       " ('vers', 4651),\n",
       " ('fois', 4633),\n",
       " ('commune', 4579),\n",
       " ('avait', 4510),\n",
       " ('français', 4449),\n",
       " ('bien', 4400),\n",
       " ('pays', 4333),\n",
       " ('sans', 4297),\n",
       " ('année', 4292),\n",
       " ('avoir', 4290),\n",
       " ('notamment', 4290),\n",
       " ('place', 4216),\n",
       " ('paris', 4156),\n",
       " ('monde', 3974),\n",
       " ('temps', 3900),\n",
       " ('grand', 3896),\n",
       " ('mort', 3861),\n",
       " ('autre', 3828),\n",
       " ('suite', 3823),\n",
       " ('saison', 3804),\n",
       " ('état', 3780),\n",
       " ('pendant', 3735),\n",
       " ('né', 3734),\n",
       " ('2', 3728),\n",
       " ('leurs', 3688),\n",
       " ('nord', 3676),\n",
       " ('nouveau', 3670),\n",
       " ('encore', 3669),\n",
       " ('tous', 3649),\n",
       " ('l', 3605),\n",
       " ('nouvelle', 3576),\n",
       " ('selon', 3568),\n",
       " ('équipe', 3535),\n",
       " ('cours', 3499),\n",
       " ('général', 3499),\n",
       " ('partir', 3484),\n",
       " ('peu', 3484),\n",
       " ('soit', 3459),\n",
       " ('sud', 3447),\n",
       " ('faire', 3431),\n",
       " ('si', 3414),\n",
       " ('devient', 3402),\n",
       " ('grande', 3371),\n",
       " ('région', 3282),\n",
       " ('donc', 3261),\n",
       " ('club', 3230),\n",
       " ('titre', 3218),\n",
       " ('politique', 3210),\n",
       " ('début', 3183),\n",
       " ('états', 3180),\n",
       " ('parti', 3162),\n",
       " ('tour', 3161),\n",
       " ('quelques', 3145),\n",
       " ('vie', 3122),\n",
       " ('moins', 3086),\n",
       " ('histoire', 3079),\n",
       " ('quatre', 3051),\n",
       " ('famille', 3043),\n",
       " ('s’', 3038),\n",
       " ('lieu', 2995),\n",
       " ('3', 2990),\n",
       " ('durant', 2965),\n",
       " ('nombre', 2945),\n",
       " ('française', 2914),\n",
       " ('ensuite', 2872),\n",
       " ('église', 2843),\n",
       " ('série', 2825),\n",
       " ('the', 2819),\n",
       " ('non', 2796),\n",
       " ('forme', 2787),\n",
       " ('société', 2757),\n",
       " ('dès', 2738),\n",
       " ('étant', 2702),\n",
       " ('cependant', 2690),\n",
       " ('époque', 2686),\n",
       " ('centre', 2686),\n",
       " ('4', 2643),\n",
       " ('qu’', 2598),\n",
       " ('chaque', 2585),\n",
       " ('dernier', 2581),\n",
       " ('nombreux', 2574),\n",
       " ('étaient', 2573),\n",
       " ('origine', 2567),\n",
       " ('part', 2561),\n",
       " ('armée', 2559),\n",
       " ('président', 2537),\n",
       " ('face', 2532),\n",
       " ('ligne', 2525),\n",
       " ('tard', 2512),\n",
       " ('septembre', 2503),\n",
       " ('école', 2487),\n",
       " ('ayant', 2482),\n",
       " ('université', 2480),\n",
       " ('trouve', 2468),\n",
       " ('fils', 2468),\n",
       " ('afin', 2450),\n",
       " ('roi', 2439),\n",
       " ('chez', 2431),\n",
       " ('population', 2428),\n",
       " ('gouvernement', 2423),\n",
       " ('juin', 2411),\n",
       " ('près', 2405),\n",
       " ('prix', 2394),\n",
       " ('mars', 2391),\n",
       " ('album', 2390),\n",
       " ('mois', 2383),\n",
       " ('film', 2380),\n",
       " ('deuxième', 2376),\n",
       " ('unis', 2347),\n",
       " ('certains', 2331),\n",
       " ('celle', 2310),\n",
       " ('souvent', 2310),\n",
       " ('mai', 2308),\n",
       " ('ensemble', 2304),\n",
       " ('permet', 2279),\n",
       " ('ancien', 2273),\n",
       " ('elles', 2271),\n",
       " ('nationale', 2270),\n",
       " ('cet', 2268),\n",
       " ('celui', 2260),\n",
       " ('située', 2249),\n",
       " ('janvier', 2242),\n",
       " ('reste', 2224),\n",
       " ('effet', 2222),\n",
       " ('décembre', 2219),\n",
       " ('tant', 2214),\n",
       " ('octobre', 2199),\n",
       " ('juillet', 2195),\n",
       " ('père', 2192),\n",
       " ('homme', 2189),\n",
       " ('cas', 2189),\n",
       " ('pierre', 2187),\n",
       " ('environ', 2186),\n",
       " ('seconde', 2186),\n",
       " ('5', 2183),\n",
       " ('département', 2183),\n",
       " ('toujours', 2165),\n",
       " ('national', 2161),\n",
       " ('service', 2147),\n",
       " ('période', 2145),\n",
       " ('doit', 2141),\n",
       " ('exemple', 2134),\n",
       " ('europe', 2133),\n",
       " ('2010', 2130),\n",
       " ('conseil', 2117),\n",
       " ('carrière', 2110),\n",
       " ('car', 2104),\n",
       " ('point', 2077),\n",
       " ('novembre', 2068),\n",
       " ('toute', 2064),\n",
       " ('2012', 2063),\n",
       " ('village', 2063),\n",
       " ('10', 2060),\n",
       " ('compte', 2046),\n",
       " ('toutes', 2046),\n",
       " ('jean', 2034),\n",
       " ('niveau', 2031),\n",
       " ('prend', 2025),\n",
       " ('août', 2018),\n",
       " ('eau', 2016),\n",
       " ('château', 2015),\n",
       " ('système', 2013),\n",
       " ('peuvent', 2012),\n",
       " ('cinq', 2009),\n",
       " ('avril', 2009),\n",
       " ('île', 2002),\n",
       " ('sein', 2000),\n",
       " ('membres', 1997),\n",
       " ('ouest', 1996),\n",
       " ('jour', 1985),\n",
       " ('sera', 1970),\n",
       " ('membre', 1950),\n",
       " ('of', 1949),\n",
       " ('2008', 1942),\n",
       " ('nommé', 1939),\n",
       " ('furent', 1929),\n",
       " ('2009', 1926),\n",
       " ('2007', 1917),\n",
       " ('rôle', 1914),\n",
       " ('parmi', 1912),\n",
       " ('nombreuses', 1911),\n",
       " ('territoire', 1909),\n",
       " ('6', 1898),\n",
       " ('troisième', 1887),\n",
       " ('âge', 1875),\n",
       " ('études', 1872),\n",
       " ('2011', 1871),\n",
       " ('droit', 1860),\n",
       " ('maison', 1860),\n",
       " ('tête', 1858),\n",
       " ('février', 1857),\n",
       " ('championnat', 1852),\n",
       " ('projet', 1850),\n",
       " ('travail', 1843),\n",
       " ('2014', 1843),\n",
       " ('match', 1834),\n",
       " ('2013', 1831),\n",
       " ('ordre', 1819),\n",
       " ('communes', 1817),\n",
       " ('jours', 1807),\n",
       " ('situé', 1805),\n",
       " ('jeu', 1797),\n",
       " ('habitants', 1768),\n",
       " ('mondiale', 1764),\n",
       " ('développement', 1761),\n",
       " ('2015', 1760),\n",
       " ('pouvoir', 1752),\n",
       " ('premiers', 1749),\n",
       " ('coupe', 1740),\n",
       " ('œuvre', 1736),\n",
       " ('grâce', 1733),\n",
       " ('succès', 1732),\n",
       " ('site', 1730),\n",
       " ('points', 1715),\n",
       " ('long', 1712),\n",
       " ('personnes', 1710),\n",
       " ('chef', 1696),\n",
       " ('jeune', 1690),\n",
       " ('ailleurs', 1679),\n",
       " ('musique', 1678),\n",
       " ('base', 1678),\n",
       " ('devant', 1665),\n",
       " ('loi', 1662),\n",
       " ('art', 1655),\n",
       " ('enfants', 1648),\n",
       " ('ministre', 1644),\n",
       " ('poste', 1642),\n",
       " ('création', 1641),\n",
       " ('union', 1635),\n",
       " ('petit', 1622),\n",
       " ('rue', 1620),\n",
       " ('mise', 1619),\n",
       " ('corps', 1616),\n",
       " ('2006', 1609),\n",
       " ('dernière', 1608),\n",
       " ('mouvement', 1607),\n",
       " ('construction', 1600),\n",
       " ('hommes', 1600),\n",
       " ('laquelle', 1597),\n",
       " ('république', 1593),\n",
       " ('s', 1593),\n",
       " ('cela', 1587),\n",
       " ('terre', 1582),\n",
       " ('finale', 1579),\n",
       " ('déjà', 1574),\n",
       " ('élections', 1573),\n",
       " ('saint', 1570),\n",
       " ('beaucoup', 1565),\n",
       " ('public', 1562),\n",
       " ('remporte', 1557),\n",
       " ('but', 1557),\n",
       " ('8', 1551),\n",
       " ('six', 1546),\n",
       " ('seul', 1545),\n",
       " (\"aujourd'hui\", 1543),\n",
       " ('louis', 1531),\n",
       " ('plan', 1530),\n",
       " ('anglais', 1530),\n",
       " ('production', 1529),\n",
       " ('présente', 1527),\n",
       " ('côté', 1521),\n",
       " ('dit', 1519),\n",
       " ('direction', 1518),\n",
       " ('division', 1515),\n",
       " ('20', 1508),\n",
       " ('n’', 1503),\n",
       " ('terme', 1494),\n",
       " ('connu', 1493),\n",
       " ('raison', 1489),\n",
       " ('15', 1480),\n",
       " ('2016', 1479),\n",
       " ('lorsque', 1478),\n",
       " ('agit', 1478),\n",
       " ('royaume', 1477),\n",
       " ('quand', 1473),\n",
       " ('porte', 1471),\n",
       " ('new', 1469),\n",
       " ('tandis', 1457),\n",
       " ('charles', 1454),\n",
       " ('communauté', 1453),\n",
       " ('musée', 1452),\n",
       " ('travaux', 1449),\n",
       " ('...', 1446),\n",
       " ('second', 1435),\n",
       " ('ancienne', 1434),\n",
       " ('jeux', 1433),\n",
       " ('association', 1432),\n",
       " ('compagnie', 1432),\n",
       " ('militaire', 1431),\n",
       " ('formation', 1427),\n",
       " ('participe', 1424),\n",
       " ('12', 1424),\n",
       " ('route', 1420),\n",
       " ('ligue', 1418),\n",
       " ('passe', 1418),\n",
       " ('va', 1418),\n",
       " ('fille', 1417),\n",
       " ('version', 1413),\n",
       " ('joue', 1411),\n",
       " ('abord', 1407),\n",
       " ('7', 1401),\n",
       " ('femme', 1395),\n",
       " ('type', 1394),\n",
       " ('zone', 1392),\n",
       " ('différents', 1391),\n",
       " ('retour', 1388),\n",
       " ('john', 1388),\n",
       " ('victoire', 1386),\n",
       " ('autour', 1386),\n",
       " ('écrit', 1385),\n",
       " ('international', 1381),\n",
       " ('siège', 1380),\n",
       " ('finalement', 1379),\n",
       " ('an', 1379),\n",
       " ('2005', 1377),\n",
       " ('rivière', 1376),\n",
       " ('ceux', 1375),\n",
       " ('particulier', 1374),\n",
       " ('ii', 1369),\n",
       " ('liste', 1368),\n",
       " ('commence', 1368),\n",
       " ('milieu', 1367),\n",
       " ('bataille', 1367),\n",
       " ('obtient', 1366),\n",
       " ('parfois', 1364),\n",
       " ('donne', 1361),\n",
       " ('or', 1360),\n",
       " ('aurait', 1360),\n",
       " ('certaines', 1358),\n",
       " ('2000', 1358),\n",
       " ('joueur', 1357),\n",
       " ('allemagne', 1344),\n",
       " ('premières', 1343),\n",
       " ('seulement', 1341),\n",
       " ('organisation', 1335),\n",
       " ('surtout', 1334),\n",
       " ('livre', 1327),\n",
       " ('malgré', 1327),\n",
       " ('cause', 1324),\n",
       " ('américain', 1324),\n",
       " ('font', 1313),\n",
       " ('existe', 1305),\n",
       " ('utilisé', 1298),\n",
       " ('mis', 1296),\n",
       " ('produit', 1290),\n",
       " ('scène', 1289),\n",
       " ('réseau', 1286),\n",
       " ('eu', 1285),\n",
       " ('grands', 1272),\n",
       " ('2004', 1271),\n",
       " ('date', 1271),\n",
       " ('langue', 1269),\n",
       " ('millions', 1269),\n",
       " ('cadre', 1267),\n",
       " ('avaient', 1261),\n",
       " ('italie', 1261),\n",
       " ('rapport', 1258),\n",
       " ('créé', 1256),\n",
       " ('œuvres', 1256),\n",
       " ('recherche', 1253),\n",
       " ('seule', 1248),\n",
       " ('petite', 1245),\n",
       " ('nous', 1241),\n",
       " ('fonction', 1237),\n",
       " ('manière', 1235),\n",
       " ('différentes', 1231),\n",
       " ('parc', 1229),\n",
       " ('gauche', 1228),\n",
       " ('important', 1226),\n",
       " ('directeur', 1224),\n",
       " ('30', 1223),\n",
       " ('françois', 1219),\n",
       " ('domaine', 1219),\n",
       " ('bois', 1216),\n",
       " ('18', 1215),\n",
       " ('femmes', 1212),\n",
       " ('marque', 1210),\n",
       " ('mer', 1210),\n",
       " ('plupart', 1210),\n",
       " ('gare', 1207),\n",
       " ('auteur', 1202),\n",
       " ('grandes', 1198),\n",
       " ('9', 1198),\n",
       " ('chanson', 1192),\n",
       " ('suivante', 1191),\n",
       " ('théâtre', 1191),\n",
       " ('jamais', 1189),\n",
       " ('culture', 1188),\n",
       " ('province', 1188),\n",
       " ('nouvelles', 1188),\n",
       " ('celui-ci', 1185),\n",
       " ('11', 1184),\n",
       " ('eux', 1183),\n",
       " ('aide', 1181),\n",
       " ('lequel', 1179),\n",
       " ('voir', 1179),\n",
       " ('rencontre', 1172),\n",
       " ('rapidement', 1169),\n",
       " ('2002', 1167),\n",
       " ('moment', 1164),\n",
       " ('16', 1163),\n",
       " ('principalement', 1162),\n",
       " ('1990', 1162),\n",
       " ('14', 1157),\n",
       " ('appelé', 1154),\n",
       " ('haut', 1152),\n",
       " ('toutefois', 1151),\n",
       " ('2017', 1147),\n",
       " ('activité', 1146),\n",
       " ('édition', 1146),\n",
       " ('britannique', 1143),\n",
       " ('voit', 1142),\n",
       " ('york', 1142),\n",
       " ('…', 1133),\n",
       " ('frère', 1131),\n",
       " ('enfin', 1130),\n",
       " ('moyenne', 1129),\n",
       " ('élu', 1129),\n",
       " ('là', 1128),\n",
       " ('suivant', 1127),\n",
       " ('25', 1127),\n",
       " ('marie', 1126),\n",
       " ('style', 1125),\n",
       " ('travers', 1122),\n",
       " ('généralement', 1119),\n",
       " ('taux', 1116),\n",
       " ('jeunes', 1112),\n",
       " ('mère', 1112),\n",
       " ('présence', 1108),\n",
       " ('demande', 1105),\n",
       " ('football', 1102),\n",
       " ('fer', 1102),\n",
       " ('17', 1100),\n",
       " ('paul', 1100),\n",
       " ('rejoint', 1099),\n",
       " ('groupes', 1099),\n",
       " ('troupes', 1095),\n",
       " ('cour', 1090),\n",
       " ('possède', 1090),\n",
       " ('tels', 1089),\n",
       " ('espagne', 1088),\n",
       " ('administration', 1085),\n",
       " ('révolution', 1084),\n",
       " ('services', 1084),\n",
       " ('allemand', 1082),\n",
       " ('propre', 1081),\n",
       " ('moyen', 1081),\n",
       " ('entreprise', 1077),\n",
       " ('communiste', 1076),\n",
       " ('angleterre', 1075),\n",
       " ('meilleur', 1074),\n",
       " ('activités', 1074),\n",
       " ('action', 1071),\n",
       " ('lac', 1068),\n",
       " ('quartier', 1065),\n",
       " ('reçoit', 1064),\n",
       " ('prendre', 1060),\n",
       " ('voie', 1058),\n",
       " ('roman', 1057),\n",
       " ('position', 1055),\n",
       " ('principal', 1053),\n",
       " ('occasion', 1052),\n",
       " ('affaires', 1050),\n",
       " ('13', 1048),\n",
       " ('1980', 1047),\n",
       " ('intérieur', 1046),\n",
       " ('canton', 1042),\n",
       " ('née', 1041),\n",
       " ('épouse', 1041),\n",
       " ('droite', 1039),\n",
       " ('populaire', 1036),\n",
       " ('retrouve', 1034),\n",
       " ('politiques', 1033),\n",
       " ('situation', 1031),\n",
       " ('2003', 1027),\n",
       " ('classement', 1027),\n",
       " ('importante', 1025),\n",
       " ('–', 1025),\n",
       " ('empire', 1024),\n",
       " ('droits', 1021),\n",
       " ('1999', 1020),\n",
       " ('professeur', 1019),\n",
       " (']', 1019),\n",
       " ('24', 1018),\n",
       " ('mesure', 1017),\n",
       " ('internationale', 1017),\n",
       " ('fort', 1017),\n",
       " ('publié', 1015),\n",
       " ('étude', 1015),\n",
       " ('[', 1013),\n",
       " ('équipes', 1011),\n",
       " ('rouge', 1011),\n",
       " ('possible', 1011),\n",
       " ('sept', 1009),\n",
       " ('objet', 1008),\n",
       " ('vue', 1007),\n",
       " ('2001', 1007),\n",
       " ('particulièrement', 1004),\n",
       " ('journal', 1004),\n",
       " ('met', 1002),\n",
       " ('créée', 1000),\n",
       " ('—', 999),\n",
       " ('construit', 999),\n",
       " ('henri', 996),\n",
       " ('espace', 990),\n",
       " ('0', 990),\n",
       " ('outre', 987),\n",
       " ('afrique', 986),\n",
       " ('éléments', 986),\n",
       " ('produits', 986),\n",
       " ('matchs', 985),\n",
       " ('aucun', 983),\n",
       " ('décide', 983),\n",
       " ('sens', 982),\n",
       " ('russie', 982),\n",
       " ('contrôle', 981),\n",
       " ('villes', 981),\n",
       " ('classe', 980),\n",
       " ('américaine', 978),\n",
       " ('forces', 977),\n",
       " ('campagne', 977),\n",
       " ('robert', 974),\n",
       " ('1970', 972),\n",
       " ('donné', 970),\n",
       " ('force', 969),\n",
       " ('jacques', 969),\n",
       " ('termine', 969),\n",
       " ('proche', 966),\n",
       " ('fit', 965),\n",
       " ('voix', 962),\n",
       " ('composé', 962),\n",
       " ('ni', 961),\n",
       " ('historique', 958),\n",
       " ('serait', 956),\n",
       " ('suisse', 953),\n",
       " ('lui-même', 949),\n",
       " ('trop', 943),\n",
       " ('départ', 942),\n",
       " ('assez', 941),\n",
       " ('comité', 939),\n",
       " ('programme', 937),\n",
       " ('entrée', 936),\n",
       " ('buts', 935),\n",
       " ('nouveaux', 935),\n",
       " ('dix', 935),\n",
       " ('al', 935),\n",
       " ('émission', 934),\n",
       " ('économique', 934),\n",
       " ('façon', 933),\n",
       " ('c’', 932),\n",
       " ('arrivée', 932),\n",
       " ('réalisé', 931),\n",
       " ('espèces', 931),\n",
       " ('atteint', 925),\n",
       " ('auprès', 925),\n",
       " ('celle-ci', 923),\n",
       " ('air', 923),\n",
       " ('mission', 917),\n",
       " ('1998', 917),\n",
       " ('22', 917),\n",
       " ('semble', 916),\n",
       " ('capitale', 913),\n",
       " ('chambre', 911),\n",
       " ('annonce', 910),\n",
       " ('faisant', 908),\n",
       " ('1960', 905),\n",
       " ('d', 903),\n",
       " ('personnage', 901),\n",
       " ('21', 899),\n",
       " ('19', 899),\n",
       " ('japon', 894),\n",
       " ('générale', 893),\n",
       " ('course', 892),\n",
       " ('attaque', 891),\n",
       " ('vallée', 889),\n",
       " ('londres', 888),\n",
       " ('!', 886),\n",
       " ('anciens', 886),\n",
       " ('combat', 886),\n",
       " ('derniers', 882),\n",
       " ('signe', 879),\n",
       " ('celles', 879),\n",
       " ('bâtiment', 879),\n",
       " ('23', 878),\n",
       " ('modèle', 878),\n",
       " ('festival', 877),\n",
       " ('prince', 877),\n",
       " ('défense', 875),\n",
       " ('sciences', 874),\n",
       " ('sort', 872),\n",
       " ('comte', 870),\n",
       " ('rome', 869),\n",
       " ('désormais', 867),\n",
       " ('mettre', 863),\n",
       " ('commission', 863),\n",
       " ('and', 862),\n",
       " ('régime', 859),\n",
       " ('sortie', 855),\n",
       " ('pris', 854),\n",
       " ('article', 851),\n",
       " ('heures', 849),\n",
       " ('mètres', 847),\n",
       " ('résultats', 844),\n",
       " ('genre', 843),\n",
       " ('sorti', 842),\n",
       " ('traité', 841),\n",
       " ('télévision', 840),\n",
       " ('devenu', 839),\n",
       " ('cité', 839),\n",
       " ('qualité', 835),\n",
       " ('honneur', 833),\n",
       " ('chemin', 833),\n",
       " ('accès', 832),\n",
       " ('canada', 830),\n",
       " ('devenir', 828),\n",
       " ('artistes', 827),\n",
       " ('enseignement', 827),\n",
       " ('salle', 825),\n",
       " ('assemblée', 825),\n",
       " ('films', 824),\n",
       " ('meurt', 824),\n",
       " ('champion', 824),\n",
       " ('majorité', 821),\n",
       " ('argent', 821),\n",
       " ('russe', 820),\n",
       " ('académie', 820),\n",
       " ('québec', 818),\n",
       " ('espèce', 818),\n",
       " ('santé', 817),\n",
       " ('valeur', 817),\n",
       " ('quatrième', 816),\n",
       " ('comprend', 816),\n",
       " ('influence', 816),\n",
       " ('total', 815),\n",
       " ('m', 814),\n",
       " ('centrale', 814),\n",
       " ('libre', 813),\n",
       " ('marché', 812),\n",
       " ('cinéma', 812),\n",
       " ('28', 810),\n",
       " ('technique', 810),\n",
       " ('lignes', 810),\n",
       " ('terrain', 809),\n",
       " ('large', 805),\n",
       " ('joseph', 804),\n",
       " ('belgique', 804),\n",
       " ('aucune', 803),\n",
       " ('?', 802),\n",
       " ('plutôt', 800),\n",
       " ('compétition', 799),\n",
       " ('conditions', 796),\n",
       " ('publique', 795),\n",
       " ('néanmoins', 794),\n",
       " ('unique', 793),\n",
       " ('européenne', 792),\n",
       " ('chine', 792),\n",
       " ('utilisation', 792),\n",
       " ('éducation', 791),\n",
       " ('26', 790),\n",
       " ('passage', 789),\n",
       " ('principaux', 789),\n",
       " ('quant', 789),\n",
       " ('faut', 788),\n",
       " ('contrat', 788),\n",
       " ('secrétaire', 787),\n",
       " ('i', 784),\n",
       " ('économie', 780),\n",
       " ('personne', 780),\n",
       " ('clubs', 780),\n",
       " ('sécurité', 779),\n",
       " ('exposition', 779),\n",
       " ('élection', 777),\n",
       " (\"lorsqu'\", 776),\n",
       " ('données', 773),\n",
       " ('san', 772),\n",
       " ('quitte', 771),\n",
       " ('moitié', 771),\n",
       " ('issue', 771),\n",
       " ('huit', 770),\n",
       " ('certain', 769),\n",
       " ('représente', 769),\n",
       " ('pratique', 769),\n",
       " ('publie', 768),\n",
       " ('vient', 768),\n",
       " ('double', 766),\n",
       " ('forte', 766),\n",
       " ('divers', 766),\n",
       " ('permettant', 766),\n",
       " ('pièces', 765),\n",
       " ('charge', 765),\n",
       " ('célèbre', 764),\n",
       " ('terres', 764),\n",
       " ('arts', 763),\n",
       " ('haute', 763),\n",
       " ('nuit', 761),\n",
       " ('radio', 761),\n",
       " ('1997', 761),\n",
       " ('maire', 760),\n",
       " ('surface', 760),\n",
       " ('inscrit', 759),\n",
       " ('côte', 758),\n",
       " ('apparaît', 757),\n",
       " ('phase', 757),\n",
       " ('vitesse', 757),\n",
       " ('source', 756),\n",
       " ('critique', 756),\n",
       " ('commerce', 755),\n",
       " ('savoie', 755),\n",
       " ('presque', 754),\n",
       " ('nature', 752),\n",
       " ('prise', 751),\n",
       " ('ouvrage', 748),\n",
       " ('amérique', 747),\n",
       " ('stade', 746),\n",
       " ('michel', 746),\n",
       " ('iii', 746),\n",
       " ('enfant', 744),\n",
       " ('référence', 744),\n",
       " ('collection', 744),\n",
       " ('1996', 743),\n",
       " ('régions', 743),\n",
       " ('fonctions', 740),\n",
       " ('structure', 739),\n",
       " ('soutien', 738),\n",
       " ('principale', 737),\n",
       " ('titres', 734),\n",
       " ('je', 734),\n",
       " ('énergie', 734),\n",
       " ('pont', 734),\n",
       " ('jusqu’', 732),\n",
       " ('rend', 732),\n",
       " ('secteur', 731),\n",
       " ('taille', 729),\n",
       " ('unité', 728),\n",
       " ('réalise', 727),\n",
       " ('longue', 727),\n",
       " ('kilomètres', 726),\n",
       " ('station', 724),\n",
       " ('joueurs', 723),\n",
       " ('palais', 722),\n",
       " ('maître', 721),\n",
       " ('mariage', 720),\n",
       " ('parties', 719),\n",
       " ('c', 719),\n",
       " ('autant', 719),\n",
       " ('allemands', 719),\n",
       " ('allemande', 718),\n",
       " ('eut', 717),\n",
       " ('évêque', 716),\n",
       " ('information', 716),\n",
       " ('in', 716),\n",
       " ('gestion', 715),\n",
       " ('etc.', 715),\n",
       " ('1995', 714),\n",
       " ('actuellement', 713),\n",
       " ('simple', 713),\n",
       " ('théorie', 713),\n",
       " ('naissance', 713),\n",
       " ('fondation', 712),\n",
       " ('doivent', 712),\n",
       " ('accord', 712),\n",
       " ('mode', 711),\n",
       " ('représentant', 710),\n",
       " ('patrimoine', 710),\n",
       " ('image', 710),\n",
       " ('passé', 710),\n",
       " ('relations', 710),\n",
       " ('reprises', 710),\n",
       " ('presse', 710),\n",
       " ('militaires', 709),\n",
       " ('propose', 709),\n",
       " ('îles', 709),\n",
       " ('james', 709),\n",
       " ('épisode', 709),\n",
       " ('italien', 706),\n",
       " ('devint', 705),\n",
       " ('travaille', 704),\n",
       " ('27', 702),\n",
       " ('chaîne', 701),\n",
       " ('connue', 701),\n",
       " ('hiver', 701),\n",
       " ('coup', 701),\n",
       " ('mot', 701),\n",
       " ('forêt', 701),\n",
       " ('considéré', 699),\n",
       " ('dispose', 699),\n",
       " ('constitue', 699),\n",
       " ('peuple', 699),\n",
       " ('ait', 698),\n",
       " ('proximité', 698),\n",
       " ('soviétique', 698),\n",
       " ('lettres', 697),\n",
       " ('1994', 697),\n",
       " ('peintre', 697),\n",
       " ('diverses', 696),\n",
       " ('continue', 695),\n",
       " ('armes', 693),\n",
       " ('bon', 693),\n",
       " ('chapelle', 692),\n",
       " ('obtenir', 692),\n",
       " ('duc', 690),\n",
       " ('évolution', 688),\n",
       " ('ministère', 687),\n",
       " ('protection', 687),\n",
       " ('pu', 687),\n",
       " ('reçu', 687),\n",
       " ('1992', 686),\n",
       " ('david', 686),\n",
       " ('choix', 686),\n",
       " ('tableau', 685),\n",
       " ('&', 685),\n",
       " ('minutes', 684),\n",
       " ('derrière', 683),\n",
       " (\"c'est-à-dire\", 683),\n",
       " ('juifs', 683),\n",
       " ('soldats', 683),\n",
       " ('°', 683),\n",
       " ('revue', 682),\n",
       " ('blanc', 681),\n",
       " ('utilisée', 681),\n",
       " ('tournoi', 680),\n",
       " ('sociale', 680),\n",
       " ('directement', 679),\n",
       " ('port', 679),\n",
       " ('pied', 678),\n",
       " ('institut', 678),\n",
       " ('idée', 678),\n",
       " ('cœur', 678),\n",
       " ('<', 676),\n",
       " ('1993', 675),\n",
       " ('police', 674),\n",
       " ('constitution', 674),\n",
       " ('faible', 673),\n",
       " ('chargé', 672),\n",
       " ('philippe', 672),\n",
       " ('royal', 672),\n",
       " ('capitaine', 672),\n",
       " ('arrive', 671),\n",
       " ('actuel', 671),\n",
       " ('empereur', 669),\n",
       " ('personnages', 668),\n",
       " ('tours', 668),\n",
       " ('offre', 668),\n",
       " ('côtés', 667),\n",
       " ('sujet', 666),\n",
       " ('1991', 665),\n",
       " ('pourrait', 665),\n",
       " ('processus', 665),\n",
       " ('livres', 664),\n",
       " ('altitude', 664),\n",
       " ('dieu', 664),\n",
       " ('supérieur', 661),\n",
       " ('front', 661),\n",
       " ('usage', 660),\n",
       " ('transport', 659),\n",
       " ('opposition', 658),\n",
       " ('objets', 658),\n",
       " ('hôtel', 657),\n",
       " ('environnement', 656),\n",
       " ('question', 656),\n",
       " ('probablement', 655),\n",
       " ('opération', 655),\n",
       " ('longueur', 654),\n",
       " ('statut', 653),\n",
       " ('via', 653),\n",
       " ('garde', 653),\n",
       " ('50', 652),\n",
       " ('noir', 652),\n",
       " ('tournée', 651),\n",
       " ('parlement', 649),\n",
       " ('industrie', 649),\n",
       " ('actuelle', 648),\n",
       " ('créer', 648),\n",
       " ('historiques', 647),\n",
       " ('zones', 647),\n",
       " ('seront', 647),\n",
       " ('sélection', 647),\n",
       " ('fondé', 646),\n",
       " ('comté', 646),\n",
       " ('dernières', 646),\n",
       " ('montre', 646),\n",
       " ('intérêt', 644),\n",
       " ('petits', 644),\n",
       " ('reprend', 642),\n",
       " ('débuts', 641),\n",
       " ('vit', 641),\n",
       " ('fondée', 640),\n",
       " ('sol', 640),\n",
       " ('vu', 640),\n",
       " ...]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab = 60000\n",
    "min_freq = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = [o for o,c in cnt.most_common(max_vocab) if c > min_freq]\n",
    "itos.insert(0,'_pad_')\n",
    "itos.insert(0,'_unk_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48446"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = collections.defaultdict(int,{s:i for (i,s) in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Numericalize each partial file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numericalize(name):\n",
    "    results = []\n",
    "    for file in tqdm(PATH.glob(f'tmp/{name}_tok*')):\n",
    "        tok = np.load(file)\n",
    "        results.append(np.array([[stoi[word] for word in sent] for sent in tok]))\n",
    "    return np.concatenate(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_ids = numericalize('trn')\n",
    "np.save(PATH/'tmp/trn_ids.npy', trn_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ids = numericalize('val')\n",
    "np.save(PATH/'tmp/trn_ids.npy', val_ids)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}