03_select_texts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from mltb2.text import (\n",
    "    has_invisible_characters,\n",
    "    has_special_whitespaces,\n",
    "    clean_all_invisible_chars_and_whitespaces,\n",
    "    TextDistance,\n",
    ")\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "tqdm.pandas()  # activate tqdm pandas progress bar support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995892</th>\n",
       "      <td>14995892</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>DB2 komplett in Richtung nativer Unterstützung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995893</th>\n",
       "      <td>14995893</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Nachdem sich die eNF-Relationen am Ende bei de...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995894</th>\n",
       "      <td>14995894</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Jim Melton, der federführende Editor aller SQL...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995895</th>\n",
       "      <td>14995895</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Auch wenn sich das ISO-SQL-Gremium nicht dazu ...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995896</th>\n",
       "      <td>14995896</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Die Entwicklung und Implementierung eines so g...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14876607 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                                              title  \\\n",
       "0                0                                      Nekrolog 2022   \n",
       "1                1                                        Deutschland   \n",
       "2                2                                        Deutschland   \n",
       "3                3                                        Deutschland   \n",
       "4                4                                        Deutschland   \n",
       "...            ...                                                ...   \n",
       "14995892  14995892  Advanced Information Management Prototype (AIM-P)   \n",
       "14995893  14995893  Advanced Information Management Prototype (AIM-P)   \n",
       "14995894  14995894  Advanced Information Management Prototype (AIM-P)   \n",
       "14995895  14995895  Advanced Information Management Prototype (AIM-P)   \n",
       "14995896  14995896  Advanced Information Management Prototype (AIM-P)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14995892  DB2 komplett in Richtung nativer Unterstützung...   \n",
       "14995893  Nachdem sich die eNF-Relationen am Ende bei de...   \n",
       "14995894  Jim Melton, der federführende Editor aller SQL...   \n",
       "14995895  Auch wenn sich das ISO-SQL-Gremium nicht dazu ...   \n",
       "14995896  Die Entwicklung und Implementierung eines so g...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14995892  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995893  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995894  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995895  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995896  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "\n",
       "          paragraph_id  langs  \n",
       "0                    0     38  \n",
       "1                    0    295  \n",
       "2                    1    295  \n",
       "3                    2    295  \n",
       "4                    3    295  \n",
       "...                ...    ...  \n",
       "14995892           124      0  \n",
       "14995893           125      0  \n",
       "14995894           126      0  \n",
       "14995895           127      0  \n",
       "14995896           128      0  \n",
       "\n",
       "[14876607 rows x 8 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle(\"./data/wikipedia-22-12-de-no-emb-clean.pkl.gz\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956787</th>\n",
       "      <td>14956787</td>\n",
       "      <td>Bomber &amp;amp; Paganini</td>\n",
       "      <td>Da naht die vermeintliche Rettung in Form eine...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=6918884</td>\n",
       "      <td>6918884</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956788</th>\n",
       "      <td>14956788</td>\n",
       "      <td>C Pfalz 71</td>\n",
       "      <td>Die Wagen dieser Blatt-Nr. hatten keine Bremse...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=6919760</td>\n",
       "      <td>6919760</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956789</th>\n",
       "      <td>14956789</td>\n",
       "      <td>Pflegamt Gundelfingen</td>\n",
       "      <td>Mit der Verwaltungsneugliederung Bayerns im Ja...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=6948216</td>\n",
       "      <td>6948216</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956790</th>\n",
       "      <td>14956790</td>\n",
       "      <td>Elberfelder Kalksenke</td>\n",
       "      <td>Die Senke wird vollständig durch dichte Siedlu...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=6948342</td>\n",
       "      <td>6948342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956791</th>\n",
       "      <td>14956791</td>\n",
       "      <td>Cotapata (Nor Yungas)</td>\n",
       "      <td>Chuspipata liegt in einer Entfernung von 52 St...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=6948429</td>\n",
       "      <td>6948429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7676126 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                  title  \\\n",
       "0                0          Nekrolog 2022   \n",
       "1                1            Deutschland   \n",
       "2                2            Deutschland   \n",
       "3                3            Deutschland   \n",
       "4                4            Deutschland   \n",
       "...            ...                    ...   \n",
       "14956787  14956787  Bomber &amp; Paganini   \n",
       "14956788  14956788             C Pfalz 71   \n",
       "14956789  14956789  Pflegamt Gundelfingen   \n",
       "14956790  14956790  Elberfelder Kalksenke   \n",
       "14956791  14956791  Cotapata (Nor Yungas)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14956787  Da naht die vermeintliche Rettung in Form eine...   \n",
       "14956788  Die Wagen dieser Blatt-Nr. hatten keine Bremse...   \n",
       "14956789  Mit der Verwaltungsneugliederung Bayerns im Ja...   \n",
       "14956790  Die Senke wird vollständig durch dichte Siedlu...   \n",
       "14956791  Chuspipata liegt in einer Entfernung von 52 St...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14956787   https://de.wikipedia.org/wiki?curid=6918884   6918884     0.000000   \n",
       "14956788   https://de.wikipedia.org/wiki?curid=6919760   6919760     0.000000   \n",
       "14956789   https://de.wikipedia.org/wiki?curid=6948216   6948216     0.000000   \n",
       "14956790   https://de.wikipedia.org/wiki?curid=6948342   6948342     0.000000   \n",
       "14956791   https://de.wikipedia.org/wiki?curid=6948429   6948429     0.000000   \n",
       "\n",
       "          paragraph_id  langs  \n",
       "0                    0     38  \n",
       "1                    0    295  \n",
       "2                    1    295  \n",
       "3                    2    295  \n",
       "4                    3    295  \n",
       "...                ...    ...  \n",
       "14956787             3      0  \n",
       "14956788             3      0  \n",
       "14956789             3      0  \n",
       "14956790             3      0  \n",
       "14956791             3      0  \n",
       "\n",
       "[7676126 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"paragraph_id\"].between(0, 3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14876607/14876607 [03:25<00:00, 72525.93it/s] \n"
     ]
    }
   ],
   "source": [
    "text_distance = TextDistance(show_progress_bar=True, max_dimensions=150)\n",
    "text_distance.fit(df[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14876607/14876607 [12:46<00:00, 19408.38it/s]\n"
     ]
    }
   ],
   "source": [
    "df[\"text_distance\"] = df[\"text\"].progress_apply(text_distance.distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "      <th>text_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>0.383651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "      <td>0.303631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "      <td>0.238855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "      <td>0.228585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "      <td>0.161922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995892</th>\n",
       "      <td>14995892</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>DB2 komplett in Richtung nativer Unterstützung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "      <td>0.252011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995893</th>\n",
       "      <td>14995893</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Nachdem sich die eNF-Relationen am Ende bei de...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "      <td>0.210647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995894</th>\n",
       "      <td>14995894</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Jim Melton, der federführende Editor aller SQL...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "      <td>0.283772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995895</th>\n",
       "      <td>14995895</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Auch wenn sich das ISO-SQL-Gremium nicht dazu ...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "      <td>0.159984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995896</th>\n",
       "      <td>14995896</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Die Entwicklung und Implementierung eines so g...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0.305180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14876607 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                                              title  \\\n",
       "0                0                                      Nekrolog 2022   \n",
       "1                1                                        Deutschland   \n",
       "2                2                                        Deutschland   \n",
       "3                3                                        Deutschland   \n",
       "4                4                                        Deutschland   \n",
       "...            ...                                                ...   \n",
       "14995892  14995892  Advanced Information Management Prototype (AIM-P)   \n",
       "14995893  14995893  Advanced Information Management Prototype (AIM-P)   \n",
       "14995894  14995894  Advanced Information Management Prototype (AIM-P)   \n",
       "14995895  14995895  Advanced Information Management Prototype (AIM-P)   \n",
       "14995896  14995896  Advanced Information Management Prototype (AIM-P)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14995892  DB2 komplett in Richtung nativer Unterstützung...   \n",
       "14995893  Nachdem sich die eNF-Relationen am Ende bei de...   \n",
       "14995894  Jim Melton, der federführende Editor aller SQL...   \n",
       "14995895  Auch wenn sich das ISO-SQL-Gremium nicht dazu ...   \n",
       "14995896  Die Entwicklung und Implementierung eines so g...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14995892  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995893  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995894  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995895  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995896  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "\n",
       "          paragraph_id  langs  text_distance  \n",
       "0                    0     38       0.383651  \n",
       "1                    0    295       0.303631  \n",
       "2                    1    295       0.238855  \n",
       "3                    2    295       0.228585  \n",
       "4                    3    295       0.161922  \n",
       "...                ...    ...            ...  \n",
       "14995892           124      0       0.252011  \n",
       "14995893           125      0       0.210647  \n",
       "14995894           126      0       0.283772  \n",
       "14995895           127      0       0.159984  \n",
       "14995896           128      0       0.305180  \n",
       "\n",
       "[14876607 rows x 9 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvsklEQVR4nO3de1RVdf7H/9cB5YAXjhrKJVFIMvNKo0U4OsU3Ev05rugyqdVorLRVX+uX0ZVWQX3rG9WkY834jak0qsl0Gs0mLbpQeEVNjUkbNSEML1zEgiNHhYL9+8Ofp06ichA9Hw/Px1p75d77vT/nvWuN5zV7f87eNsuyLAEAABgswNcNAAAAnAqBBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgvA6+bqAtNDU1ad++feratatsNpuv2wEAAC1gWZYOHjyoqKgoBQSc/BqKXwSWffv2KTo62tdtAACAVti9e7d69+590hq/CCxdu3aVdPSEQ0NDfdwNAABoCafTqejoaPf3+Mn4RWA5dhsoNDSUwAIAwDmmJdM5mHQLAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwALAWLW1tRo1apT69OmjUaNGqba21tctAfARrwJLdna2Lr30UnXt2lW9evVSamqqduzYccrj3nnnHQ0YMEDBwcEaMmSIPvjgA4/9lmUpMzNTkZGRCgkJUXJysnbu3OndmQDwK3FxcerWrZvWrFmj3bt3a82aNerWrZvi4uJ83RoAH/AqsKxYsUIzZszQunXr9Mknn+jHH3/UmDFj5HK5TnjM2rVrNXnyZN1222368ssvlZqaqtTUVG3dutVd89xzz+nFF19UTk6O1q9fr86dOyslJUVHjhxp/ZkBOGfFxcWppKREkjR27FgVFhZq7NixkqSSkhJCC9AO2SzLslp78P79+9WrVy+tWLFCv/vd75qtmThxolwul5YtW+bedvnllys+Pl45OTmyLEtRUVG67777dP/990s6ehk4PDxcubm5mjRp0in7cDqdcjgcqq2t5V1CwDmutrZW3bp1kyS5XC516tTJve/QoUPq3LmzJKmmpkYOh8MXLQJoI958f5/WHJZj95N79OhxwprCwkIlJyd7bEtJSVFhYaEkqbS0VBUVFR41DodDCQkJ7ppfq6+vl9Pp9FgA+Ifx48dLOnpl5ZdhRZI6deqkMWPGeNQBaB9aHViampo0c+ZM/fa3v9XgwYNPWFdRUaHw8HCPbeHh4aqoqHDvP7btRDW/lp2dLYfD4V6io6NbexoADFNWViZJysrKanb/o48+6lEHoH1odWCZMWOGtm7dqoULF7ZlPy2SkZGh2tpa97J79+6z3gOAM6NPnz6SpCeeeKLZ/U899ZRHHYD2oVWB5a677tKyZcv0+eefq3fv3ietjYiIUGVlpce2yspKRUREuPcf23aiml+z2+0KDQ31WAD4h+XLl0uS8vLydOjQIY99hw4d0scff+xRB6B98CqwWJalu+66S++++64+++wzxcbGnvKYxMRE5efne2z75JNPlJiYKEmKjY1VRESER43T6dT69evdNQDaD4fDoX79+kmS+xeDq1atUkpKinvCbb9+/ZhwC7QzHbwpnjFjhhYsWKD33ntPXbt2dc8xcTgcCgkJkSRNmTJF559/vrKzsyVJ99xzj6644grNmjVL48eP18KFC7Vx40a9/PLLkiSbzaaZM2fqqaee0oUXXqjY2Fg99thjioqKUmpqahueKoBzRXFxsfunzR9//LH7qop0NKwUFxf7sDsAvuBVYHnppZckSVdeeaXH9tdee0233nqrpKMT4QICfr5wM3LkSC1YsECPPvqoHnnkEV144YVaunSpx0TdBx98UC6XS7fffrtqamo0atQo5eXlKTg4uJWnBeBcV1xcrNraWo0fP15lZWXq06ePli9fzpUVoJ06reewmILnsAAAcO45a89hAQAAOBu8uiUEAGdTY2OjVq1apfLyckVGRmr06NEKDAz0dVsAfIArLACMtGTJEsXFxSkpKUk33XSTkpKSFBcXpyVLlvi6NQA+wBUWAMZZsmSJbrjhBo0fP14PPPCAQkJCdPjwYX344Ye64YYb9M9//lPXXXedr9sEcBYx6RaAURobGxUXF6ewsDBVV1dr165d7n0xMTEKCwvTgQMHtHPnTm4PAec4Jt0COGetWrVKu3bt0qZNmzRkyBAVFhbq4MGDKiws1JAhQ7Rp0yaVlpZq1apVvm4VwFlEYAFglL1790o6+rbmxYsX68iRI3r//fd15MgRLV68WGPHjvWoA9A+MIcFgFH2798v6ejtn/79+x93S+hYYDlWB6B94AoLAKP07NlT0tEnaw8ePNjjltDgwYOVk5PjUQegfSCwADDKr9/SblmWezlZHQD/xi0hAEa6+OKLtWXLFo0cOdK9LSYmRgMGDND27dt92BkAXyCwADBKVVWVJGn79u3HPYclLy9Py5cv96gD0D4QWAAYJTIyUpL09NNP629/+5uWLVvm3hcbG6v//d//1SOPPOKuA9A+EFgAGGX06NGKiYnR2rVr9c0332jNmjXudwn99re/1fXXX6/Y2FiNHj3a160COIuYdAvAKIGBgZo1a5aWLVum66+/Xna7Xb///e9lt9t1/fXXa9myZXr++ed5yi3QznCFBYBxrrvuOv3zn//Ufffd5zHpNjY2lvcIAe0U7xICYKzGxkatWrXKfUto9OjRXFkB/AjvEgIAAH6FwALASEuWLFFcXJySkpJ00003KSkpSXFxcVqyZImvWwPgAwQWAMZZsmSJbrjhhmbf1nzDDTcQWoB2iDksAIzS2NiouLg4DRkyRIsXL272Z81bt27Vzp07mc8CnOOYwwLgnLVq1Srt2rVLI0eOVP/+/T1uCfXv31+JiYkqLS3VqlWrfN0qgLOIwALAKOXl5ZKkjIyMZm8JPfLIIx51ANoHnsMCwCi9evWSJI0aNUpLly5VQMDR/191+eWXa+nSpfrd736nNWvWuOsAtA8EFgDGamxs1MqVKz3msNhsNl+3BcAHCCwAjHLsLcyrV6+Ww+HQ4cOH3fuOvbX5l3UA2gfmsAAwyi/fwvzLsPLrdd7WDLQvBBYARhk5cqR73kpwcLDHvmPrAQEBHu8YAuD/uCUEwCirVq1SU1OTJCk0NFQzZszQBRdcoG+//VZvvvmmjhw5oqamJq1atUpXXXWVj7sFcLYQWAAY5bPPPpN09JZPVVWVZs2a5d7XoUMHRUZGqry8XJ999hmBBWhHCCwAjFJWVibp6HNWxo8fr7i4OB0+fFghISEqLi7W8uXLPeoAtA8EFgBG6d27tySpc+fO2rJlizugSFKfPn3UuXNnuVwudx2A9oHAAsAoYWFhkiSXy3Xcr4T27Nnjnt9yrA5A++D1r4RWrlypCRMmKCoqSjabTUuXLj1p/a233iqbzXbcMmjQIHfN448/ftz+AQMGeH0yAM59LX2CLU+6BdoXrwOLy+XSsGHDNHfu3BbVv/DCCyovL3cvu3fvVo8ePfSHP/zBo27QoEEedatXr/a2NQB+4JcPhDt2NaW5dR4cB7QvXt8SGjdunMaNG9fieofDIYfD4V5funSpfvjhB6WlpXk20qGDIiIivG0HgJ/5/vvv27QOgH846w+OmzdvnpKTk9W3b1+P7Tt37lRUVJQuuOAC3XzzzSf9BUB9fb2cTqfHAsA/NDQ0tGkdAP9wVgPLvn379OGHH2ratGke2xMSEpSbm6u8vDy99NJLKi0t1ejRo3Xw4MFmx8nOznZfuXE4HIqOjj4b7QM4C1auXNmmdQD8g82yLKvVB9tsevfdd5Wamtqi+uzsbM2aNUv79u1TUFDQCetqamrUt29fzZ49W7fddttx++vr61VfX+9edzqdio6OVm1trUJDQ70+DwDmCA4O9vjf94nY7XYdOXLkLHQE4ExxOp1yOBwt+v4+az9rtixL8+fP1x//+MeThhVJ6tatm/r376/i4uJm99vtdtnt9jPRJgAfa0lY8aYOgH84a7eEVqxYoeLi4mavmPxaXV2dSkpKeBsrAACQ1IrAUldXp6KiIhUVFUmSSktLVVRU5J4km5GRoSlTphx33Lx585SQkKDBgwcft+/+++/XihUrtGvXLq1du1bXXnutAgMDNXnyZG/bA3COCwwMbNM6AP7B61tCGzduVFJSkns9PT1dkjR16lTl5uaqvLz8uF/41NbWavHixXrhhReaHXPPnj2aPHmyDhw4oJ49e2rUqFFat26devbs6W17AM5xwcHBcrlcLaoD0H6c1qRbU3gzaQeA2UJDQ0/4C8Ff6tq1K480AM5x3nx/n/XnsADAyTQ2NrZpHQD/QGABYJRDhw61aR0A/0BgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAEbp2LFjm9YB8A8EFgBGCQkJadM6AP6BwALAKB06dGjTOgD+wevAsnLlSk2YMEFRUVGy2WxaunTpSesLCgpks9mOWyoqKjzq5s6dq5iYGAUHByshIUEbNmzwtjUAfuDIkSNtWgfAP3gdWFwul4YNG6a5c+d6ddyOHTtUXl7uXnr16uXet2jRIqWnpysrK0ubN2/WsGHDlJKSoqqqKm/bA3COs9lsbVoHwD94fU113LhxGjdunNcf1KtXL3Xr1q3ZfbNnz9b06dOVlpYmScrJydHy5cs1f/58Pfzww15/FoBz1+HDh9u0DoB/OGtzWOLj4xUZGamrr75aa9ascW9vaGjQpk2blJyc/HNTAQFKTk5WYWFhs2PV19fL6XR6LAD8Q1NTU5vWAfAPZzywREZGKicnR4sXL9bixYsVHR2tK6+8Ups3b5YkVVdXq7GxUeHh4R7HhYeHHzfP5Zjs7Gw5HA73Eh0dfaZPAwAA+NAZn2Z/0UUX6aKLLnKvjxw5UiUlJfrzn/+sN998s1VjZmRkKD093b3udDoJLQAA+DGf/C7wsssu0+rVqyVJYWFhCgwMVGVlpUdNZWWlIiIimj3ebrfLbref8T4BAIAZfPIclqKiIkVGRkqSgoKCNHz4cOXn57v3NzU1KT8/X4mJib5oDwAAGMbrKyx1dXUqLi52r5eWlqqoqEg9evRQnz59lJGRob179+qNN96QJM2ZM0exsbEaNGiQjhw5oldffVWfffaZPv74Y/cY6enpmjp1qkaMGKHLLrtMc+bMkcvlcv9qCAAAtG9eB5aNGzcqKSnJvX5sLsnUqVOVm5ur8vJylZWVufc3NDTovvvu0969e9WpUycNHTpUn376qccYEydO1P79+5WZmamKigrFx8crLy/vuIm4APxfcHBwix4KFxwcfBa6AWAKm2VZlq+bOF1Op1MOh0O1tbUKDQ31dTsATsOgQYP0n//855R1AwcO1Ndff30WOgJwpnjz/c27hAAY5fvvv2/TOgD+gcACwCgHDhxo0zoA/oHAAsAojY2NbVoHwD8QWAAYhUfzA2gOgQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPG8DiwrV67UhAkTFBUVJZvNpqVLl560fsmSJbr66qvVs2dPhYaGKjExUR999JFHzeOPPy6bzeaxDBgwwNvWAACAn/I6sLhcLg0bNkxz585tUf3KlSt19dVX64MPPtCmTZuUlJSkCRMm6Msvv/SoGzRokMrLy93L6tWrvW0NAAD4qQ7eHjBu3DiNGzeuxfVz5szxWH/66af13nvv6f3339cll1zycyMdOigiIsLbdgAAQDtw1uewNDU16eDBg+rRo4fH9p07dyoqKkoXXHCBbr75ZpWVlZ1wjPr6ejmdTo8FAAD4r7MeWJ5//nnV1dXpxhtvdG9LSEhQbm6u8vLy9NJLL6m0tFSjR4/WwYMHmx0jOztbDofDvURHR5+t9gEAgA/YLMuyWn2wzaZ3331XqampLapfsGCBpk+frvfee0/JycknrKupqVHfvn01e/Zs3Xbbbcftr6+vV319vXvd6XQqOjpatbW1Cg0N9fo8AJjDZrO1uPY0/voCYACn0ymHw9Gi72+v57C01sKFCzVt2jS98847Jw0rktStWzf1799fxcXFze632+2y2+1nok0AAGCgs3JL6O2331ZaWprefvttjR8//pT1dXV1KikpUWRk5FnoDgAAmM7rKyx1dXUeVz5KS0tVVFSkHj16qE+fPsrIyNDevXv1xhtvSDp6G2jq1Kl64YUXlJCQoIqKCklSSEiIHA6HJOn+++/XhAkT1LdvX+3bt09ZWVkKDAzU5MmT2+IcAQDAOc7rKywbN27UJZdc4v5Jcnp6ui655BJlZmZKksrLyz1+4fPyyy/rp59+0owZMxQZGele7rnnHnfNnj17NHnyZF100UW68cYbdd5552ndunXq2bPn6Z4fAADwA6c16dYU3kzaAWA2Jt0C7Yc339+8SwgAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGM/rwLJy5UpNmDBBUVFRstlsWrp06SmPKSgo0G9+8xvZ7XbFxcUpNzf3uJq5c+cqJiZGwcHBSkhI0IYNG7xtDQAA+CmvA4vL5dKwYcM0d+7cFtWXlpZq/PjxSkpKUlFRkWbOnKlp06bpo48+ctcsWrRI6enpysrK0ubNmzVs2DClpKSoqqrK2/YAAIAfslmWZbX6YJtN7777rlJTU09Y89BDD2n58uXaunWre9ukSZNUU1OjvLw8SVJCQoIuvfRS/fWvf5UkNTU1KTo6WnfffbcefvjhU/bhdDrlcDhUW1ur0NDQ1p4OAAPYbLYW157GX18ADODN9/cZn8NSWFio5ORkj20pKSkqLCyUJDU0NGjTpk0eNQEBAUpOTnbX/Fp9fb2cTqfHAgAA/NcZDywVFRUKDw/32BYeHi6n06nDhw+rurpajY2NzdZUVFQ0O2Z2drYcDod7iY6OPmP9AwAA3zsnfyWUkZGh2tpa97J7925ftwQAAM6gDmf6AyIiIlRZWemxrbKyUqGhoQoJCVFgYKACAwObrYmIiGh2TLvdLrvdfsZ6BgAAZjnjV1gSExOVn5/vse2TTz5RYmKiJCkoKEjDhw/3qGlqalJ+fr67BgAAtG9eB5a6ujoVFRWpqKhI0tGfLRcVFamsrEzS0ds1U6ZMcdffcccd+vbbb/Xggw9q+/bt+r//+z/94x//0L333uuuSU9P1yuvvKLXX39d27Zt05133imXy6W0tLTTPD0AAOAPvL4ltHHjRiUlJbnX09PTJUlTp05Vbm6uysvL3eFFkmJjY7V8+XLde++9euGFF9S7d2+9+uqrSklJcddMnDhR+/fvV2ZmpioqKhQfH6+8vLzjJuICAID26bSew2IKnsMC+A+ewwK0H0Y9hwUAAOB0EVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOO1KrDMnTtXMTExCg4OVkJCgjZs2HDC2iuvvFI2m+24Zfz48e6aW2+99bj9Y8eObU1rAADAD3Xw9oBFixYpPT1dOTk5SkhI0Jw5c5SSkqIdO3aoV69ex9UvWbJEDQ0N7vUDBw5o2LBh+sMf/uBRN3bsWL322mvudbvd7m1rAADAT3l9hWX27NmaPn260tLSNHDgQOXk5KhTp06aP39+s/U9evRQRESEe/nkk0/UqVOn4wKL3W73qOvevXvrzggAAPgdrwJLQ0ODNm3apOTk5J8HCAhQcnKyCgsLWzTGvHnzNGnSJHXu3Nlje0FBgXr16qWLLrpId955pw4cOHDCMerr6+V0Oj0WAADgv7wKLNXV1WpsbFR4eLjH9vDwcFVUVJzy+A0bNmjr1q2aNm2ax/axY8fqjTfeUH5+vp599lmtWLFC48aNU2NjY7PjZGdny+FwuJfo6GhvTgMAAJxjvJ7DcjrmzZunIUOG6LLLLvPYPmnSJPefhwwZoqFDh6pfv34qKCjQVVddddw4GRkZSk9Pd687nU5CCwAAfsyrKyxhYWEKDAxUZWWlx/bKykpFRESc9FiXy6WFCxfqtttuO+XnXHDBBQoLC1NxcXGz++12u0JDQz0WAADgv7wKLEFBQRo+fLjy8/Pd25qampSfn6/ExMSTHvvOO++ovr5et9xyyyk/Z8+ePTpw4IAiIyO9aQ8AAPgpr38llJ6erldeeUWvv/66tm3bpjvvvFMul0tpaWmSpClTpigjI+O44+bNm6fU1FSdd955Htvr6ur0wAMPaN26ddq1a5fy8/N1zTXXKC4uTikpKa08LQAA4E+8nsMyceJE7d+/X5mZmaqoqFB8fLzy8vLcE3HLysoUEOCZg3bs2KHVq1fr448/Pm68wMBAffXVV3r99ddVU1OjqKgojRkzRk8++STPYgEAAJIkm2VZlq+bOF1Op1MOh0O1tbXMZwHOcTabrcW1fvDXF9CuefP9zbuEAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGK+DrxsA4J8ONzSqZH/dGf2MrXtrW3Vcv55dFBIU2MbdADiTCCwAzoiS/XX6/V9Wn9HPaO34y+4epcHnO9q4GwBnks2yLMvXTZwup9Mph8Oh2tpahYaG+rodAGr9FZYhvbu1uHbLnhqvx5e4wgKYwpvvb66wADgjQoICz/hVDK6SAO0Hk24BGKWlF3394OIwAC8QWAAY51RhhLACtD8EFgBGOlEoIawA7ROBBYCxLMvSlj016vvQMm3ZU0NYAdoxAgsAADAegQUAABiPwAIAAIxHYAEAAMZrVWCZO3euYmJiFBwcrISEBG3YsOGEtbm5ubLZbB5LcHCwR41lWcrMzFRkZKRCQkKUnJysnTt3tqY1AADgh7wOLIsWLVJ6erqysrK0efNmDRs2TCkpKaqqqjrhMaGhoSovL3cv3333ncf+5557Ti+++KJycnK0fv16de7cWSkpKTpy5Ij3ZwQAAPyO14Fl9uzZmj59utLS0jRw4EDl5OSoU6dOmj9//gmPsdlsioiIcC/h4eHufZZlac6cOXr00Ud1zTXXaOjQoXrjjTe0b98+LV26tFUnBQAA/ItXgaWhoUGbNm1ScnLyzwMEBCg5OVmFhYUnPK6urk59+/ZVdHS0rrnmGn399dfufaWlpaqoqPAY0+FwKCEh4YRj1tfXy+l0eiwAAMB/eRVYqqur1djY6HGFRJLCw8NVUVHR7DEXXXSR5s+fr/fee09///vf1dTUpJEjR2rPnj2S5D7OmzGzs7PlcDjcS3R0tDenAQAAzjFn/FdCiYmJmjJliuLj43XFFVdoyZIl6tmzp/72t7+1esyMjAzV1ta6l927d7dhxwAAwDReBZawsDAFBgaqsrLSY3tlZaUiIiJaNEbHjh11ySWXqLi4WJLcx3kzpt1uV2hoqMcCAAD8l1eBJSgoSMOHD1d+fr57W1NTk/Lz85WYmNiiMRobG7VlyxZFRkZKkmJjYxUREeExptPp1Pr161s8JgAA8G8dvD0gPT1dU6dO1YgRI3TZZZdpzpw5crlcSktLkyRNmTJF559/vrKzsyVJ//M//6PLL79ccXFxqqmp0Z/+9Cd99913mjZtmqSjvyCaOXOmnnrqKV144YWKjY3VY489pqioKKWmprbdmQIAgHOW14Fl4sSJ2r9/vzIzM1VRUaH4+Hjl5eW5J82WlZUpIODnCzc//PCDpk+froqKCnXv3l3Dhw/X2rVrNXDgQHfNgw8+KJfLpdtvv101NTUaNWqU8vLyjnvAHAAAaJ9slh+8r93pdMrhcKi2tpb5LICf2bq3Vr//y2otu3uUBp/v8HU7ANqQN9/fvEsIAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPE6+LoBAGYprXbJVf+Tr9twK66q8/inSTrbOyg2rLOv2wDaBQILALfSapeSni/wdRvNmrmoyNctNOvz+68ktABnAYEFgNuxKytzJsYrrlcXH3dz1JEfG7Xnh8Pq3T1EwR0Dfd2OW3FVnWYuKjLqahTgzwgsAI4T16uLBp/v8HUbbiNifN0BAF9j0i0AADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB5PugXgwdbBqVLnDgUEm/FoflOVOutk6+D0dRtAu0FgAeChY7f1emTD075u45zQsdtVkv4fX7cBtAsEFgAefqxJ0KzxN6mfIS8/NFVJVZ3+37dKfN0G0G4QWAB4sH4KVWzoRRp4njkvPzRR05FaWT/t93UbQLvBpFsAAGA8AgsAADAegQUAABivVYFl7ty5iomJUXBwsBISErRhw4YT1r7yyisaPXq0unfvru7duys5Ofm4+ltvvVU2m81jGTt2bGtaAwAAfsjrwLJo0SKlp6crKytLmzdv1rBhw5SSkqKqqqpm6wsKCjR58mR9/vnnKiwsVHR0tMaMGaO9e/d61I0dO1bl5eXu5e23327dGQEAAL/jdWCZPXu2pk+frrS0NA0cOFA5OTnq1KmT5s+f32z9W2+9pf/+7/9WfHy8BgwYoFdffVVNTU3Kz8/3qLPb7YqIiHAv3bt3b90ZAQAAv+PVz5obGhq0adMmZWRkuLcFBAQoOTlZhYWFLRrj0KFD+vHHH9WjRw+P7QUFBerVq5e6d++u//qv/9JTTz2l8847r9kx6uvrVV9f7153OnnaJNAWDv/YKEnaurfWx5387MiPjdrzw2H17h6i4I6Bvm7HrbiqztctAO2KV4GlurpajY2NCg8P99geHh6u7du3t2iMhx56SFFRUUpOTnZvGzt2rK677jrFxsaqpKREjzzyiMaNG6fCwkIFBh7/F1R2draeeOIJb1oH0AIl//+X8MNLtvi4k3NHZzuPswLOhrP6v7RnnnlGCxcuVEFBgYKDg93bJ02a5P7zkCFDNHToUPXr108FBQW66qqrjhsnIyND6enp7nWn06no6Ogz2zzQDowZFCFJ6teri0IMuZpRXFWnmYuKNGdivOIMe/puZ3sHxYZ19nUbQLvgVWAJCwtTYGCgKisrPbZXVlYqIiLipMc+//zzeuaZZ/Tpp59q6NChJ6294IILFBYWpuLi4mYDi91ul91u96Z1AC3Qo3OQJl3Wx9dtNCuuVxcNPp+n7wLtlVeTboOCgjR8+HCPCbPHJtAmJiae8LjnnntOTz75pPLy8jRixIhTfs6ePXt04MABRUZGetMeAADwU17/Sig9PV2vvPKKXn/9dW3btk133nmnXC6X0tLSJElTpkzxmJT77LPP6rHHHtP8+fMVExOjiooKVVRUqK7u6L3yuro6PfDAA1q3bp127dql/Px8XXPNNYqLi1NKSkobnSYAADiXeT2HZeLEidq/f78yMzNVUVGh+Ph45eXluSfilpWVKSDg5xz00ksvqaGhQTfccIPHOFlZWXr88ccVGBior776Sq+//rpqamoUFRWlMWPG6Mknn+S2DwAAkCTZLMuyfN3E6XI6nXI4HKqtrVVoaKiv2wHQhrburdXv/7Jay+4exRwWwM948/3Nu4QAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwXgdfNwDAPx1uaFTJ/rrTHqe4qs7jn22hX88uCgkKbLPxAJx5BBYAZ0TJ/jr9/i+r22y8mYuK2mysZXeP0uDzHW02HoAzj8AC4Izo17OLlt096rTHOfJjo/b8cFi9u4couGPbXBXp17NLm4wD4OwhsAA4I0KCAtvsKsaImDYZBsA5jEm3AADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4rQosc+fOVUxMjIKDg5WQkKANGzactP6dd97RgAEDFBwcrCFDhuiDDz7w2G9ZljIzMxUZGamQkBAlJydr586drWkNAAD4Ia8Dy6JFi5Senq6srCxt3rxZw4YNU0pKiqqqqpqtX7t2rSZPnqzbbrtNX375pVJTU5WamqqtW7e6a5577jm9+OKLysnJ0fr169W5c2elpKToyJEjrT8zAADgN2yWZVneHJCQkKBLL71Uf/3rXyVJTU1Nio6O1t13362HH374uPqJEyfK5XJp2bJl7m2XX3654uPjlZOTI8uyFBUVpfvuu0/333+/JKm2tlbh4eHKzc3VpEmTTtmT0+mUw+FQbW2tQkNDvTkdAADgI958f3t1haWhoUGbNm1ScnLyzwMEBCg5OVmFhYXNHlNYWOhRL0kpKSnu+tLSUlVUVHjUOBwOJSQknHDM+vp6OZ1OjwUAAPgvr97WXF1drcbGRoWHh3tsDw8P1/bt25s9pqKiotn6iooK9/5j205U82vZ2dl64oknjttOcAEA4Nxx7Hu7JTd7vAospsjIyFB6erp7fe/evRo4cKCio6N92BUAAGiNgwcPyuFwnLTGq8ASFhamwMBAVVZWemyvrKxUREREs8dERESctP7YPysrKxUZGelREx8f3+yYdrtddrvdvd6lSxft3r1bXbt2lc1m8+aUABjO6XQqOjpau3fvZo4a4Gcsy9LBgwcVFRV1ylqvAktQUJCGDx+u/Px8paamSjo66TY/P1933XVXs8ckJiYqPz9fM2fOdG/75JNPlJiYKEmKjY1VRESE8vPz3QHF6XRq/fr1uvPOO1vUV0BAgHr37u3NqQA4x4SGhhJYAD90qisrx3h9Syg9PV1Tp07ViBEjdNlll2nOnDlyuVxKS0uTJE2ZMkXnn3++srOzJUn33HOPrrjiCs2aNUvjx4/XwoULtXHjRr388suSJJvNppkzZ+qpp57ShRdeqNjYWD322GOKiopyhyIAANC+eR1YJk6cqP379yszM1MVFRWKj49XXl6ee9JsWVmZAgJ+/vHRyJEjtWDBAj366KN65JFHdOGFF2rp0qUaPHiwu+bBBx+Uy+XS7bffrpqaGo0aNUp5eXkKDg5ug1MEAADnOq+fwwIAZ1N9fb2ys7OVkZHhMXcNQPtCYAEAAMbj5YcAAMB4BBYAAGA8AgsAADAegQWAEQoKCmSz2VRTUyNJys3NVbdu3XzaEwBzEFiAdubKK6/0eJCjqWNOnDhR33zzTYtqCTeA/zsn3yUEwP+FhIQoJCTE120AMARXWIB25NZbb9WKFSv0wgsvyGazyWazadeuXdq6davGjRunLl26KDw8XH/84x9VXV0t6eitmqCgIK1atco9znPPPadevXqpsrLyhGOeygcffKD+/fsrJCRESUlJxx3z66sm//73v5WUlKSuXbsqNDRUw4cP18aNG1VQUKC0tDTV1ta6P//xxx+XJL355psaMWKEunbtqoiICN10002qqqpyj3nsNlR+fr5GjBihTp06aeTIkdqxY4dHL++//74uvfRSBQcHKywsTNdee617X319ve6//36df/756ty5sxISElRQUNCy/yAAWs4C0G7U1NRYiYmJ1vTp063y8nKrvLzcqq6utnr27GllZGRY27ZtszZv3mxdffXVVlJSkvu4Bx54wOrbt69VU1Njbd682QoKCrLee++9E475008/nbSPsrIyy263W+np6db27dutv//971Z4eLglyfrhhx8sy7Ks1157zXI4HO5jBg0aZN1yyy3Wtm3brG+++cb6xz/+YRUVFVn19fXWnDlzrNDQUPfnHzx40LIsy5o3b571wQcfWCUlJVZhYaGVmJhojRs3zj3m559/bkmyEhISrIKCAuvrr7+2Ro8ebY0cOdJds2zZMiswMNDKzMy0/vOf/1hFRUXW008/7d4/bdo0a+TIkdbKlSut4uJi609/+pNlt9utb775ptX/nQAcj8ACtDNXXHGFdc8997jXn3zySWvMmDEeNbt377YkWTt27LAsy7Lq6+ut+Ph468Ybb7QGDhxoTZ8+/aRjnkpGRoY1cOBAj20PPfTQSQNL165drdzc3GbH+3XtiXzxxReWJHegORZYPv30U3fN8uXLLUnW4cOHLcuyrMTEROvmm29udrzvvvvOCgwMtPbu3eux/aqrrrIyMjJO2Q+AluOWENDO/fvf/9bnn3+uLl26uJcBAwZIkkpKSiQdfVP7W2+9pcWLF+vIkSP685//fFqfuW3bNiUkJHhsO/YG9xNJT0/XtGnTlJycrGeeecbd28ls2rRJEyZMUJ8+fdS1a1ddccUVko6+8+yXhg4d6v5zZGSkJLlvHRUVFemqq65qdvwtW7aosbFR/fv39/j3t2LFihb1B6DlmHQLtHN1dXWaMGGCnn322eP2HfvylqS1a9dKkr7//nt9//336ty581nrUZIef/xx3XTTTVq+fLk+/PBDZWVlaeHChR7zSX7J5XIpJSVFKSkpeuutt9SzZ0+VlZUpJSVFDQ0NHrUdO3Z0/9lms0mSmpqaJOmkE3/r6uoUGBioTZs2KTAw0GNfly5dWnWeAJrHFRagnQkKClJjY6N7/Te/+Y2+/vprxcTEKC4uzmM5FkpKSkp077336pVXXlFCQoKmTp3q/kJvbsxTufjii7VhwwaPbevWrTvlcf3799e9996rjz/+WNddd51ee+21E37+9u3bdeDAAT3zzDMaPXq0BgwY4DHhtqWGDh2q/Pz8ZvddcsklamxsVFVV1XH/7iIiIrz+LAAnRmAB2pmYmBitX79eu3btUnV1tWbMmKHvv/9ekydP1hdffKGSkhJ99NFHSktLU2NjoxobG3XLLbcoJSVFaWlpeu211/TVV19p1qxZJxzzl2GmOXfccYd27typBx54QDt27NCCBQuUm5t7wvrDhw/rrrvuUkFBgb777jutWbNGX3zxhS6++GL359fV1Sk/P1/V1dU6dOiQ+vTpo6CgIP3lL3/Rt99+q3/961968sknvf73lZWVpbfffltZWVnatm2btmzZ4r4a1b9/f918882aMmWKlixZotLSUm3YsEHZ2dlavny5158F4CR8PYkGwNm1Y8cO6/LLL7dCQkIsSVZpaan1zTffWNdee63VrVs3KyQkxBowYIA1c+ZMq6mpyXriiSesyMhIq7q62j3G4sWLraCgIKuoqOiEY57K+++/b8XFxVl2u90aPXq0NX/+/BNOuq2vr7cmTZpkRUdHW0FBQVZUVJR11113uSfGWpZl3XHHHdZ5551nSbKysrIsy7KsBQsWWDExMZbdbrcSExOtf/3rX5Yk68svv7Qs6+dJt8c+07Is68svvzzuHBYvXmzFx8dbQUFBVlhYmHXddde59zU0NFiZmZlWTEyM1bFjRysyMtK69tprra+++qrl/1EAnJLNsizLh3kJAADglLglBAAAjEdgAdDm7rjjDo+f+f5yueOOO3zdHoBzELeEALS5qqoqOZ3OZveFhoaqV69eZ7kjAOc6AgsAADAet4QAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOP9fz7xC5oOjzfIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the box plot of text_distance\n",
    "df[\"text_distance\"].plot.box()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Frequency'>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvgklEQVR4nO3df1RVdb7/8dcBBNIEY0gQRemHOoMZTgiMTd5ipAi7TOaqseZqaObM7eJcG7K5eO+avK7mpmWZ43XP0MxNqblz05rKWmPZD5LoB4WhVOZoYlSWAtkPjuCEes7+/tHXszqCCIcD+/A5z8darJmzz4cP749bPK8++/PZ22Xbti0AAAADRThdAAAAQF8h6AAAAGMRdAAAgLEIOgAAwFgEHQAAYCyCDgAAMBZBBwAAGIugAwAAjBXldAFO83q9OnDggIYOHSqXy+V0OQAAoBts29bhw4eVkpKiiIhTz9uEfdA5cOCAUlNTnS4DAAAEYP/+/Ro1atQp3w/7oDN06FBJ3/xBxcXFOVwNAADoDrfbrdTUVN/n+KmEfdA5cbkqLi6OoAMAwABzumUnLEYGAADGIugAAABjhW3QsSxL6enpysrKcroUAADQR1y2bdtOF+Ekt9ut+Ph4tbS0sEYHAIABoruf32E7owMAAMxH0AEAAMYi6AAAAGMRdAAAgLEIOgAAwFgEHQAAYCyCDgAAMBZBBwAAGIugAwAAjGXE08sbGhp00003qampSZGRkXrjjTc0ZMgQp8saENJKN/u9/nDFVQ5VAgBA8BkRdObOnavf/OY3mjp1qr744gvFxMQ4XVJIOjnUdLcN4QcAMFAN+KDz3nvvadCgQZo6daokKSEhweGKAABAqHB8jU5VVZUKCwuVkpIil8ulTZs2dWhjWZbS0tIUGxurnJwc1dTU+N7bu3evzjzzTBUWFuqiiy7SXXfd1Y/Vh7a00s1+X073AwBAf3M86LS1tSkjI0OWZXX6/saNG1VSUqKlS5dq+/btysjIUH5+vpqbmyVJx48f1yuvvKLf/e53qq6u1gsvvKAXXnihP4cAAABClOOXrgoKClRQUHDK91etWqUFCxZo3rx5kqSysjJt3rxZ69atU2lpqUaOHKnJkycrNTVVkjR9+nTV1dXp8ssv77S/9vZ2tbe3+1673e4gjiY8sI4HADBQOD6j05WjR4+qtrZWeXl5vmMRERHKy8tTdXW1JCkrK0vNzc368ssv5fV6VVVVpe9973un7HP58uWKj4/3fZ0ISAAAwDyOz+h05dChQ/J4PEpKSvI7npSUpN27d0uSoqKidNddd+kf/uEfZNu2rrjiCv3jP/7jKftcsmSJSkpKfK/dbjdhJwjYpg4ACEUhHXS663SXv74tJiZGMTExsixLlmXJ4/H0cXX9g0XCAAB0FNKXrhITExUZGammpia/401NTUpOTu5V38XFxdq1a5e2bdvWq34AAEDoCukZnejoaGVmZqqiokIzZsyQJHm9XlVUVGjhwoXOFocusWAZABAKHA86ra2tqq+v971uaGhQXV2dEhISNHr0aJWUlKioqEiTJ09Wdna2Vq9erba2Nt8urECZdukKAAB05LJt23aygMrKSuXm5nY4XlRUpPLycknS2rVrtXLlSjU2NmrSpElas2aNcnJygvLz3W634uPj1dLSori4uKD06YSBsEaHGR0AQLB09/Pb8aDjNIKOcwg+AIBAdffzO6QXI/cly7KUnp6urKwsp0sBAAB9JGyDDruuAAAwn+OLkRGYgXip6mTszAIA9LWwndEBAADmC9ugwxodAADMF7ZBhzU6AACYjzU6CCk8HBQAEExhO6MDAADMF7ZBhzU6AACYL2yDDmt0AAAwH2t0ENK41w4AoDfCdkYHAACYj6ADAACMRdABAADGCtugw64rAADM57Jt23a6CCe53W7Fx8erpaVFcXFxTpfTbSY81DNYWJwMAOGnu5/fYTujAwAAzMf28gGA2RsAAALDjA4AADAWQQcAABiLS1cY8Lh7MgDgVMJ2Roft5QAAmC9sgw4P9QQAwHxhG3QAAID5CDoAAMBYBB0AAGAsdl3BSCfvxGIXFgCEJ2Z0AACAsQg6AADAWAQdAABgrLANOtwwEAAA84Vt0OGGgQAAmC9sgw4AADAf28sRFnjwJwCEJ2Z0AACAsQg6AADAWAQdAABgLIIOAAAwFkEHAAAYi11XIaizHUIIPh78CQDmY0YHAAAYy4gZnbS0NMXFxSkiIkJnnXWWtm7d6nRJAAAgBBgRdCTp9ddf15lnnul0GQAAIIRw6QoAABjL8aBTVVWlwsJCpaSkyOVyadOmTR3aWJaltLQ0xcbGKicnRzU1NX7vu1wuXXrppcrKytKf//znfqocAACEOseDTltbmzIyMmRZVqfvb9y4USUlJVq6dKm2b9+ujIwM5efnq7m52dfm1VdfVW1trZ5++mndddddeuedd/qrfAAAEMJctm3bThdxgsvl0pNPPqkZM2b4juXk5CgrK0tr166VJHm9XqWmpuoXv/iFSktLO/Rx++23a8KECZo7d26nP6O9vV3t7e2+1263W6mpqWppaVFcXFxQxxMotpeHDracA0Bocrvdio+PP+3nt+MzOl05evSoamtrlZeX5zsWERGhvLw8VVdXS/pmRujw4cOSpNbWVr300kuaMGHCKftcvny54uPjfV+pqal9OwgAAOCYkA46hw4dksfjUVJSkt/xpKQkNTY2SpKampp0ySWXKCMjQz/4wQ904403Kisr65R9LlmyRC0tLb6v/fv39+kYAACAcwb89vJzzz1Xb7/9drfbx8TEKCYmRpZlybIseTyePqwOAAA4KaRndBITExUZGammpia/401NTUpOTu5V38XFxdq1a5e2bdvWq34AAEDoCumgEx0drczMTFVUVPiOeb1eVVRUaMqUKQ5WBgAABgLHL121traqvr7e97qhoUF1dXVKSEjQ6NGjVVJSoqKiIk2ePFnZ2dlavXq12traNG/evF79XC5dAQBgPse3l1dWVio3N7fD8aKiIpWXl0uS1q5dq5UrV6qxsVGTJk3SmjVrlJOTE5Sf393taf2J7eWhg+3lABCauvv57XjQcRpBBz1B8AGA0GDEfXT6kmVZSk9P73IrOgAAGNjCNuiw6woAAPOFbdABAADmI+gAAABjhW3QYY0OAADmC9ugwxodAADM5/gNA4GBpLOt/2w5B4DQFbYzOgAAwHwEHQAAYKywDTosRgYAwHxhG3RYjAwAgPnCNugAAADzEXQAAICxCDoAAMBYYXsfHcuyZFmWPB6P06VggDv53jrcVwcAQkfYzuiwGBkAAPOFbdABAADmI+gAAABjhe0anVDR2bOTAABAcDCjAwAAjEXQAQAAxgrboMOzrgAAMF/YBh22lwMAYD4WIwNB1tkCc24iCADOCNsZHQAAYD6CDgAAMBZBBwAAGIugAwAAjEXQAQAAxiLoAAAAY4Vt0OGGgQAAmM9l27btdBFOcrvdio+PV0tLi+Li4vr95/NQz/DEfXUAoHe6+/kdtjM6AADAfAQdAABgLIIOAAAwFkEHAAAYi6ADAACMRdABAADGIugAAABjRTldABCOOrt/EvfWAYDgY0YHAAAYy5igc+TIEY0ZM0aLFy92uhQAABAijAk6//Vf/6Uf/OAHTpcBAABCiBFBZ+/evdq9e7cKCgqcLgUAAIQQx4NOVVWVCgsLlZKSIpfLpU2bNnVoY1mW0tLSFBsbq5ycHNXU1Pi9v3jxYi1fvryfKgYAAAOF40Gnra1NGRkZsiyr0/c3btyokpISLV26VNu3b1dGRoby8/PV3NwsSXrqqac0btw4jRs3rj/LBgAAA4Dj28sLCgq6vOS0atUqLViwQPPmzZMklZWVafPmzVq3bp1KS0v1xhtvaMOGDXrsscfU2tqqY8eOKS4uTnfccUen/bW3t6u9vd332u12B3dAAAAgZDgedLpy9OhR1dbWasmSJb5jERERysvLU3V1tSRp+fLlvstW5eXl2rlz5ylDzon2y5Yt69vCgQCcfG8d7qsDAL3n+KWrrhw6dEgej0dJSUl+x5OSktTY2BhQn0uWLFFLS4vva//+/cEoFQAAhKCQntHpqblz5562TUxMjGJiYvq+GAAA4LiQntFJTExUZGSkmpqa/I43NTUpOTm5V31blqX09HRlZWX1qh8AABC6QjroREdHKzMzUxUVFb5jXq9XFRUVmjJlSq/6Li4u1q5du7Rt27belgkAAEKU45euWltbVV9f73vd0NCguro6JSQkaPTo0SopKVFRUZEmT56s7OxsrV69Wm1tbb5dWIGyLEuWZcnj8fR2CAAAIES5bNu2nSygsrJSubm5HY4XFRWpvLxckrR27VqtXLlSjY2NmjRpktasWaOcnJyg/Hy32634+Hi1tLQoLi4uKH32RGdPsQYkdl0BQFe6+/nteNBxGkEHoYqgAwCn1t3Pb8cvXQHoXGchmPADAD0T0ouR+xK7rgAAMF/YBh12XQEAYL6wDToAAMB8YRt0uHQFAID5wjbocOkKAADzhW3QAQAA5mN7OTCAnLzlnO3mANA1ZnQAAICxwjbosBgZAADzhW3QYTEyAADmC9ugAwAAzBdQ0Pnggw+CXQcAAEDQBbTr6vzzz9ell16q+fPn69prr1VsbGyw6zIWTysHAKD/BDSjs337dl144YUqKSlRcnKyfv7zn6umpibYtfUpFiMDAGA+l23bdqDffPz4cT399NMqLy/Xli1bNG7cON10002aM2eOzj777GDW2Wfcbrfi4+PV0tKiuLi4Pv95zOigr3FvHQDhoLuf371ajBwVFaWZM2fqscce09133636+notXrxYqampuvHGG3Xw4MHedA8AANArvQo6b731lv7lX/5FI0aM0KpVq7R48WLt27dPL7zwgg4cOKCrr746WHUCAAD0WECLkVetWqX169drz549mj59uh5++GFNnz5dERHf5KZzzjlH5eXlSktLC2atAAAAPRJQ0Pn973+vm266SXPnztWIESM6bTN8+HA9+OCDvSoOAACgNwIKOnv37j1tm+joaBUVFQXSPQAAQFAEtEZn/fr1euyxxzocf+yxx/TQQw/1uqj+wPZyAADMF9D28nHjxumBBx5Qbm6u3/GXX35ZP/vZz7Rnz56gFdjX2F6OcMCWcwCm6dPt5R9//LHOOeecDsfHjBmjjz/+OJAuAQAAgi6goDN8+HC98847HY6//fbb+s53vtProgAAAIIhoKBzww036F//9V+1detWeTweeTwevfTSS1q0aJGuv/76YNcIAAAQkIB2Xd1555368MMPNW3aNEVFfdOF1+vVjTfeqLvuuiuoBQIAAAQqoKATHR2tjRs36s4779Tbb7+tM844QxMnTtSYMWOCXR8AAEDAAgo6J4wbN07jxo0LVi0A+sjJu/3YhQUgXAQUdDwej8rLy1VRUaHm5mZ5vV6/91966aWgFAcAANAbAQWdRYsWqby8XFdddZUuuOACuVyuYNcFAADQawEFnQ0bNujRRx/V9OnTg11Pv7EsS5ZlyePxOF0KAADoIwFtL4+Ojtb5558f7Fr6VXFxsXbt2qVt27Y5XQoAAOgjAQWd2267Tb/97W8VwNMjAAAA+k1Al65effVVbd26Vc8++6wmTJigQYMG+b3/xBNPBKU4AH2js2eusRMLgIkCCjrDhg3TNddcE+xaAAAAgiqgoLN+/fpg1wEAABB0Aa3RkaTjx4/rxRdf1AMPPKDDhw9Lkg4cOKDW1tagFQcAANAbAc3ofPTRR7ryyiv18ccfq729XZdffrmGDh2qu+++W+3t7SorKwt2nQAAAD0W0IzOokWLNHnyZH355Zc644wzfMevueYaVVRUBK04AACA3ghoRueVV17R66+/rujoaL/jaWlp+vTTT4NSGID+xfOwAJgooBkdr9fb6R2FP/nkEw0dOrTXRfXEV199pcmTJ2vSpEm64IIL9Mc//rFffz4AAAhdAQWdK664QqtXr/a9drlcam1t1dKlS/v9sRBDhw5VVVWV6urq9Oabb+quu+7S559/3q81AACA0BTQpav77rtP+fn5Sk9P19dff62f/vSn2rt3rxITE/XII48Eu8YuRUZGavDgwZKk9vZ22bbNHZsBAICkAGd0Ro0apbffflv//u//rl/+8pf6/ve/rxUrVmjHjh0aPnx4j/qqqqpSYWGhUlJS5HK5tGnTpg5tLMtSWlqaYmNjlZOTo5qaGr/3v/rqK2VkZGjUqFG6/fbblZiYGMiwAACAYQKa0ZGkqKgozZ49u9cFtLW1KSMjQzfddJNmzpzZ4f2NGzeqpKREZWVlysnJ0erVq5Wfn689e/b4QtWwYcP09ttvq6mpSTNnztS1116rpKSkXtcGAAAGtoCCzsMPP9zl+zfeeGO3+yooKFBBQcEp31+1apUWLFigefPmSZLKysq0efNmrVu3TqWlpX5tk5KSlJGRoVdeeUXXXnttp/21t7ervb3d99rtdne7VgAAMLAEFHQWLVrk9/rYsWM6cuSIoqOjNXjw4B4Fna4cPXpUtbW1WrJkie9YRESE8vLyVF1dLUlqamrS4MGDNXToULW0tKiqqkq33HLLKftcvny5li1bFpT6AJPx4E8AJghojc6XX37p99Xa2qo9e/bokksuCepi5EOHDsnj8XS4DJWUlKTGxkZJ39yleerUqcrIyNDUqVP1i1/8QhMnTjxln0uWLFFLS4vva//+/UGrFwAAhJaA1+icbOzYsVqxYoVmz56t3bt3B6vb08rOzlZdXV2328fExCgmJqbvCgIAACEj4Id6diYqKkoHDhwIWn+JiYmKjIxUU1OT3/GmpiYlJyf3qm/LspSenq6srKxe9QMAAEJXQDM6Tz/9tN9r27Z18OBBrV27Vj/84Q+DUpgkRUdHKzMzUxUVFZoxY4akb+7KXFFRoYULF/aq7+LiYhUXF8vtdis+Pj4I1QLm4zERAAaagILOidBxgsvl0tlnn60f/ehHuu+++3rUV2trq+rr632vGxoaVFdXp4SEBI0ePVolJSUqKirS5MmTlZ2drdWrV6utrc23CytQlmXJsqxOH2UBAADM4LIdvo1wZWWlcnNzOxwvKipSeXm5JGnt2rVauXKlGhsbNWnSJK1Zs0Y5OTlB+fknZnRaWloUFxcXlD670tlOFmCgYkYHgFO6+/nteNBxGkEHCBxBB4BTuvv5HdClq5KSkm63XbVqVSA/AgAAoNcCCjo7duzQjh07dOzYMY0fP16S9P777ysyMlIXXXSRr53L5QpOlX2ANTpA73FTQQChLqCgU1hYqKFDh+qhhx7SWWedJembmwjOmzdPU6dO1W233RbUIvsCu64AADBfQPfRue+++7R8+XJfyJGks846S7/5zW96vOsKAACgrwQ0o+N2u/XZZ591OP7ZZ5/p8OHDvS6qP3DpCugb3GsHQCgJaEbnmmuu0bx58/TEE0/ok08+0SeffKLHH39c8+fP18yZM4NdY58oLi7Wrl27tG3bNqdLAQAAfSSgGZ2ysjItXrxYP/3pT3Xs2LFvOoqK0vz587Vy5cqgFggAABCogILO4MGD9bvf/U4rV67Uvn37JEnnnXeehgwZEtTiAAAAeqNXD/U8ePCgDh48qLFjx2rIkCEK83sPAgCAEBNQ0Pn88881bdo0jRs3TtOnT9fBgwclSfPnzx8QW8slnl4OAEA4CCjo/PKXv9SgQYP08ccfa/Dgwb7js2bN0pYtW4JWXF9iMTIAAOYLaI3O888/r+eee06jRo3yOz527Fh99NFHQSkMAACgtwKa0Wlra/ObyTnhiy++UExMTK+LAgAACIaAgs7UqVP18MMP+167XC55vV7dc889ys3NDVpxAAAAvRHQpat77rlH06ZN01tvvaWjR4/qV7/6ld577z198cUXeu2114JdY5/gzshA/+DBnwCcFNCMzgUXXKD3339fl1xyia6++mq1tbVp5syZ2rFjh84777xg19gnWIwMAID5ejyjc+zYMV155ZUqKyvTf/zHf/RFTQAAAEHR46AzaNAgvfPOO31RC4AwwYM/AfSXgC5dzZ49Ww8++GCwawEAAAiqgBYjHz9+XOvWrdOLL76ozMzMDs+4WrVqVVCKAxAeWLAMoK/0KOh88MEHSktL086dO3XRRRdJkt5//32/Ni6XK3jVAQAA9EKPgs7YsWN18OBBbd26VdI3j3xYs2aNkpKS+qS4vsT2cgAAzNejNTonP5382WefVVtbW1AL6i9sLwcAwHwBLUY+4eTgAwAAEEp6FHRcLleHNTisyQEAAKGqR2t0bNvW3LlzfQ/u/Prrr/XP//zPHXZdPfHEE8GrEAAAIEA9CjpFRUV+r2fPnh3UYgAAAIKpR0Fn/fr1fVUHAABA0PVqMTIAAEAoC+jOyADQ13geFoBgYEYHAAAYK2yDjmVZSk9PV1ZWltOlAACAPhK2QYc7IwMAYL6wDToAAMB8BB0AAGAsdl31oZN3jQAAgP5F0AEwYLEFHcDpEHQADAjMkAIIBGt0AACAsQg6AADAWAQdAABgrAG/Rmf//v2aM2eOmpubFRUVpV//+te67rrrnC4LgAM6W8fDAmUgvA34oBMVFaXVq1dr0qRJamxsVGZmpqZPn64hQ4Y4XRoAAHDYgA86I0aM0IgRIyRJycnJSkxM1BdffEHQAdDv2O4OhB7H1+hUVVWpsLBQKSkpcrlc2rRpU4c2lmUpLS1NsbGxysnJUU1NTad91dbWyuPxKDU1tY+rBgAAA4HjQaetrU0ZGRmyLKvT9zdu3KiSkhItXbpU27dvV0ZGhvLz89Xc3OzX7osvvtCNN96oP/zhD/1RNgAAGAAcv3RVUFCggoKCU76/atUqLViwQPPmzZMklZWVafPmzVq3bp1KS0slSe3t7ZoxY4ZKS0t18cUXd/nz2tvb1d7e7nvtdruDMAoAABCKHJ/R6crRo0dVW1urvLw837GIiAjl5eWpurpakmTbtubOnasf/ehHmjNnzmn7XL58ueLj431fXOYCAMBcjs/odOXQoUPyeDxKSkryO56UlKTdu3dLkl577TVt3LhRF154oW99z5/+9CdNnDix0z6XLFmikpIS32u3203YAQzWnUdHsGgYMFdIB53uuOSSS+T1ervdPiYmRjExMX1YEQAACBUhfekqMTFRkZGRampq8jve1NSk5OTkXvVtWZbS09OVlZXVq34AAEDoCumgEx0drczMTFVUVPiOeb1eVVRUaMqUKb3qu7i4WLt27dK2bdt6WyYAAAhRjl+6am1tVX19ve91Q0OD6urqlJCQoNGjR6ukpERFRUWaPHmysrOztXr1arW1tfl2YQFAb7GOBzCX40HnrbfeUm5uru/1iYXCRUVFKi8v16xZs/TZZ5/pjjvuUGNjoyZNmqQtW7Z0WKDcU5ZlybIseTyeXvUDAABCl8u2bdvpIpzkdrsVHx+vlpYWxcXFBbXv7vxXIoCBoTszOjwCAug/3f38Duk1OgAAAL0RtkGHXVcAAJgvbIMOu64AADCf44uRAWAgYg0eMDCEbdBh1xUAmKGz0MlCcJzApSsuXQEAYKywDToAAMB8YXvpCgB6gjU5wMDEjA4AADBW2AYd7qMDAID5wjbosBgZAADzsUYHABzE1migbxF0ACBMhFOo6s7icVPHDn9he+kKAACYL2xndLgzMgAnsE0d6F9hG3SKi4tVXFwst9ut+Ph4p8sBgF4Jp8tSQE+EbdABgL7Wl7M3zAwB3cMaHQAAYCxmdAAgxJw8W8MlKCBwzOgAAABjMaMDACGO9ThA4MJ2RodnXQEAYL6wDTo86woAAPOFbdABAADmY40OAIQxdnj1DDdmHHgIOgBgKBYxAwQdAIBDnJ4dYTYrPBB0AABdCrVAwEwVeoLFyAAAwFgEHQAAYCwuXQEAesTptTWhJtQu7cFf2M7ocGdkAADMF7YzOsXFxSouLpbb7VZ8fLzT5QAAHMZMlZnCNugAADpiRxNME7aXrgAAgPkIOgAAwFgEHQAAYCyCDgAAMBaLkQEAOAUWZw98BB0AQK9x0zyEKi5dAQAAYxkxo3PNNdeosrJS06ZN01/+8henywEAdKI7l4FMnRkydVwDgREzOosWLdLDDz/sdBkAACDEGBF0LrvsMg0dOtTpMgAAQIhxPOhUVVWpsLBQKSkpcrlc2rRpU4c2lmUpLS1NsbGxysnJUU1NTf8XCgAABhzH1+i0tbUpIyNDN910k2bOnNnh/Y0bN6qkpERlZWXKycnR6tWrlZ+frz179mj48OEOVAwAOB22ZSNUOB50CgoKVFBQcMr3V61apQULFmjevHmSpLKyMm3evFnr1q1TaWlpj39ee3u72tvbfa/dbnfPiwYAAAOC45euunL06FHV1tYqLy/PdywiIkJ5eXmqrq4OqM/ly5crPj7e95WamhqscgEAQIgJ6aBz6NAheTweJSUl+R1PSkpSY2Oj73VeXp6uu+46PfPMMxo1alSXIWjJkiVqaWnxfe3fv7/P6gcAAM5y/NJVMLz44ovdbhsTE6OYmJg+rAYAAISKkA46iYmJioyMVFNTk9/xpqYmJScn96pvy7JkWZY8Hk+v+gEA4NtYiB1aQvrSVXR0tDIzM1VRUeE75vV6VVFRoSlTpvSq7+LiYu3atUvbtm3rbZkAACBEOT6j09raqvr6et/rhoYG1dXVKSEhQaNHj1ZJSYmKioo0efJkZWdna/Xq1Wpra/PtwgIAADgVx4POW2+9pdzcXN/rkpISSVJRUZHKy8s1a9YsffbZZ7rjjjvU2NioSZMmacuWLR0WKPcUl64AADCfy7Zt2+kinOR2uxUfH6+WlhbFxcUFtW+u0wJA73T28EsT/m3loZ69193P75BeowMAANAbjl+6cgqXrgAg9JkwewNnhe2MDruuAAAwX9gGHQAAYD6CDgAAMFbYBh3LspSenq6srCynSwEAAH0kbIMOa3QAADBf2AYdAABgPoIOAAAwFvfR4T46AAAEzcn3PnL6LtBhO6PDGh0AAMwXtkEHAACYj6ADAACMRdABAADGCtugww0DAQAwX9gGHRYjAwBgvrANOgAAwHwEHQAAYCyCDgAAMBZBBwAAGIugAwAAjBW2QYft5QAAmC9sgw7bywEAMF/YBh0AAGA+gg4AADAWQQcAABiLoAMAAIxF0AEAAMYi6AAAAGMRdAAAgLEIOgAAwFhRThfgFMuyZFmWPB6P06UAAMJMWunmgL7vwxVXBbkS84XtjA53RgYAwHxhG3QAAID5CDoAAMBYBB0AAGAsgg4AADAWQQcAABiLoAMAAIxF0AEAAMYi6AAAAGMRdAAAgLGMCDp//etfNX78eI0dO1b/8z//43Q5AAAgRAz4Z10dP35cJSUl2rp1q+Lj45WZmalrrrlG3/nOd5wuDQAAOGzAz+jU1NRowoQJGjlypM4880wVFBTo+eefd7osAAAQAhwPOlVVVSosLFRKSopcLpc2bdrUoY1lWUpLS1NsbKxycnJUU1Pje+/AgQMaOXKk7/XIkSP16aef9kfpAAAgxDkedNra2pSRkSHLsjp9f+PGjSopKdHSpUu1fft2ZWRkKD8/X83NzQH9vPb2drndbr8vAABgJsfX6BQUFKigoOCU769atUoLFizQvHnzJEllZWXavHmz1q1bp9LSUqWkpPjN4Hz66afKzs4+ZX/Lly/XsmXLgjcAAAD6SVrp5tO2+XDFVUHpJ9C+Q43jMzpdOXr0qGpra5WXl+c7FhERoby8PFVXV0uSsrOztXPnTn366adqbW3Vs88+q/z8/FP2uWTJErW0tPi+9u/f3+fjAAAAznB8Rqcrhw4dksfjUVJSkt/xpKQk7d69W5IUFRWl++67T7m5ufJ6vfrVr37V5Y6rmJgYxcTE9GndAAAgNIR00OmuH//4x/rxj3/co++xLEuWZcnj8fRRVQAAwGkhfekqMTFRkZGRampq8jve1NSk5OTkXvVdXFysXbt2adu2bb3qBwAAhK6QDjrR0dHKzMxURUWF75jX61VFRYWmTJniYGUAAGAgcPzSVWtrq+rr632vGxoaVFdXp4SEBI0ePVolJSUqKirS5MmTlZ2drdWrV6utrc23CytQXLoCAMB8jgedt956S7m5ub7XJSUlkqSioiKVl5dr1qxZ+uyzz3THHXeosbFRkyZN0pYtWzosUO6p4uJiFRcXy+12Kz4+vld9AQCA0OR40Lnssstk23aXbRYuXKiFCxf2U0UAAMAUIb1Gpy9ZlqX09HRlZWU5XQoAAOgjYRt02HUFAID5wjboAAAA8xF0AACAscI26LBGBwAA84Vt0GGNDgAA5gvboAMAAMzn+H10nHbiHj5utzvofXvbjwS9TwAAutKdz7NAP58C6bsvPl+/3e/p7sXnsk/XwlAnHgFx9OhR7du3z+lyAABAAPbv369Ro0ad8v2wDToneL1e7dmzR+np6dq/f7/i4uKcLqlPud1upaamMlbDMFYzMVYzMdbgsG1bhw8fVkpKiiIiTr0SJ+wvXUVERGjkyJGSpLi4OOP/0p3AWM3EWM3EWM3EWHuvO8+qZDEyAAAwFkEHAAAYi6AjKSYmRkuXLlVMTIzTpfQ5xmomxmomxmomxtq/wn4xMgAAMBczOgAAwFgEHQAAYCyCDgAAMBZBBwAAGMvIoGNZltLS0hQbG6ucnBzV1NR02f6xxx7Td7/7XcXGxmrixIl65pln/N63bVt33HGHRowYoTPOOEN5eXnau3dvXw6h23oy1j/+8Y+aOnWqzjrrLJ111lnKy8vr0H7u3LlyuVx+X1deeWVfD6NbejLW8vLyDuOIjY31a2PKeb3ssss6jNXlcumqq67ytQnV81pVVaXCwkKlpKTI5XJp06ZNp/2eyspKXXTRRYqJidH555+v8vLyDm16+m9Af+jpWJ944gldfvnlOvvssxUXF6cpU6boueee82vzn//5nx3O63e/+90+HEX39HSslZWVnf4dbmxs9Gtnwnnt7HfR5XJpwoQJvjahel6XL1+urKwsDR06VMOHD9eMGTO0Z8+e036f05+xxgWdjRs3qqSkREuXLtX27duVkZGh/Px8NTc3d9r+9ddf1w033KD58+drx44dmjFjhmbMmKGdO3f62txzzz1as2aNysrK9Oabb2rIkCHKz8/X119/3V/D6lRPx1pZWakbbrhBW7duVXV1tVJTU3XFFVfo008/9Wt35ZVX6uDBg76vRx55pD+G06WejlX65k6c3x7HRx995Pe+Kef1iSee8Bvnzp07FRkZqeuuu86vXSie17a2NmVkZMiyrG61b2ho0FVXXaXc3FzV1dXp1ltv1c033+wXAAL5u9IfejrWqqoqXX755XrmmWdUW1ur3NxcFRYWaseOHX7tJkyY4HdeX3311b4ov0d6OtYT9uzZ4zeW4cOH+94z5bz+9re/9Rvj/v37lZCQ0OH3NRTP68svv6zi4mK98cYbeuGFF3Ts2DFdccUVamtrO+X3hMRnrG2Y7Oxsu7i42Pfa4/HYKSkp9vLlyztt/5Of/MS+6qqr/I7l5OTYP//5z23btm2v12snJyfbK1eu9L3/1Vdf2TExMfYjjzzSByPovp6O9WTHjx+3hw4daj/00EO+Y0VFRfbVV18d7FJ7radjXb9+vR0fH3/K/kw+r/fff789dOhQu7W11XcsVM/rt0myn3zyyS7b/OpXv7InTJjgd2zWrFl2fn6+73Vv//z6Q3fG2pn09HR72bJlvtdLly61MzIygldYH+jOWLdu3WpLsr/88stTtjH1vD755JO2y+WyP/zwQ9+xgXBebdu2m5ubbUn2yy+/fMo2ofAZa9SMztGjR1VbW6u8vDzfsYiICOXl5am6urrT76murvZrL0n5+fm+9g0NDWpsbPRrEx8fr5ycnFP22R8CGevJjhw5omPHjikhIcHveGVlpYYPH67x48frlltu0eeffx7U2nsq0LG2trZqzJgxSk1N1dVXX6333nvP957J5/XBBx/U9ddfryFDhvgdD7XzGojT/b4G488vVHm9Xh0+fLjD7+vevXuVkpKic889V//0T/+kjz/+2KEKe2/SpEkaMWKELr/8cr322mu+4yaf1wcffFB5eXkaM2aM3/GBcF5bWlokqcPfyW8Lhc9Yo4LOoUOH5PF4lJSU5Hc8KSmpw7XeExobG7tsf+J/e9JnfwhkrCf7t3/7N6WkpPj9Bbvyyiv18MMPq6KiQnfffbdefvllFRQUyOPxBLX+nghkrOPHj9e6dev01FNP6X//93/l9Xp18cUX65NPPpFk7nmtqanRzp07dfPNN/sdD8XzGohT/b663W79/e9/D8rvRai699571draqp/85Ce+Yzk5OSovL9eWLVv0+9//Xg0NDZo6daoOHz7sYKU9N2LECJWVlenxxx/X448/rtTUVF122WXavn27pOD8exeKDhw4oGeffbbD7+tAOK9er1e33nqrfvjDH+qCCy44ZbtQ+IwN+6eXh6sVK1Zow4YNqqys9Fuke/311/v+/8SJE3XhhRfqvPPOU2VlpaZNm+ZEqQGZMmWKpkyZ4nt98cUX63vf+54eeOAB3XnnnQ5W1rcefPBBTZw4UdnZ2X7HTTmv4er//u//tGzZMj311FN+61YKCgp8///CCy9UTk6OxowZo0cffVTz5893otSAjB8/XuPHj/e9vvjii7Vv3z7df//9+tOf/uRgZX3roYce0rBhwzRjxgy/4wPhvBYXF2vnzp0hsXbodIya0UlMTFRkZKSampr8jjc1NSk5ObnT70lOTu6y/Yn/7Umf/SGQsZ5w7733asWKFXr++ed14YUXdtn23HPPVWJiourr63tdc6B6M9YTBg0apO9///u+cZh4Xtva2rRhw4Zu/UMYCuc1EKf6fY2Li9MZZ5wRlL8roWbDhg26+eab9eijj3a4BHCyYcOGady4cQPuvHYmOzvbNw4Tz6tt21q3bp3mzJmj6OjoLtuG2nlduHCh/vrXv2rr1q0aNWpUl21D4TPWqKATHR2tzMxMVVRU+I55vV5VVFT4/df9t02ZMsWvvSS98MILvvbnnHOOkpOT/dq43W69+eabp+yzPwQyVumb1e133nmntmzZosmTJ5/253zyySf6/PPPNWLEiKDUHYhAx/ptHo9H7777rm8cpp1X6ZstnO3t7Zo9e/Zpf04onNdAnO73NRh/V0LJI488onnz5umRRx7xu13AqbS2tmrfvn0D7rx2pq6uzjcO086r9M0Opvr6+m79h0monFfbtrVw4UI9+eSTeumll3TOOeec9ntC4jM2KEuaQ8iGDRvsmJgYu7y83N61a5f9s5/9zB42bJjd2Nho27Ztz5kzxy4tLfW1f+211+yoqCj73nvvtf/2t7/ZS5cutQcNGmS/++67vjYrVqywhw0bZj/11FP2O++8Y1999dX2OeecY//973/v9/F9W0/HumLFCjs6Otr+y1/+Yh88eND3dfjwYdu2bfvw4cP24sWL7erqaruhocF+8cUX7YsuusgeO3as/fXXXzsyxhN6OtZly5bZzz33nL1v3z67trbWvv766+3Y2Fj7vffe87Ux5byecMkll9izZs3qcDyUz+vhw4ftHTt22Dt27LAl2atWrbJ37Nhhf/TRR7Zt23Zpaak9Z84cX/sPPvjAHjx4sH377bfbf/vb32zLsuzIyEh7y5Ytvjan+/NzSk/H+uc//9mOioqyLcvy+3396quvfG1uu+02u7Ky0m5oaLBfe+01Oy8vz05MTLSbm5v7fXzf1tOx3n///famTZvsvXv32u+++669aNEiOyIiwn7xxRd9bUw5ryfMnj3bzsnJ6bTPUD2vt9xyix0fH29XVlb6/Z08cuSIr00ofsYaF3Rs27b/+7//2x49erQdHR1tZ2dn22+88YbvvUsvvdQuKirya//oo4/a48aNs6Ojo+0JEybYmzdv9nvf6/Xav/71r+2kpCQ7JibGnjZtmr1nz57+GMpp9WSsY8aMsSV1+Fq6dKlt27Z95MgR+4orrrDPPvtse9CgQfaYMWPsBQsWOP4PyQk9Geutt97qa5uUlGRPnz7d3r59u19/ppxX27bt3bt325Ls559/vkNfoXxeT2wrPvnrxPiKiorsSy+9tMP3TJo0yY6OjrbPPfdce/369R367erPzyk9Heull17aZXvb/mZr/YgRI+zo6Gh75MiR9qxZs+z6+vr+HVgnejrWu+++2z7vvPPs2NhYOyEhwb7sssvsl156qUO/JpxX2/5m+/QZZ5xh/+EPf+i0z1A9r52NU5Lf72Aofsa6/n/xAAAAxjFqjQ4AAMC3EXQAAICxCDoAAMBYBB0AAGAsgg4AADAWQQcAABiLoAMAAIxF0AEAAMYi6AAAAGMRdAAAgLEIOgAAwFgEHQAAYKz/B7CNjmmDlxANAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the histogram of text_distance on a log scale\n",
    "df[\"text_distance\"].plot.hist(logy=True, bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# order df by text_distance descending\n",
    "df = df.sort_values(by=\"text_distance\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question\n",
    "\n",
    "- Where should we put the cutoff to remove \"dirty\" texts?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feng Zhe, (; * 1920 in Tianjin oder 1921 in Hainan als \"Feng Yizhe\" ; † 2. Juni 1969) war ein chinesischer Schauspieler.\n",
      "0.6698421447280607\n",
      "\n",
      "Nach der Verstaatlichung 1884 kamen sie zu den k.k. österreichischen Staatsbahnen (kkStB); die 31–47 wurden zunächst kkStB 10.01–09, dann 1891 kkStB 22.11–19 und schließlich 1904 122.11–19.\n",
      "0.6698415254895317\n",
      "\n",
      "<onlyinclude>* Moritz Brückner (Landrat) (\"Ernst Moritz Karl Brückner\"; 1807–1887), deutscher Verwaltungsjurist und Politiker\n",
      "0.6698391603716991\n",
      "\n",
      "Das Ehepaar Binder hatte drei Kinder Oscar (* 1832 Sulz am Neckar), Fanny (* 1836 Sulz am Neckar) und Clara (* 1841 Ludwigsburg).\n",
      "0.6698359934634563\n",
      "\n",
      "Seit 2012 gibt es eine Mixed-Senior-Flag-Football-Mannschaft (ab 16 Jahren), die in der NRW-Flag-Football-Liga und der Deutschen Flag-Football-Liga (DFFL) antritt.\n",
      "0.6698354920521385\n",
      "\n",
      "Sanballat III. (hebräisch , babylonisch Sin-uballit; † 332 v. Chr.) war ein persischer Statthalter (Satrap) von Samarien.\n",
      "0.6698312203017686\n",
      "\n",
      "Die angrenzenden Gemeinden sind Altopascio, Bientina (PI), Borgo a Mozzano, Buti (PI), Calci (PI), Lucca, Montecarlo, Pescia (PT), Porcari, San Giuliano Terme (PI) und Villa Basilica.\n",
      "0.6698297736678898\n",
      "\n",
      "Benedict John Osta SJ (* 15. August 1931 in Blasurahi; † 30. Januar 2014 in Patna) war Erzbischof von Patna.\n",
      "0.6698279271051113\n",
      "\n",
      "Nikolaus Sachau (auch \"Nicolaus Sachow\", \"Czachow\"; * um 1385 in Lübeck; † 11. Oktober 1449 ebenda) war ein deutscher Rechtsgelehrter und als Nikolaus II. von 1439 bis 1449 Bischof von Lübeck.\n",
      "0.6698272515748669\n",
      "\n",
      "<onlyinclude>* Klaus Schweizer (Musikwissenschaftler) (* 1939), deutscher Komponist, Musikwissenschaftler und Hochschullehrer\n",
      "0.6698269027218344\n",
      "\n",
      "Zur Ortschaft gehören die Einzelhöfe \"Geier\", \"Großbergner\", \"Kahl\", \"Lindner\", \"Murer\", \"Oberbergner\", \"Rogges\", \"Turning\" und \"Wadlitzer\".\n",
      "0.6698251519704997\n",
      "\n",
      "In der Nähe liegen das Court Tomb von Ballymacdermot, das Portal Tomb von Ballykeel und das Wedge Tomb Clontygora East.\n",
      "0.6698243634976411\n",
      "\n",
      "Im Jahr 2011 nahm sie an der Telenovela „Morde e Assopra“ teil als „Celeste Sampaio“ und 2012 als „Malvina Tavares“ in „Gabriela“ (Telenovela).\n",
      "0.6698229174127869\n",
      "\n",
      "Koehlau heiratete am 29. April 1854 in Trier Charlotte Anna Cooch. Das Paar hatte einen Sohn (* 1856).\n",
      "0.6698225189412808\n",
      "\n",
      "<onlyinclude>* Viorel Tănase (Fußballspieler, 1908) (1908–2011), rumänischer Fußballspieler, -schiedsrichter und -funktionär\n",
      "0.6698220545438471\n",
      "\n",
      "Im Suco befinden sich die vier Aldeias \"Ailoc Laran\", \"Fatuisin\" (\"Futusin\"), \"Manequin\" und \"Sanfuc\".\n",
      "0.6698210216523157\n",
      "\n",
      "Victor Axel Hackman (* 27. April 1866 in Wyborg; † 26. November 1941 in Helsinki) war ein finnischer Geologe.\n",
      "0.6698202087795478\n",
      "\n",
      "Heinrich von Burgund (* im 10. Jahrhundert; † 21. August 1018 in Lausanne) war von 985 bis 1018 Bischof von Lausanne.\n",
      "0.6698201913497035\n",
      "\n",
      "1. Kf8? Kf6 und Schwarz ist dem b-Bauern näher gekommen, Weiß aber nicht. Daher: 1. Kh8!! Kf5 2. Kg7 Ke5 3. Kg6 Kd4 4. Kf5 Kc3 5. Ke4 Kb2 6. Kd3 Kxa2 7. Kc2 und der schwarze König ist in der Ecke eingesperrt.\n",
      "0.6698200990679778\n",
      "\n",
      "<onlyinclude>* \"Ernst Kämpfer\", Pseudonym von Karl Viererbl (1903–1945), deutscher Journalist und Politiker (DNSAP/NSDAP)\n",
      "0.6698173568732475\n",
      "\n",
      "Die Art \"Allium lusitanicum\" gehört zur Sektion \"Allium\" sect. \"Rhizirideum\" aus der Untergattung \"Allium\" subg. \"Rhizirideum\" in der Gattung \"Allium\".\n",
      "0.6698167695521208\n",
      "\n",
      "Er vermählte sich 1832 in Berlin mit Pauline Eck (1800–1869), mit der er die Töchter Johanna (* 1834), Marie (* 1837) und Martha (* 1846) hatte.\n",
      "0.6698163226164984\n",
      "\n",
      "Sir Percy Lyham Loraine, 12. Baronet KCMG, PC (* 5. November 1880 in London; † 23. Mai 1961 ebenda) war ein britischer Diplomat.\n",
      "0.6698151665544078\n",
      "\n",
      "Edivaldo Rojas Hermoza (* 17. November 1985 in Cuiabá, Brasilien) ist ein bolivianischer Fußballspieler.\n",
      "0.6698139967323585\n",
      "\n",
      "Sendefenster: Montag bis Freitag von 12:00 bis 19:00 Uhr sowie Samstag und Sonntag von 10:00 Uhr bis 17:00 Uhr\n",
      "0.6698137072166488\n",
      "\n",
      "Pedro José de Alvarado y Baeza (* 25. Juni 1767 in Cartago, Costa Rica; † 8. Juli 1839 ebenda) war Priester und Präsident von Costa Rica.\n",
      "0.6698111471234628\n",
      "\n",
      "Die Kleine Simsenlilie bildet bei gemeinsamem Vorkommen mit anderen Simsenlilien-Arten (\"Tofieldia\") Hybriden:\n",
      "0.6698099300905063\n",
      "\n",
      "Der Pelzhandel zählt das Kreuzfuchsfell zu den so genannten „Edelfuchsfellen“, wie auch das Silberfuchsfell, das Blaufuchsfell und das Polarfuchsfell.\n",
      "0.6698084698109854\n",
      "\n",
      "Weitere Orte der Gemeinde sind Gołosze, Grabina, Kępina, Ludwików, Łukaszów, Podgórze und Wólka Kadłubska.\n",
      "0.6698069525511413\n",
      "\n",
      "Werner I. (II. des Namens) Graf von Habsburg (* 1030; † 11. November 1096) war der dritte Sohn von Ita von Lothringen und Radbot von Habsburg.\n",
      "0.6698017094789951\n",
      "\n",
      "Der Radstand betrug 2790 mm und die Spurweite 1420 mm (vorne) bzw. 1440 mm (hinten). Das Fahrzeug war 4400 mm lang, 1700 mm breit und 1783 mm hoch.\n",
      "0.66980024883102\n",
      "\n",
      "Die Vorgänger von Jakobs waren Josef Klein (CDU, 2009–2014) und zuvor zwanzig Jahre lang Heinz Franke.\n",
      "0.669798372116871\n",
      "\n",
      "Temel Kotil (* 3. Dezember 1959 in Gündoğdu, Rize, Türkei) war von 2005 bis 2016 CEO der Turkish Airlines.\n",
      "0.6697947796181127\n",
      "\n",
      "Eva Maria Livia Amurri (* 15. März 1985 in New York City, New York) ist eine US-amerikanische Schauspielerin.\n",
      "0.6697935635284722\n",
      "\n",
      "Er heiratete am 24. Januar 1787 in Potsdam Philippie Sophie Wilhelmine Dorothea von Borch (* 1767; † 5. Oktober 1820).\n",
      "0.6697912290096004\n",
      "\n",
      "Die Ortschaft Würzelbrunn hatte 1817 49 Einwohner und 7 Wohngebäude, 1861 57 Einwohner und 1961 36 Einwohner und 8 Wohngebäude.\n",
      "0.6697885224611023\n",
      "\n",
      "Da formula_14 sich wie formula_15 verhält, wichtet man die formula_16-Seminorm mit formula_17 und erhält\n",
      "0.6697883085247001\n",
      "\n",
      "Tauno Kullervo Pylkkänen (* 22. März 1918 in Helsinki; † 13. März 1980 ebenda) war ein finnischer Komponist.\n",
      "0.6697867090580709\n",
      "\n",
      "Seit 2005 ist Stadtkewitz Mitglied im UHW (Unionhilfswerk e. V.) und seit 2007 Mitglied im Verein für Pankow e. V.\n",
      "0.6697862605240521\n",
      "\n",
      "Der SBFV-Pokal (auch Rothaus-Pokal oder Südbadischer Pokal genannt) ist der Verbandspokal des Südbadischen Fußballverbands.\n",
      "0.6697850763096043\n",
      "\n",
      "Augustinus Frotz (* 25. Mai 1903 in Vohwinkel; † 12. November 1994 in Köln) war Weihbischof im Erzbistum Köln.\n",
      "0.6697816556662632\n",
      "\n",
      "Sind alle drei Cevaschen Strecken formula_13, formula_14 und formula_15 gleich lang, so gilt: formula_60\n",
      "0.6697764222420715\n",
      "\n",
      "Am Stichtag 1. April 1933 hatte das BW Kaiserslautern insgesamt 13 Maschinen im Bestand (Nummern 77 001–004, 101–109).\n",
      "0.6697754158175093\n",
      "\n",
      "Die minimale Fallhöhe beträgt 17,8 m, die maximale 28,8 m. Der maximale Durchfluss liegt bei 250 m³/s je Turbine.\n",
      "0.6697751827452603\n",
      "\n",
      "Otto Stumpf (* 30. März 1940 in Celle; † 17. April 2017) war ein deutscher Kanute, Sportfunktionär und Politiker (CDU).\n",
      "0.6697748333499143\n",
      "\n",
      "Benachbart sind die Orte Cachoeirinha, Esteio, Gravataí, Nova Santa Rita, Novo Hamburgo, Portão und São Leopoldo.\n",
      "0.6697740434141555\n",
      "\n",
      "Christian Zwingmann (* 27. Mai 1827 in Riga; † 4. Mai 1891 in Kopenhagen) war ein Architekt des Eklektizismus.\n",
      "0.6697728563711877\n",
      "\n",
      "Erster 1951: Blagoja Vidinić, Stanić, Rora, Ninoslav Kapamadžija, Lazarevski, Perić, Radivoje Ognjanović, Milan Ljubenović, Miloš Milutinović, Dragoljub Župac, Luka LipošinovićZweiter 1953: Aufgebot liegt bisher nicht vorZweiter 1962: Poklepovic; Pavlovic, Kovacevic; Rasovic, Gracianin, Skrbic; Radosav, Milosavijesic, Kovac, Feric, CornDritter 1966: Detailliertes Aufgebot liegt bisher nicht vor (u. a. Radomir Vukčević) Zweiter 1974: Vlak; Radelias, Vuikov (41. Janianin), Vraneć, Popović; Vavaković, Sestić, Salov; Krinciar, Jukić, KostićZweiter 1978: Mrkun; Janković, Juricko, Milovanović, Rumora; Mehmed Baždarević (58. Smajić), Ivan Gudelj, Icin; Elzner, Stanic, MlinarićErster 1979: Ivan Pudar; Caplijc, Bosko Djurovski, Lazić, Dzanić; Ivan Gudelj, Radulović, Mehmed Baždarević; Bogdanović (48. Stojance Zlatanovski), Zdravko Čakalić (70. Malinkovic), Senad Merdanović\n",
      "0.6697722657144474\n",
      "\n",
      "Weitere Orte der Gemeinde sind Chodków (leśniczówka), Działki Brzóskie, Grabnowola-Kolonia, Helenówek, Józefów, Mała Wieś, Michałówek, Nadleśnictwo Studzianki und\n",
      "0.669772119692416\n",
      "\n",
      "José Joaquín Prieto Vial (* 20. August 1786 in Concepción, Chile; † 22. November 1854 in Santiago de Chile) war von 1831 bis 1841 Präsident von Chile.\n",
      "0.6697719838158002\n",
      "\n",
      "Tišnov (Tschechien), Brzozów (Polen), Cristuru Secuiesc (Rumänien), Karcag (Ungarn, 1998), Edelény (Ungarn), Siklós (Ungarn), Tarcal (Ungarn), Encs (Ungarn) und der XVIII. Budapester Bezirk (Ungarn).\n",
      "0.6697707271974855\n",
      "\n",
      "<onlyinclude>* Ladislaus Hunyadi (ungarisch \"László Hunyadi\"; 1433–1457), ungarischer Staatsmann und Krieger; Sohn von Johann Hunyadi\n",
      "0.6697698912374461\n",
      "\n",
      "Das Bistum Rubiataba-Mozarlândia umfasst die Gemeinden Araguapaz, Aruanã, Crixás, Faina, Matrinchã, Morro Agudo de Goiás, Mozarlândia, Mundo Novo, Nova América, Nova Crixás, Rubiataba und Uirapuru des Bundesstaates Goiás.\n",
      "0.6697696563292592\n",
      "\n",
      "Im Zeitraum 1988 bis 2018 stieg die Einwohnerzahl von 1468 auf 1497 um 29 Einwohner bzw. um 2 %. 2002 hatte die Gemeinde 1703 Einwohner.\n",
      "0.6697696563086477\n",
      "\n",
      "Davidlloydit kristallisiert im triklinen Kristallsystem in der mit den Gitterparametern \"a\" = 5,9756 Å; \"b\" = 7,6002 Å; \"c\" = 5,4471 Å; α = 84,2892°; β = 90,4920° und γ = 87,9958° sowie einer Formeleinheit pro Elementarzelle.\n",
      "0.6697668502795229\n",
      "\n",
      "Antonio Lanfranchi (* 17. Mai 1946 in Grondone di Ferriere; † 17. Februar 2015 in Modena) war Erzbischof von Modena-Nonantola.\n",
      "0.669766286894734\n",
      "\n",
      "Vladislav I. (* um 1070; † 12. April 1125) war 1109 bis 1117 und 1120 bis 1125 herrschender Fürst von Böhmen aus dem Geschlecht der Přemysliden.\n",
      "0.6697636842848751\n",
      "\n",
      "Maria Caniglia (* 5. Mai 1905 in Neapel; † 16. April 1979 in Rom) war eine italienische Opernsängerin (Sopran) der 1930er und 1940er Jahre.\n",
      "0.6697630658924286\n",
      "\n",
      "\"The Regatta Mystery\": Juni 1936 – \"Strand Magazine\" Ausgabe 546 (unter dem Titel \"Poirot and the Regatta Mystery\").\n",
      "0.6697621802585617\n",
      "\n",
      "In den Karten I und II trägt der Polarstern (α UMi) mit 2,08 mag die Nummer 1s, gefolgt von den Sternen 1 (δ UMi, 4,37) und 2 (5,28).\n",
      "0.6697604433987281\n",
      "\n",
      "<onlyinclude>* Jim Watson (Eishockeyspieler) (\"James Arthur Watson\"; * 1943), kanadischer Eishockeyspieler\n",
      "0.6697590645455384\n",
      "\n",
      "Es löste 1991 das bis dahin geltende \"Gesetz über Wasser- und Bodenverbände\" vom 10. Februar 1937 (RGBl. I S. 188) sowie die \"Erste Verordnung über Wasser- und Bodenverbände\" (WVVO) vom 3. September 1937 (RGBl. I S. 933) ab (§ 78 WVG).\n",
      "0.669757831668663\n",
      "\n",
      "Maria Herr-Beck (* 8. Juni 1928 in Alzey; † 7. Oktober 2015 in Mainz) war eine deutsche Politikerin (CDU).\n",
      "0.6697570085885018\n",
      "\n",
      "<onlyinclude>* Georg Schmitz (Politiker, 1819) (1819–1879), deutscher Jurist und Politiker, MdL Hessen\n",
      "0.6697566792096209\n",
      "\n",
      "Die Premiere erfolgte am 20. Juli 2019 im Cinedom in Köln. Der Film spielte laut Box Office Mojo 2,0 Millionen US-Dollar, rund Millionen Euro, ein.\n",
      "0.6697564163083154\n",
      "\n",
      "Döhren wurde früher \"Thornede\" (um 1220), \"Dornede\" (um 1270), \"Dornede prope Wevelige\" (1311) und im 14. Jahrhundert \"Dhornede\", \"Dornedhe\" oder \"Dornethe\" genannt.\n",
      "0.6697551559798673\n",
      "\n",
      "Bei der Landtagswahl am 14. Mai 2017 waren 110.288 Bürger wahlberechtigt. Von 77.202 abgegebenen Zweitstimmen waren 76.624 gültig.\n",
      "0.6697548652417977\n",
      "\n",
      "Spektroskopische Doppelsterne in der Leier sind (in Klammer die Periode): Sheliak (12,91 Tage), Delta Lyrae (88,35 Tage), Zeta Lyrae (4,3 Tage), Eta Lyrae (56,4 Tage) und 17 Lyrae (42,88 Tage).\n",
      "0.6697544915958027\n",
      "\n",
      "Johann Raskop (* 13. November 1911 in Pankow; † 15. Juli 1988 in Berlin) war ein deutscher SED-Funktionär.\n",
      "0.6697502586675126\n",
      "\n",
      "Die PZL Bielsko SZD-12 \"Mucha 100\" ist ein polnisches Segelflugzeug. Das Kürzel SZD steht für Szybowcowy Zakład Doświadczalny (Segelflugzeug-Entwicklungswerk).\n",
      "0.6697489289664544\n",
      "\n",
      "Erstmals erwähnt wurde der Ort im Jahr 1285 als \"Jesion\". 1484 wird der Ort als \"Jassen\" erwähnt. 1534 wird erstmal der Ortsname \"Jassen\" erwähnt.\n",
      "0.6697472251508172\n",
      "\n",
      "Brenda Castillo (* 5. Juni 1992 in San Cristóbal) ist eine dominikanische Volleyball-Nationalspielerin.\n",
      "0.6697459150837076\n",
      "\n",
      "Panabo (offiziell: \"City of Panabo\"; Cebuano: \"Dakbayan sa Panabo\"; Filipino: \"Lungsod ng Panabo\") ist eine Stadt nahe der Stadt Davao City in der Provinz Davao del Norte auf den Philippinen.\n",
      "0.6697439995316417\n",
      "\n",
      "Limburg Lions spielte in der Saison 2011/12 im EHF-Pokal, in der Saison 2012/13 im EHF Challenge Cup und in den Spielzeiten 2013/14 und 2014/15 im EHF Europa Pokal sowie 2015/16 in der EHF Champions League.\n",
      "0.6697439861836042\n",
      "\n",
      "Moti Ben-Ari, Christophe Barraud, Stéphane Magnenat, Morgane Chevalier, Gordana Gerber, Manon Briod, Maria Beltran\n",
      "0.6697417838695696\n",
      "\n",
      "Thierry Gerbier (* 21. September 1965 in Chambéry; † 13. November 2013) war ein französischer Biathlet.\n",
      "0.6697388995022748\n",
      "\n",
      "Marie „Mizzi“ Munzinger (* 10. April 1885 in Tisza Földvar; † 4. Juli 1952 in São Paulo) war eine Schweizer Lehrmittel-Autorin.\n",
      "0.6697365695090897\n",
      "\n",
      "Der Distrikt Südatambua teilt sich in die Desas Fatukbot (7.322 Einwohner 2010), Lidak (4.845), Manuaman (7.141) und Rinbesi (3.156).\n",
      "0.6697363827563146\n",
      "\n",
      "Am 5. November 2017 wurde er hinter Geoffrey Kipsang Kamworor Zweiter beim New-York-City-Marathon in 2:10:56 h.\n",
      "0.6697360157037142\n",
      "\n",
      "<onlyinclude>* Paul Zeidler (Dichter) (1548–1627), deutscher evangelischer Theologe, Dichter und Pädagoge\n",
      "0.6697339240501685\n",
      "\n",
      "Diese Faserzugspannung formula_12 kann nun zu der Betonzugspannung formula_13 dazugezählt werden und man\n",
      "0.6697339068813359\n",
      "\n",
      "Auszeichnungen: Lunning Prize (1970); Orden des Löwen von Finnland (1980), World Glass Now 85 (YEAR); Kaj Franck Design-Preis (1992);\n",
      "0.6697271637221025\n",
      "\n",
      "Hoyt „Floyd“ Ming (* 6. Oktober 1902 im Choctaw County, Mississippi; † 28. April 1985 in Ackerman, Choctaw County, Mississippi) war ein US-amerikanischer Old-Time-Musiker.\n",
      "0.6697215971332194\n",
      "\n",
      "Das Bistum Chișinău (lat.: \"Dioecesis Chisinauensis\") ist ein römisch-katholisches Bistum in der Republik Moldau.\n",
      "0.6697206639581752\n",
      "\n",
      "\"Aux Juges\", \"Beynac\", \"Bœuf Mort\", \"Bois de la Chapelle\", \"Bois de Mazaudet\", \"Bois du Pimpidour\", \"Bois de Tendeix\", \"Bonnefond\", \"Bourbon\", \"Bourneix\", \"Chapellas\", \"Châtaignier\", \"Château de Beynac\", \"Chez Farjouty\", \"Claud de la Veyrière\", \"Contussier\", \"Côte Chaude\", \"Étang de Bonnefond\", \"Étang de la Garenne\", \"Étang de Pabot\", \"Fontaine de la Goutte\", \"Forge de la Maque\", \"Grand Étang\", \"Grandcoing\", \"Jalazier\", \"L’Aubade\", \"La Beauzetie\", \"La Besse\", \"La Bucherie\", \"La Chapelle Verlaine\", \"La Fontaine de Loriot\", \"La Forêt\", \"La Forêt de Beynac\", \"La Garde\", \"La Gourgousse\", \"La Jurie\", \"La Maque\", \"La Petite Garde\", \"La Petite Sagne\", \"La Peyrière\", \"La Pinalie de Grandcoing\", \"La Pinalie de Montagnac\", \"La Porcherie\", \"La Rebière\", \"La Rouderie\", \"La Sagne\", \"La Tuilerie de la Veyrière\", \"La Valade\", \"La Veyrière\", \"La Villotte\", \"Lacoussière\", \"Lage\", \"Larret\", \"Le Breuil\", \"Le Claud du Four\", \"Le Grand Peytour\", \"Le Grand Roc\", \"Le Monteix\", \"Le Moulin de Lacoussière\", \"Le Moulin de Maziéras\", \"Le Pic\", \"Le Pimpidour\", \"Le Puy\", \"Le Roudat\", \"Le Tendeix\", \"Le Vieux Moulin\", \"Les Bordes\", \"Les Carrières\", \"Les Clèdes\", \"Les Couderchoux\", \"Les Farges\", \"Les Gargalines\", \"Les Grafouillades\", \"Les Marottes\", \"Les Paleyres\", \"Les Petites Bordes\", \"Les Peychères\", \"Les Places\", \"Les Quatre Routes\", \"Les Rivalles\", \"Les Tuileries de Vaure\", \"Maberout\", \"Maisonneuve\", \"Manaud\", \"Mazaubrun\", \"Mazaudet\", \"Maziéras\", \"Montagnac\", \"Moulin de Grandcoing\", \"Moulin du Pont\", \"Mouton\", \"Nicolas\", \"Petit Peytour\", \"Peyfreix\", \"Peyrouse\", \"Pont des Brasdoux\", \"Puydoyeux\", \"Saut du Chalard\", \"Soumagnac\", \"Trou de Philippou\", \"Vachaumard\", \"Vacher\", \"Vaure\", \"Veyrinas\" und \"Vieille Abbaye\".\n",
      "0.6697199306778647\n",
      "\n",
      "<onlyinclude>* Paul Scheuer (Regisseur) (* 1944), luxemburgischer Filmregisseur, Drehbuchautor und Schauspieler\n",
      "0.6697169367811923\n",
      "\n",
      "\"Brégnac\", \"Chanet\", \"Chanet de la Lande\", \"Chavaroche\" bzw. \"Chaveroche\", \"Chez Nardonnet\", \"Chez Noaillac\", \"Combe de l'Autel\", \"Combe de Larat\", \"Fronsac\", \"Fougères\", \"Grange Neuve\", \"L'Étang Bleu\", \"La Beynichie\", \"La Chassagne\", \"La Férédie\", \"La Forêt\", \"La Grande Pièce\", \"La Jardonnie\", \"La Moulade\", \"La Plagne\", \"La Pointerie\", \"La Roussie\", \"La Salle\", \"Le Cheyrou\", \"Le Maupas\", \"Le Pré de la Serve\", \"Les Chabannes\", \"Les Chambarrières\", \"Les Combettes\", \"Les Coufourches\", \"Les Fayes\", \"Les Ferlandes\", \"Les Gages\", \"Les Landes\", \"Les Marteilles\", \"Les Plagnes\", \"Les Potences\", \"Les Pradelles\", \"Les Tuiles\", \"Leycoussey\", \"Maraf(f)y\", \"Montozon\", \"Moulin du Roc\", \"Rouchatoux\" und \"Terme des Balances\".\n",
      "0.6697165884538789\n",
      "\n",
      "Motoyasu Takeuchi (jap. , \"Takeuchi Motoyasu\"; * 11. August 1964) ist ein ehemaliger japanischer Skispringer.\n",
      "0.6697159291420498\n",
      "\n",
      "Marcão (* 17. April 1973 in Andirá, eigentlich \"Marcos Antônio Aparecido Cipriano\") ist ein brasilianischer Fußballspieler.\n",
      "0.6697156640996855\n",
      "\n",
      "International: Deutschland, USA, Schweden, England, Holland, Frankreich, Italien, Türkei, Makedonien, Slowenien.\n",
      "0.6697150901224188\n",
      "\n",
      "„Sta pes; Sta mi pes; sta pes, ne labere mi pes! Nisi steteris, lapides hi tibi lectus erunt“, das heißt\n",
      "0.6697141143016707\n",
      "\n",
      "Krankenhausarten: Anthroposophisches Krankenhaus – Berufsgenossenschaftliche Unfallklinik – Fachkrankenhaus – Privatklinik – Psychiatrische Klinik – Psychosomatische Klinik – Sanatorium – Unfallkrankenhaus\n",
      "0.6697139750316088\n",
      "\n",
      "György Ránki [] (* 30. Oktober 1907 in Budapest; † 22. Mai 1992 ebenda) war ein ungarischer Komponist.\n",
      "0.6697130757556371\n",
      "\n",
      "Ernst von Anhalt-Bernburg (* 19. Mai 1608 in Amberg; † 3. Dezember 1632 in Naumburg) war ein Obrist im Dreißigjährigen Krieg.\n",
      "0.6697067757662112\n",
      "\n",
      "Die Ortschaft Ellenbach hatte 1817 62 Einwohner und 7 Wohngebäude, 1861 51 Einwohner und 1961 28 Einwohner und 5 Wohngebäude.\n",
      "0.6697066827335566\n",
      "\n",
      "Es gibt verschiedene Klassen, wie z. B. 1:10 Elektro Offroad, 1:10 Elektro Onroad, 1:8 Elektro Offroad, 1:8 Verbrenner Offroad, 1:8 Verbrenner Onroad und auch 1:5 Offroad.\n",
      "0.6697041371357193\n",
      "\n",
      "<onlyinclude>* Ludwig Schneller (Theologe) (1858–1953), deutscher evangelischer Theologe und Missionar\n",
      "0.6697034130810608\n",
      "\n",
      "Die Ethiopia International 2013 im Badminton fanden vom 17. bis zum 20. Oktober 2013 in Addis Abeba statt.\n",
      "0.6697028056448268\n",
      "\n",
      "Thebae Phthiotides/Halmiros/Halmirus (ital.: \"Tebe di Ftiotide/Almira/Almiros\") ist ein Titularbistum der römisch-katholischen Kirche.\n",
      "0.6697023268614912\n",
      "\n",
      "Je nach Polymerisationsgrad bzw. Molmasse reicht das Spektrum des PIB von einem viskosen Öl (Molmasse ca. 300–3000 g·mol) über plastisch klebrige Massen (Molmasse ca. 40.000–120.000 g·mol) bis zu kautschukartigen Produkten (Molmasse ca. 300.000–2.500.000 g·mol)\n",
      "0.6697018120489728\n",
      "\n"
     ]
    }
   ],
   "source": [
    "offset = 50_000\n",
    "for i in range(100):\n",
    "    print(df.iloc[i + offset][\"text\"])\n",
    "    print(df.iloc[i + offset][\"text_distance\"])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion:\n",
    "\n",
    "- we use 0.67 as a threshold for text_distance and remove all that is greater\n",
    "- remove all that is greater"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44748"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dirty_wiki_ids = df[df[\"text_distance\"] > 0.67][\"wiki_id\"].unique()\n",
    "len(dirty_wiki_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "      <th>text_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3180087</th>\n",
       "      <td>3180087</td>\n",
       "      <td>Grube Anna</td>\n",
       "      <td>Nach dem Zweiten Weltkrieg stieg die Förderung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=119732</td>\n",
       "      <td>119732</td>\n",
       "      <td>163.171616</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.669998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14184531</th>\n",
       "      <td>14184531</td>\n",
       "      <td>Isidoro Carini</td>\n",
       "      <td>Isidoro Carini (* 7. Januar 1843 in Palermo; †...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=8116081</td>\n",
       "      <td>8116081</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.669998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3939080</th>\n",
       "      <td>3939080</td>\n",
       "      <td>Horgau</td>\n",
       "      <td>1862 bis 1929 gehörte Horgau zum Bezirksamt Zu...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=466895</td>\n",
       "      <td>466895</td>\n",
       "      <td>107.911201</td>\n",
       "      <td>10</td>\n",
       "      <td>36</td>\n",
       "      <td>0.669998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7727609</th>\n",
       "      <td>7727609</td>\n",
       "      <td>Lamont Johnson (Regisseur)</td>\n",
       "      <td>Danach drehte er als Regisseur Episoden für di...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=5848859</td>\n",
       "      <td>5848859</td>\n",
       "      <td>19.931370</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0.669997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12764512</th>\n",
       "      <td>12764512</td>\n",
       "      <td>Lee Castle</td>\n",
       "      <td>Lee Castle (* 28. Februar 1915 in New York Cit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=4716640</td>\n",
       "      <td>4716640</td>\n",
       "      <td>2.484907</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.669996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938136</th>\n",
       "      <td>1938136</td>\n",
       "      <td>Fußball in London</td>\n",
       "      <td>Der Tottenham Hotspur FC wurde 1882 von jungen...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=9504218</td>\n",
       "      <td>9504218</td>\n",
       "      <td>348.901825</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.094879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6392992</th>\n",
       "      <td>6392992</td>\n",
       "      <td>Jakob Sigle</td>\n",
       "      <td>Bei seinen Reisen nach Amerika ab 1900 informi...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=9622860</td>\n",
       "      <td>9622860</td>\n",
       "      <td>34.508080</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.094291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9540017</th>\n",
       "      <td>9540017</td>\n",
       "      <td>Justus Siebein</td>\n",
       "      <td>Mit nur noch knapp 10.000 kampffähigen Soldate...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=4217097</td>\n",
       "      <td>4217097</td>\n",
       "      <td>9.821843</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.092346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4717732</th>\n",
       "      <td>4717732</td>\n",
       "      <td>Krazy Kat</td>\n",
       "      <td>Kommerziell gesehen gehörte Herrimans Strip ni...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=273569</td>\n",
       "      <td>273569</td>\n",
       "      <td>73.122513</td>\n",
       "      <td>7</td>\n",
       "      <td>18</td>\n",
       "      <td>0.092124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7822520</th>\n",
       "      <td>7822520</td>\n",
       "      <td>Schloss Naundorf (Dippoldiswalde)</td>\n",
       "      <td>Um 1473 verkaufte Rudolph von Bünau an die Her...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=10719966</td>\n",
       "      <td>10719966</td>\n",
       "      <td>19.238222</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.091380</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14431887 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                              title  \\\n",
       "3180087    3180087                         Grube Anna   \n",
       "14184531  14184531                     Isidoro Carini   \n",
       "3939080    3939080                             Horgau   \n",
       "7727609    7727609         Lamont Johnson (Regisseur)   \n",
       "12764512  12764512                         Lee Castle   \n",
       "...            ...                                ...   \n",
       "1938136    1938136                  Fußball in London   \n",
       "6392992    6392992                        Jakob Sigle   \n",
       "9540017    9540017                     Justus Siebein   \n",
       "4717732    4717732                          Krazy Kat   \n",
       "7822520    7822520  Schloss Naundorf (Dippoldiswalde)   \n",
       "\n",
       "                                                       text  \\\n",
       "3180087   Nach dem Zweiten Weltkrieg stieg die Förderung...   \n",
       "14184531  Isidoro Carini (* 7. Januar 1843 in Palermo; †...   \n",
       "3939080   1862 bis 1929 gehörte Horgau zum Bezirksamt Zu...   \n",
       "7727609   Danach drehte er als Regisseur Episoden für di...   \n",
       "12764512  Lee Castle (* 28. Februar 1915 in New York Cit...   \n",
       "...                                                     ...   \n",
       "1938136   Der Tottenham Hotspur FC wurde 1882 von jungen...   \n",
       "6392992   Bei seinen Reisen nach Amerika ab 1900 informi...   \n",
       "9540017   Mit nur noch knapp 10.000 kampffähigen Soldate...   \n",
       "4717732   Kommerziell gesehen gehörte Herrimans Strip ni...   \n",
       "7822520   Um 1473 verkaufte Rudolph von Bünau an die Her...   \n",
       "\n",
       "                                                   url   wiki_id       views  \\\n",
       "3180087     https://de.wikipedia.org/wiki?curid=119732    119732  163.171616   \n",
       "14184531   https://de.wikipedia.org/wiki?curid=8116081   8116081    0.693147   \n",
       "3939080     https://de.wikipedia.org/wiki?curid=466895    466895  107.911201   \n",
       "7727609    https://de.wikipedia.org/wiki?curid=5848859   5848859   19.931370   \n",
       "12764512   https://de.wikipedia.org/wiki?curid=4716640   4716640    2.484907   \n",
       "...                                                ...       ...         ...   \n",
       "1938136    https://de.wikipedia.org/wiki?curid=9504218   9504218  348.901825   \n",
       "6392992    https://de.wikipedia.org/wiki?curid=9622860   9622860   34.508080   \n",
       "9540017    https://de.wikipedia.org/wiki?curid=4217097   4217097    9.821843   \n",
       "4717732     https://de.wikipedia.org/wiki?curid=273569    273569   73.122513   \n",
       "7822520   https://de.wikipedia.org/wiki?curid=10719966  10719966   19.238222   \n",
       "\n",
       "          paragraph_id  langs  text_distance  \n",
       "3180087             15      1       0.669998  \n",
       "14184531             0      3       0.669998  \n",
       "3939080             10     36       0.669998  \n",
       "7727609              3     15       0.669997  \n",
       "12764512             0      4       0.669996  \n",
       "...                ...    ...            ...  \n",
       "1938136              6      8       0.094879  \n",
       "6392992              3      0       0.094291  \n",
       "9540017             10      0       0.092346  \n",
       "4717732              7     18       0.092124  \n",
       "7822520              2      0       0.091380  \n",
       "\n",
       "[14431887 rows x 9 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove \"dirty\" wiki_ids from df\n",
    "df = df[~df[\"wiki_id\"].isin(dirty_wiki_ids)]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24014"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relevant_wiki_pages = pd.read_csv(\"./data/wikipedia_pages_it2.tsv.gz\", sep=\"\\t\")\n",
    "relevant_pageids = set(relevant_wiki_pages[\"pageid\"].tolist())\n",
    "len(relevant_pageids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73491"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relevant_wiki_pages = pd.read_csv(\"./data/wp_Digitale_Welt.tsv\", sep=\"\\t\")\n",
    "relevant_pageids.update(relevant_wiki_pages[\"pageid\"].tolist())\n",
    "len(relevant_pageids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73491"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select some \"other\" page ids\n",
    "other_pageids = list(set(df[\"wiki_id\"].tolist()) - relevant_pageids)\n",
    "random.shuffle(other_pageids)\n",
    "other_pageids = other_pageids[: len(relevant_pageids)]\n",
    "len(other_pageids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "146982"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add the \"other\" page ids to the relevant page ids\n",
    "relevant_pageids.update(other_pageids)\n",
    "len(relevant_pageids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "      <th>text_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6983787</th>\n",
       "      <td>6983787</td>\n",
       "      <td>Friedrich Stromeyer</td>\n",
       "      <td>Er heiratete Luise Ludewig (* 29. September 17...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=784611</td>\n",
       "      <td>784611</td>\n",
       "      <td>26.980623</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>0.669988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13332431</th>\n",
       "      <td>13332431</td>\n",
       "      <td>Siddiqiy Ajziy</td>\n",
       "      <td>Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=5871091</td>\n",
       "      <td>5871091</td>\n",
       "      <td>1.791759</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.669982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3440127</th>\n",
       "      <td>3440127</td>\n",
       "      <td>7,5-cm-Leichtgeschütz 40</td>\n",
       "      <td>Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=5365282</td>\n",
       "      <td>5365282</td>\n",
       "      <td>141.204010</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.669980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13755596</th>\n",
       "      <td>13755596</td>\n",
       "      <td>Tiberius Claudius Claudianus</td>\n",
       "      <td>&lt;onlyinclude&gt;* Tiberius Claudius Claudianus (S...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12114918</td>\n",
       "      <td>12114918</td>\n",
       "      <td>1.386294</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.669959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8987544</th>\n",
       "      <td>8987544</td>\n",
       "      <td>Gmina Stargard</td>\n",
       "      <td>Weitere Ortschaften in der Gemeinde sind Bębni...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=7596131</td>\n",
       "      <td>7596131</td>\n",
       "      <td>12.188967</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>0.669884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7250546</th>\n",
       "      <td>7250546</td>\n",
       "      <td>Rigomagus</td>\n",
       "      <td>Nach dem Bataveraufstand und den Wirren des Vi...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=5517172</td>\n",
       "      <td>5517172</td>\n",
       "      <td>24.143497</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>0.107323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2057137</th>\n",
       "      <td>2057137</td>\n",
       "      <td>Paul Ryan (Politiker)</td>\n",
       "      <td>In seinem ersten Versuch, in den Vereinigten S...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=5589493</td>\n",
       "      <td>5589493</td>\n",
       "      <td>323.148621</td>\n",
       "      <td>11</td>\n",
       "      <td>50</td>\n",
       "      <td>0.105903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1280074</th>\n",
       "      <td>1280074</td>\n",
       "      <td>Hacker (Computersicherheit)</td>\n",
       "      <td>1988 schreibt Robert Tappan Morris aus Neugier...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1661063</td>\n",
       "      <td>1661063</td>\n",
       "      <td>550.451416</td>\n",
       "      <td>18</td>\n",
       "      <td>26</td>\n",
       "      <td>0.105857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14410373</th>\n",
       "      <td>14410373</td>\n",
       "      <td>Anna Margarethe van Delden</td>\n",
       "      <td>Anna Margarete van Delden wurde am 10. Dezembe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=11669155</td>\n",
       "      <td>11669155</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.103976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11832645</th>\n",
       "      <td>11832645</td>\n",
       "      <td>Theo Gaudig</td>\n",
       "      <td>Seine Eltern waren der Dreher Otto Gaudig, von...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=11610222</td>\n",
       "      <td>11610222</td>\n",
       "      <td>3.871201</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.103198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>993219 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                         title  \\\n",
       "6983787    6983787           Friedrich Stromeyer   \n",
       "13332431  13332431                Siddiqiy Ajziy   \n",
       "3440127    3440127      7,5-cm-Leichtgeschütz 40   \n",
       "13755596  13755596  Tiberius Claudius Claudianus   \n",
       "8987544    8987544                Gmina Stargard   \n",
       "...            ...                           ...   \n",
       "7250546    7250546                     Rigomagus   \n",
       "2057137    2057137         Paul Ryan (Politiker)   \n",
       "1280074    1280074   Hacker (Computersicherheit)   \n",
       "14410373  14410373    Anna Margarethe van Delden   \n",
       "11832645  11832645                   Theo Gaudig   \n",
       "\n",
       "                                                       text  \\\n",
       "6983787   Er heiratete Luise Ludewig (* 29. September 17...   \n",
       "13332431  Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...   \n",
       "3440127   Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...   \n",
       "13755596  <onlyinclude>* Tiberius Claudius Claudianus (S...   \n",
       "8987544   Weitere Ortschaften in der Gemeinde sind Bębni...   \n",
       "...                                                     ...   \n",
       "7250546   Nach dem Bataveraufstand und den Wirren des Vi...   \n",
       "2057137   In seinem ersten Versuch, in den Vereinigten S...   \n",
       "1280074   1988 schreibt Robert Tappan Morris aus Neugier...   \n",
       "14410373  Anna Margarete van Delden wurde am 10. Dezembe...   \n",
       "11832645  Seine Eltern waren der Dreher Otto Gaudig, von...   \n",
       "\n",
       "                                                   url   wiki_id       views  \\\n",
       "6983787     https://de.wikipedia.org/wiki?curid=784611    784611   26.980623   \n",
       "13332431   https://de.wikipedia.org/wiki?curid=5871091   5871091    1.791759   \n",
       "3440127    https://de.wikipedia.org/wiki?curid=5365282   5365282  141.204010   \n",
       "13755596  https://de.wikipedia.org/wiki?curid=12114918  12114918    1.386294   \n",
       "8987544    https://de.wikipedia.org/wiki?curid=7596131   7596131   12.188967   \n",
       "...                                                ...       ...         ...   \n",
       "7250546    https://de.wikipedia.org/wiki?curid=5517172   5517172   24.143497   \n",
       "2057137    https://de.wikipedia.org/wiki?curid=5589493   5589493  323.148621   \n",
       "1280074    https://de.wikipedia.org/wiki?curid=1661063   1661063  550.451416   \n",
       "14410373  https://de.wikipedia.org/wiki?curid=11669155  11669155    0.693147   \n",
       "11832645  https://de.wikipedia.org/wiki?curid=11610222  11610222    3.871201   \n",
       "\n",
       "          paragraph_id  langs  text_distance  \n",
       "6983787              5     21       0.669988  \n",
       "13332431             0      3       0.669982  \n",
       "3440127              0     12       0.669980  \n",
       "13755596             0      0       0.669959  \n",
       "8987544              2     15       0.669884  \n",
       "...                ...    ...            ...  \n",
       "7250546              8      2       0.107323  \n",
       "2057137             11     50       0.105903  \n",
       "1280074             18     26       0.105857  \n",
       "14410373             1      0       0.103976  \n",
       "11832645             1      0       0.103198  \n",
       "\n",
       "[993219 rows x 9 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected = df[df[\"wiki_id\"].isin(relevant_pageids)].copy()\n",
    "df_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>paragraph_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6983787</th>\n",
       "      <td>6983787</td>\n",
       "      <td>Friedrich Stromeyer</td>\n",
       "      <td>Er heiratete Luise Ludewig (* 29. September 17...</td>\n",
       "      <td>784611</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13332431</th>\n",
       "      <td>13332431</td>\n",
       "      <td>Siddiqiy Ajziy</td>\n",
       "      <td>Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...</td>\n",
       "      <td>5871091</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3440127</th>\n",
       "      <td>3440127</td>\n",
       "      <td>7,5-cm-Leichtgeschütz 40</td>\n",
       "      <td>Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...</td>\n",
       "      <td>5365282</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13755596</th>\n",
       "      <td>13755596</td>\n",
       "      <td>Tiberius Claudius Claudianus</td>\n",
       "      <td>&lt;onlyinclude&gt;* Tiberius Claudius Claudianus (S...</td>\n",
       "      <td>12114918</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8987544</th>\n",
       "      <td>8987544</td>\n",
       "      <td>Gmina Stargard</td>\n",
       "      <td>Weitere Ortschaften in der Gemeinde sind Bębni...</td>\n",
       "      <td>7596131</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7250546</th>\n",
       "      <td>7250546</td>\n",
       "      <td>Rigomagus</td>\n",
       "      <td>Nach dem Bataveraufstand und den Wirren des Vi...</td>\n",
       "      <td>5517172</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2057137</th>\n",
       "      <td>2057137</td>\n",
       "      <td>Paul Ryan (Politiker)</td>\n",
       "      <td>In seinem ersten Versuch, in den Vereinigten S...</td>\n",
       "      <td>5589493</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1280074</th>\n",
       "      <td>1280074</td>\n",
       "      <td>Hacker (Computersicherheit)</td>\n",
       "      <td>1988 schreibt Robert Tappan Morris aus Neugier...</td>\n",
       "      <td>1661063</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14410373</th>\n",
       "      <td>14410373</td>\n",
       "      <td>Anna Margarethe van Delden</td>\n",
       "      <td>Anna Margarete van Delden wurde am 10. Dezembe...</td>\n",
       "      <td>11669155</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11832645</th>\n",
       "      <td>11832645</td>\n",
       "      <td>Theo Gaudig</td>\n",
       "      <td>Seine Eltern waren der Dreher Otto Gaudig, von...</td>\n",
       "      <td>11610222</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>993219 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                         title  \\\n",
       "6983787    6983787           Friedrich Stromeyer   \n",
       "13332431  13332431                Siddiqiy Ajziy   \n",
       "3440127    3440127      7,5-cm-Leichtgeschütz 40   \n",
       "13755596  13755596  Tiberius Claudius Claudianus   \n",
       "8987544    8987544                Gmina Stargard   \n",
       "...            ...                           ...   \n",
       "7250546    7250546                     Rigomagus   \n",
       "2057137    2057137         Paul Ryan (Politiker)   \n",
       "1280074    1280074   Hacker (Computersicherheit)   \n",
       "14410373  14410373    Anna Margarethe van Delden   \n",
       "11832645  11832645                   Theo Gaudig   \n",
       "\n",
       "                                                       text   wiki_id  \\\n",
       "6983787   Er heiratete Luise Ludewig (* 29. September 17...    784611   \n",
       "13332431  Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...   5871091   \n",
       "3440127   Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...   5365282   \n",
       "13755596  <onlyinclude>* Tiberius Claudius Claudianus (S...  12114918   \n",
       "8987544   Weitere Ortschaften in der Gemeinde sind Bębni...   7596131   \n",
       "...                                                     ...       ...   \n",
       "7250546   Nach dem Bataveraufstand und den Wirren des Vi...   5517172   \n",
       "2057137   In seinem ersten Versuch, in den Vereinigten S...   5589493   \n",
       "1280074   1988 schreibt Robert Tappan Morris aus Neugier...   1661063   \n",
       "14410373  Anna Margarete van Delden wurde am 10. Dezembe...  11669155   \n",
       "11832645  Seine Eltern waren der Dreher Otto Gaudig, von...  11610222   \n",
       "\n",
       "          paragraph_id  \n",
       "6983787              5  \n",
       "13332431             0  \n",
       "3440127              0  \n",
       "13755596             0  \n",
       "8987544              2  \n",
       "...                ...  \n",
       "7250546              8  \n",
       "2057137             11  \n",
       "1280074             18  \n",
       "14410373             1  \n",
       "11832645             1  \n",
       "\n",
       "[993219 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.drop(\"url\", axis=1, inplace=True)\n",
    "df_selected.drop(\"views\", axis=1, inplace=True)\n",
    "df_selected.drop(\"langs\", axis=1, inplace=True)\n",
    "df_selected.drop(\"text_distance\", axis=1, inplace=True)\n",
    "df_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_selected.to_pickle(\"./data/wikipedia-22-12-de-no-emb-clean-selected.pkl.gz\", compression=\"gzip\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}