Generating dataset - real examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bef6e88f-b39f-49f3-b715-c3cd16cdad66",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Generating Correct Names\n",
    "using real data collected from student name - [dataset url](https://forums.madrsa-online.com/threads/3264/)\n",
    "#### Rules for data proivded (it has males and females separated not applicable for my collected data)\n",
    "- 3 words\n",
    "- First word can be male/female\n",
    "- Second and third words must be males"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac9d1715-87b7-48fb-b8aa-a4a3aae6e907",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "import pandas as pd\n",
    "import random\n",
    "import string\n",
    "\n",
    "from collections import Counter\n",
    "from tqdm import tqdm\n",
    "\n",
    "random.seed(23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7cc470de-9119-4e08-b6d6-0cee745ed931",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_name = 'both_generated_using_correct_names_40k_better_repeatedfunction'\n",
    "ds_size = 40_000"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7872b43e-5e4d-47c7-9efc-2a14b5d2f054",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Generate Correct names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1f7238d7-29b7-4974-97aa-04afa5ce4b32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3729, 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>ياسر</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1823</th>\n",
       "      <td>العتبانى</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3108</th>\n",
       "      <td>مندور</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3373</th>\n",
       "      <td>قاسم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3116</th>\n",
       "      <td>شيماء</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>عبدالعليم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>655</th>\n",
       "      <td>هاتم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>عيناء</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1101</th>\n",
       "      <td>جنيدى</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2855</th>\n",
       "      <td>لبيب</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           name\n",
       "1838       ياسر\n",
       "1823   العتبانى\n",
       "3108      مندور\n",
       "3373       قاسم\n",
       "3116      شيماء\n",
       "639   عبدالعليم\n",
       "655        هاتم\n",
       "257       عيناء\n",
       "1101      جنيدى\n",
       "2855       لبيب"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names_df = pd.read_csv(\"data/all_names.txt\", names=['name'])\n",
    "print(names_df.shape)\n",
    "names_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0d7b1d5-3b40-450d-8257-eb0c67b62016",
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_names = names_df['name'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a1f3c78e-5594-485e-9b39-bb1cd5d39c5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_fullnames(male_names: List[str], female_list: List[str] = None, num_names = 1000) -> List[str]:\n",
    "    \"\"\"\n",
    "    Generates full names where second and third names must always be males.\n",
    "    \"\"\"\n",
    "    names = []\n",
    "\n",
    "    for _ in range(num_names):\n",
    "        # males\n",
    "        names.append(\" \".join(random.sample(male_names, 3)))\n",
    "        # females\n",
    "        if female_list:\n",
    "            names.append(random.sample(female_names, 1)[0] + \" \" + \" \".join(random.sample(male_names, 2)))\n",
    "    \n",
    "    return names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7de68207-c025-4e9d-ab21-21794fd95dbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of names generated: 20000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['حمايه تيسير لاوند', 'سعداحمد ابراهبم برى', 'مكة ولاء كريمه']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = generate_fullnames(correct_names, num_names=ds_size//2)\n",
    "print(f\"Number of names generated: {len(names)}\")\n",
    "names[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8db6f0e7-23cb-49b6-9fca-d04f15b721de",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(names, columns=[\"Name\"]).to_csv(\"data/full_correct_names.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1d4e2364-abfa-4161-adbc-759df41264a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>حمايه تيسير لاوند</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>سعداحمد ابراهبم برى</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>مكة ولاء كريمه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>زايد ثراء سحاق</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>الاسمر الشندى العوضى</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Name\n",
       "0     حمايه تيسير لاوند\n",
       "1   سعداحمد ابراهبم برى\n",
       "2        مكة ولاء كريمه\n",
       "3        زايد ثراء سحاق\n",
       "4  الاسمر الشندى العوضى"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df = pd.read_csv(\"data/full_correct_names.csv\")\n",
    "temp_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf332f35-c569-44d4-abeb-b1fea84b1577",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Stats about our names "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5ce31170-1d7b-494c-a4cb-b8609583cc04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Max, Min, Mean)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3, 3, 3.0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGYCAYAAACgQ/O7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAp60lEQVR4nO3dfXBUZZ638W8n2B3QdHgzbyVCFIcQCeHFMbYjDEg2DaacycruKqCgBhiYxBHiAMZhmQRmNxQUIFPyUpZKrBpYgS3NaHADTRAYJq1IMGLQpBRhgiUdZlW6JWITSD9/TOWs/QBKNLHJzfWp6irOOb8+fZ9YZa7qPklsoVAoJAAAAMNERXoBAAAAnYHIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGCkbpFeQCS1trbq008/VWxsrGw2W6SXAwAALkMoFNKXX36p5ORkRUVd+v2aqzpyPv30U/Xr1y/SywAAAN/D8ePHdcMNN1zy+FUdObGxsZL+8UVyOp0RXg0AALgcgUBA/fr1s76PX8pVHTltH1E5nU4iBwCALua7bjXhxmMAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYqV2RU1paqp/+9KeKjY1VfHy8cnNz1dDQEDbz9ddfKz8/X3369NF1112niRMnqqmpKWymsbFROTk56tGjh+Lj4zVv3jydO3cubGb37t0aMWKEHA6HBg4cqLKysgvWs2bNGg0YMEAxMTHKzMzU/v3723M5AADAYO2KnD179ig/P19vvvmmPB6PWlpalJ2drebmZmtm7ty5eu2117R161bt2bNHn376qe677z7r+Pnz55WTk6OzZ8+qurpaL774osrKyrRo0SJr5ujRo8rJydHYsWNVW1urOXPmaPr06dq+fbs1s3nzZhUWFur3v/+9Dh48qIyMDLndbp08efKHfD0AAIApQj/AyZMnQ5JCe/bsCYVCodCpU6dC11xzTWjr1q3WzAcffBCSFPJ6vaFQKBR6/fXXQ1FRUSGfz2fNrFu3LuR0OkPBYDAUCoVC8+fPD916661hr3X//feH3G63tX377beH8vPzre3z58+HkpOTQ6WlpZe9fr/fH5IU8vv97bhqAAAQSZf7/fsH3ZPj9/slSb1795Yk1dTUqKWlRVlZWdZMamqqbrzxRnm9XkmS1+tVenq6EhISrBm3261AIKDDhw9bM988R9tM2znOnj2rmpqasJmoqChlZWVZMwAA4OrW7fs+sbW1VXPmzNHPfvYzDRkyRJLk8/lkt9vVs2fPsNmEhAT5fD5r5puB03a87di3zQQCAZ05c0ZffPGFzp8/f9GZ+vr6S645GAwqGAxa24FAoB1XDAAAupLvHTn5+fmqq6vTvn37OnI9naq0tFQlJSWRXgYiaMCT2yK9BACd5NjSnEgvAVeY7/VxVUFBgSoqKvTGG2/ohhtusPYnJibq7NmzOnXqVNh8U1OTEhMTrZn//6et2ra/a8bpdKp79+7q27evoqOjLzrTdo6LKSoqkt/vtx7Hjx9v34UDAIAuo12REwqFVFBQoFdeeUW7du1SSkpK2PGRI0fqmmuuUVVVlbWvoaFBjY2NcrlckiSXy6X33nsv7KegPB6PnE6n0tLSrJlvnqNtpu0cdrtdI0eODJtpbW1VVVWVNXMxDodDTqcz7AEAAMzUro+r8vPztWnTJv35z39WbGysdQ9NXFycunfvrri4OOXl5amwsFC9e/eW0+nUY489JpfLpTvuuEOSlJ2drbS0ND300ENatmyZfD6fFi5cqPz8fDkcDknSrFmz9Mwzz2j+/Pl69NFHtWvXLm3ZskXbtv3fRw2FhYWaNm2abrvtNt1+++16+umn1dzcrEceeaSjvjYAAKALa1fkrFu3TpI0ZsyYsP0bNmzQww8/LElatWqVoqKiNHHiRAWDQbndbq1du9aajY6OVkVFhWbPni2Xy6Vrr71W06ZN0+LFi62ZlJQUbdu2TXPnztXq1at1ww036LnnnpPb7bZm7r//fv3973/XokWL5PP5NGzYMFVWVl5wMzIAALg62UKhUCjSi4iUQCCguLg4+f1+Prq6SnDjMWAubjy+elzu92/+dhUAADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwUrsjZ+/evbr33nuVnJwsm82m8vLysOM2m+2ij+XLl1szAwYMuOD40qVLw85z6NAhjRo1SjExMerXr5+WLVt2wVq2bt2q1NRUxcTEKD09Xa+//np7LwcAABiq3ZHT3NysjIwMrVmz5qLHT5w4EfZ44YUXZLPZNHHixLC5xYsXh8099thj1rFAIKDs7Gz1799fNTU1Wr58uYqLi/Xss89aM9XV1Zo0aZLy8vL0zjvvKDc3V7m5uaqrq2vvJQEAAAN1a+8TJkyYoAkTJlzyeGJiYtj2n//8Z40dO1Y33XRT2P7Y2NgLZtts3LhRZ8+e1QsvvCC73a5bb71VtbW1WrlypWbOnClJWr16tcaPH6958+ZJkpYsWSKPx6NnnnlG69evb+9lAQAAw3TqPTlNTU3atm2b8vLyLji2dOlS9enTR8OHD9fy5ct17tw565jX69Xo0aNlt9utfW63Ww0NDfriiy+smaysrLBzut1ueb3eTroaAADQlbT7nZz2ePHFFxUbG6v77rsvbP9vfvMbjRgxQr1791Z1dbWKiop04sQJrVy5UpLk8/mUkpIS9pyEhATrWK9eveTz+ax935zx+XyXXE8wGFQwGLS2A4HAD7o+AABw5erUyHnhhRc0ZcoUxcTEhO0vLCy0/j106FDZ7Xb96le/UmlpqRwOR6etp7S0VCUlJZ12fgAAcOXotI+r/vKXv6ihoUHTp0//ztnMzEydO3dOx44dk/SP+3qamprCZtq22+7judTMpe7zkaSioiL5/X7rcfz48fZcEgAA6EI6LXKef/55jRw5UhkZGd85W1tbq6ioKMXHx0uSXC6X9u7dq5aWFmvG4/Fo0KBB6tWrlzVTVVUVdh6PxyOXy3XJ13E4HHI6nWEPAABgpnZHzunTp1VbW6va2lpJ0tGjR1VbW6vGxkZrJhAIaOvWrRd9F8fr9erpp5/Wu+++q48//lgbN27U3Llz9eCDD1oBM3nyZNntduXl5enw4cPavHmzVq9eHfYx1+OPP67KykqtWLFC9fX1Ki4u1oEDB1RQUNDeSwIAAAZq9z05Bw4c0NixY63ttvCYNm2aysrKJEkvvfSSQqGQJk2adMHzHQ6HXnrpJRUXFysYDColJUVz584NC5i4uDjt2LFD+fn5GjlypPr27atFixZZPz4uSXfeeac2bdqkhQsX6qmnntItt9yi8vJyDRkypL2XBAAADGQLhUKhSC8iUgKBgOLi4uT3+/no6iox4MltkV4CgE5ybGlOpJeAH8nlfv/mb1cBAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAI7U7cvbu3at7771XycnJstlsKi8vDzv+8MMPy2azhT3Gjx8fNvP5559rypQpcjqd6tmzp/Ly8nT69OmwmUOHDmnUqFGKiYlRv379tGzZsgvWsnXrVqWmpiomJkbp6el6/fXX23s5AADAUO2OnObmZmVkZGjNmjWXnBk/frxOnDhhPf7rv/4r7PiUKVN0+PBheTweVVRUaO/evZo5c6Z1PBAIKDs7W/3791dNTY2WL1+u4uJiPfvss9ZMdXW1Jk2apLy8PL3zzjvKzc1Vbm6u6urq2ntJAADAQLZQKBT63k+22fTKK68oNzfX2vfwww/r1KlTF7zD0+aDDz5QWlqa3n77bd12222SpMrKSt1zzz365JNPlJycrHXr1ul3v/udfD6f7Ha7JOnJJ59UeXm56uvrJUn333+/mpubVVFRYZ37jjvu0LBhw7R+/frLWn8gEFBcXJz8fr+cTuf3+Aqgqxnw5LZILwFAJzm2NCfSS8CP5HK/f3fKPTm7d+9WfHy8Bg0apNmzZ+uzzz6zjnm9XvXs2dMKHEnKyspSVFSU3nrrLWtm9OjRVuBIktvtVkNDg7744gtrJisrK+x13W63vF5vZ1wSAADoYrp19AnHjx+v++67TykpKTpy5IieeuopTZgwQV6vV9HR0fL5fIqPjw9fRLdu6t27t3w+nyTJ5/MpJSUlbCYhIcE61qtXL/l8PmvfN2faznExwWBQwWDQ2g4EAj/oWgEAwJWrwyPngQcesP6dnp6uoUOH6uabb9bu3bs1bty4jn65diktLVVJSUlE1wAAAH4cnf4j5DfddJP69u2rjz76SJKUmJiokydPhs2cO3dOn3/+uRITE62ZpqamsJm27e+aaTt+MUVFRfL7/dbj+PHjP+ziAADAFavTI+eTTz7RZ599pqSkJEmSy+XSqVOnVFNTY83s2rVLra2tyszMtGb27t2rlpYWa8bj8WjQoEHq1auXNVNVVRX2Wh6PRy6X65JrcTgccjqdYQ8AAGCmdkfO6dOnVVtbq9raWknS0aNHVVtbq8bGRp0+fVrz5s3Tm2++qWPHjqmqqkq//OUvNXDgQLndbknS4MGDNX78eM2YMUP79+/XX//6VxUUFOiBBx5QcnKyJGny5Mmy2+3Ky8vT4cOHtXnzZq1evVqFhYXWOh5//HFVVlZqxYoVqq+vV3FxsQ4cOKCCgoIO+LIAAICurt2Rc+DAAQ0fPlzDhw+XJBUWFmr48OFatGiRoqOjdejQIf3iF7/QT37yE+Xl5WnkyJH6y1/+IofDYZ1j48aNSk1N1bhx43TPPfforrvuCvsdOHFxcdqxY4eOHj2qkSNH6oknntCiRYvCfpfOnXfeqU2bNunZZ59VRkaG/vu//1vl5eUaMmTID/l6AAAAQ/yg35PT1fF7cq4+/J4cwFz8npyrR0R/Tw4AAECkETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAI7U7cvbu3at7771XycnJstlsKi8vt461tLRowYIFSk9P17XXXqvk5GRNnTpVn376adg5BgwYIJvNFvZYunRp2MyhQ4c0atQoxcTEqF+/flq2bNkFa9m6datSU1MVExOj9PR0vf766+29HAAAYKh2R05zc7MyMjK0Zs2aC4599dVXOnjwoP793/9dBw8e1Msvv6yGhgb94he/uGB28eLFOnHihPV47LHHrGOBQEDZ2dnq37+/ampqtHz5chUXF+vZZ5+1ZqqrqzVp0iTl5eXpnXfeUW5urnJzc1VXV9feSwIAAAbq1t4nTJgwQRMmTLjosbi4OHk8nrB9zzzzjG6//XY1NjbqxhtvtPbHxsYqMTHxoufZuHGjzp49qxdeeEF2u1233nqramtrtXLlSs2cOVOStHr1ao0fP17z5s2TJC1ZskQej0fPPPOM1q9f397LAgAAhun0e3L8fr9sNpt69uwZtn/p0qXq06ePhg8fruXLl+vcuXPWMa/Xq9GjR8tut1v73G63Ghoa9MUXX1gzWVlZYed0u93yer2ddzEAAKDLaPc7Oe3x9ddfa8GCBZo0aZKcTqe1/ze/+Y1GjBih3r17q7q6WkVFRTpx4oRWrlwpSfL5fEpJSQk7V0JCgnWsV69e8vl81r5vzvh8vkuuJxgMKhgMWtuBQOAHXyMAALgydVrktLS06N/+7d8UCoW0bt26sGOFhYXWv4cOHSq73a5f/epXKi0tlcPh6KwlqbS0VCUlJZ12fgAAcOXolI+r2gLnb3/7mzweT9i7OBeTmZmpc+fO6dixY5KkxMRENTU1hc20bbfdx3OpmUvd5yNJRUVF8vv91uP48ePtvTQAANBFdHjktAXOhx9+qJ07d6pPnz7f+Zza2lpFRUUpPj5ekuRyubR37161tLRYMx6PR4MGDVKvXr2smaqqqrDzeDweuVyuS76Ow+GQ0+kMewAAADO1++Oq06dP66OPPrK2jx49qtraWvXu3VtJSUn6l3/5Fx08eFAVFRU6f/68dY9M7969Zbfb5fV69dZbb2ns2LGKjY2V1+vV3Llz9eCDD1oBM3nyZJWUlCgvL08LFixQXV2dVq9erVWrVlmv+/jjj+vnP/+5VqxYoZycHL300ks6cOBA2I+ZAwCAq5ctFAqF2vOE3bt3a+zYsRfsnzZtmoqLiy+4YbjNG2+8oTFjxujgwYP69a9/rfr6egWDQaWkpOihhx5SYWFh2P04hw4dUn5+vt5++2317dtXjz32mBYsWBB2zq1bt2rhwoU6duyYbrnlFi1btkz33HPPZV9LIBBQXFyc/H4/7+pcJQY8uS3SSwDQSY4tzYn0EvAjudzv3+2OHJMQOVcfIgcwF5Fz9bjc79/87SoAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgpHZHzt69e3XvvfcqOTlZNptN5eXlYcdDoZAWLVqkpKQkde/eXVlZWfrwww/DZj7//HNNmTJFTqdTPXv2VF5enk6fPh02c+jQIY0aNUoxMTHq16+fli1bdsFatm7dqtTUVMXExCg9PV2vv/56ey8HAAAYqt2R09zcrIyMDK1Zs+aix5ctW6Y//vGPWr9+vd566y1de+21crvd+vrrr62ZKVOm6PDhw/J4PKqoqNDevXs1c+ZM63ggEFB2drb69++vmpoaLV++XMXFxXr22Wetmerqak2aNEl5eXl65513lJubq9zcXNXV1bX3kgAAgIFsoVAo9L2fbLPplVdeUW5urqR/vIuTnJysJ554Qr/97W8lSX6/XwkJCSorK9MDDzygDz74QGlpaXr77bd12223SZIqKyt1zz336JNPPlFycrLWrVun3/3ud/L5fLLb7ZKkJ598UuXl5aqvr5ck3X///WpublZFRYW1njvuuEPDhg3T+vXrL2v9gUBAcXFx8vv9cjqd3/fLgC5kwJPbIr0EAJ3k2NKcSC8BP5LL/f7doffkHD16VD6fT1lZWda+uLg4ZWZmyuv1SpK8Xq969uxpBY4kZWVlKSoqSm+99ZY1M3r0aCtwJMntdquhoUFffPGFNfPN12mbaXsdAABwdevWkSfz+XySpISEhLD9CQkJ1jGfz6f4+PjwRXTrpt69e4fNpKSkXHCOtmO9evWSz+f71te5mGAwqGAwaG0HAoH2XB4AAOhCrqqfriotLVVcXJz16NevX6SXBAAAOkmHRk5iYqIkqampKWx/U1OTdSwxMVEnT54MO37u3Dl9/vnnYTMXO8c3X+NSM23HL6aoqEh+v996HD9+vL2XCAAAuogOjZyUlBQlJiaqqqrK2hcIBPTWW2/J5XJJklwul06dOqWamhprZteuXWptbVVmZqY1s3fvXrW0tFgzHo9HgwYNUq9evayZb75O20zb61yMw+GQ0+kMewAAADO1O3JOnz6t2tpa1dbWSvrHzca1tbVqbGyUzWbTnDlz9Ic//EGvvvqq3nvvPU2dOlXJycnWT2ANHjxY48eP14wZM7R//3799a9/VUFBgR544AElJydLkiZPniy73a68vDwdPnxYmzdv1urVq1VYWGit4/HHH1dlZaVWrFih+vp6FRcX68CBAyooKPjhXxUAANDltfvG4wMHDmjs2LHWdlt4TJs2TWVlZZo/f76am5s1c+ZMnTp1SnfddZcqKysVExNjPWfjxo0qKCjQuHHjFBUVpYkTJ+qPf/yjdTwuLk47duxQfn6+Ro4cqb59+2rRokVhv0vnzjvv1KZNm7Rw4UI99dRTuuWWW1ReXq4hQ4Z8ry8EAAAwyw/6PTldHb8n5+rD78kBzMXvybl6ROT35AAAAFwpiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkTo8cgYMGCCbzXbBIz8/X5I0ZsyYC47NmjUr7ByNjY3KyclRjx49FB8fr3nz5uncuXNhM7t379aIESPkcDg0cOBAlZWVdfSlAACALqxbR5/w7bff1vnz563turo6/dM//ZP+9V//1do3Y8YMLV682Nru0aOH9e/z588rJydHiYmJqq6u1okTJzR16lRdc801+s///E9J0tGjR5WTk6NZs2Zp48aNqqqq0vTp05WUlCS3293RlwQAALqgDo+c66+/Pmx76dKluvnmm/Xzn//c2tejRw8lJiZe9Pk7duzQ+++/r507dyohIUHDhg3TkiVLtGDBAhUXF8tut2v9+vVKSUnRihUrJEmDBw/Wvn37tGrVKiIHAABI6uR7cs6ePas//elPevTRR2Wz2az9GzduVN++fTVkyBAVFRXpq6++so55vV6lp6crISHB2ud2uxUIBHT48GFrJisrK+y13G63vF5vZ14OAADoQjr8nZxvKi8v16lTp/Twww9b+yZPnqz+/fsrOTlZhw4d0oIFC9TQ0KCXX35ZkuTz+cICR5K17fP5vnUmEAjozJkz6t69+0XXEwwGFQwGre1AIPCDrxEAAFyZOjVynn/+eU2YMEHJycnWvpkzZ1r/Tk9PV1JSksaNG6cjR47o5ptv7szlqLS0VCUlJZ36GgAA4MrQaR9X/e1vf9POnTs1ffr0b53LzMyUJH300UeSpMTERDU1NYXNtG233cdzqRmn03nJd3EkqaioSH6/33ocP368fRcFAAC6jE6LnA0bNig+Pl45OTnfOldbWytJSkpKkiS5XC699957OnnypDXj8XjkdDqVlpZmzVRVVYWdx+PxyOVyfetrORwOOZ3OsAcAADBTp0ROa2urNmzYoGnTpqlbt//7ROzIkSNasmSJampqdOzYMb366quaOnWqRo8eraFDh0qSsrOzlZaWpoceekjvvvuutm/froULFyo/P18Oh0OSNGvWLH388ceaP3++6uvrtXbtWm3ZskVz587tjMsBAABdUKdEzs6dO9XY2KhHH300bL/dbtfOnTuVnZ2t1NRUPfHEE5o4caJee+01ayY6OloVFRWKjo6Wy+XSgw8+qKlTp4b9Xp2UlBRt27ZNHo9HGRkZWrFihZ577jl+fBwAAFhsoVAoFOlFREogEFBcXJz8fj8fXV0lBjy5LdJLANBJji399tsjYI7L/f7N364CAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARiJyAACAkYgcAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJGIHAAAYCQiBwAAGInIAQAARurwyCkuLpbNZgt7pKamWse//vpr5efnq0+fPrruuus0ceJENTU1hZ2jsbFROTk56tGjh+Lj4zVv3jydO3cubGb37t0aMWKEHA6HBg4cqLKyso6+FAAA0IV1yjs5t956q06cOGE99u3bZx2bO3euXnvtNW3dulV79uzRp59+qvvuu886fv78eeXk5Ojs2bOqrq7Wiy++qLKyMi1atMiaOXr0qHJycjR27FjV1tZqzpw5mj59urZv394ZlwMAALqgbp1y0m7dlJiYeMF+v9+v559/Xps2bdLdd98tSdqwYYMGDx6sN998U3fccYd27Nih999/Xzt37lRCQoKGDRumJUuWaMGCBSouLpbdbtf69euVkpKiFStWSJIGDx6sffv2adWqVXK73Z1xSQAAoIvplHdyPvzwQyUnJ+umm27SlClT1NjYKEmqqalRS0uLsrKyrNnU1FTdeOON8nq9kiSv16v09HQlJCRYM263W4FAQIcPH7ZmvnmOtpm2cwAAAHT4OzmZmZkqKyvToEGDdOLECZWUlGjUqFGqq6uTz+eT3W5Xz549w56TkJAgn88nSfL5fGGB03a87di3zQQCAZ05c0bdu3e/6NqCwaCCwaC1HQgEftC1AgCAK1eHR86ECROsfw8dOlSZmZnq37+/tmzZcsn4+LGUlpaqpKQkomsAAAA/jk7/EfKePXvqJz/5iT766CMlJibq7NmzOnXqVNhMU1OTdQ9PYmLiBT9t1bb9XTNOp/NbQ6qoqEh+v996HD9+/IdeHgAAuEJ1euScPn1aR44cUVJSkkaOHKlrrrlGVVVV1vGGhgY1NjbK5XJJklwul9577z2dPHnSmvF4PHI6nUpLS7NmvnmOtpm2c1yKw+GQ0+kMewAAADN1eOT89re/1Z49e3Ts2DFVV1frn//5nxUdHa1JkyYpLi5OeXl5Kiws1BtvvKGamho98sgjcrlcuuOOOyRJ2dnZSktL00MPPaR3331X27dv18KFC5Wfny+HwyFJmjVrlj7++GPNnz9f9fX1Wrt2rbZs2aK5c+d29OUAAIAuqsPvyfnkk080adIkffbZZ7r++ut111136c0339T1118vSVq1apWioqI0ceJEBYNBud1urV271np+dHS0KioqNHv2bLlcLl177bWaNm2aFi9ebM2kpKRo27Ztmjt3rlavXq0bbrhBzz33HD8+DgAALLZQKBSK9CIiJRAIKC4uTn6/n4+urhIDntwW6SUA6CTHluZEegn4kVzu92/+dhUAADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwUodHTmlpqX76058qNjZW8fHxys3NVUNDQ9jMmDFjZLPZwh6zZs0Km2lsbFROTo569Oih+Ph4zZs3T+fOnQub2b17t0aMGCGHw6GBAweqrKysoy8HAAB0UR0eOXv27FF+fr7efPNNeTwetbS0KDs7W83NzWFzM2bM0IkTJ6zHsmXLrGPnz59XTk6Ozp49q+rqar344osqKyvTokWLrJmjR48qJydHY8eOVW1trebMmaPp06dr+/btHX1JAACgC+rW0SesrKwM2y4rK1N8fLxqamo0evRoa3+PHj2UmJh40XPs2LFD77//vnbu3KmEhAQNGzZMS5Ys0YIFC1RcXCy73a7169crJSVFK1askCQNHjxY+/bt06pVq+R2uzv6sgAAQBfT6ffk+P1+SVLv3r3D9m/cuFF9+/bVkCFDVFRUpK+++so65vV6lZ6eroSEBGuf2+1WIBDQ4cOHrZmsrKywc7rdbnm93s66FAAA0IV0+Ds539Ta2qo5c+boZz/7mYYMGWLtnzx5svr376/k5GQdOnRICxYsUENDg15++WVJks/nCwscSda2z+f71plAIKAzZ86oe/fuF6wnGAwqGAxa24FAoGMuFAAAXHE6NXLy8/NVV1enffv2he2fOXOm9e/09HQlJSVp3LhxOnLkiG6++eZOW09paalKSko67fwAAODK0WkfVxUUFKiiokJvvPGGbrjhhm+dzczMlCR99NFHkqTExEQ1NTWFzbRtt93Hc6kZp9N50XdxJKmoqEh+v996HD9+vP0XBgAAuoQOj5xQKKSCggK98sor2rVrl1JSUr7zObW1tZKkpKQkSZLL5dJ7772nkydPWjMej0dOp1NpaWnWTFVVVdh5PB6PXC7XJV/H4XDI6XSGPQAAgJk6PHLy8/P1pz/9SZs2bVJsbKx8Pp98Pp/OnDkjSTpy5IiWLFmimpoaHTt2TK+++qqmTp2q0aNHa+jQoZKk7OxspaWl6aGHHtK7776r7du3a+HChcrPz5fD4ZAkzZo1Sx9//LHmz5+v+vp6rV27Vlu2bNHcuXM7+pIAAEAX1OGRs27dOvn9fo0ZM0ZJSUnWY/PmzZIku92unTt3Kjs7W6mpqXriiSc0ceJEvfbaa9Y5oqOjVVFRoejoaLlcLj344IOaOnWqFi9ebM2kpKRo27Zt8ng8ysjI0IoVK/Tcc8/x4+MAAECSZAuFQqFILyJSAoGA4uLi5Pf7+ejqKjHgyW2RXgKATnJsaU6kl4AfyeV+/+ZvVwEAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjETkAAMBIRA4AADASkQMAAIxE5AAAACMROQAAwEhEDgAAMBKRAwAAjETkAAAAIxE5AADASEQOAAAwEpEDAACMROQAAAAjdfnIWbNmjQYMGKCYmBhlZmZq//79kV4SAAC4AnTpyNm8ebMKCwv1+9//XgcPHlRGRobcbrdOnjwZ6aUBAIAI69KRs3LlSs2YMUOPPPKI0tLStH79evXo0UMvvPBCpJcGAAAirMtGztmzZ1VTU6OsrCxrX1RUlLKysuT1eiO4MgAAcCXoFukFfF//+7//q/PnzyshISFsf0JCgurr6y/6nGAwqGAwaG37/X5JUiAQ6LyF4orSGvwq0ksA0En4f/nVo+2/dSgU+ta5Lhs530dpaalKSkou2N+vX78IrAYA0JHino70CvBj+/LLLxUXF3fJ4102cvr27avo6Gg1NTWF7W9qalJiYuJFn1NUVKTCwkJru7W1VZ9//rn69Okjm83WqesF8OMKBALq16+fjh8/LqfTGenlAOhAoVBIX375pZKTk791rstGjt1u18iRI1VVVaXc3FxJ/4iWqqoqFRQUXPQ5DodDDocjbF/Pnj07eaUAIsnpdBI5gIG+7R2cNl02ciSpsLBQ06ZN02233abbb79dTz/9tJqbm/XII49EemkAACDCunTk3H///fr73/+uRYsWyefzadiwYaqsrLzgZmQAAHD1sYW+69ZkAOiCgsGgSktLVVRUdMHH1ACuDkQOAAAwUpf9ZYAAAADfhsgBAABGInIAAICRiBwAAGAkIgcAABiJyAFghA8++EAbNmyw/kBvfX29Zs+erUcffVS7du2K8OoARAI/Qg6gy6usrNQvf/lLXXfddfrqq6/0yiuvaOrUqcrIyFBra6v27NmjHTt26O677470UgH8iIgcAF3enXfeqbvvvlt/+MMf9NJLL+nXv/61Zs+erf/4j/+Q9I8/zltTU6MdO3ZEeKUAfkxEDoAuLy4uTjU1NRo4cKBaW1vlcDi0f/9+DR8+XJJUV1enrKws+Xy+CK8UwI+Je3IAGMFms0mSoqKiFBMTE/YXimNjY+X3+yO1NAARQuQA6PIGDBigDz/80Nr2er268cYbre3GxkYlJSVFYmkAIqhL/xVyAJCk2bNn6/z589b2kCFDwo7/z//8DzcdA1ch7skBAABG4uMqAABgJCIHAAAYicgBAABGInIAAICRiBwAAGAkIgcAABiJyAEAAEYicgAAgJH+H4T1MbizesK2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp_df['Name'].str.split().map(len).value_counts().plot.bar()\n",
    "print(\"(Max, Min, Mean)\")\n",
    "temp_df['Name'].str.split().map(len).max(), temp_df['Name'].str.split().map(len).min(), temp_df['Name'].str.split().map(len).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a8f85d5-f2c0-4f8f-b659-f1ef2b2d24cd",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Generating Bad Names (over engineered)\n",
    "generate individual full names:\n",
    "- total garbage\n",
    "  - we can randomly shuffle the correct names\n",
    "- mistype names\n",
    "  - repeat random characters\n",
    "  - remove random characters\n",
    "- other correct words but not names\n",
    "- mix of all of above\n",
    "- mix bad and good names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e36f797-371a-4c91-93f6-e744fa8d5878",
   "metadata": {},
   "source": [
    "### Helper functions "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0dd85de-6ae7-4a74-8b67-d629ecdc0c1a",
   "metadata": {},
   "source": [
    "### Generate names with repeated and or mistyped characters between 1, 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "57217579-4900-4e1e-aff3-57bb07b9b1a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mistype_characters(name: str, num_repeats: int) -> str:\n",
    "    \"\"\"\n",
    "    Repeat random characters for a num of repeats, randomly turned off(to reduce the number of repeats) so words are not recognizably long.\n",
    "    \"\"\"\n",
    "    new_name: str = \"\"\n",
    "    for idx, ch in enumerate(name):\n",
    "        new_name += ch * random.randint(0, num_repeats)\n",
    "    \n",
    "    return new_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "46873888-4552-4dc0-b655-3fbc3a1c0077",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'مرييم'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mistype_characters(\"مريم\", 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3eb0ab27-37df-4e8d-b0ff-bd36803d48b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ززيير',\n",
       " 'صصففوح',\n",
       " 'عللا',\n",
       " 'بر',\n",
       " 'زاا',\n",
       " 'ننررةة',\n",
       " 'ععببددللنااعع',\n",
       " 'خخللو',\n",
       " 'مريم',\n",
       " 'اابب']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mistyped_names = []\n",
    "for name in names_df.iloc[:, 0].values:\n",
    "    mistyped_names.append(mistype_characters(name, 2))\n",
    "    \n",
    "mistyped_names[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6e65d54-0e98-44c7-bff5-a35330fb3a69",
   "metadata": {},
   "source": [
    "### Get some random but correct words from the internet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e210beb7-66de-4900-9a95-820b42b23cc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Positive</td>\n",
       "      <td>ممتاز نوعا ما . النظافة والموقع والتجهيز والشا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Positive</td>\n",
       "      <td>أحد أسباب نجاح الإمارات أن كل شخص في هذه الدول...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Positive</td>\n",
       "      <td>هادفة .. وقوية. تنقلك من صخب شوارع القاهرة الى...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Positive</td>\n",
       "      <td>خلصنا .. مبدئيا اللي مستني ابهار زي الفيل الاز...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Positive</td>\n",
       "      <td>ياسات جلوريا جزء لا يتجزأ من دبي . فندق متكامل...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                               text\n",
       "0  Positive  ممتاز نوعا ما . النظافة والموقع والتجهيز والشا...\n",
       "1  Positive  أحد أسباب نجاح الإمارات أن كل شخص في هذه الدول...\n",
       "2  Positive  هادفة .. وقوية. تنقلك من صخب شوارع القاهرة الى...\n",
       "3  Positive  خلصنا .. مبدئيا اللي مستني ابهار زي الفيل الاز...\n",
       "4  Positive  ياسات جلوريا جزء لا يتجزأ من دبي . فندق متكامل..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/ar_reviews_100k.tsv\", sep=\"\\t\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dbd21e4e-93b3-44ff-8b8b-f7f39b12d489",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 99999/99999 [00:03<00:00, 25394.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words: 351992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "word_counter = Counter()\n",
    "\n",
    "for sent in tqdm(df.iloc[:, 1], total=df.shape[0]):\n",
    "    # remove Punctuation first\n",
    "    sent = sent.translate(str.maketrans('', '', string.punctuation))\n",
    "    word_counter.update(sent.split())\n",
    "\n",
    "print(f\"Number of words: {len(word_counter)}\")\n",
    "some_common_words = word_counter.most_common()[10_000: 15_000]\n",
    "some_common_words = [i[0] for i in some_common_words]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "658c9342-1229-4076-abef-13d2ba7e465e",
   "metadata": {},
   "source": [
    "### Total Garabage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a68e7150-52a4-47bb-9c1c-723147c00856",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['رهزي',\n",
       " 'فوحص',\n",
       " 'عيلاا',\n",
       " 'رجوب',\n",
       " 'ازهى',\n",
       " 'ورنة',\n",
       " 'عدانفبعال',\n",
       " 'لوخد',\n",
       " 'ممري',\n",
       " 'عارب']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "garbage_names = []\n",
    "\n",
    "for name in names_df.iloc[:, 0].values:\n",
    "    name = list(name)\n",
    "    random.shuffle(name)\n",
    "    garbage_names.append(\"\".join(name))\n",
    "    \n",
    "garbage_names[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35d3f983-8b38-4c31-858c-6dc1e7e9f15f",
   "metadata": {},
   "source": [
    "## Generating Bad Names\n",
    "- [x] total garbage\n",
    "- [x] mistype\n",
    "- [x] correct words but not names\n",
    "- [x] mix bad names\n",
    "- [x] mix good and bad names\n",
    "    - 1 or 2 good names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a172bd24-fed0-47e9-83d8-3348fff36259",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mix(bad_names: List[str], good_names: List[str], num_names: int = 500) -> List[str]:\n",
    "    \"\"\"\n",
    "    Generate 3-words names that have 1 or 2 correct names in it.\n",
    "    \"\"\"\n",
    "    \n",
    "    names = []\n",
    "    for _ in range(num_names):\n",
    "        # pick 1 or 2 good names\n",
    "        num_good = random.randint(1, 2)\n",
    "        num_bad = 3 - num_good\n",
    "\n",
    "        name_words = []\n",
    "        name_words.extend(random.sample(good_names, num_good)) # good names\n",
    "        name_words.extend(random.sample(bad_names, num_bad)) # bad names\n",
    "        random.shuffle(name_words)\n",
    "        names.append(\" \".join(name_words))\n",
    "        \n",
    "    return names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d6ad45a5-175b-4a09-a8f6-27d921a87670",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of bad names: 20000\n"
     ]
    }
   ],
   "source": [
    "# garbage\n",
    "n = ds_size//2//5\n",
    "\n",
    "bad_names = []\n",
    "bad_names.extend(generate_fullnames(garbage_names, num_names=n))\n",
    "\n",
    "# mistyped\n",
    "bad_names.extend(generate_fullnames(mistyped_names, num_names=n))\n",
    "\n",
    "# correct but not names\n",
    "bad_names.extend(generate_fullnames(some_common_words, num_names=n))\n",
    "\n",
    "# mix bad names\n",
    "bad_names.extend(generate_fullnames(garbage_names+mistyped_names+some_common_words, num_names=n))\n",
    "\n",
    "# mix good and bad names\n",
    "bad_names.extend(generate_mix(garbage_names+mistyped_names+some_common_words, correct_names, num_names=n))\n",
    "\n",
    "print(f\"Number of bad names: {len(bad_names)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2c7b20fe-0fbb-4de9-8530-c7429322fcaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(bad_names, columns=[\"Name\"]).to_csv(\"data/full_bad_names.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "55a0d64c-efbd-46c0-8a98-e966aac986e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14193</th>\n",
       "      <td>بحصى دالل نحيسه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19965</th>\n",
       "      <td>شوف عمرمنصور جزر</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18974</th>\n",
       "      <td>اابهسىنلو مصلحى حنفى</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5328</th>\n",
       "      <td>ووههدىى ييسسميننه اا</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>ادهسع شباع مىعر</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Name\n",
       "14193       بحصى دالل نحيسه\n",
       "19965      شوف عمرمنصور جزر\n",
       "18974  اابهسىنلو مصلحى حنفى\n",
       "5328   ووههدىى ييسسميننه اا\n",
       "447         ادهسع شباع مىعر"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df = pd.read_csv(\"data/full_bad_names.csv\")\n",
    "temp_df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "577ada31-e1e0-4f43-a3cd-89d4ebe7083d",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Making the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d6ca067f-a4d1-43dd-b4c2-20adbf6ad04f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_correct = pd.read_csv(\"data/full_correct_names.csv\")\n",
    "df_incorrect = pd.read_csv(\"data/full_bad_names.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2a671937-45dc-41a1-a8bb-36160a3e74d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>حمايه تيسير لاوند</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>سعداحمد ابراهبم برى</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>مكة ولاء كريمه</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>زايد ثراء سحاق</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>الاسمر الشندى العوضى</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39995</th>\n",
       "      <td>حامد قطار عزيزة</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39996</th>\n",
       "      <td>ععددوو خليفة اسماء</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39997</th>\n",
       "      <td>العسوى رحاب وه</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39998</th>\n",
       "      <td>خادم بشاره لب</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39999</th>\n",
       "      <td>تتأثر الباز الورادنى</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Name  class\n",
       "0         حمايه تيسير لاوند    1.0\n",
       "1       سعداحمد ابراهبم برى    1.0\n",
       "2            مكة ولاء كريمه    1.0\n",
       "3            زايد ثراء سحاق    1.0\n",
       "4      الاسمر الشندى العوضى    1.0\n",
       "...                     ...    ...\n",
       "39995       حامد قطار عزيزة    0.0\n",
       "39996    ععددوو خليفة اسماء    0.0\n",
       "39997        العسوى رحاب وه    0.0\n",
       "39998         خادم بشاره لب    0.0\n",
       "39999  تتأثر الباز الورادنى    0.0\n",
       "\n",
       "[40000 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_df = df_correct.copy()\n",
    "full_df['class'] = 1\n",
    "full_df = pd.concat([full_df, df_incorrect], ignore_index=True)\n",
    "full_df.fillna(0, inplace=True)\n",
    "full_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b022bf4b-9e78-4ac7-9ccd-4d302e39ee33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32958</th>\n",
       "      <td>حزم الاختلاط لسسعع</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14925</th>\n",
       "      <td>غيداء سعاد مشيره</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10017</th>\n",
       "      <td>باسل معاز السلاوى</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12709</th>\n",
       "      <td>عبدالعالم اسمى امال</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38670</th>\n",
       "      <td>وروعة راوية جوهرة</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Name  class\n",
       "32958   حزم الاختلاط لسسعع    0.0\n",
       "14925     غيداء سعاد مشيره    1.0\n",
       "10017    باسل معاز السلاوى    1.0\n",
       "12709  عبدالعالم اسمى امال    1.0\n",
       "38670    وروعة راوية جوهرة    0.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_df.to_csv(f\"data/{ds_name}.csv\", index=False)\n",
    "temp_df = pd.read_csv(f\"data/{ds_name}.csv\")\n",
    "print(temp_df.shape)\n",
    "temp_df.sample(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}