0_3_clean_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\RistoHinno\\AppData\\Local\\Temp\\ipykernel_37632\\940619647.py:7: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from collections import Counter\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('data/interim/stenos.csv')\n",
    "df2023=pd.read_csv('data/interim/stenos2023.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(213689, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(47236, 13)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2023.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2021    22656\n",
       "2016    20406\n",
       "2017    19997\n",
       "2020    19135\n",
       "2010    17255\n",
       "2014    17082\n",
       "2018    16466\n",
       "2012    16351\n",
       "2015    15225\n",
       "2022    13242\n",
       "2011    12972\n",
       "2019    12130\n",
       "2013    10772\n",
       "Name: year, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.year.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2022    31676\n",
       "2023    15560\n",
       "Name: year, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2023.year.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\RistoHinno\\AppData\\Local\\Temp\\ipykernel_37632\\1817094240.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df=df[df.year!=2022].append(df2023)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(247683, 13)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=df[df.year!=2022].append(df2023)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>heading</th>\n",
       "      <th>speaker</th>\n",
       "      <th>link_video</th>\n",
       "      <th>link_steno</th>\n",
       "      <th>index_pk</th>\n",
       "      <th>index_snd</th>\n",
       "      <th>text</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>time</th>\n",
       "      <th>seconds_from_start</th>\n",
       "      <th>video_link_wo_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10:00  Istungi rakendamine</td>\n",
       "      <td>Aseesimees Siim Kallas</td>\n",
       "      <td>https://www.youtube.com/watch?v=GaFL2Zc4OMg&amp;t=0</td>\n",
       "      <td>https://stenogrammid.riigikogu.ee/201911191000</td>\n",
       "      <td>PKP-25040</td>\n",
       "      <td>SND-559399</td>\n",
       "      <td>Auväärt kolleegid, auväärt Riigikogu, tere hommikust! Kui te nüüd korraks jälle keskendute meie suurepärase istungi rakendamisele ja räägite omavahelisi jutte edasi pärast, siis on mul suur rõõm avada Riigikogu täiskogu II istungjärgu 9. töönädala teisipäevane istung. Kõigepealt on eelnõude ja arupärimiste üleandmine, kui selleks on soovi. Eelnõude ja arupärimiste üleandmise soovi ma ei näe. Siis teeme palun kohaloleku kontrolli.  Kohaloleku kontroll Kohal on 75 Riigikogu liiget, puudub 26. Sellest piisab täielikult, et asuda meie päevakorra arutelu juurde.</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>19</td>\n",
       "      <td>10:00</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.youtube.com/watch?v=GaFL2Zc4OMg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      heading                 speaker  \\\n",
       "0  10:00  Istungi rakendamine  Aseesimees Siim Kallas   \n",
       "\n",
       "                                        link_video  \\\n",
       "0  https://www.youtube.com/watch?v=GaFL2Zc4OMg&t=0   \n",
       "\n",
       "                                       link_steno   index_pk   index_snd  \\\n",
       "0  https://stenogrammid.riigikogu.ee/201911191000  PKP-25040  SND-559399   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  text  \\\n",
       "0  Auväärt kolleegid, auväärt Riigikogu, tere hommikust! Kui te nüüd korraks jälle keskendute meie suurepärase istungi rakendamisele ja räägite omavahelisi jutte edasi pärast, siis on mul suur rõõm avada Riigikogu täiskogu II istungjärgu 9. töönädala teisipäevane istung. Kõigepealt on eelnõude ja arupärimiste üleandmine, kui selleks on soovi. Eelnõude ja arupärimiste üleandmise soovi ma ei näe. Siis teeme palun kohaloleku kontrolli.  Kohaloleku kontroll Kohal on 75 Riigikogu liiget, puudub 26. Sellest piisab täielikult, et asuda meie päevakorra arutelu juurde.   \n",
       "\n",
       "   year  month  day   time seconds_from_start  \\\n",
       "0  2019  11     19   10:00  0                   \n",
       "\n",
       "                            video_link_wo_time  \n",
       "0  https://www.youtube.com/watch?v=GaFL2Zc4OMg  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6317"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.text.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(241366, 13)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove cases where text is missing - might be some data collection errors,\n",
    "#but not much compared to overall amount of data (less than 2%)\n",
    "df=df[~pd.isnull(df.text)]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## See chars we have - is there something funny"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt_char=Counter(''.join(df.text).lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "158"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cnt_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(' ', 24400713),\n",
       " ('e', 18924027),\n",
       " ('a', 17088228),\n",
       " ('i', 14892231),\n",
       " ('s', 13622917),\n",
       " ('t', 11672334),\n",
       " ('l', 9130965),\n",
       " ('u', 9007766),\n",
       " ('n', 7411497),\n",
       " ('k', 7058496),\n",
       " ('o', 5956143),\n",
       " ('m', 5880225),\n",
       " ('d', 5637951),\n",
       " ('r', 4276758),\n",
       " ('v', 3224725),\n",
       " ('g', 2941112),\n",
       " ('p', 2375498),\n",
       " ('j', 2359137),\n",
       " ('h', 2333069),\n",
       " (',', 2311920),\n",
       " ('ä', 2056157),\n",
       " ('õ', 1761167),\n",
       " ('.', 1594811),\n",
       " ('ü', 1039977),\n",
       " ('b', 1011381),\n",
       " ('ö', 416892),\n",
       " ('!', 341567),\n",
       " ('\\xa0', 319491),\n",
       " ('0', 219697),\n",
       " ('f', 169592),\n",
       " ('1', 164217),\n",
       " ('-', 159462),\n",
       " ('2', 148869),\n",
       " ('?', 123519),\n",
       " ('–', 85216),\n",
       " ('5', 68026),\n",
       " ('3', 59229),\n",
       " ('\"', 57323),\n",
       " (':', 52576),\n",
       " ('4', 49329),\n",
       " ('6', 42252),\n",
       " ('7', 41100),\n",
       " ('9', 40711),\n",
       " ('8', 37162),\n",
       " ('%', 36779),\n",
       " ('c', 22125),\n",
       " ('[', 21258),\n",
       " (']', 21258),\n",
       " (')', 14607),\n",
       " ('(', 14231),\n",
       " ('‑', 11775),\n",
       " ('§', 9967),\n",
       " ('y', 6223),\n",
       " ('š', 5266),\n",
       " ('x', 3532),\n",
       " ('ž', 3512),\n",
       " ('w', 3364),\n",
       " ('z', 2685),\n",
       " (';', 2675),\n",
       " ('/', 2131),\n",
       " (\"'\", 2067),\n",
       " ('+', 1360),\n",
       " ('…', 1210),\n",
       " ('q', 359),\n",
       " ('&', 196),\n",
       " ('é', 145),\n",
       " ('\\u202f', 132),\n",
       " ('à', 116),\n",
       " ('á', 82),\n",
       " ('´', 54),\n",
       " ('č', 45),\n",
       " ('’', 38),\n",
       " ('“', 34),\n",
       " ('å', 29),\n",
       " ('×', 27),\n",
       " ('è', 25),\n",
       " ('„', 24),\n",
       " ('р', 23),\n",
       " ('с', 20),\n",
       " ('ó', 19),\n",
       " ('и', 19),\n",
       " ('=', 16),\n",
       " ('°', 16),\n",
       " ('@', 14),\n",
       " ('ï', 14),\n",
       " ('\\xad', 13),\n",
       " ('í', 11),\n",
       " ('у', 11),\n",
       " ('к', 11),\n",
       " ('ê', 10),\n",
       " ('”', 10),\n",
       " ('й', 10),\n",
       " ('ł', 9),\n",
       " ('ç', 9),\n",
       " ('ā', 8),\n",
       " ('±', 8),\n",
       " ('€', 7),\n",
       " ('²', 7),\n",
       " ('_', 7),\n",
       " ('м', 7),\n",
       " ('е', 7),\n",
       " ('â', 6),\n",
       " ('ø', 6),\n",
       " ('‰', 6),\n",
       " ('`', 6),\n",
       " ('ñ', 6),\n",
       " ('а', 6),\n",
       " ('о', 6),\n",
       " ('«', 6),\n",
       " ('»', 6),\n",
       " ('ú', 5),\n",
       " ('ń', 5),\n",
       " ('ğ', 5),\n",
       " ('т', 5),\n",
       " ('̈', 4),\n",
       " ('ë', 4),\n",
       " ('ò', 4),\n",
       " ('\\n', 4),\n",
       " ('н', 4),\n",
       " ('ō', 3),\n",
       " ('ė', 3),\n",
       " ('в', 3),\n",
       " ('−', 2),\n",
       " ('#', 2),\n",
       " ('¹', 2),\n",
       " ('ý', 2),\n",
       " ('̃', 2),\n",
       " ('ς', 2),\n",
       " ('д', 2),\n",
       " ('п', 2),\n",
       " ('ш', 2),\n",
       " ('ǟ', 2),\n",
       " ('ṣ', 2),\n",
       " ('з', 2),\n",
       " ('я', 2),\n",
       " ('\\x9f', 1),\n",
       " ('ę', 1),\n",
       " ('\\x84', 1),\n",
       " ('‒', 1),\n",
       " ('º', 1),\n",
       " ('\\x9b', 1),\n",
       " ('*', 1),\n",
       " ('\\x98', 1),\n",
       " ('α', 1),\n",
       " ('æ', 1),\n",
       " ('″', 1),\n",
       " ('β', 1),\n",
       " ('ē', 1),\n",
       " ('ß', 1),\n",
       " ('г', 1),\n",
       " ('~', 1),\n",
       " ('̀', 1),\n",
       " ('\\u200e', 1),\n",
       " ('ʼ', 1),\n",
       " ('ь', 1),\n",
       " ('щ', 1),\n",
       " ('ф', 1),\n",
       " ('ī', 1)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt_char.most_common(200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make text without punctuation (migth be useful for topic modelling)\n",
    "## Also remove some junk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'austatud riigikogu tere päevast teate mis'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_punct(text):\n",
    "    \"\"\"remove punctuation, junk from text\n",
    "    -INPUT:\n",
    "        -text: str of text to be cleaned\n",
    "    -OUTPUT:\n",
    "        -cleaned text str\"\"\"\n",
    "    text=re.sub('(?<=[^0-9])\\. ', ' ', text) #keep . if they are between characters (no space) or after numbers\n",
    "    text_clean=text.lower().replace('\\x98', ' ').replace('\\x9f', ' ').replace('\\x84', ' ').replace('\\x9b', ' ')\\\n",
    "    .replace('\\xad', ' ')\\\n",
    "    .replace('\\xa0', ' ').replace('...', ' ')\\\n",
    "    .replace('!',' ').replace('\\?',' ').replace('?',' ').replace(', ',' ').replace(')',' ')\\\n",
    "    .replace('(',' ').replace('\"',' ')\\\n",
    "    .replace('ºC', ' kraadi ').replace(' – ', ' ').replace(':', ' ').replace('–', ' ').\\\n",
    "    replace('[---]', ' ').replace('[', '').replace(']', '')\\\n",
    "    .replace(';', ' ').replace('+', ' pluss ').replace(' & ', '&').replace('”', ' ').\\\n",
    "    replace('„', ' ').replace('=', ' võrdub ')\\\n",
    "    .replace('´', \"'\").replace('’', \"'\").replace('“', ' ').replace(' ±', ' pluss-miinus ').\\\n",
    "    replace(' ..',' ').replace('`', \"'\")\\\n",
    "    .replace('…', ' ').replace('″', ' ').replace(' = ', ' võrdub ')\n",
    "    text_clean=' '.join(text_clean.split())\n",
    "    if text_clean.endswith('.'):\n",
    "        text_clean=text_clean[:-1]\n",
    "    return text_clean\n",
    "\n",
    "remove_punct('Austatud Riigikogu, tere päevast! Teate mis, ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_wo_punct']=df.text.apply(remove_punct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "129\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(' ', 24415246),\n",
       " ('e', 18924027),\n",
       " ('a', 17088228),\n",
       " ('i', 14892247),\n",
       " ('s', 13625661),\n",
       " ('t', 11672334),\n",
       " ('l', 9132333),\n",
       " ('u', 9009158),\n",
       " ('n', 7411505),\n",
       " ('k', 7058496),\n",
       " ('o', 5956143),\n",
       " ('m', 5880233),\n",
       " ('d', 5637967),\n",
       " ('r', 4276774),\n",
       " ('v', 3224741),\n",
       " ('g', 2941112),\n",
       " ('p', 2376866),\n",
       " ('j', 2359137),\n",
       " ('h', 2333069),\n",
       " ('ä', 2056157),\n",
       " ('õ', 1761183),\n",
       " ('ü', 1039977),\n",
       " ('b', 1011397),\n",
       " ('ö', 416892),\n",
       " ('0', 219697),\n",
       " ('f', 169592),\n",
       " ('1', 164217),\n",
       " ('-', 158633),\n",
       " ('2', 148869),\n",
       " ('.', 148026),\n",
       " ('5', 68026),\n",
       " ('3', 59229),\n",
       " ('4', 49329),\n",
       " ('6', 42252),\n",
       " ('7', 41100),\n",
       " ('9', 40711),\n",
       " ('8', 37162),\n",
       " ('%', 36779),\n",
       " ('c', 22125),\n",
       " (',', 19763),\n",
       " ('‑', 11775),\n",
       " ('§', 9967),\n",
       " ('y', 6223),\n",
       " ('š', 5266),\n",
       " ('x', 3532),\n",
       " ('ž', 3512),\n",
       " ('w', 3364),\n",
       " ('z', 2685),\n",
       " (\"'\", 2165),\n",
       " ('/', 2131),\n",
       " ('q', 359),\n",
       " ('&', 196),\n",
       " ('é', 145),\n",
       " ('à', 116),\n",
       " ('á', 82),\n",
       " ('č', 45),\n",
       " ('å', 29),\n",
       " ('×', 27),\n",
       " ('è', 25),\n",
       " ('р', 23),\n",
       " ('с', 20),\n",
       " ('ó', 19),\n",
       " ('и', 19),\n",
       " ('°', 16),\n",
       " ('@', 14),\n",
       " ('ï', 14),\n",
       " ('í', 11),\n",
       " ('у', 11),\n",
       " ('к', 11),\n",
       " ('ê', 10),\n",
       " ('й', 10),\n",
       " ('ł', 9),\n",
       " ('ç', 9),\n",
       " ('ā', 8),\n",
       " ('€', 7),\n",
       " ('²', 7),\n",
       " ('_', 7),\n",
       " ('м', 7),\n",
       " ('е', 7),\n",
       " ('â', 6),\n",
       " ('ø', 6),\n",
       " ('‰', 6),\n",
       " ('ñ', 6),\n",
       " ('а', 6),\n",
       " ('о', 6),\n",
       " ('«', 6),\n",
       " ('»', 6),\n",
       " ('ú', 5),\n",
       " ('ń', 5),\n",
       " ('ğ', 5),\n",
       " ('т', 5),\n",
       " ('̈', 4),\n",
       " ('ë', 4),\n",
       " ('ò', 4),\n",
       " ('н', 4),\n",
       " ('ō', 3),\n",
       " ('ė', 3),\n",
       " ('в', 3),\n",
       " ('−', 2),\n",
       " ('#', 2),\n",
       " ('¹', 2),\n",
       " ('ý', 2),\n",
       " ('̃', 2),\n",
       " ('ς', 2),\n",
       " ('д', 2),\n",
       " ('п', 2),\n",
       " ('ш', 2),\n",
       " ('ǟ', 2),\n",
       " ('ṣ', 2),\n",
       " ('з', 2),\n",
       " ('я', 2),\n",
       " ('ę', 1),\n",
       " ('‒', 1),\n",
       " ('º', 1),\n",
       " ('*', 1),\n",
       " ('α', 1),\n",
       " ('æ', 1),\n",
       " ('β', 1),\n",
       " ('ē', 1),\n",
       " ('ß', 1),\n",
       " ('г', 1),\n",
       " ('~', 1),\n",
       " ('̀', 1),\n",
       " ('\\u200e', 1),\n",
       " ('ʼ', 1),\n",
       " ('ь', 1),\n",
       " ('щ', 1),\n",
       " ('ф', 1),\n",
       " ('ī', 1)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we should have less funny characters now\n",
    "cnt_char_clean=Counter(''.join(df.text_wo_punct).lower())\n",
    "print(len(cnt_char_clean))\n",
    "cnt_char_clean.most_common(200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove \"v a h e a e g\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#not really said but indicates when vaheaeg started\n",
    "df['text_wo_punct']=df.text_wo_punct.str.replace('v a h e a e g', '')\n",
    "df['text_wo_punct']=df.text_wo_punct.str.replace('v a h e a eg', '')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/interim/stenos_cleaned.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}