01_CleaningText.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fcbd1ca2-2547-4d05-a0f1-8ffec77ac76c",
   "metadata": {},
   "source": [
    "# Cleaning text collected from Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "420e9e82-c0f0-48e8-8ea4-9be4a3de9c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "### This notebook is in charge of preparing the corpus for the text extraction process\n",
    "#### This preparation process uses the Wikipedia biographies collected as text files and runs the following process:\n",
    "#### For each file\n",
    "#### Cleaning process\n",
    "#### - Read files from text_dataset/\n",
    "#### - Clean blank lines\n",
    "#### - Delete information after \"=== See also section ===\"\n",
    "#### - Separate paragraphs into sentences\n",
    "#### - Create clean text file\n",
    "#### - Add file in cleanText/\n",
    "\n",
    "#### Indexing process\n",
    "#### - Read files from cleanText/\n",
    "#### - Add index to each section in the biography\n",
    "#### - Add index to each paragraph in the section\n",
    "#### - Create CSV file with information of the indexes\n",
    "#### - Add file in indexedParagraphs/\n",
    "#### - Add index to each sentence in the paragraph\n",
    "#### - Create a CSV file with information of the indexes\n",
    "#### - Add file in indexedSentences/\n",
    "\n",
    "#### It uses nltk and spacy libraries\n",
    "#### Directories information:\n",
    "#### text_dataset/ : collection of biographies. Each file is a biography. The file name is the same as the authors number ID in Wikipedia\n",
    "#### cleanText/ : collection of biographies after the cleaning process\n",
    "#### indexedParagraphs/ : collection of biographies in CSV format. Each row of the file represents a paragraph. Each row has a section name and paragraph index\n",
    "#### indexedSentences/ : collection of biographies in CSV format. Each row of the file represents a sentence. Each row has a section name and paragraph index, and sentence index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "175e8963-91f4-41cb-8056-dd8ed4e30861",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from _datetime import date\n",
    "import nltk.tokenize as nt\n",
    "import os\n",
    "import re\n",
    "# Update to improve sentences segmentation\n",
    "#import spacy library\n",
    "import spacy\n",
    "#load core english library\n",
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6de88234-f035-4d38-807c-70f3637a33b7",
   "metadata": {},
   "source": [
    "### 1. Cleaning text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2311d18f-1af7-4d64-a329-c4587787de45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 33309 entries, 0 to 33308\n",
      "Data columns (total 1 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   file_name  33309 non-null  object\n",
      "dtypes: object(1)\n",
      "memory usage: 260.4+ KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10002116.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10004137.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10006387.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000684.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10009278.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_name\n",
       "0  10002116.txt\n",
       "1  10004137.txt\n",
       "2  10006387.txt\n",
       "3   1000684.txt\n",
       "4  10009278.txt"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read all files in sample folder\n",
    "# return a list object of files in the given folder\n",
    "files_list = [f for f in os.listdir('text_dataset') if not f.startswith('.')]\n",
    "# parse to dataframe\n",
    "df_files = pd.DataFrame(files_list, columns=['file_name'])\n",
    "# df_files = df_files.query(\"file_name=='10085.txt'\")\n",
    "df_files.to_csv('totalBiographies.csv',index=False)\n",
    "\n",
    "df_files.info()\n",
    "df_files.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3688c567-3ede-497f-92c4-86a6aca2bbad",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_text(filename,folder):\n",
    "    fileObject = open(folder+filename, \"r\")\n",
    "    data = fileObject.readlines()\n",
    "    return data\n",
    "\n",
    "# save the file after cleaning\n",
    "def writeTextFile(directory, filename, content):\n",
    "    f = open(directory+filename, \"w\")\n",
    "    f.writelines(content)\n",
    "    f.close()\n",
    "\n",
    "# cleaning text, get rid of blank lines and section \"==See also\"\n",
    "# receives a paragraph\n",
    "def cleanText(text):\n",
    "    ctext=[]\n",
    "    for line in text:\n",
    "        if \"== See also ==\" in line:\n",
    "            break\n",
    "        if not len(line.strip()) < 1 :\n",
    "            ctext.append(re.sub(r'([^\\.]\\.)([A-Z][^\\.,;:])', r'\\1 \\2', line))\n",
    "    return ctext\n",
    "\n",
    "def identifySection(txt):\n",
    "    '''\n",
    "    args: text, the paragraph to identify\n",
    "    return: section and subsection\n",
    "    '''\n",
    "    typeSection = 0\n",
    "    if txt.startswith(\"==\", 0, 2):\n",
    "        typeSection = 1\n",
    "    if txt.startswith(\"===\", 0, 3):\n",
    "        typeSection = 2\n",
    "    \n",
    "    return typeSection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1ffce5d6-9e5a-42d5-8cdf-a7f42af244b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "\n",
    "# for biography_file in df_files.itertuples():\n",
    "for chunk in pd.read_csv('totalBiographies.csv', chunksize=1000):\n",
    "    df_file_name = pd.DataFrame()\n",
    "    df_file_name['file_name'] = chunk['file_name']\n",
    "    for df_row in df_file_name.itertuples():\n",
    "        # Adding a control to check if a file already exist    \n",
    "        # read text\n",
    "        text = read_text(df_row.file_name,\"text_dataset/\")\n",
    "        # clean empty lines\n",
    "        clean_text= cleanText(text)\n",
    "        # print number of paragraphs\n",
    "        # print(len(clean_text))\n",
    "        # save the file after cleaning\n",
    "        writeTextFile('cleanText/', df_row.file_name,clean_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "770be941-3602-4d09-b9c9-c82126789c2d",
   "metadata": {},
   "source": [
    "### 2. Adding index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b8d22c1a-9987-4e3e-b114-3f42c57d52ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def identifySection(txt):\n",
    "    '''\n",
    "    args: text, the paragraph to identify\n",
    "    return: section and subsection\n",
    "    '''\n",
    "    typeSection = 0\n",
    "    if txt.startswith(\"==\", 0, 2):\n",
    "        typeSection = 1\n",
    "    if txt.startswith(\"===\", 0, 3):\n",
    "        typeSection = 2\n",
    "    \n",
    "    return typeSection\n",
    "\n",
    "# used to extract the sections and subsections\n",
    "def setSection(biography_df):# row section\n",
    "    sectionTitle_list = []\n",
    "    lastTitle = \"\"\n",
    "    sectionTitle = \"\"\n",
    "    # for each paragraph in the dataframe\n",
    "    for prg in biography_df.itertuples():\n",
    "        #return the type of section: level 0, level 1 (belongs to previous section)\n",
    "        typeSection = identifySection(prg.paragraph)\n",
    "        if typeSection==0 and lastTitle == \"\":\n",
    "            sectionTitle = \"N/A\"\n",
    "        elif typeSection==1:\n",
    "            lastTitle=prg.paragraph.replace('\\n', '')\n",
    "            # sectionTitle = prg.paragraph\n",
    "            sectionTitle = lastTitle\n",
    "        elif typeSection==2:\n",
    "            sectionTitle = lastTitle+\" | \"+prg.paragraph.replace('\\n', '')\n",
    "        # add to a list\n",
    "        sectionTitle_list.append(sectionTitle)\n",
    "    return sectionTitle_list\n",
    "    # row subsection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fab4c20e-8411-493e-b105-4eecbad67113",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Structure: biography as a collection sections, sections as a collection of paragraphs\n",
    "## and paragraphs as a collection of sentences\n",
    "# Biography <- Section (s) <- Paragraph(s) <- Sentence(s)\n",
    "# for text_file_name in df_files.itertuples():\n",
    "for chunk in pd.read_csv('totalBiographies.csv', chunksize=500):\n",
    "    df_file_name = pd.DataFrame()\n",
    "    df_file_name['file_name'] = chunk['file_name']\n",
    "    for text_file_name in df_file_name.itertuples():\n",
    "        df_coded_result = pd.DataFrame()\n",
    "\n",
    "        #read text\n",
    "        text = read_text(text_file_name.file_name,\"cleanText/\")\n",
    "        biography_df = pd.DataFrame({'paragraph':text})\n",
    "        #index number\n",
    "        biography_df['paragraphIndex'] = list(range(len(biography_df)))\n",
    "\n",
    "        section_list = setSection(biography_df)\n",
    "        #section name\n",
    "        biography_df['section'] = section_list\n",
    "        # save information\n",
    "        biography_df.to_csv('indexedParagraphs/'+text_file_name.file_name.replace(\".txt\",\"\")+'.csv',index=False)\n",
    "\n",
    "        # for each paragraph in the biography\n",
    "        for paragraph_row in biography_df.itertuples():\n",
    "            # divide the paragraph into sentences\n",
    "            # ss=nt.sent_tokenize(paragraph_row.paragraph)\n",
    "            # biography_df_per_sent = pd.DataFrame({'sentences':ss})\n",
    "            # UPDATE: improve sentences segmentation\n",
    "            # print(paragraph_row.paragraph)\n",
    "            ss = nlp(u\"{}\".format(paragraph_row.paragraph.strip()))\n",
    "            nlp_text = [sent.text.strip() for sent in ss.sents]\n",
    "            biography_df_per_sent = pd.DataFrame({'sentences':nlp_text})\n",
    "            #\n",
    "            # add an index for sentences\n",
    "            biography_df_per_sent['sentenceIndex'] = list(range(len(biography_df_per_sent)))\n",
    "            biography_df_per_sent['paragraphIndex'] = paragraph_row.paragraphIndex\n",
    "            biography_df_per_sent['section'] = paragraph_row.section\n",
    "            biography_df_per_sent['wikiId'] = text_file_name.file_name.replace(\".txt\",\"\")\n",
    "\n",
    "            df_coded_result = df_coded_result.append(biography_df_per_sent)\n",
    "\n",
    "        df_coded_result.to_csv('indexedSentences/'+text_file_name.file_name.replace(\".txt\",\"\")+'.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4575948b-b419-4f92-8106-3f447658e20b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}