BPC_populateDB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a3ae4472",
   "metadata": {},
   "source": [
    "# BPCorpus: Generating corpus assets\n",
    "\n",
    "Project repository: https://github.com/k-nem/bpcorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2391,
   "id": "2f3180cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aab2205",
   "metadata": {},
   "source": [
    "## Original files\n",
    "\n",
    "Original HTML files were scraped from knihi.com based on author list.\n",
    "\n",
    "The files and scraping scripts can be found in ['bpcorpus-collect' repository](https://github.com/k-nem/bpcorpus-collect).\n",
    "\n",
    "Local path:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2428,
   "id": "d710da55",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../../parsed'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06c7f1a5",
   "metadata": {},
   "source": [
    "## Collect full file list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2389,
   "id": "c18a7a03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFiles(path):\n",
    "    \"\"\"Walk through subfolders and retrieve HTML file list\"\"\"\n",
    "    htmls = []\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for name in files:\n",
    "            if name.endswith('.html') and 'page' not in name:\n",
    "                htmls.append(root+'/'+name)\n",
    "                \n",
    "    htmls = sorted(htmls)\n",
    "                \n",
    "    return htmls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2390,
   "id": "0a175ef6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../../../parsed/003_Ян Баршчэўскі/003_0004.html', '../../../parsed/003_Ян Баршчэўскі/003_0005.html', '../../../parsed/003_Ян Баршчэўскі/003_0007.html', '../../../parsed/003_Ян Баршчэўскі/003_0008.html', '../../../parsed/003_Ян Баршчэўскі/003_0009.html'] \n",
      "\n",
      " 5948\n"
     ]
    }
   ],
   "source": [
    "allFiles = getFiles(path)\n",
    "print(allFiles[:5], '\\n\\n', len(allFiles))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6710a210",
   "metadata": {},
   "source": [
    "## Original location URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2392,
   "id": "d229fc9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get original URL from CSV download log\n",
    "with open('../../../parsed/bpc_alllinks.csv', newline='') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    links = [row for row in reader]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2393,
   "id": "e3273ed8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'003_0003'"
      ]
     },
     "execution_count": 2393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links[3][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1503,
   "id": "8d4e8489",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getUrl(path):\n",
    "    \"\"\"Get original location URL\"\"\"\n",
    "    nm = path.split('/')[-1].split('.')[0]\n",
    "    url = 'https://knihi.com/'\n",
    "    for l in links:\n",
    "        if l[3] == nm:\n",
    "            url = 'https://knihi.com/' + l[10]\n",
    "    \n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2298,
   "id": "76f08d62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://knihi.com//Jan_Barsceuski/Dzvie_biarozy.html'"
      ]
     },
     "execution_count": 2298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getUrl('../../../parsed/003_Ян Баршчэўскі/003_0007.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1508,
   "id": "0444afec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://knihi.com/'"
      ]
     },
     "execution_count": 1508,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getUrl('incorrect file name')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "802d66b4",
   "metadata": {},
   "source": [
    "## Filter out irrelevant files "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce71ee80",
   "metadata": {},
   "source": [
    "Not all files are relevant for the corpus, since only poetic forms need to be added. To determine the genres and other metadata I parse the HTML with regular expression and extract the values from JS/HTML style comments like these (the format is determined by the knihi.com source files):\n",
    "<!-- HEADER_FIELD Authors: Ян Чачот -->\n",
    "<!-- HEADER_FIELD CreationYear: 1825-1846? -->"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2394,
   "id": "b8c2e308",
   "metadata": {},
   "outputs": [],
   "source": [
    "def metaP(file): \n",
    "    \"\"\"Get metadata from the files with relevant genre value to determine the final file list\"\"\"\n",
    "    \n",
    "    # genre list\n",
    "    genres = ['Верш', 'Паэма', 'Басня', 'Балада']\n",
    "    \n",
    "    with open(file,'r') as f:\n",
    "        raw = f.read()\n",
    "    \n",
    "    # the string below shows that there is no table of contents (as in a collection of works), so only singular texts are retrieved\n",
    "    if '<!-- TOC_BEGIN -->\\n<!-- TOC_END -->' in raw:\n",
    "\n",
    "        metadict = {}\n",
    "        meta = re.findall('(?<=<!-- HEADER_FIELD ).*(?= -->)', raw)\n",
    "\n",
    "        if meta: \n",
    "            for m in meta:\n",
    "                metadict[m.split(':')[0]] = m.split(':')[1].lstrip()\n",
    "\n",
    "        if 'StyleGenre' in metadict:\n",
    "\n",
    "            if '/' in metadict['StyleGenre']:\n",
    "\n",
    "                if ',' in metadict['StyleGenre']:\n",
    "                    metadict['StyleGenre'] = metadict['StyleGenre'].split(',')\n",
    "                    metadict['StyleGenre'] = [item.split('/')[1].capitalize() for item in metadict['StyleGenre'] if len(item.split('/')) > 1]\n",
    "\n",
    "                elif ';' in metadict['StyleGenre']:\n",
    "                    metadict['StyleGenre'] = metadict['StyleGenre'].split(';')\n",
    "                    metadict['StyleGenre'] = [item.split('/')[1].capitalize() for item in metadict['StyleGenre'] if len(item.split('/')) > 1]\n",
    "\n",
    "                else:\n",
    "                    metadict['StyleGenre'] = metadict['StyleGenre'].split('/')[1].capitalize()\n",
    "\n",
    "            if not isinstance(metadict['StyleGenre'], list):     \n",
    "                if metadict['StyleGenre'] in genres:\n",
    "                    \n",
    "                    return metadict, file\n",
    "\n",
    "            else:\n",
    "                styleList = [item for item in metadict['StyleGenre'] if item in genres]\n",
    "                if len(styleList) > 0:\n",
    "                    metadict['StyleGenre'] = styleList[0]\n",
    "                    \n",
    "                    return metadict, file\n",
    "                \n",
    "def getAllP(filelist):\n",
    "    \"\"\"Apply getP() to a file list\"\"\"\n",
    "    \n",
    "    allMetaList = []\n",
    "    finalFileList = []\n",
    "    \n",
    "    for file in filelist:\n",
    "        item = metaP(file)\n",
    "        if item:\n",
    "            allMetaList.append(item[0])\n",
    "            finalFileList.append(item[1])\n",
    "        else:\n",
    "            continue\n",
    "            \n",
    "    return allMetaList, finalFileList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2395,
   "id": "17dcbb8a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'Authors': 'Уладзіслаў Сыракомля',\n",
       "  'CreationYear': '[1911-1912]?',\n",
       "  'FirstPublicationYear': '1913; 1932',\n",
       "  'LangOrig': 'pol',\n",
       "  'Pravapis': 'A2008',\n",
       "  'PublicationYear': '1997',\n",
       "  'StyleGenre': 'Балада',\n",
       "  'Title': 'Каралі',\n",
       "  'Title2': '(Пацеркі)',\n",
       "  'Translation': 'Янка Купала'},\n",
       " '../../../parsed/013_Уладзіслаў Сыракомля/013_0004.html')"
      ]
     },
     "execution_count": 2395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metaP('../../../parsed/013_Уладзіслаў Сыракомля/013_0004.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2396,
   "id": "70d00032",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Authors': 'Ян Баршчэўскі',\n",
       " 'CreationYear': '13.07.1841',\n",
       " 'Edition': 'Ян Баршчэўскі. Выбраныя творы. Менск, МФ «Беларускі кнігазбор», 1998.',\n",
       " 'FirstPublicationYear': '1842',\n",
       " 'LangOrig': 'pol',\n",
       " 'Pravapis': 'A1957',\n",
       " 'PublicationYear': '1998',\n",
       " 'SectionAuthor': 'Балады',\n",
       " 'StyleGenre': 'Балада',\n",
       " 'Title': 'Дзве бярозы',\n",
       " 'Title2': 'З народных паданняў',\n",
       " 'Translation': 'Кастусь Цьвірка'}"
      ]
     },
     "execution_count": 2396,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pList = getAllP(allFiles)\n",
    "pList[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2403,
   "id": "82add775",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../../../parsed/261_Яўгенія Янішчыц/261_0012.html',\n",
       " '../../../parsed/261_Яўгенія Янішчыц/261_0014.html',\n",
       " '../../../parsed/261_Яўгенія Янішчыц/261_0015.html',\n",
       " '../../../parsed/265_Леанід Галубовіч/265_0001.html',\n",
       " '../../../parsed/265_Леанід Галубовіч/265_0002.html']"
      ]
     },
     "execution_count": 2403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pList[1][-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2404,
   "id": "99161ac6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3068"
      ]
     },
     "execution_count": 2404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pList[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c850675",
   "metadata": {},
   "source": [
    "This list will be used for corpus generation. Folder structure is arbitrary since metadata is located in the file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1356,
   "id": "7418e6e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in pList[0]:\n",
    "#     if 'LangOrig' in i:\n",
    "#         if i['LangOrig'] == 'yid':\n",
    "#             print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1357,
   "id": "bb25ba46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'deu', 'lat', 'pol', 'rus', 'ukr', 'yid'}"
      ]
     },
     "execution_count": 1357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# all original languages for translated works\n",
    "set([i['LangOrig'] for i in pList[0] if 'LangOrig' in i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84a8786b",
   "metadata": {},
   "source": [
    "## Author index ID\n",
    "This dictionary will be used to provide author IDs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1363,
   "id": "3a91ff1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def allAuthors(metalist):\n",
    "    \"\"\"Get author work count\"\"\"\n",
    "    \n",
    "    authors = {}\n",
    "\n",
    "    for item in metalist:\n",
    "        if 'Authors' in item.keys():\n",
    "            if item['Authors'] not in authors.keys():\n",
    "                authors[item['Authors']] = 1\n",
    "                \n",
    "            else:\n",
    "                authors[item['Authors']] += 1\n",
    "                \n",
    "        if 'Translation' in item.keys():\n",
    "            if item['Translation'] not in authors.keys():\n",
    "                authors[item['Translation']] = 1\n",
    "                \n",
    "            else:\n",
    "                authors[item['Translation']] += 1\n",
    "\n",
    "    return authors\n",
    "\n",
    "def authIds(authlist):\n",
    "    \"\"\"Generate author IDs and name dictionaries for tree population\"\"\"\n",
    "    \n",
    "    order = sorted(authlist, key=lambda x: str(x.split()[-1]))\n",
    "    authIds = {}\n",
    "    \n",
    "    for i, name in enumerate(order):\n",
    "\n",
    "        names = name.split()\n",
    "\n",
    "        if len(names) == 2:\n",
    "            forename = names[0]\n",
    "            surname = names[1]\n",
    "            names = {'Forename': forename, 'Surname': surname}\n",
    "            authIds[name] = {'name': names, 'id': i + 1}\n",
    "            \n",
    "        else:\n",
    "            authIds[name] = {'name': name, 'id': i + 1}\n",
    "        \n",
    "    return authIds "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2405,
   "id": "d7e305fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 2405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authCounts = allAuthors(pList[0])\n",
    "authCounts['Цётка']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2408,
   "id": "7e0f2fab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Ян Баршчэўскі': 8,\n",
       " 'Кастусь Цьвірка': 1,\n",
       " 'Уладзімір Мархель': 8,\n",
       " 'Рыгор Барадулін': 23,\n",
       " 'Ян Чачот': 3,\n",
       " 'Уладзіслаў Сыракомля': 11,\n",
       " 'Янка Купала': 860,\n",
       " 'Кастусь Каліноўскі': 1,\n",
       " 'Францішак Багушэвіч': 36,\n",
       " 'Міхась Машара': 1,\n",
       " 'Сяржук Сокалаў-Воюш': 2,\n",
       " 'Максім Танк': 24,\n",
       " 'Куба Паўтаржыцкі': 1,\n",
       " 'Пятро Бітэль': 1,\n",
       " 'Алег Лойка': 4,\n",
       " 'Янка Лучына': 2,\n",
       " 'Генадзь Тумаш': 1,\n",
       " 'Цётка': 4,\n",
       " 'Кароль Балінскі': 1,\n",
       " 'Герман Гесэ': 1,\n",
       " 'М. Ісакоўскі': 1,\n",
       " 'Аляксей Кальцоў': 1,\n",
       " 'Марыя Канапніцкая': 11,\n",
       " 'Ян Каспровіч': 1,\n",
       " 'М. Красільнікаў': 1,\n",
       " 'Юзаф Крашэўскі': 2,\n",
       " 'Іван Крылоў': 1,\n",
       " 'Раіса Кудашава': 1,\n",
       " 'Адам Міцкевіч': 9,\n",
       " 'Невядомы': 6,\n",
       " 'Мікалай Някрасаў': 5,\n",
       " \"Валяр'ян Паляшчук\": 1,\n",
       " 'Іда Пілецкая': 1,\n",
       " 'Карнель Уейскі': 1,\n",
       " 'Дзмітрый Цэнзар': 1,\n",
       " 'Фёдар Цютчаў': 1,\n",
       " 'Грыцька Чупрынка': 2,\n",
       " 'Тарас Шаўчэнка': 35,\n",
       " 'Піліп Шкулёў': 1,\n",
       " 'Л. Явалкоўская-Кашуцкая': 1,\n",
       " 'Яскулка': 1,\n",
       " 'Якуб Колас': 808,\n",
       " 'Сяргей Гарадзецкі': 2,\n",
       " 'Максім Рыльскі': 1,\n",
       " 'Паўло Тычына': 1,\n",
       " 'Георг Хэрвег': 1,\n",
       " 'Максім Багдановіч': 21,\n",
       " 'Алесь Гарун': 159,\n",
       " 'Уладзіслаў Галубок': 6,\n",
       " 'Цішка Гартны': 77,\n",
       " 'Янка Журба': 1,\n",
       " 'Канстанцыя Буйло': 23,\n",
       " 'Зоська Верас': 1,\n",
       " 'Міхась Чарот': 14,\n",
       " 'Кандрат Крапіва': 150,\n",
       " 'Уладзімір Жылка': 1,\n",
       " 'Уладзімір Дубоўка': 9,\n",
       " 'Язэп Пушча': 6,\n",
       " 'Мікола Гусоўскі': 1,\n",
       " 'Натальля Арсеньнева; Віктар Дарашкевіч': 1,\n",
       " 'Адам Русак': 1,\n",
       " 'Пятрусь Броўка': 18,\n",
       " 'Пятро Глебка': 17,\n",
       " 'Алесь Звонак': 4,\n",
       " 'Максім Лужанін': 22,\n",
       " 'Ларыса Геніюш': 2,\n",
       " 'Васіль Вітка': 7,\n",
       " 'Аляксей Русецкі': 1,\n",
       " 'Сяргей Грахоўскі': 6,\n",
       " 'Аркадзь Куляшоў': 57,\n",
       " 'Пімен Панчанка': 292,\n",
       " 'Алесь Бачыла': 1,\n",
       " 'Аляксей Пысін': 18,\n",
       " 'Анатоль Вялюгін': 1,\n",
       " 'Еўдакія Лось': 5,\n",
       " 'Уладзімір Караткевіч': 320,\n",
       " 'Ніл Гілевіч': 12,\n",
       " 'Анатоль Вярцінскі': 4,\n",
       " 'Юрась Свірка': 2,\n",
       " 'Марк Шагал': 6,\n",
       " 'Васіль Зуёнак': 5,\n",
       " 'Янка Сіпакоў': 2,\n",
       " 'Данута Бічэль-Загнетава': 8,\n",
       " 'Анатоль Грачанікаў': 7,\n",
       " 'Ніна Мацяш': 5,\n",
       " 'Раіса Баравікова': 3,\n",
       " 'Алесь Разанаў': 3,\n",
       " 'Яўгенія Янішчыц': 8,\n",
       " 'Леанід Галубовіч': 2}"
      ]
     },
     "execution_count": 2408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authCounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2411,
   "id": "2d5b851d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': {'Forename': 'Кастусь', 'Surname': 'Цьвірка'}, 'id': 77},\n",
       " {'name': 'Цётка', 'id': 80}]"
      ]
     },
     "execution_count": 2411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authDict = authIds(authCounts.keys())\n",
    "[authDict['Кастусь Цьвірка'], authDict['Цётка']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a80c4cb",
   "metadata": {},
   "source": [
    "## Check the most used tags within texts (optional)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6b1405a",
   "metadata": {},
   "source": [
    "The following code doesn't serve any function in the generation of the corpus apart from collecting information about the most used tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1368,
   "id": "2a8fd68c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def allTags(path):\n",
    "    \"\"\"Full tag list for a file\"\"\"\n",
    "    \n",
    "    with open(path,'r') as f:\n",
    "        raw = f.read()\n",
    "        \n",
    "    pat = re.compile('(?<=(<!-- BOOK_BEGIN -->))(\\s.*\\s?)*(?=(<!-- BOOK_END -->))')\n",
    "    tags = re.search(pat, raw)\n",
    "    \n",
    "    if tags: \n",
    "        soup = BeautifulSoup(tags.group(0),'html.parser')   \n",
    "        taglist = [[tag.name, tag.text, tag.attrs] for tag in soup.find_all()]\n",
    "    \n",
    "    return taglist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1673,
   "id": "b3c12cfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['p', 'З польскай — У. Сыракомлі', {}],\n",
       " ['i', 'З польскай — У. Сыракомлі', {}],\n",
       " ['p', '\\xa0', {}],\n",
       " ['p', 'Як ішоў я ў бой кіпячы,', {}],\n",
       " ['p', 'Як прашчаўся з хаткай,', {}],\n",
       " ['p', 'Тут Гануля мая з плачам:', {}],\n",
       " ['p', '«Пойдзеш гінуць, братка!', {}],\n",
       " ['p', 'Але буду я маліцца', {}],\n",
       " ['p', 'Па табе з трывогі,', {}],\n",
       " ['p', 'Ты ж прынось за то гасцінца —', {}]]"
      ]
     },
     "execution_count": 1673,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allTags('../../../parsed/013_Уладзіслаў Сыракомля/013_0004.html')[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2412,
   "id": "87fa1aec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tagCounts(files):  \n",
    "    \"\"\"Count non-empty tags\"\"\"\n",
    "    tagCount = {}\n",
    "    divs = []\n",
    "    sups = []\n",
    "    links = []\n",
    "    cen = []\n",
    "    bs = []\n",
    "    ps = []\n",
    "\n",
    "    for file in files:\n",
    "        with open(file,'r') as f:\n",
    "            raw = f.read()\n",
    "\n",
    "        pat = re.compile('(?<=(<!-- BOOK_BEGIN -->))(\\s.*\\s?)*(?=(<!-- BOOK_END -->))')\n",
    "        tags = re.search(pat, raw)\n",
    "\n",
    "        if tags:\n",
    "            soup = BeautifulSoup(tags.group(0),'html.parser')   \n",
    "            taglist = [[tag.name, tag.text, tag.attrs] for tag in soup.find_all()]\n",
    "\n",
    "            for tag in taglist:\n",
    "                if tag[1].strip():\n",
    "                    if tag[0] == 'div':\n",
    "                        if tag[2]['class']:\n",
    "                            if tag[2]['class'] == ['POETRY']:\n",
    "                                continue  \n",
    "\n",
    "                            else:\n",
    "                                divs.append([file, tag])\n",
    "\n",
    "                                if tag[0] not in tagCount.keys():\n",
    "                                    tagCount[tag[0]] = 1\n",
    "\n",
    "                                else:\n",
    "                                    tagCount[tag[0]] += 1\n",
    "\n",
    "                        else:\n",
    "                                divs.append([file, tag])\n",
    "\n",
    "                                if tag[0] not in tagCount.keys():\n",
    "                                    tagCount[tag[0]] = 1\n",
    "\n",
    "                                else:\n",
    "                                    tagCount[tag[0]] += 1\n",
    "\n",
    "                    elif tag[0] == 'sup':\n",
    "                        sups.append([file, tag])\n",
    "\n",
    "                        if tag[0] not in tagCount.keys():\n",
    "                            tagCount[tag[0]] = 1\n",
    "\n",
    "                        else:\n",
    "                            tagCount[tag[0]] += 1\n",
    "\n",
    "                    elif tag[0] == 'a':\n",
    "                        links.append([file, tag])\n",
    "\n",
    "                        if tag[0] not in tagCount.keys():\n",
    "                            tagCount[tag[0]] = 1\n",
    "\n",
    "                        else:\n",
    "                            tagCount[tag[0]] += 1\n",
    "\n",
    "                    elif tag[0] == 'center':\n",
    "                        cen.append([file, tag])\n",
    "\n",
    "                        if tag[0] not in tagCount.keys():\n",
    "                            tagCount[tag[0]] = 1\n",
    "\n",
    "                        else:\n",
    "                            tagCount[tag[0]] += 1\n",
    "\n",
    "                    elif tag[0] == 'b':\n",
    "                        bs.append([file, tag])\n",
    "\n",
    "                        if tag[0] not in tagCount.keys():\n",
    "                            tagCount[tag[0]] = 1\n",
    "\n",
    "                        else:\n",
    "                            tagCount[tag[0]] += 1\n",
    "\n",
    "                    elif tag[0] == 'p':\n",
    "                        if tag[1] != '\\xa0':\n",
    "                            ps.append([file, tag])\n",
    "\n",
    "                        if tag[0] not in tagCount.keys():\n",
    "                            tagCount[tag[0]] = 1\n",
    "\n",
    "                        else:\n",
    "                            tagCount[tag[0]] += 1\n",
    "\n",
    "    return tagCount, divs, sups, links, cen, bs, ps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2413,
   "id": "a3a67bcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tagInfo = tagCounts(pList[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2414,
   "id": "6aa96f0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'p': 127450, 'sup': 101, 'b': 159, 'a': 94, 'div': 163, 'center': 42}"
      ]
     },
     "execution_count": 2414,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2422,
   "id": "ae673dc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/028_Янка Купала/028_0493.html',\n",
       "  ['div', '\\n1 Суомі — Фінляндыя.\\n\\n', {'class': ['endnote-block']}]],\n",
       " ['../../../parsed/028_Янка Купала/028_0493.html',\n",
       "  ['div', '1 Суомі — Фінляндыя.\\n', {'class': ['footnote']}]],\n",
       " ['../../../parsed/028_Янка Купала/028_0561.html',\n",
       "  ['div',\n",
       "   '\\nЕўрапейскія дзяржавы пастанавілі запрасіць на міжнародную нараду ў Геную т. Леніна.\\n\\xa0\\nЗ газет 1921 г.\\n',\n",
       "   {'class': ['EPIGRAPH']}]]]"
      ]
     },
     "execution_count": 2422,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[1][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2423,
   "id": "ba6b7c26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/003_Ян Баршчэўскі/003_0007.html', ['sup', '1', {}]],\n",
       " ['../../../parsed/028_Янка Купала/028_0493.html', ['sup', '1', {}]],\n",
       " ['../../../parsed/028_Янка Купала/028_0493.html', ['sup', '1', {}]]]"
      ]
     },
     "execution_count": 2423,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[2][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2424,
   "id": "3b43bbc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/028_Янка Купала/028_0493.html',\n",
       "  ['a', '1', {'href': '#footnote-1'}]],\n",
       " ['../../../parsed/028_Янка Купала/028_0493.html',\n",
       "  ['a', '1', {'id': 'footnote-1'}]],\n",
       " ['../../../parsed/032_Алесь Гарун/032_0016.html',\n",
       "  ['a', '42', {'href': '#footnote-1'}]]]"
      ]
     },
     "execution_count": 2424,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[3][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2425,
   "id": "483c172a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/029_Якуб Колас/029_0401.html', ['center', '*', {}]],\n",
       " ['../../../parsed/029_Якуб Колас/029_0401.html', ['center', '*', {}]],\n",
       " ['../../../parsed/029_Якуб Колас/029_0401.html', ['center', '*', {}]]]"
      ]
     },
     "execution_count": 2425,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[4][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2430,
   "id": "53af84a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/003_Ян Баршчэўскі/003_0007.html', ['b', 'Каментары', {}]],\n",
       " ['../../../parsed/003_Ян Баршчэўскі/003_0008.html', ['b', 'Каментары', {}]],\n",
       " ['../../../parsed/003_Ян Баршчэўскі/003_0009.html', ['b', 'Каментары', {}]]]"
      ]
     },
     "execution_count": 2430,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[5][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2420,
   "id": "33758599",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/003_Ян Баршчэўскі/003_0007.html',\n",
       "  ['p', 'Ведае вёска: так шчыра Марылю', {}]],\n",
       " ['../../../parsed/003_Ян Баршчэўскі/003_0007.html',\n",
       "  ['p', 'Ясь, маладую, кахае!', {}]],\n",
       " ['../../../parsed/003_Ян Баршчэўскі/003_0007.html',\n",
       "  ['p', 'З Полацка возіць ёй стужкі штохвілю,', {}]]]"
      ]
     },
     "execution_count": 2420,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagInfo[6][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2421,
   "id": "f315181b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['../../../parsed/019_Янка Лучына/019_0001.html', '* * *'],\n",
       " ['../../../parsed/019_Янка Лучына/019_0001.html', '* * *'],\n",
       " ['../../../parsed/019_Янка Лучына/019_0001.html', '* * *'],\n",
       " ['../../../parsed/019_Янка Лучына/019_0001.html', '* * *'],\n",
       " ['../../../parsed/019_Янка Лучына/019_0001.html', '* * *']]"
      ]
     },
     "execution_count": 2421,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[p[0], p[1][1]] for p in tagInfo[6] if p[1][1] == '* * *'][:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9ec6895",
   "metadata": {},
   "source": [
    "## Build XML tree I"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e68a7167",
   "metadata": {},
   "source": [
    "Generation a XML tree with TEI header and verse structure on stanza level `<lg>` (tokenization and lemmatization will be performed at a later stage and a wider context is necessary for higher accuracy of tagging). Meta elements like epigraphs, footnotes and headings are placed into `<seg>` tags so that they are not included in lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1390,
   "id": "c10130f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRaw(path):\n",
    "    \"\"\"Open file\"\"\"\n",
    "    with open(path,'r') as f:\n",
    "        raw = f.read()   \n",
    "    return raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2307,
   "id": "a472d0b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def meta(raw): \n",
    "    \"\"\"Get metadata (without filtering)\"\"\"\n",
    "\n",
    "    metadict = {}\n",
    "    meta = re.findall('(?<=<!-- HEADER_FIELD ).*(?= -->)', raw)\n",
    "\n",
    "    if meta: \n",
    "        for m in meta:\n",
    "            metadict[m.split(':')[0]] = m.split(':')[1].lstrip()\n",
    "\n",
    "            \n",
    "    if 'Authors' in metadict:\n",
    "        if isinstance(authDict[metadict['Authors']]['name'], dict):\n",
    "            metadict['Forename'] = authDict[metadict['Authors']]['name']['Forename']\n",
    "            metadict['Surname'] = authDict[metadict['Authors']]['name']['Surname']\n",
    "            \n",
    "        metadict['aid'] = authDict[metadict['Authors']]['id']\n",
    "        \n",
    "        \n",
    "    if 'Translation' in metadict:\n",
    "        if isinstance(authDict[metadict['Translation']]['name'], dict):\n",
    "            metadict['trForename'] = authDict[metadict['Translation']]['name']['Forename']\n",
    "            metadict['trSurname'] = authDict[metadict['Translation']]['name']['Surname']\n",
    "            \n",
    "        metadict['tid'] = authDict[metadict['Translation']]['id']\n",
    "\n",
    "    if 'StyleGenre' in metadict:\n",
    "\n",
    "        if '/' in metadict['StyleGenre']:\n",
    "\n",
    "            if ',' in metadict['StyleGenre']:\n",
    "                metadict['StyleGenre'] = metadict['StyleGenre'].split(',')\n",
    "                metadict['StyleGenre'] = [item.split('/')[1].capitalize() for item in metadict['StyleGenre'] if len(item.split('/')) > 1]\n",
    "\n",
    "            elif ';' in metadict['StyleGenre']:\n",
    "                metadict['StyleGenre'] = metadict['StyleGenre'].split(';')\n",
    "                metadict['StyleGenre'] = [item.split('/')[1].capitalize() for item in metadict['StyleGenre'] if len(item.split('/')) > 1]\n",
    "\n",
    "            else:\n",
    "                metadict['StyleGenre'] = metadict['StyleGenre'].split('/')[1].capitalize()\n",
    "\n",
    "        else:\n",
    "            styleList = [item for item in metadict['StyleGenre'] if item in genres]\n",
    "            metadict['StyleGenre'] = styleList[0]\n",
    "            \n",
    "        if isinstance(metadict['StyleGenre'], list):\n",
    "            metadict['StyleGenre'] = metadict['StyleGenre'][0]\n",
    "            \n",
    "    attrs = ['Authors', 'CreationYear', 'Edition', 'FirstPublicationYear', 'LangOrig', 'PublicationYear', 'StyleGenre', 'Title', 'Translation', 'Forename', 'Surname', 'trForename', 'trSurname', 'tid']\n",
    "    for a in attrs:\n",
    "        if a not in metadict.keys():\n",
    "            metadict[a] = None\n",
    "    \n",
    "    return metadict\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2308,
   "id": "2d9a7b57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Authors': 'Уладзіслаў Сыракомля',\n",
       " 'CreationYear': '[1911-1912]?',\n",
       " 'FirstPublicationYear': '1913; 1932',\n",
       " 'LangOrig': 'pol',\n",
       " 'Pravapis': 'A2008',\n",
       " 'PublicationYear': '1997',\n",
       " 'StyleGenre': 'Балада',\n",
       " 'Title': 'Каралі',\n",
       " 'Title2': '(Пацеркі)',\n",
       " 'Translation': 'Янка Купала',\n",
       " 'Forename': 'Уладзіслаў',\n",
       " 'Surname': 'Сыракомля',\n",
       " 'aid': 70,\n",
       " 'trForename': 'Янка',\n",
       " 'trSurname': 'Купала',\n",
       " 'tid': 47,\n",
       " 'Edition': None}"
      ]
     },
     "execution_count": 2308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta(getRaw('../../../parsed/013_Уладзіслаў Сыракомля/013_0004.html'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2309,
   "id": "17cbe0b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Authors': 'Цётка',\n",
       " 'CreationYear': '1905',\n",
       " 'Edition': 'Беларуская літаратура ',\n",
       " 'FirstPublicationYear': '1905?',\n",
       " 'Pravapis': 'A1957',\n",
       " 'PublicationYear': '2004',\n",
       " 'SectionAuthor': 'Вершы',\n",
       " 'Source': 'скан',\n",
       " 'StyleGenre': 'Верш',\n",
       " 'Title': 'Вера беларуса',\n",
       " 'Uploaded': '2011-08-04T12',\n",
       " 'Year': '1905',\n",
       " 'aid': 80,\n",
       " 'LangOrig': None,\n",
       " 'Translation': None,\n",
       " 'Forename': None,\n",
       " 'Surname': None,\n",
       " 'trForename': None,\n",
       " 'trSurname': None,\n",
       " 'tid': None}"
      ]
     },
     "execution_count": 2309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta(getRaw('../../../parsed/026_Цётка/026_0002.html'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2388,
   "id": "4f6357c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def genTree(raw, treemeta, fileName):\n",
    "    \"\"\"Step 1. Generate TEI header and line group structure after collecting metadata\"\"\"\n",
    "    \n",
    "    #raw = getRaw(file)\n",
    "    #treemeta = meta(raw)\n",
    "    newtree = BeautifulSoup('', 'xml')\n",
    "    newtree.append(newtree.new_tag('TEI'))\n",
    "    \n",
    "    # teiHeader\n",
    "    \n",
    "    head = newtree.new_tag('teiHeader')\n",
    "    head.append(newtree.new_tag('fileDesc'))\n",
    "    \n",
    "    # titleStmt\n",
    "    \n",
    "    titlest = newtree.new_tag('titleStmt')\n",
    "    \n",
    "    if 'Title' in treemeta:\n",
    "        if treemeta['Title']:\n",
    "            titlest.append(newtree.new_tag('title'))\n",
    "            titlest.title.string = treemeta['Title']\n",
    "        \n",
    "    if 'Title2' in treemeta: \n",
    "        if treemeta['Title2']:\n",
    "            titlest.append(newtree.new_tag('subtitle'))\n",
    "            titlest.subtitle.string = treemeta['Title2']\n",
    "    \n",
    "    titlest.append(newtree.new_tag('author'))\n",
    "    titlest.author.append(newtree.new_tag('persName'))\n",
    "    \n",
    "    if 'Authors' in treemeta:\n",
    "        if treemeta['Authors']:\n",
    "            titlest.author.persName['ident'] = treemeta['aid']\n",
    "            if treemeta['Forename']:\n",
    "                titlest.author.persName.append(newtree.new_tag('forename'))\n",
    "                titlest.author.persName.forename.string = treemeta['Forename']\n",
    "                titlest.author.persName.append(newtree.new_tag('surname'))\n",
    "                titlest.author.persName.surname.string = treemeta['Surname']\n",
    "\n",
    "            else:\n",
    "                nm = newtree.new_tag('name')\n",
    "                nm.string = treemeta['Authors']\n",
    "                titlest.author.persName.append(nm)\n",
    "        \n",
    "    head.fileDesc.append(titlest)\n",
    "        \n",
    "    # sourceDesc\n",
    "    \n",
    "    src = newtree.new_tag('sourceDesc')\n",
    "    \n",
    "    if 'Edition' in treemeta:\n",
    "        if treemeta['Edition']:\n",
    "            origs = newtree.new_tag('bibl')\n",
    "            origs['type'] = 'originalSource'\n",
    "            origs.string = treemeta['Edition']\n",
    "            src.append(origs)\n",
    "        \n",
    "    digs = newtree.new_tag('bibl')\n",
    "    digs['type'] = 'digitalSource'\n",
    "    digs.append(newtree.new_tag('ptr'))\n",
    "    digs.ptr['target'] = getUrl(fileName)\n",
    "    src.append(digs)\n",
    "    \n",
    "    head.fileDesc.append(src)\n",
    "    \n",
    "    # profileDesc\n",
    "    \n",
    "    prof = newtree.new_tag('profileDesc')\n",
    "    tc = newtree.new_tag('textClass')\n",
    "    prof.append(tc)\n",
    "    kw = newtree.new_tag('keywords')\n",
    "    tc.append(kw)\n",
    "    \n",
    "    lu = newtree.new_tag('langUsage')\n",
    "    lang = newtree.new_tag('language')\n",
    "    lang['ident'] = 'be'\n",
    "    lang.string = 'Беларуская'\n",
    "    lu.append(lang)\n",
    "    prof.append(lu)\n",
    "    \n",
    "    form = newtree.new_tag('term')\n",
    "    form['type'] = 'form'\n",
    "    form.string = 'Паэзія'\n",
    "    kw.append(form)\n",
    "    \n",
    "    if 'StyleGenre' in treemeta:  \n",
    "        if treemeta['StyleGenre']: \n",
    "            genre = newtree.new_tag('term')\n",
    "            genre['type'] = 'genre'\n",
    "            genre.string = treemeta['StyleGenre']\n",
    "            kw.append(genre)\n",
    "        \n",
    "    head.fileDesc.append(prof)\n",
    "    \n",
    "    # textDesc\n",
    "    \n",
    "    txtd = newtree.new_tag('textDesc')\n",
    "    \n",
    "    if 'creationYear' in treemeta:\n",
    "        if treemeta['creationYear']: \n",
    "            txtd.append(newtree.new_tag('creationYear'))\n",
    "            txtd.creationYear['when'] = treemeta['CreationYear']\n",
    "            txtd.creationYear.string = treemeta['CreationYear']\n",
    "        \n",
    "    if 'publicationYear' in treemeta:\n",
    "        if treemeta['publicationYear']: \n",
    "            txtd.append(newtree.new_tag('publicationYear'))\n",
    "            txtd.publicationYear['when'] = treemeta['PublicationYear']\n",
    "            txtd.publicationYear.string = treemeta['PublicationYear']\n",
    "\n",
    "    if 'FirstPublicationYear' in treemeta:\n",
    "        if treemeta['FirstPublicationYear']: \n",
    "            txtd.append(newtree.new_tag('firstPublicationYear'))\n",
    "            txtd.firstPublicationYear['when'] = treemeta['FirstPublicationYear']\n",
    "            txtd.firstPublicationYear.string = treemeta['FirstPublicationYear']\n",
    "    \n",
    "    if 'Translation' in treemeta:\n",
    "        if treemeta['Translation']:\n",
    "            tra = newtree.new_tag('translator')\n",
    "            tra['ident'] = treemeta['tid']\n",
    "            if treemeta['trForename']:\n",
    "                tra.append(newtree.new_tag('forename'))\n",
    "                tra.forename.string = treemeta['trForename']\n",
    "                tra.append(newtree.new_tag('surname'))\n",
    "                tra.surname.string = treemeta['trSurname']\n",
    "\n",
    "            else:\n",
    "                nm = newtree.new_tag('name')\n",
    "                nm.string = treemeta['Translation']\n",
    "                tra.append(nm)\n",
    "                \n",
    "            txtd.append(tra)\n",
    "        \n",
    "    if 'LangOrig' in treemeta:\n",
    "        if treemeta['LangOrig']:\n",
    "            lNames = {'deu':['de', 'Нямецкая' ], 'lat': ['la', 'Лацінская'], 'pol': ['pl', 'Польская'], 'rus': ['ru', 'Руская'], 'ukr': ['uk', 'Украінская'], 'yid': ['yi', 'Ідыш']}\n",
    "\n",
    "            lo = newtree.new_tag('originalLanguage')\n",
    "            lo['ident'] = lNames[treemeta['LangOrig']][0]\n",
    "            lo.string = lNames[treemeta['LangOrig']][1]\n",
    "            txtd.append(lo)\n",
    "    \n",
    "    head.fileDesc.append(txtd)\n",
    "\n",
    "    newtree.TEI.append(head)\n",
    "    \n",
    "    body = newtree.new_tag('body')\n",
    "    newtree.TEI.append(body)\n",
    "    \n",
    "    # parse body\n",
    "    \n",
    "    pat = re.compile('(?<=(<!-- BOOK_BEGIN -->))(\\s.*\\s?)*(?=(<!-- BOOK_END -->))')\n",
    "    textbody = re.search(pat, raw)\n",
    "\n",
    "    soup = BeautifulSoup(textbody.group(0),'html.parser')\n",
    "    tags = [[tag.name, tag.text, tag.attrs] for tag in soup.find_all()]\n",
    "    \n",
    "    lg = newtree.new_tag('lg')\n",
    "    lg.string = ''\n",
    "    \n",
    "    if 'EPIGRAPH' in [tag[2]['class'][0] for tag in tags if tag[0]== 'div' and tag[2]['class']]:\n",
    "        for i,item in enumerate(tags):\n",
    "            if item[2]:\n",
    "                if 'class' in item[2]:\n",
    "                    if item[2]['class'][0] == 'CLEAR':\n",
    "                        for tag in tags[:i]:\n",
    "                            if tag[2]:\n",
    "                                if 'class' in tag[2]:\n",
    "                                    if tag[2]['class'][0] == 'EPIGRAPH':\n",
    "                                        seg = newtree.new_tag('seg')\n",
    "                                        seg['type'] = 'epigraph'\n",
    "\n",
    "                                        for line in tag[1].strip().split('\\n'):\n",
    "                                            l = newtree.new_tag('l')\n",
    "                                            l['type'] = 'ep'\n",
    "                                            l.string = line\n",
    "                                            seg.append(l)\n",
    "\n",
    "                                        newtree.body.append(seg)\n",
    "\n",
    "                        tags = tags[i+1:]\n",
    "                    \n",
    "    cseg = None\n",
    "    \n",
    "    if ['b','Каментары', {}] in tags:\n",
    "        for i,item in enumerate (tags):\n",
    "            if ['b','Каментары', {}] == item:\n",
    "                coms = [tag[1] for tag in tags[i+1:] if tag[0] == 'p']\n",
    "                cseg = newtree.new_tag('seg')\n",
    "                cseg['type'] = 'footnotes'\n",
    "                \n",
    "                for c in [c for c in coms if c != '\\xa0']:\n",
    "                    l = newtree.new_tag('l')\n",
    "                    dig = re.search('^\\d{1,2}[\\.|\\)]?(?=[^\\d])', c)\n",
    "                        \n",
    "                    if dig:\n",
    "                        s = newtree.new_tag('num')\n",
    "                        s.string = dig.group().strip('.)')\n",
    "                        l.append(s)\n",
    "\n",
    "                        com = newtree.new_tag('footnote')\n",
    "                        com.string = c.replace(dig.group(),'').strip()\n",
    "                        l.append(com)\n",
    "\n",
    "                    else:\n",
    "                        com = newtree.new_tag('footnote')\n",
    "                        com.string = c\n",
    "                        l.append(com)\n",
    "\n",
    "                    cseg.append(l) \n",
    "\n",
    "                tags = tags[:i-1]\n",
    "  \n",
    "\n",
    "    for i,tag in enumerate(tags): \n",
    "        if i == len(tags) - 1 and lg.string != '':        \n",
    "            newtree.body.append(lg)\n",
    "            \n",
    "        else:\n",
    "            if tag[0] == 'p':   \n",
    "                if tag[1] == '\\xa0':\n",
    "                    if lg.string:\n",
    "                        newtree.body.append(lg)\n",
    "                        lg = newtree.new_tag('lg')\n",
    "                        lg.string = ''\n",
    "                    \n",
    "                elif tag[1] == '*' or tag[1] == '* * *':\n",
    "                    if lg.string:\n",
    "                        newtree.body.append(lg)\n",
    "                        lg = newtree.new_tag('lg')\n",
    "                        lg.string = ''\n",
    "                        \n",
    "                    seg = newtree.new_tag('seg')\n",
    "                    seg.string = tag[1]\n",
    "                    seg['type'] = 'divider'\n",
    "                    newtree.body.append(seg)\n",
    "                    \n",
    "                elif tags[i + 1][0] == 'i' and tags[i + 1][1] == tag[1]:\n",
    "                    if lg.string:\n",
    "                        newtree.body.append(lg)\n",
    "                        lg = newtree.new_tag('lg')\n",
    "                        lg.string = ''\n",
    "                            \n",
    "                    seg = newtree.new_tag('seg')\n",
    "                    seg.string = tag[1]\n",
    "                    seg['type'] = 'italic'\n",
    "                    newtree.body.append(seg)\n",
    "                    \n",
    "                elif tags[i + 1][0] == 'b' and tags[i + 1][1] == tag[1]:\n",
    "                    if lg.string:\n",
    "                        newtree.body.append(lg)\n",
    "                        lg = newtree.new_tag('lg')\n",
    "                        lg.string = ''\n",
    "                            \n",
    "                    seg = newtree.new_tag('seg')\n",
    "                    seg.string = tag[1]\n",
    "                    seg['type'] = 'header'\n",
    "                    newtree.body.append(seg)\n",
    "                    \n",
    "                elif tag[1] == ' ':\n",
    "                    continue\n",
    "\n",
    "                else:\n",
    "                    year = re.findall('[1|2]\\d{3}', tag[1])\n",
    "                    \n",
    "                    if year:\n",
    "                        if lg.string:\n",
    "                            newtree.body.append(lg)\n",
    "                            lg = newtree.new_tag('lg')\n",
    "                            lg.string = ''\n",
    "                        \n",
    "                        seg = newtree.new_tag('seg')\n",
    "                        seg.string = tag[1]\n",
    "                        seg['type'] = 'timestamp'\n",
    "                        newtree.body.append(seg)\n",
    "                        \n",
    "                    else: \n",
    "                        lg.string += tag[1] + '\\n'\n",
    "     \n",
    "\n",
    "            elif tag[0] == 'div':\n",
    "                \n",
    "                if tag[2]['class'][0] == 'endnote-block' and tag[1].strip():\n",
    "                    if lg.string:\n",
    "                        newtree.body.append(lg)\n",
    "                        \n",
    "                    coms = [tag[1] for tag in tags[i+1:] if tag[0] == 'p']\n",
    "\n",
    "                    cseg = newtree.new_tag('seg')\n",
    "                    cseg['type'] = 'footnotes'\n",
    "\n",
    "                    for c in [c for c in coms if c != '\\xa0']:\n",
    "                        l = newtree.new_tag('l')\n",
    "                        dig = re.search('^\\d{1,2}[\\.|\\)]?(?=[^\\d])', c)\n",
    "\n",
    "                        if dig:\n",
    "                            s = newtree.new_tag('num')\n",
    "                            s.string = dig.group().strip('.)')\n",
    "                            l.append(s)\n",
    "\n",
    "                            com = newtree.new_tag('footnote')\n",
    "                            com.string = c.replace(dig.group(),'').strip()\n",
    "                            l.append(com)\n",
    "\n",
    "                        else:\n",
    "                            com = newtree.new_tag('footnote')\n",
    "                            com.string = c\n",
    "                            l.append(com)\n",
    "\n",
    "\n",
    "                        cseg.append(l) \n",
    "\n",
    "                    newtree.body.append(cseg)\n",
    "                    break\n",
    "                    \n",
    "            elif tag[0] == 'center':\n",
    "                if lg.string:\n",
    "                    newtree.body.append(lg)\n",
    "                    lg = newtree.new_tag('lg')\n",
    "                    lg.string = ''\n",
    "\n",
    "                seg = newtree.new_tag('seg')\n",
    "                seg.string = tag[1]\n",
    "                \n",
    "                if tag[1] == '*' or tag[1] == '* * *':\n",
    "                    seg['type'] = 'divider'\n",
    "                    newtree.body.append(seg)\n",
    "                    \n",
    "#             elif tag[0] == 'b':\n",
    "                \n",
    "#                 if lg.string:\n",
    "#                     newtree.body.append(lg)\n",
    "#                     lg = newtree.new_tag('lg')\n",
    "#                     lg.string = ''\n",
    "\n",
    "#                 b = newtree.new_tag('seg')\n",
    "#                 b.string = tag[1]\n",
    "#                 b['type'] = 'heading'\n",
    "#                 newtree.body.append(b)\n",
    "                    \n",
    "    if cseg != None:\n",
    "        newtree.body.append(cseg)\n",
    "    \n",
    "    return newtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2376,
   "id": "05630b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<TEI><teiHeader><fileDesc><titleStmt><title>«Сталася здарэнне дый каля крыніцы...»</title><author><persName ident=\"21\"><forename>Алесь</forename><surname>Гарун</surname></persName></author></titleStmt><sourceDesc><bibl type=\"originalSource\">Гарун А. Сэрцам пачуты звон</bibl><bibl type=\"digitalSource\"><ptr target=\"https://knihi.com//Ales_Harun/Stalasia_zdarennie_dyj_kala_krynicy_spz.html\"/></bibl></sourceDesc><profileDesc><textClass><keywords><term type=\"form\">Паэзія</term><term type=\"genre\">Верш</term></keywords></textClass><langUsage><language ident=\"be\">Беларуская</language></langUsage></profileDesc><textDesc><firstPublicationYear when=\"1929\">1929</firstPublicationYear></textDesc></fileDesc></teiHeader><body><seg type=\"epigraph\"><l type=\"ep\">Ja? konia poi?31.</l></seg><lg>Сталася здарэнне дый каля крыніцы\n",
       "У хлопца-баламута, ў сіроткі дзявіцы.\n",
       "Ясь спаткаў Марысю, што вадзіцу брала,\n",
       "Ды і ну дурыці, каб памандравала.\n",
       "«Забяры, Марыля, мамчына багацце,—\n",
       "Разам у чужыне мы дазнаем шчасця».\n",
       "«Што ж ты намаўляеш, як мне мандраваці,\n",
       "Ў доме ўсё замкнута, а ключы у маці,\n",
       "Ды і грэх вялікі мне за гэта будзе,\n",
       "Пан бог пакарае, свет увесь асудзе».\n",
       "«Ой, ключы нядоўга, дзевачка, дастаці,\n",
       "Папрасіся толькі у камору спаці.\n",
       "Людзі не пазнаюць і не будуць чуці,—\n",
       "Мы з табой здалеем на край свет умкнуці».\n",
       "Ну і збаламуціў хлопец той дзявіцу,\n",
       "Покі гаманілі, стоя ля крыніцы.\n",
       "Цёмнай ноччу ў вёсцы людзі моцна спалі,\n",
       "Ясечка з Марыляй ў свет памандравалі.\n",
       "Ведала ж Марыля ці сабе гадала,\n",
       "Што праз тры дзянёчкі Ясю абрыдала?\n",
       "Дзеўка ашуканца шчыра пакахала,\n",
       "А што ён надумаў, дык не спрачувала.\n",
       "Праз Дунай праз ціхі як пераплывалі,\n",
       "Скінуў Ясь Марылю з коніка у хвалі.\n",
       "Ускрыкнула дзяўчына ў страху, у распачы,—\n",
       "Ясь не аглянецца, ад злачынства скача.\n",
       "Нераты паблізу рыбакі стаўлялі,\n",
       "От яны нябозе ды ратунак далі.\n",
       "«Скуль, скажы, дзяўчынка, што каму зрабіла?»\n",
       "Плакала Марыля: «Я яго любіла».\n",
       "З свету баламуты каб усе спрагліся\n",
       "От за тыя слёзкі, што тут праліліся.\n",
       "</lg><seg type=\"footnotes\"><l><num>31</num><footnote>Ясь паіў каня (польск.).</footnote></l></seg></body></TEI>"
      ]
     },
     "execution_count": 2376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test1 = '../../../parsed/003_Ян Баршчэўскі/003_0007.html'\n",
    "test2 = '../../../parsed/028_Янка Купала/028_0493.html'\n",
    "test3 = '../../../parsed/031_Максім Багдановіч/031_0012.html'\n",
    "test4 = '../../../parsed/032_Алесь Гарун/032_0016.html'\n",
    "test5 = '../../../parsed/032_Алесь Гарун/032_0028.html'\n",
    "test6 = '../../../parsed/032_Алесь Гарун/032_0147.html'\n",
    "\n",
    "genTree(getRaw(test6), meta(getRaw(test6)), test6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2049,
   "id": "aebcf609",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['div',\n",
       "  '\\nЦветы последние милей\\nРоскошных  первенцев полей.\\nА. Пушкін\\n',\n",
       "  {'class': ['EPIGRAPH']}],\n",
       " ['p', 'Цветы последние милей', {}],\n",
       " ['p', 'Роскошных  первенцев полей.', {}],\n",
       " ['p', 'А. Пушкін', {'class': ['sign']}],\n",
       " ['div', '', {'class': ['CLEAR']}],\n",
       " ['p', '\\xa0', {}],\n",
       " ['p', 'Плакала лета, зямлю пакідаючы;', {}],\n",
       " ['p', 'Ціха ліліся сьлязінкі на поле.', {}],\n",
       " ['p', 'Але прыгожаю восеньню яснаю', {}],\n",
       " ['p', 'Там, дзе упалі яны, вырасталі', {}]]"
      ]
     },
     "execution_count": 2049,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getAllTags(test3)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16845aaa",
   "metadata": {},
   "source": [
    "## Generate simplified TXT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8d76a5a",
   "metadata": {},
   "source": [
    "Generation a simplified TXT file for NLP processing, including POS-tagging. Only verses proper are included in this file, headings, epigraphs, footnotes are removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2321,
   "id": "c8879b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = genTree(getRaw('../../../parsed/028_Янка Купала/028_0493.html'), meta('../../../parsed/028_Янка Купала/028_0493.html'), '../../../parsed/028_Янка Купала/028_0493.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2314,
   "id": "31aa72fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def simpleTxt(tree):\n",
    "    \"\"\"Generate simple TXT from <lg> only. Meta elements are omitted\"\"\"\n",
    "    \n",
    "    pmarks = '.,;:!?-–—()\\\"[]«»0123456789' # punctuation marks\n",
    "    txt = ''    \n",
    "    for lg in tree.body.find_all('lg'):\n",
    "        for char in lg.string.lower():\n",
    "            if char in pmarks:\n",
    "                continue\n",
    "            else:\n",
    "                txt += char\n",
    "        txt += '\\n'\n",
    "        \n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2320,
   "id": "bccf50e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'на поўначы сумнай у фіншчыне дзіўнай\\nракой з вадаспадам заліўся пакат\\nвуоксаю рэчка завецца у фінаў\\nіматрай завуць вадаспад\\n\\nклакочуць рагочуць іматрыны хвалі\\nна цэлыя вёрсты шумгоман стаіць\\nнем толечы каменны бераг як з сталі\\nі зараснік хвойны маўчыць\\n\\nшалее іматра між каменных глыбаў\\nза хваляю хвалю імчыць к нізіне\\nяк хмар недаступных махнатыя скібы\\nадна адну схопіць піргне\\n\\nадна з адной рынуцца ўглыб як магілу\\nтам скруцяцца ўзнімуцца клубам дугой\\nрассыплюцца пухам рассеюцца пылам\\nзноў выскачаць к небу гарой\\n\\nзірнуць ззіхануцца сыпнуцца на скалы\\nўсім дантаўскім процьмам на здзіў\\nі люнуць на волю забыўшыся шалаў\\nплывуць паміж пустак і ніў\\n\\nдругія іх зменяць і пеняцца ў зломе\\nадвечністым шумам калышуць прастор\\nсвабодай сваёю і роднай суомі\\nсягнуць быццам хочуць да зор\\n\\nстаіш і глядзіш на бунтоўныя воды\\nі сэрца лялеецца ў сцішнай жальбе\\nўсё слухаеш нема як стогнуць нягодай\\nды як бы ўсё клічуць цябе\\n\\nхадзі к нам бяспутнік кінь долю на свеце\\nспачын векавечны дамо ў забыцці\\nнязведанай воляю будзем шумеці\\nі гутарку з сонцам вясці\\n\\nхадзі ў нашы хвалі спаўём твае грудзі\\nвадзіцай сцюдзёнай кіпучай як вар\\nнарод аб нас казку злажыць не забудзе\\nі песню нам зложыць пясняр\\n\\nтак сумна на поўначы ў фіншчыне дзіўнай\\nракой з вадаспадам заліўся пакат\\nвуоксаю рэчка завецца у фінаў\\nіматрай завуць вадаспад\\n\\n'"
      ]
     },
     "execution_count": 2320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simpleTxt(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1667,
   "id": "17c56c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../simplified.txt', 'w') as s:\n",
    "    s.write(simpleTxt(tree))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba5e957c",
   "metadata": {},
   "source": [
    "## POS-tagging"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6654056f",
   "metadata": {},
   "source": [
    "Tagging is done by UDPipe/UFAL with the use of [UD Belarusian HSE](https://github.com/UniversalDependencies/UD_Belarusian-HSE) tagger model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 570,
   "id": "ea299114",
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "id": "24db3a83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'udpipe' already exists and is not an empty directory.\r\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/ufal/udpipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2431,
   "id": "f7581f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %cd ./udpipe/src\n",
    "# !make "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2429,
   "id": "3bd398f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tagger file to be used for processing\n",
    "tagger = '../../tagger/be-ud-2.7-tagger-20210115.udpipe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "id": "383c18a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading UDPipe model: done.\n",
      "# newdoc\n",
      "# newpar\n",
      "# sent_id = 1\n",
      "# text = Гэта\n",
      "1\tГэта\tгэта\tPRON\t_\tAnimacy=Inan|Case=Nom|Gender=Neut|Number=Sing|PronType=Dem\t_\t_\t_\tSpacesAfter=\\n\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!echo \"Гэта\" | ./udpipe --tokenize --tag '../../tagger/be-ud-2.7-tagger-20210115.udpipe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 572,
   "id": "e7667b5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = subprocess.check_output([\"./udpipe\", \"--tokenize\", \"--tag\", tagger], input=txt[0], text=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "id": "78cab419",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'# newdoc\\n# newpar\\n# sent_id = 1\\n# text = На поўначы сумнай, у Фіншчыне дзіўнай, Ракой з вадаспадам заліўся пакат; Вуоксаю рэчка завецца у фінаў, Іматрай завуць вадаспад.\\n1\\tНа\\tна\\tADP\\tIN\\t_\\t_\\t_\\t_\\t_\\n2\\tпоўначы\\tпоўнач\\tNOUN\\tNN\\tAnimacy=Inan|Case=Loc|Gender=Fem|Number=Sing\\t_\\t_\\t_\\t_\\n3\\tсумнай\\tсумны\\tADJ\\tJJL\\tCase=Gen|Degree=Pos|Gender=Fem|Number=Sing\\t_\\t_\\t_\\tSpaceAfter=No\\n4\\t,\\t,\\tPUNCT\\tPUNCT\\t_\\t_\\t_\\t_\\t_\\n5\\tу\\tу\\tADP\\tIN\\t_\\t_\\t_\\t_\\t_\\n6\\tФіншчыне\\tФіншчына\\tPROPN\\tNNP\\tAnimacy=Inan|Case=Loc|Gender=Fem|Number=Sing\\t_\\t_\\t_\\t_\\n7\\tдзіўнай\\tдзіўны\\tADJ\\tJJL\\tCase=Ins|Degree=Pos|Gender=Fem|Number=Sing\\t_\\t_\\t_\\tSpaceAfter=No\\n8\\t,\\t,\\tPUNCT\\tPUNCT\\t_\\t_\\t_\\t_\\tSpacesAfter=\\\\n\\n9\\tРакой\\tрака\\tNOUN\\tNN\\tAnimacy=Inan|Case=Ins|Gender=Fem|Number=Sing\\t_\\t_\\t_\\t_\\n10\\tз\\tз\\tADP\\tIN\\t_\\t_\\t_\\t_\\t_\\n11\\tвадаспадам\\tвадаспад\\tNOUN\\tNN\\tAnimacy=Inan|Case=Ins|Gender=Masc|Number=Sing\\t_\\t_\\t_\\t_\\n12\\tзаліўся\\tзаліцца\\tVERB\\tVBC\\tAspect=Perf|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid\\t_\\t_\\t_\\t_\\n13\\tпакат\\tпакат\\tNOUN\\tNN\\tAnimacy=Inan|Case=Nom|Gender=Masc|Number=Sing\\t_\\t_\\t_\\tSpaceAfter=No\\n14\\t;\\t;\\tPUNCT\\tPUNCT\\t_\\t_\\t_\\t_\\tSpacesAfter=\\\\n\\n15\\tВуоксаю\\tВуоксы\\tVERB\\tVBC\\tAspect=Imp|Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin|Voice=Act\\t_\\t_\\t_\\t_\\n16\\tрэчка\\tрэчка\\tNOUN\\tNN\\tAnimacy=Inan|Case=Nom|Gender=Fem|Number=Sing\\t_\\t_\\t_\\t_\\n17\\tзавецца\\tзвацца\\tVERB\\tVBC\\tAspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Mid\\t_\\t_\\t_\\t_\\n18\\tу\\tу\\tADP\\tIN\\t_\\t_\\t_\\t_\\t_\\n19\\tфінаў\\tфіна\\tNOUN\\tNN\\tAnimacy=Inan|Case=Gen|Gender=Masc|Number=Plur\\t_\\t_\\t_\\tSpaceAfter=No\\n20\\t,\\t,\\tPUNCT\\tPUNCT\\t_\\t_\\t_\\t_\\tSpacesAfter=\\\\n\\n21\\tІматрай\\tІматра\\tNOUN\\tNN\\tAnimacy=Inan|Case=Nom|Gender=Masc|Number=Sing\\t_\\t_\\t_\\t_\\n22\\tзавуць\\tзваць\\tVERB\\tVBC\\tAspect=Perf|Mood=Ind|Number=Plur|Person=3|Tense=Fut|VerbForm=Fin|Voice=Act\\t_\\t_\\t_\\t_\\n23\\tвадаспад\\tвадаспад\\tNOUN\\tNN\\tAnimacy=Inan|Case=Nom|Gender=Masc|Number=Sing\\t_\\t_\\t_\\tSpaceAfter=No\\n24\\t.\\t.\\tPUNCT\\tPUNCT\\t_\\t_\\t_\\t_\\tSpacesAfter=\\\\n\\n\\n'"
      ]
     },
     "execution_count": 575,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46d9d364",
   "metadata": {},
   "source": [
    "The output is produced in tab-separated UD [CoNLL-U format](https://universaldependencies.org/format.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1440,
   "id": "e3952afe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPos(text):\n",
    "    \"\"\"Transform ConNLL-U format into Python data structures\"\"\"\n",
    "    \n",
    "    output = subprocess.check_output([\"./udpipe\", \"--tokenize\", \"--tag\", tagger], input=text, text=True) \n",
    "    lemlist = [line.strip().split('\\t') for line in output.split('\\n') if '\\t' in line]\n",
    "    posTags = []\n",
    "\n",
    "    for l in lemlist:\n",
    "        if '=' in l[9]:\n",
    "            space = l[9].split('=')[1]\n",
    "        else:\n",
    "            space = None\n",
    "\n",
    "        posTags.append([l[1], l[2].lower(), l[3], l[4], l[5], space])\n",
    "        \n",
    "    return posTags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1442,
   "id": "1be1fa72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['лепшы',\n",
       "  'лепшы',\n",
       "  'ADJ',\n",
       "  'ORD',\n",
       "  'Case=Nom|Degree=Pos|Gender=Masc|Number=Sing',\n",
       "  '\\\\n']]"
      ]
     },
     "execution_count": 1442,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getPos('лепшы')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2316,
   "id": "2af100ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def posTag(tree):\n",
    "    \"\"\"Tokenize previously generated simple tree and populate it with POS tags\"\"\"\n",
    "    \n",
    "    newtree = tree\n",
    "    lgStr = ''\n",
    "    lgs = [lg for lg in tree.body.find_all_next('lg')]\n",
    "    \n",
    "    for lg in lgs: \n",
    "        lgStr += lg.string + '\\n\\n'\n",
    "        \n",
    "    posTags = getPos(lgStr)\n",
    "    \n",
    "    posLgs = [] #stanza list\n",
    "    posLg = [] #stanza\n",
    "    posL = [] #single line\n",
    "    \n",
    "    for pos in posTags:         \n",
    "        if pos[5] == '\\\\n':\n",
    "            posL.append([pos[0], pos[1], pos[2], pos[3], pos[4], None])\n",
    "            posLg.append(posL)\n",
    "            posL = []\n",
    "            \n",
    "        elif pos[5] == '\\\\n\\\\n\\\\n':\n",
    "            posL.append([pos[0], pos[1], pos[2], pos[3], pos[4], None])\n",
    "            posLg.append(posL)\n",
    "            posLgs.append(posLg)\n",
    "            posL = []\n",
    "            posLg = []\n",
    "            \n",
    "        else:\n",
    "            posL.append(pos)\n",
    "            \n",
    "    if len(posLgs) == len(lgs):        \n",
    "\n",
    "        for i,lg in enumerate(lgs):\n",
    "            lg.string = ''\n",
    "            \n",
    "            for posTags in posLgs[i]:\n",
    "                \n",
    "                line = newtree.new_tag('l')\n",
    "\n",
    "                for pos in posTags:                     \n",
    "                    if pos[2] == 'PUNCT':\n",
    "                        pc = newtree.new_tag('pc')\n",
    "                        pc.string = pos[0]\n",
    "                        \n",
    "                        if pos[5]:\n",
    "                            if 'No' in pos[5]: \n",
    "                                pc['join'] = 'right'\n",
    "                        \n",
    "                        line.append(pc)\n",
    "\n",
    "                    else:\n",
    "                        dig = re.search('(?=[\\w]*)\\d{1,2}', pos[0])\n",
    "                        if not dig:\n",
    "                            w = newtree.new_tag('w')\n",
    "                            w.string = pos[0]\n",
    "                            w['lemma'] = pos[1]\n",
    "                            w['pos'] = pos[2]\n",
    "                            w['ppos'] = pos[3]\n",
    "                            if pos[4] != '_':\n",
    "                                w['msd'] = pos[4]\n",
    "\n",
    "                            if pos[5]:\n",
    "                                if 'No' in pos[5]: \n",
    "                                    w['join'] = 'right'\n",
    "                                    \n",
    "                            line.append(w)\n",
    "                                    \n",
    "                        else:\n",
    "                            s = newtree.new_tag('sup')\n",
    "                            s.string = dig.group()\n",
    "                            \n",
    "                            w = newtree.new_tag('w')\n",
    "                            w.string = pos[0].replace(dig.group(),'')\n",
    "                            w['lemma'] = pos[0].replace(dig.group(),'')\n",
    "                            w['pos'] = pos[2]\n",
    "                            w['ppos'] = pos[3]\n",
    "                            if pos[4] != '_':\n",
    "                                w['msd'] = pos[4]\n",
    "                                \n",
    "                            if pos[5]:\n",
    "                                if 'No' in pos[5]: \n",
    "                                    s['join'] = 'right'\n",
    "                            \n",
    "                            line.append(w)\n",
    "                            line.append(s)\n",
    "                        \n",
    "                lg.append(line)\n",
    "                    \n",
    "    \n",
    "    return newtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2432,
   "id": "c9bbb912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<teiHeader><fileDesc><titleStmt><author><persName/></author></titleStmt><sourceDesc><bibl type=\"digitalSource\"><ptr target=\"https://knihi.com//Uladzislau_Syrakomla/Karali.html\"/></bibl></sourceDesc><profileDesc><textClass><keywords><term type=\"form\">Паэзія</term></keywords></textClass><langUsage><language ident=\"be\">Беларуская</language></langUsage></profileDesc><textDesc/></fileDesc></teiHeader>"
      ]
     },
     "execution_count": 2432,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpath = '../../../parsed/013_Уладзіслаў Сыракомля/013_0004.html'\n",
    "testtree = posTag(genTree(getRaw(tpath), meta(tpath), tpath))\n",
    "testtree.teiHeader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2319,
   "id": "649bc556",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open ('../../postest.xml', 'w') as s:\n",
    "    s.write(str(testtree))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed8f3abd",
   "metadata": {},
   "source": [
    "## Generate search index string\n",
    "Index string is the text that will be stored in FTS5 virtual table in database for search purposes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1818,
   "id": "7948c3e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "inpath = '../../../parsed/028_Янка Купала/028_0493.html'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1819,
   "id": "9dd6ea29",
   "metadata": {},
   "outputs": [],
   "source": [
    "testtree = posTag(genTree(inpath)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1820,
   "id": "b87365ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def searchInd(fulltree):\n",
    "    \"\"\"Collect tokens and lemmas\"\"\"\n",
    "    indexList = []\n",
    "    \n",
    "    for w in fulltree.body.find_all(['w']):\n",
    "        if w.string:\n",
    "            token = w.string.lower()\n",
    "            if w['lemma']:\n",
    "                lemma = w['lemma'].lower()\n",
    "                indexList.extend([token, lemma])\n",
    "\n",
    "            else:\n",
    "                indexList.extend(token)\n",
    "\n",
    "    indexString = ' '.join(set(indexList))      \n",
    "    return indexString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2434,
   "id": "94dc9501",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'цэркаўкай не плач спелых трэба цяжэй укленчыць здаля ды ў прынось па зламаць хутка радзімай людзі радзіўся марыі свяцейшай за бы тут маліцца дукатаў загудзелі укленчыў паручыла хутчэй зваць гасцінец мы заенчыць смела ісці далей шчасці замігціцца куль'"
      ]
     },
     "execution_count": 2434,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "searchInd(testtree)[:250]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "331d162a",
   "metadata": {},
   "source": [
    "## Generate HTML for website view\n",
    "New HTML markups includes styling of semantic elements and POS-tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1928,
   "id": "985a39c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mkHtml(tree):\n",
    "    \"\"\"Generate HTML for website viewing\"\"\"\n",
    "\n",
    "    htmltree = BeautifulSoup('', 'html')\n",
    "\n",
    "    for i,item in enumerate(tree.body.find_all(['lg', 'seg'])):\n",
    "\n",
    "        br = htmltree.new_tag('br')\n",
    "\n",
    "        if item.name == 'lg':  \n",
    "            st = htmltree.new_tag('span')\n",
    "            st['class'] = 'stanza'\n",
    "            st.string = ''\n",
    "\n",
    "            for line in item.find_all('l'):\n",
    "                p = htmltree.new_tag('p')\n",
    "                p.string = ''\n",
    "\n",
    "                for token in line.find_all():\n",
    "                    ssp = htmltree.new_tag('span')\n",
    "                    ssp['class'] = 'space'\n",
    "                    ssp.string = ' '\n",
    "\n",
    "                    if token.name == 'w':\n",
    "                        wsp = htmltree.new_tag('span')\n",
    "                        wsp.string = token.text\n",
    "                        wsp['class'] = 'token'\n",
    "                        wsp['title'] = ' '.join(token.attrs.values())\n",
    "                        p.append(wsp)\n",
    "\n",
    "                        if 'join' not in token.attrs:\n",
    "                            p.append(ssp)\n",
    "\n",
    "\n",
    "                    elif token.name == 'pc':\n",
    "                        psp = htmltree.new_tag('span')\n",
    "                        psp.string = token.text\n",
    "                        psp['class'] = 'pc'\n",
    "                        p.append(psp)\n",
    "\n",
    "                        if 'join' not in token.attrs:\n",
    "                            p.append(ssp)\n",
    "\n",
    "                    elif token.name == 'sup':\n",
    "                        sup = htmltree.new_tag('sup')\n",
    "                        sup.string = token.text\n",
    "                        p.append(sup)\n",
    "\n",
    "                        if 'join' not in token.attrs:\n",
    "                            p.append(ssp)\n",
    "\n",
    "                st.append(p)\n",
    "\n",
    "            htmltree.append(st)\n",
    "            htmltree.append(br)\n",
    "\n",
    "        if item.name == 'seg':\n",
    "            seg = htmltree.new_tag('span')\n",
    "\n",
    "            if 'type' in item.attrs:\n",
    "                seg['class'] = item['type']\n",
    "                \n",
    "            if seg['class'] == 'footnotes':\n",
    "                p = htmltree.new_tag('p')\n",
    "                p.string = ''\n",
    "                for item in tree.body.find_all(['num','footnote']):\n",
    "                    if item.name == 'num':\n",
    "                        sup = htmltree.new_tag('sup')\n",
    "                        sup.string = item.text\n",
    "                        p.append(sup)\n",
    "                        p.append(ssp)\n",
    "\n",
    "                    elif item.name == 'footnote':\n",
    "                        ftnt = htmltree.new_tag('span')\n",
    "                        ftnt['class'] = 'footnote'\n",
    "                        ftnt.string = item.text\n",
    "                        p.append(ftnt)\n",
    "                        seg.append(p)\n",
    "                        p = htmltree.new_tag('p')\n",
    "                        p.string = ''\n",
    "                        \n",
    "            elif seg['class'] == 'epigraph': \n",
    "                for line in item.find_all('l'):\n",
    "                    p = htmltree.new_tag('p')\n",
    "                    p.string = line.text\n",
    "                    seg.append(p)\n",
    "                \n",
    "            else:\n",
    "                seg.string = item.text\n",
    "\n",
    "            htmltree.append(seg)\n",
    "            htmltree.append(br)\n",
    "            \n",
    "    return htmltree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2014,
   "id": "5ad12b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "html = mkHtml(posTag(genTree('../../../parsed/019_Янка Лучына/019_0001.html')[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2015,
   "id": "7e991d82",
   "metadata": {},
   "outputs": [],
   "source": [
    "html.find('span', {'class' : 'epigraph'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2016,
   "id": "5f0f90b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../html2.html', 'w', encoding='utf-8') as h:\n",
    "    h.write(str(html))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2e45bb1",
   "metadata": {},
   "source": [
    "# Database\n",
    "## Create database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1935,
   "id": "0458be69",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "from sqlite3 import Error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1938,
   "id": "236413bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dbfile = '../../bpc.sqlite3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1941,
   "id": "c1798989",
   "metadata": {},
   "outputs": [],
   "source": [
    "def createDb(dbfile):\n",
    "    \"\"\"Create DB\"\"\"\n",
    "    conn = None\n",
    "    try:\n",
    "        conn = sqlite3.connect(dbfile)\n",
    "        print(sqlite3.version)\n",
    "    except Error as e:\n",
    "        print(e)\n",
    "    finally:\n",
    "        if conn:\n",
    "            conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1940,
   "id": "45397a37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.6.0\n"
     ]
    }
   ],
   "source": [
    "createDb(dbfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2089,
   "id": "da6a5faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def connect(dbfile):\n",
    "    \"\"\"DB connection\"\"\"\n",
    "    \n",
    "    conn = None\n",
    "    try:\n",
    "        conn = sqlite3.connect(dbfile, isolation_level = None)\n",
    "        return conn\n",
    "    except Error as e:\n",
    "        print(e)\n",
    "\n",
    "    return conn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2090,
   "id": "7d47c1ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def execute(dbfile, sql):\n",
    "    \"\"\"Execute request\"\"\"\n",
    "    \n",
    "    cur = connect(dbfile).cursor()\n",
    "    cur.execute(sql) \n",
    "    cur.close()\n",
    "    connect(dbfile).close() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2091,
   "id": "0bc8dca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData(dbfile, sql):\n",
    "    \"\"\"Retrieve DB rows\"\"\"\n",
    "    \n",
    "    conn = sqlite3.connect(dbfile)\n",
    "    cur = conn.cursor()\n",
    "    cur.execute(sql) \n",
    "    rows = cur.fetchall()\n",
    "    cur.close()\n",
    "    connect(dbfile).close() \n",
    "    \n",
    "    if rows:\n",
    "        return rows"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "804085f4",
   "metadata": {},
   "source": [
    "## Database schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d704621a",
   "metadata": {},
   "source": [
    "### `authors` table\n",
    "\n",
    "Schema:\n",
    "- id - primary key\n",
    "- name\n",
    "- forename\n",
    "- surname\n",
    "- life_born\n",
    "- life_died\n",
    "- gender\n",
    "- wiki_url\n",
    "\n",
    "4 first values will be populated from `authDict`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1937,
   "id": "71a6ffe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': {'Forename': 'Максім', 'Surname': 'Багдановіч'}, 'id': 2}"
      ]
     },
     "execution_count": 1937,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authDict['Максім Багдановіч']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1948,
   "id": "b9d38d38",
   "metadata": {},
   "outputs": [],
   "source": [
    "schA = 'CREATE TABLE IF NOT EXISTS authors (id integer PRIMARY KEY, \\\n",
    "                                            name text NOT NULL, \\\n",
    "                                            forename text, \\\n",
    "                                            surname text, \\\n",
    "                                            life_born integer, \\\n",
    "                                            life_died integer, \\\n",
    "                                            gender integer, \\\n",
    "                                            wiki_url text);'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1951,
   "id": "121600df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create authors table\n",
    "execute(dbfile, schA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1958,
   "id": "e050a9a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'id', 'INTEGER', 0, None, 1),\n",
       " (1, 'name', 'TEXT', 1, None, 0),\n",
       " (2, 'forename', 'TEXT', 0, None, 0),\n",
       " (3, 'surname', 'TEXT', 0, None, 0),\n",
       " (4, 'life_born', 'INTEGER', 0, None, 0),\n",
       " (5, 'life_died', 'INTEGER', 0, None, 0),\n",
       " (6, 'gender', 'INTEGER', 0, None, 0),\n",
       " (7, 'wiki_url', 'TEXT', 0, None, 0)]"
      ]
     },
     "execution_count": 1958,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getData(dbfile, 'PRAGMA table_info(authors)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1982,
   "id": "888d3291",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': {'Forename': 'Францішак', 'Surname': 'Багушэвіч'}, 'id': 3}"
      ]
     },
     "execution_count": 1982,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authDict['Францішак Багушэвіч']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1979,
   "id": "f22d47d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def auTab(adict):\n",
    "    \"\"\"Populate authors table\"\"\"\n",
    "    \n",
    "    conn = sqlite3.connect(dbfile, isolation_level = None)\n",
    "    cur = conn.cursor()\n",
    "    cur.execute('delete from authors') # reset table \n",
    "\n",
    "    for a in [(adict[i]['id'], i, adict[i]['name']) for i in adict.keys()]:\n",
    "        aid = a[0]\n",
    "        name = a[1]\n",
    "        if isinstance(a[2], dict):\n",
    "            forename = a[2]['Forename']\n",
    "            surname = a[2]['Surname'] \n",
    "            cur.execute('insert into authors (id, name, forename, surname) values (?, ?, ?, ?)', \\\n",
    "                        (aid, name, forename, surname))\n",
    "        else:\n",
    "            cur.execute('insert into authors (id, name) values (?, ?)', (aid, name))\n",
    "            \n",
    "    print('+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1980,
   "id": "e0861c46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+\n"
     ]
    }
   ],
   "source": [
    "auTab(authDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2086,
   "id": "8136b44e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 'Францішак Багушэвіч', 'Францішак', 'Багушэвіч', None, None, None, None)]"
      ]
     },
     "execution_count": 2086,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getData(dbfile, 'SELECT * FROM authors WHERE id = 3;')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "053e524d",
   "metadata": {},
   "source": [
    "### `text_meta` table\n",
    "\n",
    "- id - primary key\n",
    "- author_id - foreign key `authors (id)`\n",
    "- author_name\n",
    "- title\n",
    "- editions - bibl. information\n",
    "- genre\n",
    "- pub_year - publication year\n",
    "- cr_year - creation year\n",
    "- fp_year - first published\n",
    "- tr_author_id - translator, foreign key `authors (id)`\n",
    "- tr_author_name\n",
    "- tr_lang - original language, translation only\n",
    "- orfl - original file name for debugging\n",
    "\n",
    "This table will be populated with `meta()` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2435,
   "id": "95bd24e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "lNames = {'deu':['de', 'Нямецкая' ], \\\n",
    "          'lat': ['la', 'Лацінская'], \\\n",
    "          'pol': ['pl', 'Польская'], \\\n",
    "          'rus': ['ru', 'Руская'], \\\n",
    "          'ukr': ['uk', 'Украінская'], \\\n",
    "          'yid': ['yi', 'Ідыш']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2059,
   "id": "8f83e49e",
   "metadata": {},
   "outputs": [],
   "source": [
    "schTm = 'CREATE TABLE IF NOT EXISTS text_meta (id integer PRIMARY KEY, \\\n",
    "                                                author_id integer, \\\n",
    "                                                author_name text, \\\n",
    "                                                title text, \\\n",
    "                                                edition text, \\\n",
    "                                                genre text, \\\n",
    "                                                pub_year integer, \\\n",
    "                                                cr_year integer, \\\n",
    "                                                fp_year integer, \\\n",
    "                                                tr_author_id integer, \\\n",
    "                                                tr_author_name text, \\\n",
    "                                                tr_lang text, \\\n",
    "                                                orfl text, \\\n",
    "                                                FOREIGN KEY (author_id) REFERENCES authors (id), \\\n",
    "                                                FOREIGN KEY (tr_author_id) REFERENCES authors (id));'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2060,
   "id": "81c417f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create meta table\n",
    "execute(dbfile, schTm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2061,
   "id": "1ff3abd4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'id', 'INTEGER', 0, None, 1),\n",
       " (1, 'author_id', 'INTEGER', 0, None, 0),\n",
       " (2, 'author_name', 'TEXT', 0, None, 0),\n",
       " (3, 'title', 'TEXT', 0, None, 0),\n",
       " (4, 'edition', 'TEXT', 0, None, 0),\n",
       " (5, 'genre', 'TEXT', 0, None, 0),\n",
       " (6, 'pub_year', 'INTEGER', 0, None, 0),\n",
       " (7, 'cr_year', 'INTEGER', 0, None, 0),\n",
       " (8, 'fp_year', 'INTEGER', 0, None, 0),\n",
       " (9, 'tr_author_id', 'INTEGER', 0, None, 0),\n",
       " (10, 'tr_author_name', 'TEXT', 0, None, 0),\n",
       " (11, 'tr_lang', 'TEXT', 0, None, 0),\n",
       " (12, 'orfl', 'TEXT', 0, None, 0)]"
      ]
     },
     "execution_count": 2061,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getData(dbfile, 'PRAGMA table_info(text_meta)')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e2ee46",
   "metadata": {},
   "source": [
    "### `text_file` table\n",
    "\n",
    "- id - primary key, foreign key `text_meta (id)`\n",
    "- xml\n",
    "- txt \n",
    "- html\n",
    "\n",
    "This table contains all file assets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2002,
   "id": "1c39bcac",
   "metadata": {},
   "outputs": [],
   "source": [
    "schTf = 'CREATE TABLE IF NOT EXISTS text_files (id integer NOT NULL PRIMARY KEY, \\\n",
    "                                                xml text, \\\n",
    "                                                txt text, \\\n",
    "                                                html text, \\\n",
    "                                                FOREIGN KEY (id) REFERENCES text_meta (id));'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2003,
   "id": "eac7c669",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create file table\n",
    "execute(dbfile, schTf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2004,
   "id": "67ad5fb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'id', 'INTEGER', 1, None, 1),\n",
       " (1, 'xml', 'TEXT', 0, None, 0),\n",
       " (2, 'txt', 'TEXT', 0, None, 0),\n",
       " (3, 'html', 'TEXT', 0, None, 0)]"
      ]
     },
     "execution_count": 2004,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getData(dbfile, 'PRAGMA table_info(text_files)')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "273901fe",
   "metadata": {},
   "source": [
    "### Virtual search index table\n",
    "\n",
    "The table will use SQLite3 FTS5 extention.\n",
    "\n",
    "- id - foreign key `text_meta (id)`\n",
    "- author\n",
    "- title\n",
    "- wordlist - string created by `searchInd()` function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2005,
   "id": "9471a7cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "vt = 'CREATE VIRTUAL TABLE search USING fts5(id, author, title, wordlist);'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2006,
   "id": "33d147f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "execute(dbfile, vt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2085,
   "id": "3804f5e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'id', '', 0, None, 0),\n",
       " (1, 'author', '', 0, None, 0),\n",
       " (2, 'title', '', 0, None, 0),\n",
       " (3, 'wordlist', '', 0, None, 0)]"
      ]
     },
     "execution_count": 2085,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getData(dbfile, 'PRAGMA table_info(search)')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75ffce37",
   "metadata": {},
   "source": [
    "# Corpus database population\n",
    "\n",
    "The results of the previously defined functions are generated for every file and inserted into database.\n",
    "\n",
    "- `meta()` is the source of `text_meta` table values\n",
    "- `genTree() -> posTag()`, `simpleTxt()`, `mkHtml()` are written to `text_files` as well as to OS file system\n",
    "- `searchInd()` and some `meta()` values are stored in `search` virtual table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2052,
   "id": "048cda30",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2055,
   "id": "b347ca69",
   "metadata": {},
   "outputs": [],
   "source": [
    "folders = ['../../assets', '../../assets/xml', '../../assets/txt', '../../assets/html']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2338,
   "id": "7e04479e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in folders:\n",
    "    if not os.path.isdir(f):\n",
    "        os.mkdir(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2111,
   "id": "9aaf5353",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test values\n",
    "execute(dbfile, 'INSERT INTO text_meta (id, orfl) values  (2, \"031_0012\")')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2112,
   "id": "7de23c68",
   "metadata": {},
   "outputs": [],
   "source": [
    "execute(dbfile, 'delete from text_meta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2278,
   "id": "4a18f69c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def addedFiles(dbfile):\n",
    "    \"\"\"Get last ID and added files list\"\"\"\n",
    "    rows = getData(dbfile, 'select * from text_meta order by id desc')\n",
    "    \n",
    "    if rows:\n",
    "        lastId = rows[0][0]\n",
    "        files = [row[12] for row in rows]\n",
    "        \n",
    "    else:\n",
    "        lastId = 0\n",
    "        files = []\n",
    "    \n",
    "    return lastId, files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2279,
   "id": "faa17dc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20,\n",
       " ['003_0011',\n",
       "  '003_0010',\n",
       "  '003_0009',\n",
       "  '003_0008',\n",
       "  '003_0007',\n",
       "  '013_0008',\n",
       "  '013_0006',\n",
       "  '013_0005',\n",
       "  '013_0004',\n",
       "  '013_0003',\n",
       "  '013_0002',\n",
       "  '013_0001',\n",
       "  '007_0004',\n",
       "  '007_0002',\n",
       "  '007_0001',\n",
       "  '003_0011',\n",
       "  '003_0010',\n",
       "  '003_0009',\n",
       "  '003_0008',\n",
       "  '003_0007'])"
      ]
     },
     "execution_count": 2279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "addedFiles(dbfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2323,
   "id": "912c6d04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Authors': 'Ян Баршчэўскі',\n",
       " 'CreationYear': '13.07.1841',\n",
       " 'Edition': 'Ян Баршчэўскі. Выбраныя творы. Менск, МФ «Беларускі кнігазбор», 1998.',\n",
       " 'FirstPublicationYear': '1842',\n",
       " 'LangOrig': 'pol',\n",
       " 'Pravapis': 'A1957',\n",
       " 'PublicationYear': '1998',\n",
       " 'SectionAuthor': 'Балады',\n",
       " 'StyleGenre': 'Балада',\n",
       " 'Title': 'Дзве бярозы',\n",
       " 'Title2': 'З народных паданняў',\n",
       " 'Translation': 'Кастусь Цьвірка',\n",
       " 'Forename': 'Ян',\n",
       " 'Surname': 'Баршчэўскі',\n",
       " 'aid': 7,\n",
       " 'trForename': 'Кастусь',\n",
       " 'trSurname': 'Цьвірка',\n",
       " 'tid': 77}"
      ]
     },
     "execution_count": 2323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta(getRaw(test1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2337,
   "id": "0ce8a636",
   "metadata": {},
   "outputs": [],
   "source": [
    "def populate(filelist, dbfile):\n",
    "    \n",
    "    added = addedFiles(dbfile)\n",
    "    \n",
    "    txid = added[0] + 1\n",
    "    done = added[1]\n",
    "    \n",
    "    cur = connect(dbfile).cursor()\n",
    "    \n",
    "    for file in filelist:\n",
    "\n",
    "        fname = None\n",
    "        fname = file.split('/')[-1].split('.')[0]\n",
    "        \n",
    "        raw = None\n",
    "        metadict = None\n",
    "        tree = None\n",
    "        txt = None\n",
    "        xml = None\n",
    "        search = None\n",
    "        html = None\n",
    "\n",
    "        if fname not in done:\n",
    "\n",
    "            # generate files\n",
    "            \n",
    "            raw = getRaw(file)\n",
    "            metadict = meta(raw)\n",
    "            tree = genTree(raw, metadict, file)\n",
    "            txt = simpleTxt(tree)\n",
    "            xml = posTag(tree)\n",
    "            search = searchInd(xml)\n",
    "            html = mkHtml(xml)\n",
    "            \n",
    "            # text_meta\n",
    "            \n",
    "            metaTuple = (txid, \\\n",
    "                            metadict['aid'], \\\n",
    "                            metadict['Authors'], \\\n",
    "                            metadict['Title'], \\\n",
    "                            metadict['Edition'], \\\n",
    "                            metadict['StyleGenre'], \\\n",
    "                            metadict['PublicationYear'], \\\n",
    "                            metadict['CreationYear'], \\\n",
    "                            metadict['FirstPublicationYear'], \\\n",
    "                            metadict['tid'], \\\n",
    "                            metadict['Translation'], \\\n",
    "                            metadict['LangOrig'], \\\n",
    "                            fname)\n",
    "\n",
    "            cur.execute('INSERT INTO text_meta ( id, \\\n",
    "                                                author_id, \\\n",
    "                                                author_name, \\\n",
    "                                                title, \\\n",
    "                                                edition, \\\n",
    "                                                genre, \\\n",
    "                                                pub_year, \\\n",
    "                                                cr_year, \\\n",
    "                                                fp_year, \\\n",
    "                                                tr_author_id, \\\n",
    "                                                tr_author_name, \\\n",
    "                                                tr_lang, \\\n",
    "                                                orfl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', metaTuple)\n",
    "            \n",
    "            \n",
    "            # text_files\n",
    "            \n",
    "            fileTuple = (txid, str(xml), str(txt), str(html))\n",
    "            \n",
    "            cur.execute('INSERT INTO text_files ( id, \\\n",
    "                                                xml, \\\n",
    "                                                txt, \\\n",
    "                                                html ) values (?, ?, ?, ?)', fileTuple)\n",
    "            # search\n",
    "            \n",
    "            searchTuple = (txid, metadict['Authors'].lower(), metadict['Title'].lower(), search)\n",
    "            \n",
    "            cur.execute('INSERT INTO search ( id, \\\n",
    "                                                author, \\\n",
    "                                                title, \\\n",
    "                                                wordlist ) values (?, ?, ?, ?)', searchTuple)\n",
    "            \n",
    "            \n",
    "            # save files\n",
    "            \n",
    "            with open('../../assets/xml/' + str(txid) + '.xml', 'w') as xmlfile:\n",
    "                xmlfile.write(str(xml))\n",
    "                \n",
    "            with open('../../assets/txt/' + str(txid) + '.txt', 'w') as txtfile:\n",
    "                txtfile.write(str(txt))\n",
    "                \n",
    "            with open('../../assets/html/' + str(txid) + '.html', 'w') as htmlfile:\n",
    "                htmlfile.write(str(html))\n",
    "            \n",
    "            \n",
    "            txid += 1\n",
    "            done.append(fname)\n",
    "            \n",
    "\n",
    "    cur.close()\n",
    "    connect(dbfile).close() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2387,
   "id": "521f617f",
   "metadata": {},
   "outputs": [],
   "source": [
    "populate(pList[1][:3068], dbfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cedf0148",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "052ddb6a",
   "metadata": {},
   "source": [
    "#### Debug zone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2355,
   "id": "a54ac9e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3068"
      ]
     },
     "execution_count": 2355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts = pList[1][:3068]\n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2373,
   "id": "384ed8b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../../../parsed/032_Алесь Гарун/032_0147.html'"
      ]
     },
     "execution_count": 2373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pList[1][1869]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2328,
   "id": "4bce103a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../../../parsed/032_Алесь Гарун/032_0016.html 1754\n",
      "../../../parsed/selected/032_0016.html 3123\n"
     ]
    }
   ],
   "source": [
    "for i,f in enumerate(pList[1]):\n",
    "    if '032_0015' in f:\n",
    "        print(pList[1][i + 1], i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2358,
   "id": "ed48c8dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,f in enumerate(pList[1]):\n",
    "    if '032_0015' in f:\n",
    "        print(pList[1][i + 1], i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "209e1767",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b51ea16d",
   "metadata": {},
   "source": [
    "# Search\n",
    "SQL requests with `MATCH` predicate can be used to find the texts that contain a word.\n",
    "Requesting a lemma returns the instances of all its forms, and requesting a particular form will return the this form only.\n",
    "Queries bigger than one word will return invalid results (search indices have arbitrary order)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2436,
   "id": "177e9017",
   "metadata": {},
   "outputs": [],
   "source": [
    "req = 'SELECT id FROM search WHERE wordlist MATCH \"суомі\";'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2465,
   "id": "a6043c8c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dbfile = '../../bpc.sqlite3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2466,
   "id": "7b15b76f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(query):\n",
    "    \n",
    "    conn = sqlite3.connect(dbfile)\n",
    "    cur = conn.cursor()\n",
    "    cur.execute('SELECT id FROM search WHERE wordlist MATCH (?);', (query,))\n",
    "    rows = cur.fetchall()\n",
    "    cur.close()\n",
    "    connect(dbfile).close() \n",
    "    \n",
    "    if rows:\n",
    "        return [r[0] for r in rows]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2467,
   "id": "ae76e76e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[490, 747]"
      ]
     },
     "execution_count": 2467,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search('суомі')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2468,
   "id": "0e695e98",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getInfo(ids):\n",
    "    \n",
    "    conn = sqlite3.connect(dbfile)\n",
    "    cur = conn.cursor()\n",
    "    txInfo = []\n",
    "    \n",
    "    for txid in ids:\n",
    "        cur.execute('SELECT id, author_name, title, cr_year FROM text_meta WHERE id = (?);', (txid,))\n",
    "        rows = cur.fetchall()\n",
    "        if rows:\n",
    "            txInfo.append(rows[0])\n",
    "        \n",
    "    \n",
    "    cur.close()\n",
    "    connect(dbfile).close() \n",
    "    \n",
    "    return txInfo "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2469,
   "id": "7018055f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(490, 'Янка Купала', 'Над Іматрай', 1910),\n",
       " (747, 'Янка Купала', 'Фінляндыі', 1939)]"
      ]
     },
     "execution_count": 2469,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getInfo(search('суомі'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}