lm_cv8be.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vh6RwgBjxiRX"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_y2Y3EeJxi8K"
   },
   "source": [
    "## build KenLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B51xr44hxkGE",
    "outputId": "9c824e86-27b3-416f-a151-68f2159b4b5b"
   },
   "outputs": [],
   "source": [
    "!sudo apt install \\\n",
    "    build-essential cmake libboost-system-dev libboost-thread-dev \\\n",
    "    libboost-program-options-dev libboost-test-dev libeigen3-dev zlib1g-dev \\\n",
    "    libbz2-dev liblzma-dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SXwewbbFxm8m",
    "outputId": "2e0f8147-fd4a-473f-a976-31c6e284c0f6"
   },
   "outputs": [],
   "source": [
    "!wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5wxF6hDixogx",
    "outputId": "63455dc3-ff0b-48f5-f054-32fb4f105ea6"
   },
   "outputs": [],
   "source": [
    "!mkdir kenlm/build && cd kenlm/build && cmake .. && make -j2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bzpr2-HkxqBe"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2RcJIGPaxiJP"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5MDwlpDdkZlz"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import shutil\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "96Pe4dnekZfz",
    "outputId": "ce77fa13-0bc3-485b-f3a4-b278a38560e3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive/; to attempt to forcibly remount, call drive.mount(\"/content/gdrive/\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "gdrive_mount_dp = '/content/gdrive/'\n",
    "drive.mount(gdrive_mount_dp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aCAnx2oipYLw"
   },
   "outputs": [],
   "source": [
    "out_dp = 'lm'\n",
    "os.makedirs(out_dp, exist_ok=True)\n",
    "\n",
    "local_data_dp = 'data'\n",
    "os.makedirs(local_data_dp, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PMBDc36ZknHP"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2oiYkjCVyVd2"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IzsRu036yVyf"
   },
   "source": [
    "## build LM from CommonVoice 8 belarusian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ka0xAqYc0Lyt"
   },
   "outputs": [],
   "source": [
    "gdrive_exp_root_dp = input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "38c37Pohgy83",
    "outputId": "4a87463a-b53f-4fe5-a71f-e57ec0dd2048"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sentences_for_lm.tsv']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_gdrive_fp = os.path.join(gdrive_exp_root_dp, 'data', 'sentences_for_lm.tsv'\n",
    "shutil.copy2(data_gdrive_fp, local_data_dp)\n",
    "os.listdir(local_data_dp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "85XZI13nhJq1",
    "outputId": "6f92a396-e517-4cc7-8351-e0fcc2f7779c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(314676, 2)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join(local_data_dp, 'sentences_for_lm.tsv'), sep='\\t')\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "id": "bwSXZ-I-hWyD",
    "outputId": "c7cdd46a-0d50-4481-ff0c-c07d85e6ea5c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-d1a93a66-b615-4b77-9193-99d1a4556d0b\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Вы ўжо ўсе, відавочна, адчулі, што нешта не так?</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Зойдзе да яе, а яна не ведае, з чаго пачынаць ...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d1a93a66-b615-4b77-9193-99d1a4556d0b')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-d1a93a66-b615-4b77-9193-99d1a4556d0b button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-d1a93a66-b615-4b77-9193-99d1a4556d0b');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                            sentence  split\n",
       "0   Вы ўжо ўсе, відавочна, адчулі, што нешта не так?  train\n",
       "1  Зойдзе да яе, а яна не ведае, з чаго пачынаць ...  train"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AE16r_ZOhlGQ",
    "outputId": "74b26549-5682-4a08-cb3f-297b12834edf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train    314305\n",
       "other       371\n",
       "Name: split, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['split'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "V41lEnyvhn7C"
   },
   "outputs": [],
   "source": [
    "assert df.duplicated('sentence').sum() == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5jRBtP0nhJjp"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pLkVS5besl4o"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oK69SRRDsm88"
   },
   "source": [
    "## kenlm input format:\n",
    "* 1 sentence per line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "O19mdx7Hspid"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GSmPkJdfhvbM"
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "otmII1M4iWPi",
    "outputId": "70eb1b40-6628-4d3c-c6d6-df80be2177ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "sWSAwrbLiM1b",
    "outputId": "f192f7dc-a369-49f1-e6af-6009a072261c"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Вы ўжо ўсе, відавочна, адчулі, што нешта не так? Зойдзе да яе, а яна не ведае, з чаго пачынаць гаворку.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line = df.iloc[0]['sentence'] + ' ' + df.iloc[1]['sentence']\n",
    "line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "93OmuPp2hwTL",
    "outputId": "dac8505e-cc35-43f8-c9e2-6ac89b467898"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Вы ўжо ўсе , відавочна , адчулі , што нешта не так ?\n",
      "Зойдзе да яе , а яна не ведае , з чаго пачынаць гаворку .\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for sentence in nltk.sent_tokenize(line):\n",
    "    res.append(nltk.word_tokenize(sentence))\n",
    "    print(' '.join(res[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0hO7YlIgtG93"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MgBtittwtG1C"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FHKATl8utEyQ"
   },
   "source": [
    "## preprocess texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rFhroccLhwIZ",
    "outputId": "42dbdaa9-9c33-45fe-adb5-5504348ede94"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{110: 'н', 1169: 'г'}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_map = {'n': 'н', 'ґ': 'г'}\n",
    "char_map = str.maketrans(char_map)\n",
    "char_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XGqAYg0okUbA"
   },
   "outputs": [],
   "source": [
    "def preprocess_text(text, char_map):\n",
    "    text = text.lower()\n",
    "    # remove non-word chars\n",
    "    text = re.sub(r\"[^\\w\\s']\", '', text)    \n",
    "    # sub multiple sequential space chars with single space\n",
    "    text = re.sub(r\"\\s+\", ' ', text)\n",
    "    text = text.translate(char_map)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IacW4jlCtC41"
   },
   "outputs": [],
   "source": [
    "df['sentence_p'] = df['sentence'].apply(preprocess_text, char_map=char_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Kq2C2oKftYVo",
    "outputId": "31b4705d-965e-4004-ef61-e1ba1965e25e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['суседнія аўто з беларускай рэгістрацыяй засталіся цэлымі', 'у гэтым вялікая небяспека', 'аднак па словах эксперта нельга сцвярджаць што інфіцыраваных стала нашмат больш', 'а гэта вялікая перавага', 'чытаю на беларускай і расейскай мове']\n",
      "['Суседнія аўто з беларускай рэгістрацыяй засталіся цэлымі.', 'У гэтым вялікая небяспека.', 'Аднак, па словах эксперта, нельга сцвярджаць, што інфіцыраваных стала нашмат больш.', 'А гэта вялікая перавага.', 'Чытаю на беларускай і расейскай мове.']\n"
     ]
    }
   ],
   "source": [
    "s = df.sample(5)\n",
    "print(s['sentence_p'].tolist())\n",
    "print(s['sentence'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1m8AODcatCv8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RojZayJiq5tt",
    "outputId": "fcb04303-4f69-4139-b042-0605eb321162"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text_dump_fp: lm/sentences_for_lm_processed_1PerLine.txt\n"
     ]
    }
   ],
   "source": [
    "text_dump_fp = os.path.join('sentences_for_lm_processed_1PerLine.txt')\n",
    "print(f'text_dump_fp: {text_dump_fp}')\n",
    "\n",
    "with open(text_dump_fp, 'w') as fout:\n",
    "    joined_text = '\\n'.join(df['sentence_p'])\n",
    "    fout.write(joined_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DX9zX41TxSPT",
    "outputId": "22f50ea7-adb1-470b-f957-3b8966ced225"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  314675  2385268 28378057 lm/sentences_for_lm_processed_1PerLine.txt\n"
     ]
    }
   ],
   "source": [
    "!wc $text_dump_fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dpR7JT85uiCV",
    "outputId": "1ebe0251-1522-40b1-ffef-2676609ca037"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28M\tlm/sentences_for_lm_processed_1PerLine.txt\n"
     ]
    }
   ],
   "source": [
    "!du -sh $out_dp/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CCH25TUSuM0E"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mgMHuNuxuu-j"
   },
   "source": [
    "## build LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QwKi402AxYgK",
    "outputId": "b69c5b09-93a0-4029-b610-3b209cefd9eb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== 1/5 Counting and sorting n-grams ===\n",
      "Reading /content/lm/sentences_for_lm_processed_1PerLine.txt\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "tcmalloc: large alloc 1918697472 bytes == 0x55f2e110a000 @  0x7fe9516651e7 0x55f2dfbbd7e2 0x55f2dfb584fe 0x55f2dfb372eb 0x55f2dfb23066 0x7fe94f7fec87 0x55f2dfb24baa\n",
      "tcmalloc: large alloc 8953896960 bytes == 0x55f3536da000 @  0x7fe9516651e7 0x55f2dfbbd7e2 0x55f2dfbac80a 0x55f2dfbad248 0x55f2dfb37308 0x55f2dfb23066 0x7fe94f7fec87 0x55f2dfb24baa\n",
      "****************************************************************************************************\n",
      "Unigram tokens 2385268 types 144389\n",
      "=== 2/5 Calculating and sorting adjusted counts ===\n",
      "Chain sizes: 1:1732668 2:1063026240 3:1993174400 4:3189078784 5:4650740224\n",
      "tcmalloc: large alloc 4650745856 bytes == 0x55f2e110a000 @  0x7fe9516651e7 0x55f2dfbbd7e2 0x55f2dfbac80a 0x55f2dfbad248 0x55f2dfb378d7 0x55f2dfb23066 0x7fe94f7fec87 0x55f2dfb24baa\n",
      "tcmalloc: large alloc 1993179136 bytes == 0x55f435bce000 @  0x7fe9516651e7 0x55f2dfbbd7e2 0x55f2dfbac80a 0x55f2dfbad248 0x55f2dfb37cdd 0x55f2dfb23066 0x7fe94f7fec87 0x55f2dfb24baa\n",
      "tcmalloc: large alloc 3189080064 bytes == 0x55f569aa0000 @  0x7fe9516651e7 0x55f2dfbbd7e2 0x55f2dfbac80a 0x55f2dfbad248 0x55f2dfb37cdd 0x55f2dfb23066 0x7fe94f7fec87 0x55f2dfb24baa\n",
      "Statistics:\n",
      "1 144389 D1=0.602032 D2=1.06492 D3+=1.48123\n",
      "2 1141265 D1=0.817204 D2=1.12771 D3+=1.35486\n",
      "3 1862784 D1=0.918932 D2=1.23835 D3+=1.39376\n",
      "4 1931256 D1=0.966785 D2=1.31773 D3+=1.44989\n",
      "5 1721734 D1=0.982753 D2=1.47302 D3+=1.55809\n",
      "Memory estimate for binary LM:\n",
      "type     MB\n",
      "probing 146 assuming -p 1.5\n",
      "probing 174 assuming -r models -p 1.5\n",
      "trie     73 without quantization\n",
      "trie     41 assuming -q 8 -b 8 quantization \n",
      "trie     65 assuming -a 22 array pointer compression\n",
      "trie     32 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
      "=== 3/5 Calculating and sorting initial probabilities ===\n",
      "Chain sizes: 1:1732668 2:18260240 3:37255680 4:46350144 5:48208552\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "####################################################################################################\n",
      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
      "Chain sizes: 1:1732668 2:18260240 3:37255680 4:46350144 5:48208552\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "####################################################################################################\n",
      "=== 5/5 Writing ARPA model ===\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "****************************************************************************************************\n",
      "Name:lmplz\tVmPeak:14182272 kB\tVmRSS:873324 kB\tRSSMax:2054540 kB\tuser:10.1701\tsys:7.00626\tCPU:17.1764\treal:17.0476\n"
     ]
    }
   ],
   "source": [
    "!kenlm/build/bin/lmplz -o 5 --discount_fallback < \"lm/sentences_for_lm_processed_1PerLine.txt\" > \"lm/cv8be_5gram.arpa\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YJLh-AHMu_G9"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zrv_K1XClG2J",
    "outputId": "6a90c428-7a36-47ae-fc6d-61611e98b3f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "403M\tlm/cv8be_5gram.arpa\n",
      "28M\tlm/sentences_for_lm_processed_1PerLine.txt\n"
     ]
    }
   ],
   "source": [
    "!du -sh lm/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "T1xhuJJIvFzC",
    "outputId": "f588d64f-9c16-4442-c1c2-cbad7f34e50e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  6801446  36230104 422360493 lm/cv8be_5gram.arpa\n"
     ]
    }
   ],
   "source": [
    "!wc lm/cv8be_5gram.arpa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Y7Zyb7XZzOkB",
    "outputId": "ce268a3f-dff6-4f36-8f73-183122186ac6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\data\\\n",
      "ngram 1=144389\n",
      "ngram 2=1141265\n",
      "ngram 3=1862784\n",
      "ngram 4=1931256\n",
      "ngram 5=1721734\n",
      "\n",
      "\\1-grams:\n",
      "-6.058665\t<unk>\t0\n",
      "0\t<s>\t-1.0664405\n",
      "-1.2717469\t</s>\t0\n",
      "-3.3857362\tвы\t-0.36858183\n",
      "-2.9503407\tўжо\t-0.35378957\n",
      "-3.331054\tўсе\t-0.27460152\n",
      "-3.998521\tвідавочна\t-0.24098675\n",
      "-4.704899\tадчулі\t-0.257407\n",
      "-2.484223\tшто\t-0.57535625\n",
      "-3.2316258\tнешта\t-0.3649858\n",
      "-2.1907144\tне\t-0.51273227\n",
      "-2.8073287\tтак\t-0.4867947\n"
     ]
    }
   ],
   "source": [
    "!head -20 lm/cv8be_5gram.arpa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0nHyb_1pzOcJ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SLqtvWbZ_meZ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HV_wZ42qzveL"
   },
   "source": [
    "## convert .arpa to .bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RqRgFME4_Fdq",
    "outputId": "88ca0190-928e-4ab8-c106-0eddae0e74ce"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lm/cv8be_5gram.arpa\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "****************************************************************************************************\n",
      "SUCCESS\n"
     ]
    }
   ],
   "source": [
    "!kenlm/build/bin/build_binary \"lm/cv8be_5gram.arpa\" \"lm/cv8be_5gram.bin\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IsRBf8-iAxFZ",
    "outputId": "9ac027c0-df5e-4f6e-ab76-ca68ecb7869b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "403M\tlm/cv8be_5gram.arpa\n",
      "149M\tlm/cv8be_5gram.bin\n",
      "28M\tlm/sentences_for_lm_processed_1PerLine.txt\n"
     ]
    }
   ],
   "source": [
    "!du -sh lm/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D3u9FmHmAH_V"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hI4Mn76sz8fD"
   },
   "source": [
    "## copy to gdrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3lIx3CKr0Tf5"
   },
   "outputs": [],
   "source": [
    "lm_gdrive_dp = os.path.join(gdrive_exp_root_dp, 'lm')\n",
    "os.makedirs(lm_gdrive_dp, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "fPaF7CyoAnZr",
    "outputId": "4c8d0aff-15dc-4623-9aa7-37ed45a4dfd3"
   },
   "outputs": [],
   "source": [
    "shutil.copy2('lm/cv8be_5gram.bin', lm_gdrive_dp);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4OeBdZC0AKmw",
    "outputId": "265fcac1-7807-4670-bed3-1f936a442045"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cv8be_5gram.bin']"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(lm_gdrive_dp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qDIBOMbl0jft"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pGJuyuCu_FWk"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "lm_cv8be",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}