FastText.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "10eb841c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('corpus.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11c70cc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "\n",
    "# NLP stuff\n",
    "import string\n",
    "from hazm import *\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "50a5e83f",
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = Normalizer()\n",
    "stopwords = [normalizer.normalize(x.strip()) for x in codecs.open('stopwords.dat','r','utf-8').readlines()]\n",
    "lemmatizer = Lemmatizer()\n",
    "\n",
    "def preprocess(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [lemmatizer.lemmatize(t) for t in tokens if t not in stopwords]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "184713ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_preprocessed = df['beyt'].apply(preprocess)\n",
    "data_preprocessed = data_preprocessed.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "534414cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['با', 'نصرت', 'و', 'فتح', 'و', 'ظفر', 'و', 'دولت', 'والا', 'نگریست#نگر', 'علم', 'شاه', 'جهان', 'بر', 'سر', 'بالا']\n",
      "['لشکر', 'شده', 'آسوده', 'و', 'ترمذ', 'شده', 'ایمن', 'نصرت', 'شده', 'پیوسته', 'و', 'دولت', 'شده', 'والا']\n",
      "['فتح', 'آمده', 'و', 'تهنیت', 'آورده', 'جهان', 'را', 'سلطان', 'جهانگیر', 'به', 'این', 'فتح', 'مهنا']\n",
      "['بشکفته', 'به', 'دین', 'داشت#دار', 'او', 'جان', 'پیمبر', 'نازنده', 'به', 'فرزند', 'او', 'آدم', 'و', 'حوا']\n",
      "['بهروزی', 'او', 'در', 'همه', 'گیتی', 'شده', 'معروف', 'پیروزی', 'او', 'در', 'همه', 'عالم', 'شده', 'پیدا']\n",
      "['رزم', 'همه', 'با', 'نصرت', 'و', 'رسم', 'همه', 'نیکو', 'روز', 'همه', 'با', 'دولت', 'و', 'کار', 'همه', 'زیبا']\n",
      "['ای', 'شاه', 'غلا', 'تو', 'داشت#دار', 'به', 'اقطاع', 'چین', 'و', 'ختن', 'و', 'کاشغر', 'و', 'خلخ', 'و', 'یغما']\n",
      "['بر', 'بیعت', 'و', 'پیمان', 'تو', 'صد', 'نامه', 'رسیدست', 'از', 'مکه', 'و', 'غزنین', 'و', 'سمرقند', 'و', 'بخارا']\n",
      "['از', 'موکب', 'تو', 'کوه', 'نمود#نما', 'همه', 'هامون', 'وز', 'لشکر', 'تو', 'شهر', 'نمود#نما', 'همه', 'صحرا']\n",
      "['آنجاکه', 'تف', 'توس', 'چه', 'جیحون', 'و', 'چه', 'هامون', 'وانجا', 'که', 'صف', 'توس', 'چه', 'جنگ', 'و', 'چه', 'تماشا']\n",
      "['تاگرد', 'سپاه', 'تو', 'برآمد', 'ز', 'خراسان', 'یک', 'باره', 'به', 'ادبار', 'فرو', 'شد#شو', 'سر', 'اعدا']\n",
      "['زین', 'نصرت', 'و', 'زین', 'فتح', 'که', 'دید#بین', 'و', 'شنید#شنو', 'دیگر', 'به', 'خراسان', 'بود#باش', 'غارت', 'و', 'غوغا']\n",
      "['نشگفت', 'اگر', 'از', 'بیم', 'توشیران', 'بگریزند', 'کز', 'هیبت', 'تو', 'موم', 'شد#شو', 'آهن', 'و', 'خارا']\n",
      "['تا', 'دست', 'تو', 'دریا', 'بود#باش', 'و', 'تیغ', 'تو', 'آتش', 'نشگفت', 'نهیب', 'و', 'خطر', 'از', 'آتش', 'و', 'دریا']\n",
      "['هر', 'شاه', 'که', 'یک', 'راه', 'زتیغ', 'تو', 'ترسید#ترس', 'از', 'ملک', 'و', 'ولایت', 'بود#باش', 'نیز', 'شکیبا']\n",
      "['سو', 'دش', 'کرد#کن', 'تعبیه', 'قلعه', 'و', 'لشکر', 'آن', 'به', 'که', 'کند', 'با', 'سر', 'تیغ', 'تو', 'مدارا']\n",
      "['گر', 'تعبیه', 'سازی', 'به', 'سو', 'روم', 'دگر', 'بار', 'زنار', 'چو', 'افسار', 'کرد#کن', 'بر', 'سر', 'ترسا']\n",
      "['فرمان', 'تو', 'مسجد', 'کند', 'از', 'خانه', 'رهبان', 'شمشیر', 'تو', 'خرزین', 'کند', 'از', 'چوب', 'چلیپا']\n",
      "['شاها', 'ملکا', 'جمله', 'آفاق', 'تو', 'داشت#دار', 'شد#شو', 'دیده', 'دین', 'از', 'ظفر', 'و', 'فتح', 'تو', 'بینا']\n",
      "['بیم', '#است', 'ز', 'شیر', 'جهان', 'وز', 'تو', 'رعایت', 'عذرست', 'ز', 'شاه', 'جهان', 'وز', 'تو', 'محابا']\n",
      "doc len:  590882\n"
     ]
    }
   ],
   "source": [
    "print(*data_preprocessed[:20], sep='\\n')\n",
    "print('doc len: ', len(data_preprocessed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98b24027",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(filename='gensim.log',\n",
    "                    format=\"%(asctime)s:%(levelname)s:%(message)s\",\n",
    "                    level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f5c13ac",
   "metadata": {},
   "source": [
    "## Using pre-trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20d4696a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.test.utils import datapath\n",
    "import fasttext.util\n",
    "from gensim.models.fasttext import load_facebook_model, save_facebook_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c6d04c78",
   "metadata": {},
   "outputs": [],
   "source": [
    "fasttext.util.download_model('fa', if_exists='ignore')\n",
    "model = load_facebook_model(datapath(\"/home/poems/final_project/cc.fa.300.bin\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8b7d0292",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.build_vocab(data_preprocessed, update=True)\n",
    "model.train(sentences=data_preprocessed, total_examples = len(data_preprocessed), epochs=100, workers = 11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f2826449",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./results/fasttext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f8523cdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "def identity(x):\n",
    "    return x\n",
    "\n",
    "words_cv = CountVectorizer(max_features=10000, min_df=0.00001, max_df=0.3, tokenizer=identity, preprocessor=identity)\n",
    "word_count_vector = words_cv.fit_transform(data_preprocessed)\n",
    "word_tfidf = TfidfTransformer(smooth_idf=True)\n",
    "word_tfidf = word_tfidf.fit(word_count_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7fea4a94",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill\n",
    "\n",
    "with open('./results/cv_tfidf.pkl', 'wb') as fout:\n",
    "    dill.dump((words_cv, word_tfidf) , fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1efb87dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/poems/.local/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import normalize as vec_normalize\n",
    "\n",
    "words = words_cv.get_feature_names()\n",
    "words_rep = model[words]\n",
    "words_rep = vec_normalize(words_rep, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3bccd66c",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_rep = word_count_vector @ words_rep \n",
    "norm_factor = np.sum(word_count_vector,axis=1)\n",
    "norm_factor[norm_factor==0.]=1\n",
    "doc_rep /= norm_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "821e269b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./results/fasttext_doc_word_rep.pkl', 'wb') as fout:\n",
    "    dill.dump((doc_rep, words_rep) , fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "196b89a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_match(query):\n",
    "    query = word_tokenize(query)\n",
    "    query = [lemmatizer.lemmatize(normalizer.normalize(y)) for y in query]\n",
    "    query = words_cv.transform([query])\n",
    "    query = word_tfidf.transform(query)\n",
    "    query = query @ words_rep\n",
    "    scores = doc_rep @ query.T\n",
    "    return np.argsort(scores[:,0])[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "daca06be",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes = find_best_match('شهریار زیر آفتاب خورشید ماهی خورد')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "88ebc1b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "362288    همی آفتاب فلک فروتاب زتاج تو گیردچو مه زآفتاب                  \n",
       "2674      ماه است وزیر و ملک مشرق خورشید خورشید فروزنده بر ماه منیرست    \n",
       "290317    برافرازنده چرخ مدور برافروزنده خورشید انور                     \n",
       "492147    آسمانی و آفتابت صاحبست آفتابی کاسمانی چون تو کرد               \n",
       "549123    خداوند خورشید و گردنده ماه فرازنده تاج و تخت و کلاه            \n",
       "435039    ز پیشانیش تابان تیر و ناهید زر خسارش فروزان ماه و خورشید       \n",
       "282285    چو خورد آن ماه را در آب ماهی ز ماهی ماه را چون بازخواهی        \n",
       "222814    شاه بتانی و عاشقانت سپاهند ماه زمینی و آسمانت کلاهست           \n",
       "108169    زیر دست چاکران شاه ماه و آفتاب زیر دست آفتاب و ماه چرخ و روزگار\n",
       "557828    بیامد تهمتن بگسترد بر بخواهش بر شاه خورشید فر                  \n",
       "127389    در سایه شاه آسمان قدر مه طلعت آفتاب پرتو                       \n",
       "562566    بپرسیدش از شهریار و سپاه ز گردنده خورشید و تابنده ماه          \n",
       "496439    آفتاب آسمانت باد تاج و آسمان آفتابت باد گاه                    \n",
       "465399    ماه را از رخ چون خورشیدت در شب چاردهم نقصانست                  \n",
       "314078    چنانست ای مه خورشید تابان که چون ذره سوی خورشید تابان          \n",
       "451387    مه سایه پرور شب خورشید مسکنش شب سایه گستر مه خورشید منظرش      \n",
       "541275    خداوند کیوان و گردان سپهر فروزنده ماه و ناهید و مهر            \n",
       "517790    ز برج آسمان تابنده ماهی چو انجم گردش از خوبان سیپاهی           \n",
       "438916    نماید خور فروغ و شب سیاهی بتابد مهر و ماه آسمانی               \n",
       "496349    آسمانیست آفتابش رای آفتابیست آسمانش گاه                        \n",
       "Name: beyt, dtype: object"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.width', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', 0)\n",
    "\n",
    "df['beyt'][indexes[:20]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d00a9cc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_precision_at_k_score(ground_truth, preds, k):\n",
    "    ground_truth = set(ground_truth)\n",
    "    preds = set(preds[:k])\n",
    "    intersection = ground_truth.intersection(preds)\n",
    "    return len(intersection) / min(k, len(ground_truth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "79bb8996",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "57fc2abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./eval/queries1.txt', encoding='utf-8') as f:\n",
    "    queries1 = f.read().splitlines()\n",
    "ranks1 = pickle.load(open('./eval/eval_ranks1.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5aedc3ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./eval/queries2.txt', encoding='utf-8') as f:\n",
    "    queries2 = f.read().splitlines()\n",
    "ranks2 = pickle.load(open('./eval/eval_ranks2.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "cfed1405",
   "metadata": {},
   "outputs": [],
   "source": [
    "queries = queries1 + queries2\n",
    "ranks = ranks1 + ranks2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f9ce4eae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision_at_k_score for k=1 is 0.06\n",
      "precision_at_k_score for k=21 is 0.0870168165168165\n",
      "precision_at_k_score for k=41 is 0.1179357569104179\n",
      "precision_at_k_score for k=61 is 0.13810687438528121\n",
      "precision_at_k_score for k=81 is 0.16506132211017593\n",
      "precision_at_k_score for k=101 is 0.1863821661948636\n"
     ]
    }
   ],
   "source": [
    "for k in range(1, 102, 20):\n",
    "    prec = 0.0\n",
    "    for i, q in enumerate(queries):\n",
    "        prec+= get_precision_at_k_score(ranks[i], find_best_match(q), k)\n",
    "    print(f\"precision_at_k_score for k={k} is {prec/len(queries)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1dd1893",
   "metadata": {},
   "source": [
    "## Training from Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "cf1e75ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import FastText\n",
    "\n",
    "model = FastText(data_preprocessed, size=100, window=5, min_count=5, workers=11,sg=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f6b4682b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/poems/.local/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "words = words_cv.get_feature_names()\n",
    "words_rep = model[words]\n",
    "words_rep = vec_normalize(words_rep, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "34517802",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_rep = word_count_vector @ words_rep \n",
    "norm_factor = np.sum(word_count_vector,axis=1)\n",
    "norm_factor[norm_factor==0.]=1\n",
    "doc_rep /= norm_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "88cb65ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_match(query):\n",
    "    query = word_tokenize(query)\n",
    "    query = [lemmatizer.lemmatize(normalizer.normalize(y)) for y in query]\n",
    "    query = words_cv.transform([query])\n",
    "    query = word_tfidf.transform(query)\n",
    "    query = query @ words_rep\n",
    "    scores = doc_rep @ query.T\n",
    "    return np.argsort(scores[:,0])[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "154cfb90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision_at_k_score for k=1 is 0.03\n",
      "precision_at_k_score for k=21 is 0.06106360306360305\n",
      "precision_at_k_score for k=41 is 0.09265255340832988\n",
      "precision_at_k_score for k=61 is 0.09590750442351309\n",
      "precision_at_k_score for k=81 is 0.10587866795079709\n",
      "precision_at_k_score for k=101 is 0.12962096239333176\n"
     ]
    }
   ],
   "source": [
    "for k in range(1, 102, 20):\n",
    "    prec = 0.0\n",
    "    for i, q in enumerate(queries):\n",
    "        prec+= get_precision_at_k_score(ranks[i], find_best_match(q), k)\n",
    "    print(f\"precision_at_k_score for k={k} is {prec/len(queries)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}