notebooks/embeddings_trainer.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "embeddings_trainer.ipynb",
   "provenance": [],
   "collapsed_sections": [],
   "mount_file_id": "1lppvohK2TyjT3MPdRWJflMHmNt3YgEsW",
   "authorship_tag": "ABX9TyNPJnYNdf4vCFVdkuQwZfuo"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gDQfErWuyo9E"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ZTZij0aEMxcd"
   },
   "source": [
    "import multiprocessing, os, ast, nltk, pickle\n",
    "\n",
    "nltk.download(\"punkt\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "_tp7tyjS24Av"
   },
   "source": [
    "INPUT_FILE_PATH = \n",
    "TOKENIZED_PATH = \n",
    "\n",
    "INPUT_FILE = \n",
    "DOC2VEC_INPUT_FILES_PATH = \n",
    "D2V_MODEL_OUTPUT_PATH =\n",
    "W2V_MODEL_OUTPUT_PATH = \n",
    "FT_MODEL_OUTPUT_PATH =\n",
    "\n",
    "MODEL_PREFIX = \n",
    "DOC2VEC_PREFIX = \n",
    "SIZE = 300\n",
    "EPOCHS = 50\n",
    "WINDOW = 30\n",
    "IDENTIFIER_NO = 1\n",
    "MIN_COUNT = 10\n",
    "\n",
    "LANGUAGE = 'english'"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7mlq0lHa3ceQ"
   },
   "source": [
    "# Word embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yCdmFWnvx3hQ"
   },
   "source": [
    "### Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "DVxOFsJT2P6_"
   },
   "source": [
    "with open(f\"{INPUT_FILE_PATH}/{INPUT_FILE}\", encoding=\"UTF-8\") as file:\n",
    "    sentences = nltk.sent_tokenize(file.read(), language=LANGUAGE)\n",
    "    tokenized_text = []\n",
    "    for sentence in sentences:\n",
    "        if \" \" in sentence == False:\n",
    "            continue\n",
    "        if len(sentence) <= 20:\n",
    "            continue\n",
    "        tokenized_text.append(nltk.word_tokenize(sentence, language=LANGUAGE))\n",
    "    print(f\"Created {len(tokenized_text)} tokens.\")\n",
    "    print(\"Preview:\")\n",
    "    print(tokenized_text[1])\n",
    "\n",
    "file_name = INPUT_FILE.replace(\".txt\", \"\")\n",
    "with open(f\"{TOKENIZED_PATH}/_{file_name}_tokenized.txt\", \"w\") as outfile:\n",
    "    for entry in tokenized_text:\n",
    "        outfile.write(\"\".join(str(entry)) + \"\\n\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iIcuB3hV4JX0"
   },
   "source": [
    "## FastText"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X7Cq2DtQx98X"
   },
   "source": [
    "### Load tokens and build vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "8FSzNqArrgIG"
   },
   "source": [
    "from gensim.models import FastText\n",
    "\n",
    "file_name = INPUT_FILE.replace(\".txt\", \"\")\n",
    "\n",
    "loaded_tokenized_text = []\n",
    "with open(f\"{TOKENIZED_PATH}/_{file_name}_tokenized.txt\", \"r\") as infile:\n",
    "    for line in infile:\n",
    "        line = ast.literal_eval(line)\n",
    "        loaded_tokenized_text.append(line)\n",
    "\n",
    "print(f\"Loaded from file: {loaded_tokenized_text[:2]} ...\")\n",
    "\n",
    "model = FastText(size=SIZE, window=WINDOW, min_count=MIN_COUNT)\n",
    "model.build_vocab(sentences=loaded_tokenized_text)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bdjVoC_cyF0q"
   },
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "6eyv60nH6xSh"
   },
   "source": [
    "model.train(\n",
    "    sentences=loaded_tokenized_text,\n",
    "    total_examples=len(loaded_tokenized_text),\n",
    "    epochs=EPOCHS,\n",
    ")\n",
    "file_name = f\"{MODEL_PREFIX}_{SIZE}_iter{EPOCHS}_win{WINDOW}_{IDENTIFIER_NO}-FT.model\"\n",
    "model.save(f\"{FT_MODEL_OUTPUT_PATH}/{file_name}\")\n",
    "model.wv.save_word2vec_format(f\"{FT_MODEL_OUTPUT_PATH}/{file_name}-bin.kv\", binary=True)\n",
    "model.wv.save_word2vec_format(\n",
    "    f\"{FT_MODEL_OUTPUT_PATH}/{file_name}-txt.kv\", binary=False\n",
    ")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DvPflEECMNJb"
   },
   "source": [
    "## Word2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sqAVCCGcODJY"
   },
   "source": [
    "### Load tokens"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "IYTAaATEMRbI"
   },
   "source": [
    "file_name = INPUT_FILE.replace(\".txt\", \"\")\n",
    "\n",
    "loaded_tokenized_text = []\n",
    "with open(f\"{TOKENIZED_PATH}/_{file_name}_tokenized.txt\", \"r\") as infile:\n",
    "    for line in infile:\n",
    "        line = ast.literal_eval(line)\n",
    "        loaded_tokenized_text.append(line)\n",
    "\n",
    "print(f\"Loaded from file: {loaded_tokenized_text[:2]} ...\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oMXUr_TdOH3-"
   },
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "IVCdQ9t3NmeT"
   },
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "model = Word2Vec(\n",
    "    loaded_tokenized_text,\n",
    "    size=SIZE,\n",
    "    window=WINDOW,\n",
    "    min_count=MIN_COUNT,\n",
    "    workers=multiprocessing.cpu_count(),\n",
    ")\n",
    "file_name = f\"{MODEL_PREFIX}_{SIZE}_iter{EPOCHS}_win{WINDOW}_{IDENTIFIER_NO}-W2V.model\"\n",
    "model.save(f\"{W2V_MODEL_OUTPUT_PATH}/{file_name}\")\n",
    "model.wv.save_word2vec_format(\n",
    "    f\"{W2V_MODEL_OUTPUT_PATH}/{file_name}-bin.kv\", binary=True\n",
    ")\n",
    "model.wv.save_word2vec_format(\n",
    "    f\"{W2V_MODEL_OUTPUT_PATH}/{file_name}-txt.kv\", binary=False\n",
    ")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wV8yJpbk2xTw"
   },
   "source": [
    "# Doc2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uz6gSgGmLb8k"
   },
   "source": [
    "## Sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WuojDYmW4Bgc"
   },
   "source": [
    "### Preparation"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "XnYv28s420HT"
   },
   "source": [
    "from gensim.models.doc2vec import TaggedDocument\n",
    "\n",
    "documents = []\n",
    "for file in os.listdir(DOC2VEC_INPUT_FILES_PATH):\n",
    "    if file == \".DS_Store\":\n",
    "        continue\n",
    "    with open(f\"{DOC2VEC_INPUT_FILES_PATH}/{file}\", \"r\", encoding=\"UTF-8\") as file:\n",
    "        file_name = file.name.replace(DOC2VEC_INPUT_FILES_PATH, \"\")\n",
    "        sentences = nltk.sent_tokenize(file.read(), language=LANGUAGE)\n",
    "        for sentence in sentences:\n",
    "            if \" \" in sentence == False:\n",
    "                continue\n",
    "            if len(sentence) <= 20:\n",
    "                continue\n",
    "            if sentence[0] == \"-\":\n",
    "                sentence = sentence[1:]\n",
    "\n",
    "            tagged_document = TaggedDocument(\n",
    "                nltk.word_tokenize(sentence, language=LANGUAGE), [int(file_name[1])]\n",
    "            )\n",
    "            documents.append(tagged_document)\n",
    "\n",
    "print(f\"Found {len(documents)} Sentences.\")\n",
    "\n",
    "with open(\n",
    "    f\"{TOKENIZED_PATH}/_{DOC2VEC_PREFIX}_doc2vec_sentences_tagged\", \"wb\"\n",
    ") as outfile:\n",
    "    pickle.dump(documents, outfile)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QdgSr-OBA98o"
   },
   "source": [
    "### Load"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "xLnReSJzBBR9"
   },
   "source": [
    "loaded_documents = []\n",
    "with open(\n",
    "    f\"{TOKENIZED_PATH}/_{DOC2VEC_PREFIX}_doc2vec_sentences_tagged\", \"rb\"\n",
    ") as infile:\n",
    "    loaded_documents = pickle.load(infile)\n",
    "\n",
    "print(f\"Loaded from file: {loaded_documents[:3]} ...\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vFZpmoSpLiHR"
   },
   "source": [
    "## Documents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mCsJ7dfeLwwr"
   },
   "source": [
    "### Preparation"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Uuip1tELoh0i"
   },
   "source": [
    "from gensim.models.doc2vec import TaggedDocument\n",
    "\n",
    "documents = []\n",
    "for file in os.listdir(DOC2VEC_INPUT_FILES_PATH):\n",
    "    if file == \".DS_Store\":\n",
    "        continue\n",
    "    with open(f\"{DOC2VEC_INPUT_FILES_PATH}/{file}\", \"r\", encoding=\"UTF-8\") as file:\n",
    "        file_name = file.name.replace(DOC2VEC_INPUT_FILES_PATH, \"\")\n",
    "        tagged_document = TaggedDocument(\n",
    "            nltk.word_tokenize(file.read(), language=LANGUAGE), [int(file_name[1])]\n",
    "        )\n",
    "        documents.append(tagged_document)\n",
    "\n",
    "print(f\"Found {len(documents)} texts.\")\n",
    "\n",
    "with open(f\"{TOKENIZED_PATH}/_{DOC2VEC_PREFIX}_doc2vec_doc_tagged\", \"wb\") as outfile:\n",
    "    pickle.dump(documents, outfile)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WCSmIT97MgEI"
   },
   "source": [
    "### Load"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "LTQ7SwI5MhVu"
   },
   "source": [
    "loaded_documents = []\n",
    "with open(f\"{TOKENIZED_PATH}/_{DOC2VEC_PREFIX}_doc2vec_doc_tagged\", \"rb\") as infile:\n",
    "    loaded_documents = pickle.load(infile)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n1Cq-Qo7EHd-"
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "MQp2F6mjDxQt"
   },
   "source": [
    "from gensim.models.doc2vec import Doc2Vec\n",
    "\n",
    "model = Doc2Vec(\n",
    "    loaded_documents,\n",
    "    vector_size=SIZE,\n",
    "    window=WINDOW,\n",
    "    min_count=MIN_COUNT,\n",
    "    workers=multiprocessing.cpu_count(),\n",
    "    dm=0,\n",
    "    dbow_words=1,\n",
    ")\n",
    "file_name = (\n",
    "    f\"{DOC2VEC_PREFIX}_{SIZE}_iter{EPOCHS}_win{WINDOW}_{IDENTIFIER_NO}-D2V.model\"\n",
    ")\n",
    "model.save(f\"{D2V_MODEL_OUTPUT_PATH}/{file_name}\")\n",
    "model.docvecs.save_word2vec_format(\n",
    "    f\"{D2V_MODEL_OUTPUT_PATH}/{file_name}-bin.kv\", binary=True\n",
    ")\n",
    "model.docvecs.save_word2vec_format(\n",
    "    f\"{D2V_MODEL_OUTPUT_PATH}/{file_name}-txt.kv\", binary=False\n",
    ")"
   ],
   "execution_count": null,
   "outputs": []
  }
 ]
}