scripts/CatTrainRu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 325
    },
    "colab_type": "code",
    "id": "UK8wI0PruXGv",
    "outputId": "a7338b53-f88d-45cf-a344-4fef945dbc48"
   },
   "outputs": [],
   "source": [
    "!pip install pyonmttok fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "KwYvDl48uj4h",
    "outputId": "1477a63a-df83-4eeb-ed30-40cc3acd7f7a"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/facebookresearch/fastText.git\n",
    "!cd fastText && mkdir build && cd build && cmake .. && make && make install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "wvr9CVo5pZH5",
    "outputId": "a0426777-c5bb-49e7-fb96-2e308f9f7a35"
   },
   "outputs": [],
   "source": [
    "!rm -f ru_tg_train.tar.gz\n",
    "!wget https://www.dropbox.com/s/1ecl9orr2tagcgi/ru_tg_train.tar.gz\n",
    "!rm -f ru_tg_train.json\n",
    "!tar -xzvf ru_tg_train.tar.gz\n",
    "!rm ru_tg_train.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "igj91yXzBPjU",
    "outputId": "ce33d8a3-880a-4002-b7a9-51002ca60e5f"
   },
   "outputs": [],
   "source": [
    "!rm -f ru_tg_test.tar.gz\n",
    "!wget https://www.dropbox.com/s/gvfk6t4g7kxw9ae/ru_tg_test.tar.gz\n",
    "!rm -f ru_tg_test.json\n",
    "!tar -xzvf ru_tg_test.tar.gz\n",
    "!rm ru_tg_test.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 793
    },
    "colab_type": "code",
    "id": "d0x5tSFapjkO",
    "outputId": "3a8101ca-695c-4e9e-976d-15abe7705732"
   },
   "outputs": [],
   "source": [
    "!wget https://www.dropbox.com/s/amua7p1rt1dcvy0/ru_cat_train_raw_markup.tsv\n",
    "!wget https://www.dropbox.com/s/xia50d1h28e87x4/ru_cat_test_raw_markup.tsv\n",
    "!head -n 2 ru_cat_train_raw_markup.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QGvbTS192j9P"
   },
   "outputs": [],
   "source": [
    "import pyonmttok\n",
    "tokenizer = pyonmttok.Tokenizer(\"conservative\", joiner_annotate=False)\n",
    "\n",
    "def preprocess(text):\n",
    "    text = str(text).strip().replace(\"\\n\", \" \").replace(\"\\xa0\", \" \").lower()\n",
    "    tokens, _ = tokenizer.tokenize(text)\n",
    "    text = \" \".join(tokens)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "id": "lQ3mPWlrsvp7",
    "outputId": "6f003341-f165-4049-9934-2a42aa01c2f7"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from collections import Counter\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "def normalize(text):\n",
    "    return text.replace(\"\\t\", \" \").replace(\"\\n\", \" \").replace('\"', '').replace(\"\\xa0\", \" \")\n",
    "\n",
    "def convert_to_ft(answers_file_name, original_json, output_file_name, min_votes=3, use_preprocess=True):\n",
    "    with open(answers_file_name, \"r\") as r:\n",
    "        header = tuple(next(r).strip().split(\"\\t\"))\n",
    "        records = []\n",
    "        for line in r:\n",
    "            fields = line.strip().split(\"\\t\")\n",
    "            assert len(fields) == len(header), fields\n",
    "            records.append(dict(zip(header, fields)))\n",
    "\n",
    "    # Filter honeypots out\n",
    "    records = [r for r in records if not r[\"GOLDEN:res\"]]\n",
    "\n",
    "    # Normalize fields\n",
    "    for r in records:\n",
    "        r.pop(\"GOLDEN:res\", None)\n",
    "        r.pop(\"HINT:text\", None)\n",
    "        for key, value in r.items():\n",
    "            new_key = key.split(\":\")[-1]\n",
    "            r[new_key] = r.pop(key)\n",
    "\n",
    "    # Restore original urls (to fix a bug)\n",
    "    with open(original_json, \"r\") as r:\n",
    "        data = json.load(r)\n",
    "        title2url = {normalize(d[\"title\"]): d[\"url\"] for d in data}\n",
    "        for r in records:\n",
    "            title = normalize(r[\"title\"])\n",
    "            if title not in title2url:\n",
    "                continue\n",
    "            r[\"url\"] = title2url[title]\n",
    "\n",
    "    # Calc inter-annotator agreement\n",
    "    annotator2labels = defaultdict(dict)\n",
    "    unique_keys = list(set([r[\"url\"] for r in records]))\n",
    "    unique_workers = list(set([r[\"worker_id\"] for r in records]))\n",
    "    unique_res = list(set([r[\"res\"] for r in records]))\n",
    "    res2num = {res: i for i, res in enumerate(unique_res)}\n",
    "    for r in records:\n",
    "        annotator2labels[r[\"worker_id\"]][r[\"url\"]] = r[\"res\"]\n",
    "    worker2labels = {}\n",
    "    for worker_id in unique_workers:\n",
    "        worker_labels = []\n",
    "        worker_res = annotator2labels[worker_id]\n",
    "        for key in unique_keys:\n",
    "            if key not in worker_res:\n",
    "                worker_labels.append(-1)\n",
    "                continue\n",
    "            worker_labels.append(res2num[worker_res[key]])\n",
    "        worker2labels[worker_id] = worker_labels\n",
    "    scores = []\n",
    "    for w1, labels1 in worker2labels.items():\n",
    "        for w2, labels2 in worker2labels.items():\n",
    "            if w1 == w2:\n",
    "                continue\n",
    "            fixed_labels1 = []\n",
    "            fixed_labels2 = []\n",
    "            for l1, l2 in zip(labels1, labels2):\n",
    "                if l1 == -1 or l2 == -1:\n",
    "                    continue\n",
    "                fixed_labels1.append(l1)\n",
    "                fixed_labels2.append(l2)\n",
    "            if fixed_labels1 and fixed_labels2:\n",
    "                score = cohen_kappa_score(fixed_labels1, fixed_labels2)\n",
    "                if -1.0 <= score <= 1.0:\n",
    "                    scores.append(score)\n",
    "    print(\"Avg kappa score: {}\".format(sum(scores)/len(scores)))\n",
    "\n",
    "    results = defaultdict(list)\n",
    "    for r in records:\n",
    "        results[r[\"url\"]].append(r[\"res\"])\n",
    "\n",
    "    data = {r[\"url\"]: r for r in records}\n",
    "    for url, res in results.items():\n",
    "        res_count = Counter(res)\n",
    "        if res_count.most_common(1)[0][1] < min_votes:\n",
    "            data.pop(url)\n",
    "\n",
    "    rub_cnt = Counter()\n",
    "    for _, d in data.items():\n",
    "        rub_cnt[d[\"res\"]] += 1\n",
    "    print(rub_cnt.most_common())\n",
    "\n",
    "    with open(output_file_name, \"w\") as w:\n",
    "        records = list(data.values())\n",
    "        random.shuffle(records)\n",
    "        for d in records:\n",
    "            title = d[\"title\"] if not use_preprocess else preprocess(d[\"title\"])\n",
    "            text = d[\"text\"] if not use_preprocess else preprocess(d[\"text\"])\n",
    "            w.write(\"__label__{} {} {}\\n\".format(d[\"res\"], title, text))\n",
    "\n",
    "convert_to_ft(\"ru_cat_train_raw_markup.tsv\", \"ru_tg_train.json\", \"ru_cat_train_markup.txt\", min_votes=2, use_preprocess=True)\n",
    "convert_to_ft(\"ru_cat_test_raw_markup.tsv\", \"ru_tg_test.json\", \"ru_cat_test_markup.txt\", min_votes=4, use_preprocess=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "soJgINOfuSe7",
    "outputId": "32dc0d6e-1248-4027-bb2d-bfd78cc476b6"
   },
   "outputs": [],
   "source": [
    "!cat ru_cat_train_markup.txt | wc -l\n",
    "!cat ru_cat_test_markup.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 325
    },
    "colab_type": "code",
    "id": "tyrAfrWg24TK",
    "outputId": "e415d610-c7b0-42b0-c5a1-599a5459250a"
   },
   "outputs": [],
   "source": [
    "!rm -f lenta-ru-news.csv.gz\n",
    "!wget https://github.com/yutkin/Lenta.Ru-News-Dataset/releases/download/v1.0/lenta-ru-news.csv.gz\n",
    "!rm -f lenta-ru-news.csv\n",
    "!gzip -d lenta-ru-news.csv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 91
    },
    "colab_type": "code",
    "id": "bdnCAOvxvnQ7",
    "outputId": "d0600db1-74f6-40f4-c3d1-639c2a0bddb3"
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import random\n",
    "from collections import Counter\n",
    "\n",
    "def parse_lenta(input_file, output_file, use_preprocess=True):\n",
    "    parts = {\n",
    "        \"society\": 0.02,\n",
    "        \"economy\": 0.02,\n",
    "        \"sports\": 0.02,\n",
    "        \"technology\": 0.02,\n",
    "        \"science\": 0.02,\n",
    "        \"other\": 0.02,\n",
    "        \"entertainment\": 0.02\n",
    "    }\n",
    "    topics_mapping = {\n",
    "        \"Экономика\": \"economy\",\n",
    "        \"Спорт\": \"sports\",\n",
    "        \"Силовые структуры\": \"society\",\n",
    "        \"Бизнес\": \"economy\",\n",
    "        \"Культпросвет\": \"entertainment\",\n",
    "        (\"Наука и техника\", \"Игры\"): \"entertainment\",\n",
    "        (\"Наука и техника\", \"Наука\"): \"science\",\n",
    "        (\"Наука и техника\", \"Космос\"): \"science\",\n",
    "        (\"Наука и техника\", \"Жизнь\"): \"science\",\n",
    "        (\"Наука и техника\", \"История\"): \"science\",\n",
    "        (\"Наука и техника\", \"Оружие\"): \"society\",\n",
    "        (\"Наука и техника\", \"Гаджеты\"): \"technology\",\n",
    "        (\"Наука и техника\", \"Софт\"): \"technology\",\n",
    "        (\"Наука и техника\", \"Техника\"): \"technology\",\n",
    "        (\"Мир\", \"Общество\"): \"society\",\n",
    "        (\"Мир\", \"Политика\"): \"society\",\n",
    "        (\"Мир\", \"Происшествия\"): \"society\",\n",
    "        (\"Мир\", \"Конфликты\"): \"society\",\n",
    "        (\"Мир\", \"Преступность\"): \"society\",\n",
    "        (\"Россия\", \"Политика\"): \"society\",\n",
    "        (\"Россия\", \"Общество\"): \"society\",\n",
    "        (\"Россия\", \"Происшествия\"): \"society\",\n",
    "        (\"Интернет и СМИ\", \"Мемы\"): \"technology\",\n",
    "        (\"Интернет и СМИ\", \"Киберпреступность\"): \"technology\",\n",
    "        (\"Интернет и СМИ\", \"Интернет\"): \"technology\",\n",
    "        (\"Интернет и СМИ\", \"Вирусные ролики\"): \"technology\",\n",
    "        (\"Ценности\", \"Стиль\"): \"other\",\n",
    "        (\"Ценности\", \"Явления\"): \"other\",\n",
    "        (\"Ценности\", \"Внешний вид\"): \"other\",\n",
    "        (\"Ценности\", \"Движение\"): \"technology\",\n",
    "        (\"Из жизни\", \"Происшествия\"): \"society\",\n",
    "        (\"Путешествия\", \"Происшествия\"): \"society\",\n",
    "    }\n",
    "    with open(input_file, \"r\") as r:\n",
    "        next(r)\n",
    "        reader = csv.reader(r, delimiter=',')\n",
    "        records = []\n",
    "        for row in reader:\n",
    "            url, title, text, topic, tag = row\n",
    "            topic = topic.strip()\n",
    "            tag = tag.strip()\n",
    "            true_topic = None\n",
    "            if topic in topics_mapping:\n",
    "                true_topic = topics_mapping[topic]\n",
    "            elif (topic, tag) in topics_mapping:\n",
    "                true_topic = topics_mapping[(topic, tag)]\n",
    "            else:\n",
    "                continue\n",
    "            records.append({\"url\": url, \"title\": title, \"text\": text, \"res\": true_topic})\n",
    "        print(len(records))\n",
    "        rub_cnt = Counter()\n",
    "        for d in records:\n",
    "            rub_cnt[d[\"res\"]] += 1\n",
    "        print(rub_cnt.most_common())\n",
    "        with open(output_file, \"w\") as w:\n",
    "            for r in records:\n",
    "                if random.random() > parts[r[\"res\"]]:\n",
    "                    continue\n",
    "                title = preprocess(r[\"title\"]) if use_preprocess else r[\"title\"]\n",
    "                text = preprocess(r[\"text\"]) if use_preprocess else r[\"text\"]\n",
    "                w.write(\"__label__{} {} {}\\n\".format(r[\"res\"], title, text))\n",
    "\n",
    "parse_lenta(\"lenta-ru-news.csv\", \"lenta_markup.txt\")\n",
    "!cat lenta_markup.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 397
    },
    "colab_type": "code",
    "id": "Wb7XHuEiRjcl",
    "outputId": "7fc0ac4c-51a4-46b3-ab46-59944f440869"
   },
   "outputs": [],
   "source": [
    "!rm -f ru_not_news.txt\n",
    "!wget https://www.dropbox.com/s/wwptzqhgxvtjhbd/ru_not_news.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4hI6JmLlRnqq"
   },
   "outputs": [],
   "source": [
    "with open(\"ru_not_news.txt\", \"r\") as r, open(\"ru_not_news_fixed.txt\", \"w\") as w:\n",
    "    for line in r:\n",
    "        words = line.strip().split(\" \")\n",
    "        text = \" \".join(words[1:])\n",
    "        text = preprocess(text)\n",
    "        w.write(\"__label__{} {}\\n\".format(\"not_news\", text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 469
    },
    "colab_type": "code",
    "id": "T07Ri1yMDg_W",
    "outputId": "3b8c2a00-fc80-4d58-f150-773480cead8f"
   },
   "outputs": [],
   "source": [
    "!wget https://www.dropbox.com/s/2nx97d8nzbzusee/ru_vectors_v2.bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 233
    },
    "colab_type": "code",
    "id": "rGcdwkt4EmXD",
    "outputId": "1fbbda00-ab28-4025-fbeb-24d77b7336d6"
   },
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/facebookresearch/fastText/master/python/doc/examples/bin_to_vec.py\n",
    "!python bin_to_vec.py ru_vectors_v2.bin > ru_vectors_v2.vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aytxXtVZDnb8"
   },
   "outputs": [],
   "source": [
    "!cat ru_cat_train_markup.txt > ru_cat_train_all.txt\n",
    "!cat lenta_markup.txt >> ru_cat_train_all.txt\n",
    "!cat ru_not_news_fixed.txt >> ru_cat_train_all.txt\n",
    "!shuf ru_cat_train_all.txt > ru_cat_train_shuf.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "LHxIIIG0It5I",
    "outputId": "487cbd22-89a3-4416-b9e4-e9f6535259ea"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "with open(\"ru_cat_train_shuf.txt\", \"r\") as r, open(\"ru_cat_train_train.txt\", \"w\") as train, open(\"ru_cat_train_val.txt\", \"w\") as val:\n",
    "    for line in r:\n",
    "        if random.random() < 0.1:\n",
    "            val.write(line)\n",
    "        else:\n",
    "            train.write(line)\n",
    "!cat ru_cat_train_val.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 161
    },
    "colab_type": "code",
    "id": "PvaHhIP23Cir",
    "outputId": "a8c60393-9f21-47fc-e4f6-18010a58db20"
   },
   "outputs": [],
   "source": [
    "!fasttext supervised -input ru_cat_train_train.txt -pretrainedVectors ru_vectors_v2.vec -dim 50 -autotune-validation ru_cat_train_val.txt -output ru_cat -autotune-modelsize 10M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "0iwWA0PwEevw",
    "outputId": "d0fd327b-5da3-46f9-e952-837b5f943000"
   },
   "outputs": [],
   "source": [
    "!fasttext test ru_cat.ftz ru_cat_test_markup.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 881
    },
    "colab_type": "code",
    "id": "yErIMGxLJbzx",
    "outputId": "0e5cbc6e-44f5-42b0-c53e-049437bc9267"
   },
   "outputs": [],
   "source": [
    "import fasttext\n",
    "model = fasttext.load_model(\"ru_cat.ftz\")\n",
    "true_labels = []\n",
    "predicted_labels = []\n",
    "errors = []\n",
    "with open(\"ru_cat_test_markup.txt\", \"r\") as r:\n",
    "    for line in r:\n",
    "        words = line.strip().split(\" \")\n",
    "        label = words[0][9:]\n",
    "        true_labels.append(label)\n",
    "        text = \" \".join(words[1:])\n",
    "        predicted_label = model.predict([text])[0][0][0][9:]\n",
    "        if label != predicted_label:\n",
    "            errors.append((label, predicted_label, text[:100]))\n",
    "        predicted_labels.append(predicted_label)\n",
    "for label, predicted_label, text in errors:\n",
    "    print(\"T: {} P: {} | {}\".format(label, predicted_label, text))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "CatTrainRu.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}