scripts/CatTrainEn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "id": "UK8wI0PruXGv",
    "outputId": "3132a24b-509d-4fc1-a0db-f9eb15c37a42"
   },
   "outputs": [],
   "source": [
    "!pip install pyonmttok fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "GW3-AWd5D1pj",
    "outputId": "75926478-d3d2-4e31-b4c8-80ec865aa80c"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/facebookresearch/fastText.git\n",
    "!cd fastText && mkdir build && cd build && cmake .. && make && make install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "wvr9CVo5pZH5",
    "outputId": "48abfd1c-8879-4615-dd0d-d1b8cd54c4a9"
   },
   "outputs": [],
   "source": [
    "!rm -f en_tg_train.tar.gz\n",
    "!wget https://www.dropbox.com/s/umd8tyx4wz1wquq/en_tg_train.tar.gz\n",
    "!rm -f en_tg_train.json\n",
    "!tar -xzvf en_tg_train.tar.gz\n",
    "!rm en_tg_train.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "igj91yXzBPjU",
    "outputId": "a3b14b9f-f9cd-4e17-e6d1-543736e1e8ec"
   },
   "outputs": [],
   "source": [
    "!rm -f en_tg_test.tar.gz\n",
    "!wget https://www.dropbox.com/s/rw674iic8x5udb3/en_tg_test.tar.gz\n",
    "!rm -f en_tg_test.json\n",
    "!tar -xzvf en_tg_test.tar.gz\n",
    "!rm en_tg_test.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 793
    },
    "colab_type": "code",
    "id": "d0x5tSFapjkO",
    "outputId": "411a6cb1-6c57-4b00-fdb4-eba040ccc6d2"
   },
   "outputs": [],
   "source": [
    "!wget https://www.dropbox.com/s/7qpfgf8bz77h2ss/en_cat_train_raw_markup.tsv\n",
    "!wget https://www.dropbox.com/s/bszwshgwbrt328k/en_cat_test_raw_markup.tsv\n",
    "!head -n 2 en_cat_train_raw_markup.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 505
    },
    "colab_type": "code",
    "id": "UbapkM0gHtew",
    "outputId": "5ab55b77-3a39-4483-dc64-d2daf97a8fd4"
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/rmisra/news-category-dataset\n",
    "\n",
    "!rm -f news-category-dataset.zip\n",
    "!wget https://www.dropbox.com/s/ua18htwqrkwnfpg/news-category-dataset.zip\n",
    "!unzip news-category-dataset.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QGvbTS192j9P"
   },
   "outputs": [],
   "source": [
    "import pyonmttok\n",
    "tokenizer = pyonmttok.Tokenizer(\"conservative\")\n",
    "\n",
    "def preprocess(text):\n",
    "    text = str(text).strip().replace(\"\\n\", \" \").replace(\"\\xa0\", \" \").lower()\n",
    "    tokens, _ = tokenizer.tokenize(text)\n",
    "    text = \" \".join(tokens)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T1amO2NuIch4"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def save_to_ft(records, output_file_name, use_preprocess=True):\n",
    "    with open(output_file_name, \"w\") as w:\n",
    "        random.shuffle(records)\n",
    "        for d in records:\n",
    "            title = d[\"title\"] if not use_preprocess else preprocess(d[\"title\"])\n",
    "            text = d[\"text\"] if not use_preprocess else preprocess(d[\"text\"])\n",
    "            w.write(\"__label__{} {} {}\\n\".format(d[\"res\"], title, text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "id": "lQ3mPWlrsvp7",
    "outputId": "591c6152-a1fc-49a8-b4ec-3515876e35f4"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import random\n",
    "from collections import Counter, defaultdict\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "def normalize(text):\n",
    "    return text.replace(\"\\t\", \" \").replace(\"\\n\", \" \").replace('\"', '').replace(\"\\xa0\", \" \")\n",
    "\n",
    "def convert_to_ft(answers_file_name, original_json, output_file_name, min_votes=3, use_preprocess=True):\n",
    "    with open(answers_file_name, \"r\") as r:\n",
    "        header = tuple(next(r).strip().split(\"\\t\"))\n",
    "        records = []\n",
    "        for line in r:\n",
    "            fields = line.strip().split(\"\\t\")\n",
    "            assert len(fields) == len(header), fields\n",
    "            records.append(dict(zip(header, fields)))\n",
    "\n",
    "    # Filter honeypots out\n",
    "    records = [r for r in records if not r[\"GOLDEN:res\"]]\n",
    "\n",
    "    # Normalize fields\n",
    "    for r in records:\n",
    "        r.pop(\"GOLDEN:res\", None)\n",
    "        r.pop(\"HINT:text\", None)\n",
    "        for key, value in r.items():\n",
    "            new_key = key.split(\":\")[-1]\n",
    "            r[new_key] = r.pop(key)\n",
    "\n",
    "    # Restore original urls (to fix a bug)\n",
    "    with open(original_json, \"r\") as r:\n",
    "        data = json.load(r)\n",
    "        title2url = {normalize(d[\"title\"]): d[\"url\"] for d in data}\n",
    "        for r in records:\n",
    "            title = normalize(r[\"title\"])\n",
    "            if title not in title2url:\n",
    "                continue\n",
    "            r[\"url\"] = title2url[title]\n",
    "\n",
    "    # Calc inter-annotator agreement\n",
    "    annotator2labels = defaultdict(dict)\n",
    "    unique_keys = list(set([r[\"url\"] for r in records]))\n",
    "    unique_workers = list(set([r[\"worker_id\"] for r in records]))\n",
    "    unique_res = list(set([r[\"res\"] for r in records]))\n",
    "    res2num = {res: i for i, res in enumerate(unique_res)}\n",
    "    for r in records:\n",
    "        annotator2labels[r[\"worker_id\"]][r[\"url\"]] = r[\"res\"]\n",
    "    worker2labels = {}\n",
    "    for worker_id in unique_workers:\n",
    "        worker_labels = []\n",
    "        worker_res = annotator2labels[worker_id]\n",
    "        for key in unique_keys:\n",
    "            if key not in worker_res:\n",
    "                worker_labels.append(-1)\n",
    "                continue\n",
    "            worker_labels.append(res2num[worker_res[key]])\n",
    "        worker2labels[worker_id] = worker_labels\n",
    "    scores = []\n",
    "    for w1, labels1 in worker2labels.items():\n",
    "        for w2, labels2 in worker2labels.items():\n",
    "            if w1 == w2:\n",
    "                continue\n",
    "            fixed_labels1 = []\n",
    "            fixed_labels2 = []\n",
    "            for l1, l2 in zip(labels1, labels2):\n",
    "                if l1 == -1 or l2 == -1:\n",
    "                    continue\n",
    "                fixed_labels1.append(l1)\n",
    "                fixed_labels2.append(l2)\n",
    "            if fixed_labels1 and fixed_labels2:\n",
    "                score = cohen_kappa_score(fixed_labels1, fixed_labels2)\n",
    "                if -1.0 <= score <= 1.0:\n",
    "                    scores.append(score)\n",
    "    print(\"Avg kappa score: {}\".format(sum(scores)/len(scores)))\n",
    "\n",
    "    results = defaultdict(list)\n",
    "    for r in records:\n",
    "        results[r[\"url\"]].append(r[\"res\"])\n",
    "\n",
    "    data = {r[\"url\"]: r for r in records}\n",
    "    for url, res in results.items():\n",
    "        res_count = Counter(res)\n",
    "        if res_count.most_common(1)[0][1] < min_votes:\n",
    "            data.pop(url)\n",
    "\n",
    "    rub_cnt = Counter()\n",
    "    for _, d in data.items():\n",
    "        rub_cnt[d[\"res\"]] += 1\n",
    "    print(rub_cnt.most_common())\n",
    "\n",
    "    save_to_ft(list(data.values()), output_file_name, use_preprocess)\n",
    "\n",
    "convert_to_ft(\"en_cat_train_raw_markup.tsv\", \"en_tg_train.json\", \"en_cat_train_markup.txt\", min_votes=2, use_preprocess=True)\n",
    "convert_to_ft(\"en_cat_test_raw_markup.tsv\", \"en_tg_test.json\", \"en_cat_test_markup.txt\", min_votes=4, use_preprocess=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "soJgINOfuSe7",
    "outputId": "bab639ff-3464-4830-8a07-7cd3fa90e81e"
   },
   "outputs": [],
   "source": [
    "!cat en_cat_train_markup.txt | wc -l\n",
    "!cat en_cat_test_markup.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 253
    },
    "colab_type": "code",
    "id": "OBgdjnUJFpcq",
    "outputId": "f076dbc9-8d5d-4f61-c854-6eb95a2f7d88"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "def read_news_category_dataset(input_file, output_file, use_preprocess=True):\n",
    "    assert os.path.exists(input_file)\n",
    "    records = []\n",
    "    cat2res = {\n",
    "        \"POLITICS\": (\"society\", 100.0/32739),\n",
    "        \"ENTERTAINMENT\": (\"entertainment\", 100.0/16058),\n",
    "        \"BUSINESS\": (\"economy\", 300.0/5937),\n",
    "        \"CRIME\": (\"society\", 100.0/3405),\n",
    "        \"ARTS & CULTURE\": (\"entertainment\", 100.0/700),\n",
    "        \"CULTURE & ARTS\": (\"entertainment\", 100.0/700),\n",
    "        \"TECH\": (\"technology\", 300.0/2082),\n",
    "        \"SCIENCE\": (\"science\", 300.0/2178),\n",
    "        \"SPORTS\": (\"sports\", 300.0/4884),\n",
    "        \"HEALTHY LIVING\": (\"not_news\", 300.0/6694),\n",
    "        \"THE WORLDPOST\": (\"society\", 100.0/3405),\n",
    "        \"FOOD & DRINK\": (\"other\", 150.0/6226),\n",
    "        \"STYLE & BEAUTY\": (\"other\", 150.0/9649)\n",
    "    }\n",
    "\n",
    "    with open(input_file, \"r\") as r:\n",
    "        for line in r:\n",
    "            data = json.loads(line)\n",
    "            title = data[\"headline\"]\n",
    "            text = data[\"short_description\"]\n",
    "            data[\"title\"] = title\n",
    "            data[\"text\"] = text\n",
    "            category = data[\"category\"]\n",
    "            if category in cat2res:\n",
    "                res, prob = cat2res[category]\n",
    "                data[\"res\"] = res\n",
    "                if random.random() < prob:\n",
    "                    records.append(data)\n",
    "            # else:\n",
    "                # print(\"Skipping: \", category, title)\n",
    "    \n",
    "    rub_cnt = Counter()\n",
    "    for d in records:\n",
    "        rub_cnt[d[\"res\"]] += 1\n",
    "    print(rub_cnt.most_common())\n",
    "    \n",
    "    save_to_ft(records, output_file, use_preprocess)\n",
    "    return records\n",
    "\n",
    "read_news_category_dataset(\"News_Category_Dataset_v2.json\", \"nc_markup.txt\")\n",
    "!head nc_markup.txt\n",
    "!cat nc_markup.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 469
    },
    "colab_type": "code",
    "id": "T07Ri1yMDg_W",
    "outputId": "f6b81555-5fd3-40f2-b1e7-3dea076f1fe6"
   },
   "outputs": [],
   "source": [
    "!wget https://www.dropbox.com/s/no7x1n8acl5ykif/en_vectors_v2.bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 253
    },
    "colab_type": "code",
    "id": "rGcdwkt4EmXD",
    "outputId": "66424bb8-6049-4a6d-9956-ef4a31c821c1"
   },
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/facebookresearch/fastText/master/python/doc/examples/bin_to_vec.py\n",
    "!python bin_to_vec.py en_vectors_v2.bin > en_vectors_v2.vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aytxXtVZDnb8"
   },
   "outputs": [],
   "source": [
    "!cat en_cat_train_markup.txt > en_cat_train_all.txt\n",
    "!cat nc_markup.txt >> en_cat_train_all.txt\n",
    "!shuf en_cat_train_all.txt > en_cat_train_shuf.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "LHxIIIG0It5I",
    "outputId": "bde8a280-dacf-4ed8-d472-031767d1cc80"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "with open(\"en_cat_train_shuf.txt\", \"r\") as r, open(\"en_cat_train_train.txt\", \"w\") as train, open(\"en_cat_train_val.txt\", \"w\") as val:\n",
    "    for line in r:\n",
    "        if random.random() < 0.1:\n",
    "            val.write(line)\n",
    "        else:\n",
    "            train.write(line)\n",
    "!cat en_cat_train_val.txt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "PvaHhIP23Cir",
    "outputId": "cd85fc60-aa69-45ef-9213-c68bfd4ff2bc"
   },
   "outputs": [],
   "source": [
    "!fasttext supervised -input en_cat_train_train.txt -pretrainedVectors en_vectors_v2.vec -dim 50 -autotune-validation en_cat_train_val.txt -output en_cat_v2 -autotune-modelsize 10M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "0iwWA0PwEevw",
    "outputId": "c5446e6f-e35a-46b0-8f1b-b17a5886d9e1"
   },
   "outputs": [],
   "source": [
    "!fasttext test en_cat_v2.ftz en_cat_test_markup.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "yErIMGxLJbzx",
    "outputId": "15c95547-7227-4140-d023-12fbbd59bab8"
   },
   "outputs": [],
   "source": [
    "import fasttext\n",
    "model = fasttext.load_model(\"en_cat_v2.ftz\")\n",
    "true_labels = []\n",
    "predicted_labels = []\n",
    "errors = []\n",
    "with open(\"en_cat_test_markup.txt\", \"r\") as r:\n",
    "    for line in r:\n",
    "        words = line.strip().split(\" \")\n",
    "        label = words[0][9:]\n",
    "        true_labels.append(label)\n",
    "        text = \" \".join(words[1:])\n",
    "        predicted_label = model.predict([text])[0][0][0][9:]\n",
    "        if label != predicted_label:\n",
    "            errors.append((label, predicted_label, text[:100]))\n",
    "        predicted_labels.append(predicted_label)\n",
    "for label, predicted_label, text in errors:\n",
    "    print(\"T: {} P: {} | {}\".format(label, predicted_label, text))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "CatTrainEn.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}