IMDb_AL_example_selection_step1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['PYTHONHASHSEED'] = str(2019)\n",
    "os.environ['TRANSFORMERS_CACHE'] = 'D:\\\\python_pkg_data\\\\huggingface\\\\transformers'\n",
    "\n",
    "import json\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "import numpy as np \n",
    "np.random.seed(2019)\n",
    "import random\n",
    "random.seed(2019)\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(2019)\n",
    "torch.cuda.manual_seed_all(2019)\n",
    "torch.backends.cudnn.benchmark = False\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "import transformers\n",
    "\n",
    "\n",
    "\n",
    "from datasets import load_metric,load_dataset,Value\n",
    "import csv\n",
    "\n",
    "\n",
    "import nltk\n",
    "nltk.data.path.append('D:\\\\python_pkg_data\\\\nltk_data')\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
    "\n",
    "import ast\n",
    "import glob\n",
    "import shutil\n",
    "\n",
    "import importlib\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import Softmax\n",
    "from termcolor import colored\n",
    "from itertools import groupby\n",
    "from operator import itemgetter\n",
    "import html\n",
    "from IPython.core.display import display, HTML\n",
    "import more_itertools as mit\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {\n",
    "\n",
    "    'ori_train_dir':'./datasets/IMDb/orig/train.tsv',\n",
    "    'gpu_device':0,\n",
    "    'tokenizer': transformers.AutoTokenizer.from_pretrained('roberta-base'),\n",
    "    'dataset_cache_dir':\"D:\\\\python_pkg_data\\\\huggingface\\\\Datasets\", ## local directory for datasets\n",
    "    'train_random_seed':2019,                                        ## random seed for subsampling training set\n",
    "    'num_per_class': 25,                                             ## number of examples per class for initial training set\n",
    "    'save_dir': './AL_results/IMDb_step0_al_trainer',                                   ##directory for saving models\n",
    "    'num_per_step':50,                                                ##num labelled data per step\n",
    "    'num_per_example':7\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class CustomerDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    " ## import training set\n",
    "IMDb_data = {}\n",
    "\n",
    "with open(args['ori_train_dir'],errors='ignore') as file:\n",
    "    file = csv.reader(file, delimiter=\"\\t\")\n",
    "    for idx,row in enumerate(file):\n",
    "        if len(row)>0:\n",
    "\n",
    "            if row[0] == 'Negative':\n",
    "                IMDb_data[row[2]] = {'text':row[1],'label':0}\n",
    "            else:\n",
    "                IMDb_data[row[2]] = {'text':row[1],'label':1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_texts = {}\n",
    "imdb_labels = {}\n",
    "for key in IMDb_data.keys():\n",
    "    imdb_texts[key] = IMDb_data[key]['text']\n",
    "    imdb_labels[key] = IMDb_data[key]['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1] [850 857]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(list(imdb_labels.values())),np.bincount(list(imdb_labels.values())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Most uncertainty examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IMDb unlabelled data statistics -----------------------\n",
      "[0 1] [825 832]\n",
      "previous model ./AL_results/IMDb_step0_al_trainer_2019_25_7\\checkpoint-13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:27: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "758f881ca5a74d8ab7fde91954ecc6dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/52 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_keys = []\n",
    "with open(f\"{args['save_dir']}_{args['train_random_seed']}_{args['num_per_class']}_{args['num_per_example']}/keys.txt\") as f:\n",
    "    for line in f.readlines():\n",
    "        train_keys.append(line[:-1])\n",
    "          \n",
    "unlabelled_keys = list(set(list(imdb_labels.keys()))-set(train_keys))\n",
    "\n",
    "len(train_keys)\n",
    "\n",
    "unlabelled_texts = [imdb_texts[key] for key in unlabelled_keys]\n",
    "unlabelled_labels = [imdb_labels[key] for key in unlabelled_keys]\n",
    "unlabelled_encodings = args['tokenizer'](unlabelled_texts, truncation=True, padding=True)\n",
    "\n",
    "print('IMDb unlabelled data statistics -----------------------')\n",
    "print(np.unique(unlabelled_labels),np.bincount(unlabelled_labels))\n",
    "\n",
    "unlabelled_dataset = CustomerDataset(unlabelled_encodings, unlabelled_labels)\n",
    "unlabelled_dataloader = torch.utils.data.DataLoader(unlabelled_dataset, batch_size=32,shuffle=False)\n",
    "\n",
    "model_dir = glob.glob(f\"{args['save_dir']}_{args['train_random_seed']}_{args['num_per_class']}_{args['num_per_example']}/checkpoint*\")[0]\n",
    "print(f\"previous model {model_dir}\")\n",
    "model = transformers.AutoModelForSequenceClassification.from_pretrained(model_dir, num_labels=2).cuda(args['gpu_device'])\n",
    "\n",
    "model.eval()\n",
    "logits_list = []\n",
    "sm = torch.nn.Softmax(dim=1)\n",
    "for batch in tqdm_notebook(unlabelled_dataloader):\n",
    "    batch = {k: v.cuda(args['gpu_device']) for k, v in batch.items()}\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**batch)\n",
    "\n",
    "    logits = outputs.logits\n",
    "    logits = sm(logits)\n",
    "    logits_list = logits_list + logits.tolist()\n",
    "\n",
    "difference = np.array([abs(result[0]-result[1]) for result in logits_list])\n",
    "\n",
    "uncertainty_index = difference.argsort()[:args['num_per_step']]\n",
    "# certainty_index = difference.argsort()[-args['num_per_step']:]\n",
    "\n",
    "\n",
    "new_keys = np.array(unlabelled_keys)[uncertainty_index]\n",
    "# new_keys = np.array(unlabelled_keys)[certainty_index]\n",
    "train_keys = np.append(train_keys, new_keys)\n",
    "\n",
    "with open(f\"{args['save_dir']}_{args['train_random_seed']}_{args['num_per_class']}_{args['num_per_example']}/new_keys.txt\", \"w\") as fp:\n",
    "    for k in new_keys:\n",
    "        fp.write(str(k) +\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}