NLP_VDCNN_final_project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WA9v2a94lxsN"
   },
   "source": [
    "# Program Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "r_xECcW2mHAM",
    "outputId": "268e0ded-289e-45de-d35c-e47dee71dc0b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from google.colab import drive\n",
    "\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "IC7grzc_ltkU"
   },
   "outputs": [],
   "source": [
    "#https://drive.google.com/drive/folders/1NCLqicCgxQ8zx4mlXi9r-fl2Sd6k1uVQ?usp=sharing\n",
    "#You can download datasets from this link.(use ryerson account)\n",
    "\n",
    "DATASET='yelp_review_full'#['ag_news','yelp_review_full','yelp_review_polarity']\n",
    "PREPROCES_TYPE='add_pos'#['lower','denoiser','add_pos','add_hashtag','add_NOT']\n",
    "DATA_FOLDER = 'drive/MyDrive/NLP/datasets'\n",
    "MODELS_FOLDER = 'drive/MyDrive/NLP/models/vdcnn'\n",
    "def get_args():\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument(\"--dataset\", type=str, default=DATASET, help=\"'ag_news' or 'yelp_review_full' or 'yelp_review_polarity'\")\n",
    "    parser.add_argument(\"--preproces_type\", type=str, default=PREPROCES_TYPE, help=\"'lower' or 'denoiser' or 'add_pos' or 'add_hashtag' or 'add_NOT'\")\n",
    "    parser.add_argument(\"--model_folder\", type=str, default=MODELS_FOLDER+\"/\"+DATASET, help=\"result directory\")\n",
    "    parser.add_argument(\"--data_folder\", type=str, default=DATA_FOLDER+\"/\"+DATASET, help=\"address of datasets directory\")\n",
    "    parser.add_argument(\"--depth\", type=int, choices=[9, 17, 29, 49], default=29, help=\"Depth of the network tested in the paper (9, 17, 29, 49)\")\n",
    "    parser.add_argument(\"--maxlen\", type=int, default=1024, help=\"max lentgh of input string\")\n",
    "    parser.add_argument('--shortcut', action='store_true', default=False)\n",
    "    parser.add_argument(\"--batch_size\", type=int, default=128, help=\"number of example read by the gpu\")\n",
    "    parser.add_argument(\"--epochs\", type=int, default=10)\n",
    "    parser.add_argument(\"--solver\", type=str, default=\"sgd\", help=\"'sgd' or 'adam'\")\n",
    "    parser.add_argument(\"--lr\", type=float, default=0.01)\n",
    "    parser.add_argument(\"--lr_halve_interval\", type=float, default=10, help=\"Number of iterations before halving learning rate\")\n",
    "    parser.add_argument(\"--momentum\", type=float, default=0.9, help=\"Number of iterations before halving learning rate\")\n",
    "    parser.add_argument(\"--snapshot_interval\", type=int, default=5)\n",
    "    parser.add_argument(\"--gamma\", type=float, default=0.9)\n",
    "    parser.add_argument(\"--nthreads\", type=int, default=4)\n",
    "   \n",
    "    args,_ = parser.parse_known_args()\n",
    "    return args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "SQYyrfqYl7SW"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nxxBKDRetTYt"
   },
   "source": [
    "#import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "f4PZbTOuT79_"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import lmdb\n",
    "import pickle\n",
    "import sys\n",
    "import csv\n",
    "import tarfile\n",
    "import shutil\n",
    "import hashlib\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "import tqdm\n",
    "import argparse\n",
    "import tarfile\n",
    "import gzip\n",
    "\n",
    "from tqdm import tqdm\n",
    "from urllib.request import urlretrieve\n",
    "from urllib.error import URLError\n",
    "from urllib.error import HTTPError\n",
    "from collections import Counter\n",
    "from sklearn import utils, metrics\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import os, subprocess\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "\n",
    "#multiprocessing workaround\n",
    "import resource\n",
    "rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)\n",
    "resource.setrlimit(resource.RLIMIT_NOFILE, (2048, rlimit[1]))\n",
    "# get device to calculate on (either CPU or GPU with minimum memory load)\n",
    "def get_gpu_memory_map():\n",
    "    \n",
    "    result = subprocess.check_output(\n",
    "        [\n",
    "            'nvidia-smi', '--query-gpu=memory.used',\n",
    "            '--format=csv,nounits,noheader'\n",
    "        ], encoding='utf-8')\n",
    "    \n",
    "    # Convert lines into a dictionary\n",
    "    gpu_memory = [int(x) for x in result.strip().split('\\n')]\n",
    "    gpu_memory_map = dict(zip(range(len(gpu_memory)), gpu_memory))\n",
    "    \n",
    "    return gpu_memory_map\n",
    "\n",
    "def get_device():\n",
    "    if torch.cuda.is_available():\n",
    "        memory_map = get_gpu_memory_map()\n",
    "        device = \"cuda:%d\" % min(memory_map, key=memory_map.get)\n",
    "    else:\n",
    "        device = \"cpu\"\n",
    "    \n",
    "    print(\"Device:\", device)\n",
    "    return device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wqu-VenOUuZQ"
   },
   "source": [
    "#text preprosesing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wL_SOoJHSC7B",
    "outputId": "9e7b8b85-19a1-4c75-c51a-0eb3189f5b1b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /root/nltk_data...\n",
      "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
      "[nltk_data] Downloading package tagsets to /root/nltk_data...\n",
      "[nltk_data]   Unzipping help/tagsets.zip.\n",
      "[nltk_data] Downloading package treebank to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/treebank.zip.\n",
      "[nltk_data] Downloading package brown to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/brown.zip.\n",
      "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n",
      "[nltk_data]   Unzipping taggers/universal_tagset.zip.\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "import string\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download(\"averaged_perceptron_tagger\")\n",
    "nltk.download('tagsets')\n",
    "nltk.download('treebank')\n",
    "nltk.download('brown')\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "class Preprocessing():\n",
    "\n",
    "\n",
    "    def __init__(self,Preprocess_Type='lower'):\n",
    "        self.Preprocess_Type = Preprocess_Type\n",
    "        self.stop_list = set(nltk.corpus.stopwords.words('english')+[\"#\",\"@\"])#english stop words\n",
    "        self.temp_tag  = ['PRP','VBZ','CC','POS','IN','DT','TO','PRP$']#noise words\n",
    "        self.stemmer   = nltk.stem.PorterStemmer()\n",
    "        self.word_tokenize  = nltk.tokenize.word_tokenize\n",
    "        self.pos_tag   = nltk.pos_tag\n",
    "        self.punctuation = list(string.punctuation)\n",
    "        self.neg_words = [\"n't\", \"not\", \"no\", \"never\"]#used in Add_Not preprocessing\n",
    "\n",
    "    def transform(self, sentences):\n",
    "        \"\"\"\n",
    "        sentences: list(str) \n",
    "        output: list(str)\n",
    "        \"\"\"\n",
    "        if self.Preprocess_Type=='lower':\n",
    "          return [s.lower() for s in sentences]\n",
    "        elif self.Preprocess_Type=='denoiser':\n",
    "          return [self.denoiser(s) for s in sentences]\n",
    "        elif self.Preprocess_Type=='add_pos':\n",
    "          return [self.add_pos(s) for s in sentences]\n",
    "        elif self.Preprocess_Type=='add_hashtag':\n",
    "          return [self.add_hashtag(s) for s in sentences]\n",
    "        elif self.Preprocess_Type== 'add_NOT':\n",
    "          return [self.add_Not(s) for s in sentences]\n",
    "\n",
    "\n",
    "    def denoiser(self,text):\n",
    "        new_text=\"\"\n",
    "        words=self.word_tokenize(text)\n",
    "        words=nltk.pos_tag(words)\n",
    "        words=[word.lower() for word,tag in words if tag not in self.temp_tag]#remove some extra words depends on their pos_tag\n",
    "        words=[self.stemmer.stem(word) for word in words if word not in self.stop_list]#remove stop words and stemming\n",
    "        for word in words:\n",
    "            new_text=new_text+word+\" \"\n",
    "        new_text=new_text[:-1]\n",
    "        return new_text\n",
    "\n",
    "    def add_pos(self,text):\n",
    "        new_text=\"\"\n",
    "        words=self.word_tokenize(text)\n",
    "        words=nltk.pos_tag(words)\n",
    "        #concate pos_tag to the end of words,remove some extra words depends on their pos_tag,remove stop words and stemming\n",
    "        words=[(self.stemmer.stem(word)+'@'+tag).lower() for word,tag in words if ((tag not in self.temp_tag) and (word.lower() not in self.stop_list))]\n",
    "        for word in words:\n",
    "            new_text=new_text+word+\" \"\n",
    "        new_text=new_text[:-1]\n",
    "        return new_text\n",
    "\n",
    "    def add_hashtag(self,text):\n",
    "        hashtaged = lambda word : '#'+word\n",
    "        new_text=\"\"\n",
    "        words=self.word_tokenize(text)\n",
    "        words=nltk.pos_tag(words)\n",
    "        words2=[]\n",
    "        hashtag=False\n",
    "        for word,tag in words:\n",
    "            if word=='#': \n",
    "                hashtag=True # if a previous token is '#' next token concated with '#' \n",
    "            if ((tag not in self.temp_tag) and (word not in self.stop_list)) or (word!='#' and hashtag==True):#remove some extra words depends on their pos_tag,remove stop words \n",
    "                if word[0].isupper():\n",
    "                    hashtag=True # if a word is captalize will concate with '#'\n",
    "                new_word=self.stemmer.stem(word).lower()# stemming\n",
    "                if hashtag==False:\n",
    "                    words2.append(new_word)\n",
    "                if hashtag:\n",
    "                    words2.append(hashtaged(new_word))\n",
    "                    hashtag=False\n",
    "\n",
    "\n",
    "        words=words2\n",
    "        for word in words:\n",
    "            new_text=new_text+word+\" \"\n",
    "        new_text=new_text[:-1]\n",
    "        return new_text\n",
    "\n",
    "    def add_Not(self,text):\n",
    "        new_text=\"\"\n",
    "        words= self.word_tokenize(text)\n",
    "        words=self.pos_tag(words)\n",
    "        words=[word.lower() for word,tag in words if tag not in self.temp_tag]#remove some extra words depends on their pos_tag \n",
    "        words=[self.stemmer.stem(word) for word in words ] \n",
    "        flag = 0  # start with the flag in the off position\n",
    "        not_stem=[]\n",
    "        for word in words:\n",
    "            # if flag is on then append word with \"NOT_\"\n",
    "            if flag == 1:\n",
    "                # check if the word is a punctuation (this is where we need to stop if flag==1)\n",
    "                if word in  self.punctuation:\n",
    "                    # don't append anything to a punctuation\n",
    "                    # if we reached here then it means the flag is to be turned off\n",
    "                    not_stem.append(word)\n",
    "                elif(word not in  self.neg_words):\n",
    "                    not_stem.append(\"not_\"+word)\n",
    "                    \n",
    "            # otherwise add the word without making any changes\n",
    "            else:\n",
    "                not_stem.append(word)\n",
    "            \n",
    "            # if the word is a negative word then turn on the flag\n",
    "            if word in  self.neg_words:\n",
    "                flag=1\n",
    "            # if word is a punctuation then word off the flag\n",
    "            if word in  self.punctuation:\n",
    "                flag=0\n",
    "                \n",
    "        for word in not_stem:\n",
    "            new_text=new_text+word+\" \"\n",
    "        new_text=new_text[:-1]\n",
    "        return new_text\n",
    "\n",
    " # valid alpabet or charactars\n",
    "class CharVectorizer():\n",
    "    def __init__(self, maxlen=1024, padding='pre', truncating='pre', alphabet=\"\"\"abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:’\"/|$%ˆ&*˜‘+=<>()[]{}#@\"\"\"):\n",
    "        \n",
    "        self.alphabet = alphabet\n",
    "        self.maxlen = maxlen\n",
    "        self.padding = padding\n",
    "        self.truncating = truncating\n",
    "\n",
    "        self.char_dict = {'_pad_': 0, '_unk_': 1, ' ': 0} \n",
    "        for i, k in enumerate(self.alphabet, start=2):\n",
    "            self.char_dict[k] = i\n",
    "\n",
    "    def transform(self,sentences):\n",
    "        \"\"\"\n",
    "        sentences: list of string\n",
    "        list of review, review is a list of sequences, sequences is a list of int\n",
    "        \"\"\"\n",
    "        sequences = []\n",
    "\n",
    "        for sentence in sentences:\n",
    "            seq = [self.char_dict.get(char, self.char_dict[\"_unk_\"]) for char in sentence]\n",
    "            \n",
    "            if self.maxlen:\n",
    "                length = len(seq)\n",
    "\n",
    "                if length > self.maxlen:# we need to crop the sequence\n",
    "\n",
    "                    if self.truncating == 'pre':# we crope from the end of the sequence\n",
    "                        seq = seq[-self.maxlen:]\n",
    "                    elif self.truncating == 'post':# we crop the beggining of the sequence\n",
    "                        seq = seq[:self.maxlen]\n",
    "\n",
    "                if length < self.maxlen:# we need to pad the sequence\n",
    "\n",
    "                    diff = np.abs(length - self.maxlen)\n",
    "                    if self.padding == 'pre':#We pad in the beggining\n",
    "                        seq = [self.char_dict['_pad_']] * diff + seq\n",
    "                    elif self.padding == 'post':#We pad at the end\n",
    "                        seq = seq + [self.char_dict['_pad_']] * diff\n",
    "\n",
    "            sequences.append(seq)                \n",
    "\n",
    "        return sequences        \n",
    "    \n",
    "    def get_params(self):\n",
    "        params = vars(self)\n",
    "        return params\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9suUkpG1e_4t"
   },
   "source": [
    "#Detasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "VxajptuRVZxC"
   },
   "outputs": [],
   "source": [
    "class AgNews(object):\n",
    "    \"\"\"\n",
    "    credit goes to Xiang Zhang:\n",
    "    https://scholar.google.com/citations?hl=en&user=n4QjVfoAAAAJ&view_op=list_works&sortby=pubdate\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.data_name = 'ag_news'\n",
    "        self.data_folder = \"{}/{}\".format(DATA_FOLDER, self.data_name)\n",
    "        self.n_classes = 4        \n",
    "        self.epoch_size = 15\n",
    "\n",
    "    def _generator(self, file_name):\n",
    "        DataPath=os.path.join(self.data_folder, file_name)\n",
    "        with open(DataPath, mode='r', encoding='utf-8') as f:\n",
    "            reader = csv.DictReader(f, fieldnames=['label', 'title', 'description'], quotechar='\"')\n",
    "            for i,line in enumerate(reader):\n",
    "                #if(i%10==0):\n",
    "                    sentence = \"{} {}\".format(line['title'], line['description'])\n",
    "                    label = int(line['label']) - 1\n",
    "                    yield sentence, label\n",
    "\n",
    "    def load_train_data(self):\n",
    "        return self._generator(file_name=\"train.csv\")\n",
    "\n",
    "    def load_test_data(self):\n",
    "        return self._generator(file_name=\"test.csv\")\n",
    "\n",
    "class YelpReview(object):\n",
    "    \"\"\"\n",
    "    credit goes to Xiang Zhang:\n",
    "    https://scholar.google.com/citations?hl=en&user=n4QjVfoAAAAJ&view_op=list_works&sortby=pubdate\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.data_name = 'yelp_review_full'\n",
    "        \n",
    "        self.data_folder = \"{}/{}\".format(DATA_FOLDER, self.data_name)\n",
    "        self.n_classes = 5\n",
    "        self.epoch_size = 15\n",
    "\n",
    "    def _generator(self, file_name):\n",
    "        DataPath=os.path.join(self.data_folder, file_name)\n",
    "        with open(DataPath, mode='r', encoding='utf-8') as f:\n",
    "            reader = csv.DictReader(f, fieldnames=['label', 'title', 'description'], quotechar='\"')\n",
    "            for i,line in enumerate(reader):\n",
    "              #if(i%10==0):\n",
    "                sentence = \"{} {}\".format(line['title'], line['description'])\n",
    "                label = int(line['label']) - 1\n",
    "                yield sentence, label\n",
    "\n",
    "    def load_train_data(self):\n",
    "        return self._generator(\"train.csv\")\n",
    "\n",
    "    def load_test_data(self):\n",
    "        return self._generator(\"test.csv\")\n",
    "\n",
    "class YelpPolarity(object):\n",
    "    \"\"\"\n",
    "    credit goes to Xiang Zhang:\n",
    "    https://scholar.google.com/citations?hl=en&user=n4QjVfoAAAAJ&view_op=list_works&sortby=pubdate\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "\n",
    "        self.data_name ='yelp_review_polarity'\n",
    "        self.data_folder = \"{}/{}\".format(DATA_FOLDER, self.data_name)\n",
    "        self.n_classes = 2        \n",
    "        self.epoch_size = 15\n",
    "\n",
    "    def _generator(self, file_name):\n",
    "        DataPath=os.path.join(self.data_folder, file_name)\n",
    "        with open(DataPath, mode='r', encoding='utf-8') as f:\n",
    "            reader = csv.DictReader(f, fieldnames=['label', 'title', 'description'], quotechar='\"')\n",
    "            for i,line in enumerate(reader):\n",
    "                #if(i%10==0):\n",
    "                    sentence = \"{} {}\".format(line['title'], line['description'])\n",
    "                    label = int(line['label']) - 1\n",
    "                    yield sentence, label\n",
    "\n",
    "    def load_train_data(self):\n",
    "        return self._generator(\"train.csv\")\n",
    "\n",
    "    def load_test_data(self):\n",
    "        return self._generator(\"test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CONupt7umje_"
   },
   "source": [
    "#load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "b9k_Y4kUmpk2"
   },
   "outputs": [],
   "source": [
    "def load_datasets(name=\"ag_news\"):\n",
    "    dataset =None\n",
    "    if name=='ag_news':\n",
    "        dataset=AgNews()\n",
    "    if name=='yelp_review_full':\n",
    "        dataset=YelpReview()\n",
    "    if name=='yelp_review_polarity':\n",
    "        dataset=YelpPolarity()\n",
    "    return dataset\n",
    "\n",
    "\n",
    "class TupleLoader(Dataset): #torch.utils.dat.Dataset\n",
    "\n",
    "    def __init__(self, path=\"\"):\n",
    "        self.path = path\n",
    "\n",
    "        self.env = lmdb.open(path, max_readers=opt.nthreads, readonly=True, lock=False, readahead=False, meminit=False)\n",
    "        self.txn = self.env.begin(write=False)\n",
    "\n",
    "    def __len__(self):\n",
    "        return list_from_bytes(self.txn.get('nsamples'.encode()))[0]\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        xtxt = list_from_bytes(self.txn.get(('txt-%09d' % i).encode()), np.int)\n",
    "        lab = list_from_bytes(self.txn.get(('lab-%09d' % i).encode()), np.int)[0]\n",
    "        return xtxt, lab\n",
    "\n",
    "def list_to_bytes(l):\n",
    "    return np.array(l).tobytes()\n",
    "\n",
    "\n",
    "def list_from_bytes(string, dtype=np.int):\n",
    "    return np.frombuffer(string, dtype=dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VhX1FYkpmY4f"
   },
   "source": [
    "# VDCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "5Ym0uw0_mXC-"
   },
   "outputs": [],
   "source": [
    "class BasicConvResBlock(nn.Module):\n",
    "\n",
    "    def __init__(self, input_dim=128, n_filters=256, kernel_size=3, padding=1, stride=1, shortcut=False, downsample=None):\n",
    "        super(BasicConvResBlock, self).__init__()\n",
    "\n",
    "        self.downsample = downsample\n",
    "        self.shortcut = shortcut\n",
    "\n",
    "        self.conv1 = nn.Conv1d(input_dim, n_filters, kernel_size=kernel_size, padding=padding, stride=stride)\n",
    "        self.bn1 = nn.BatchNorm1d(n_filters)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.conv2 = nn.Conv1d(n_filters, n_filters, kernel_size=kernel_size, padding=padding, stride=stride)\n",
    "        self.bn2 = nn.BatchNorm1d(n_filters)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        if self.shortcut:\n",
    "            if self.downsample is not None:\n",
    "                residual = self.downsample(x)\n",
    "            out += residual\n",
    "\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "\n",
    "class VDCNN(nn.Module):\n",
    "\n",
    "    def __init__(self, n_classes=2, num_embedding=141, embedding_dim=16, depth=9, n_fc_neurons=2048, shortcut=False):\n",
    "        super(VDCNN, self).__init__()\n",
    "\n",
    "        layers = []\n",
    "        fc_layers = []\n",
    "        self.embed = nn.Embedding(num_embedding,embedding_dim, padding_idx=0, max_norm=None, norm_type=2, scale_grad_by_freq=False, sparse=False)\n",
    "        layers.append(nn.Conv1d(embedding_dim, 64, kernel_size=3, padding=1))\n",
    "\n",
    "        if depth == 9:\n",
    "            n_conv_block_64, n_conv_block_128, n_conv_block_256, n_conv_block_512 = 1, 1, 1, 1\n",
    "        elif depth == 17:\n",
    "            n_conv_block_64, n_conv_block_128, n_conv_block_256, n_conv_block_512 = 2, 2, 2, 2\n",
    "        elif depth == 29:\n",
    "            n_conv_block_64, n_conv_block_128, n_conv_block_256, n_conv_block_512 = 5, 5, 2, 2\n",
    "        elif depth == 49:\n",
    "            n_conv_block_64, n_conv_block_128, n_conv_block_256, n_conv_block_512 = 8, 8, 5, 3\n",
    "\n",
    "        layers.append(BasicConvResBlock(input_dim=64, n_filters=64, kernel_size=3, padding=1, shortcut=shortcut))\n",
    "        for _ in range(n_conv_block_64-1):\n",
    "            layers.append(BasicConvResBlock(input_dim=64, n_filters=64, kernel_size=3, padding=1, shortcut=shortcut))  \n",
    "        layers.append(nn.MaxPool1d(kernel_size=3, stride=2, padding=1)) # l = initial length / 2\n",
    "\n",
    "        ds = nn.Sequential(nn.Conv1d(64, 128, kernel_size=1, stride=1, bias=False), nn.BatchNorm1d(128))\n",
    "        layers.append(BasicConvResBlock(input_dim=64, n_filters=128, kernel_size=3, padding=1, shortcut=shortcut, downsample=ds))\n",
    "        for _ in range(n_conv_block_128-1):\n",
    "            layers.append(BasicConvResBlock(input_dim=128, n_filters=128, kernel_size=3, padding=1, shortcut=shortcut))\n",
    "        layers.append(nn.MaxPool1d(kernel_size=3, stride=2, padding=1)) # l = initial length / 4\n",
    "\n",
    "        ds = nn.Sequential(nn.Conv1d(128, 256, kernel_size=1, stride=1, bias=False), nn.BatchNorm1d(256))\n",
    "        layers.append(BasicConvResBlock(input_dim=128, n_filters=256, kernel_size=3, padding=1, shortcut=shortcut, downsample=ds))\n",
    "        for _ in range(n_conv_block_256 - 1):\n",
    "            layers.append(BasicConvResBlock(input_dim=256, n_filters=256, kernel_size=3, padding=1, shortcut=shortcut))\n",
    "        layers.append(nn.MaxPool1d(kernel_size=3, stride=2, padding=1))\n",
    "\n",
    "        ds = nn.Sequential(nn.Conv1d(256, 512, kernel_size=1, stride=1, bias=False), nn.BatchNorm1d(512))\n",
    "        layers.append(BasicConvResBlock(input_dim=256, n_filters=512, kernel_size=3, padding=1, shortcut=shortcut, downsample=ds))\n",
    "        for _ in range(n_conv_block_512 - 1):\n",
    "            layers.append(BasicConvResBlock(input_dim=512, n_filters=512, kernel_size=3, padding=1, shortcut=shortcut))\n",
    "\n",
    "        layers.append(nn.AdaptiveMaxPool1d(8))\n",
    "        fc_layers.extend([nn.Linear(8*512, n_fc_neurons), nn.ReLU()])\n",
    "        # layers.append(nn.MaxPool1d(kernel_size=8, stride=2, padding=0))\n",
    "        # fc_layers.extend([nn.Linear(61*512, n_fc_neurons), nn.ReLU()])\n",
    "\n",
    "        fc_layers.extend([nn.Linear(n_fc_neurons, n_fc_neurons), nn.ReLU()])\n",
    "        fc_layers.extend([nn.Linear(n_fc_neurons, n_classes)])\n",
    "\n",
    "        self.layers = nn.Sequential(*layers)\n",
    "        self.fc_layers = nn.Sequential(*fc_layers)\n",
    "\n",
    "        self.__init_weights()\n",
    "\n",
    "    def __init_weights(self):\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv1d):\n",
    "                nn.init.kaiming_normal_(m.weight, mode='fan_in')\n",
    "                if m.bias is not None:\n",
    "                    nn.init.constant_(m.bias, 0)\n",
    "\n",
    "    def forward(self, x):# 128x1024x69 (b X s X f0)\n",
    "\n",
    "        out = self.embed(x)         # 128x1024x16\n",
    "        out = out.transpose(1, 2)   #128x1024x16-->128x16x1024\n",
    "\n",
    "        out = self.layers(out)       #covelutional layers (feature extraction)\n",
    "\n",
    "        out = out.view(out.size(0), -1)   #flatten (After training this output can be used for any ML method)\n",
    "\n",
    "        out = self.fc_layers(out)     #fully connected layers(prediction layer)\n",
    "\n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "JM0CxPsMncdT"
   },
   "outputs": [],
   "source": [
    "####################################################################################################################################\n",
    "def train(epoch,net,dataset,device,msg=\"val/test\",optimize=False,optimizer=None,scheduler=None,criterion=None):\n",
    "    \n",
    "    net.train() if optimize else net.eval()\n",
    "\n",
    "    epoch_loss = 0\n",
    "    nclasses = len(list(net.parameters())[-1])\n",
    "    cm = np.zeros((nclasses,nclasses), dtype=int)\n",
    "\n",
    "    with tqdm(total=len(dataset),desc=\"Epoch {} - {}\".format(epoch, msg)) as pbar:\n",
    "        for iteration, (tx, ty) in enumerate(dataset):\n",
    "            \n",
    "            data = (tx, ty)\n",
    "            data = [x.to(device) for x in data]\n",
    "\n",
    "            if optimize:\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "            out = net(data[0])\n",
    "            ty_prob = F.softmax(out, 1) # probabilites\n",
    "\n",
    "            #metrics\n",
    "            y_true = data[1].detach().cpu().numpy()\n",
    "            y_pred = ty_prob.max(1)[1].cpu().numpy()\n",
    "\n",
    "            cm += metrics.confusion_matrix(y_true, y_pred, labels=range(nclasses))\n",
    "            dic_metrics = get_metrics(cm, list_metrics)\n",
    "            \n",
    "            loss =  criterion(out, data[1]) \n",
    "            epoch_loss += loss.item()\n",
    "            dic_metrics['logloss'] = epoch_loss/(iteration+1)\n",
    "\n",
    "            if optimize:\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                dic_metrics['lr'] = optimizer.state_dict()['param_groups'][0]['lr']\n",
    "\n",
    "            pbar.update(1)\n",
    "            pbar.set_postfix(dic_metrics)\n",
    "\n",
    "    if scheduler:\n",
    "        scheduler.step()\n",
    "####################################################################################################################\n",
    "def predict(net,dataset,device,msg=\"prediction\"):\n",
    "    \n",
    "    net.eval()\n",
    "\n",
    "    y_probs, y_trues = [], []\n",
    "\n",
    "    for iteration, (tx, ty) in tqdm(enumerate(dataset), total=len(dataset), desc=\"{}\".format(msg)):\n",
    "\n",
    "        data = (tx, ty)\n",
    "        data = [x.to(device) for x in data]\n",
    "        out = net(data[0])\n",
    "        ty_prob = F.softmax(out, 1) # probabilites\n",
    "        y_probs.append(ty_prob.detach().cpu().numpy())\n",
    "        y_trues.append(data[1].detach().cpu().numpy())\n",
    "\n",
    "    return np.concatenate(y_probs, 0), np.concatenate(y_trues, 0).reshape(-1, 1)\n",
    "###############################################################################################################\n",
    "def save(net, path):\n",
    "    \"\"\"\n",
    "    Saves a model's state and it's embedding dic by piggybacking torch's save function\n",
    "    \"\"\"\n",
    "    dict_m = net.state_dict()\n",
    "    torch.save(dict_m,path)\n",
    "#################################################################################################################\n",
    "def get_metrics(cm, list_metrics):\n",
    "    \"\"\"Compute metrics from a confusion matrix (cm)\n",
    "    cm: sklearn confusion matrix\n",
    "    returns:\n",
    "    dict: {metric_name: score}\n",
    "\n",
    "    \"\"\"\n",
    "    dic_metrics = {}\n",
    "    total = np.sum(cm)\n",
    "\n",
    "    if 'accuracy' in list_metrics:\n",
    "        out = np.sum(np.diag(cm))\n",
    "        dic_metrics['accuracy'] = out/total\n",
    "\n",
    "    if 'pres_0' in list_metrics:\n",
    "        num = cm[0, 0]\n",
    "        den = cm[:, 0].sum()\n",
    "        dic_metrics['pres_0'] =  num/den if den > 0 else 0\n",
    "\n",
    "    if 'pres_1' in list_metrics:\n",
    "        num = cm[1, 1]\n",
    "        den = cm[:, 1].sum()\n",
    "        dic_metrics['pres_1'] = num/den if den > 0 else 0\n",
    "\n",
    "    if 'recall_0' in list_metrics:\n",
    "        num = cm[0, 0]\n",
    "        den = cm[0, :].sum()\n",
    "        dic_metrics['recall_0'] = num/den if den > 0 else 0\n",
    "\n",
    "    if 'recall_1' in list_metrics:\n",
    "        num = cm[1, 1]\n",
    "        den = cm[1, :].sum()\n",
    "        dic_metrics['recall_1'] =  num/den if den > 0 else 0\n",
    "\n",
    "    return dic_metrics\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_doWY1nXLyoK"
   },
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QfD_Ehnrrb63"
   },
   "outputs": [],
   "source": [
    "opt = get_args()\n",
    "print(\"parameters: {}\".format(vars(opt)))\n",
    "    \n",
    "os.makedirs(opt.model_folder, exist_ok=True)\n",
    "os.makedirs(opt.data_folder, exist_ok=True)\n",
    "\n",
    "dataset = load_datasets(opt.dataset)\n",
    "dataset_name = dataset.data_name\n",
    "n_classes = dataset.n_classes\n",
    "print(\"dataset: {}, n_classes: {}\".format(dataset_name, n_classes))\n",
    "\n",
    "tr_path =  \"{}/train.lmdb\".format(opt.data_folder)\n",
    "te_path = \"{}/test.lmdb\".format(opt.data_folder)\n",
    "    \n",
    "# check if datasets exis\n",
    "all_exist = True if (os.path.exists(tr_path) and os.path.exists(te_path)) else False\n",
    "all_exist=False\n",
    "preprocessor = Preprocessing(opt.preproces_type)\n",
    "vectorizer = CharVectorizer(maxlen=opt.maxlen, padding='post', truncating='pre')\n",
    "n_tokens = len(vectorizer.char_dict)\n",
    "\n",
    "if not all_exist:\n",
    "    print(\"Creating datasets\")\n",
    "    tr_sentences = [txt for txt,lab in tqdm(dataset.load_train_data(), desc=\"counting train samples\")]\n",
    "    te_sentences = [txt for txt,lab in tqdm(dataset.load_test_data(), desc=\"counting test samples\")]\n",
    "            \n",
    "    n_tr_samples = len(tr_sentences)\n",
    "    n_te_samples = len(te_sentences)\n",
    "    del tr_sentences\n",
    "    del te_sentences\n",
    "\n",
    "    print(\"[{}/{}] train/test samples\".format(n_tr_samples, n_te_samples))\n",
    "\n",
    "    ###################\n",
    "    # transform train #\n",
    "    ###################\n",
    "    with lmdb.open(tr_path, map_size=1099511627776) as env:\n",
    "        with env.begin(write=True) as txn:\n",
    "            for i, (sentence, label) in enumerate(tqdm(dataset.load_train_data(), desc=\"transform train...\", total= n_tr_samples)):\n",
    "\n",
    "                xtxt = vectorizer.transform(preprocessor.transform([sentence]))[0]\n",
    "                lab = label\n",
    "\n",
    "                txt_key = 'txt-%09d' % i\n",
    "                lab_key = 'lab-%09d' % i\n",
    "                    \n",
    "                txn.put(lab_key.encode(), list_to_bytes([lab]))\n",
    "                txn.put(txt_key.encode(), list_to_bytes(xtxt))\n",
    "\n",
    "            txn.put('nsamples'.encode(), list_to_bytes([i+1]))\n",
    "\n",
    "    ##################\n",
    "    # transform test #\n",
    "    ##################\n",
    "    with lmdb.open(te_path, map_size=1099511627776) as env:  #\n",
    "        with env.begin(write=True) as txn:\n",
    "            for i, (sentence, label) in enumerate(tqdm(dataset.load_test_data(), desc=\"transform test...\", total= n_te_samples)):\n",
    "\n",
    "                xtxt = vectorizer.transform(preprocessor.transform([sentence]))[0]\n",
    "                lab = label\n",
    "\n",
    "                txt_key = 'txt-%09d' % i\n",
    "                lab_key = 'lab-%09d' % i\n",
    "                    \n",
    "                txn.put(lab_key.encode(), list_to_bytes([lab]))\n",
    "                txn.put(txt_key.encode(), list_to_bytes(xtxt))\n",
    "\n",
    "            txn.put('nsamples'.encode(), list_to_bytes([i+1]))\n",
    "\n",
    "                \n",
    "tr_loader = DataLoader(TupleLoader(tr_path), batch_size=opt.batch_size, shuffle=True, num_workers=opt.nthreads, pin_memory=True)\n",
    "te_loader = DataLoader(TupleLoader(te_path), batch_size=opt.batch_size, shuffle=False, num_workers=opt.nthreads, pin_memory=False) #num_workers=opt.nthreads\n",
    "\n",
    "# select cpu or gpu\n",
    "device = get_device()\n",
    "list_metrics = ['accuracy']\n",
    "\n",
    "\n",
    "print(\"Creating model...\")\n",
    "net = VDCNN(n_classes=n_classes, num_embedding=int(n_tokens + 1), embedding_dim=16, depth=opt.depth, n_fc_neurons=2048, shortcut=opt.shortcut)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "net.to(device)\n",
    "\n",
    "assert opt.solver in ['sgd', 'adam']\n",
    "if opt.solver == 'sgd':\n",
    "    print(\" - optimizer: sgd\")\n",
    "    optimizer = torch.optim.SGD(net.parameters(), lr = opt.lr, momentum=opt.momentum)\n",
    "elif opt.solver == 'adam':\n",
    "    print(\" - optimizer: adam\")\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr = opt.lr)    \n",
    "        \n",
    "scheduler = None\n",
    "if opt.lr_halve_interval and  opt.lr_halve_interval > 0:\n",
    "    print(\" - lr scheduler: {}\".format(opt.lr_halve_interval))\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, opt.lr_halve_interval, gamma=opt.gamma, last_epoch=-1)\n",
    "        \n",
    "for epoch in range(1, opt.epochs + 1):\n",
    "    train(epoch,net, tr_loader, device, msg=\"training\", optimize=True, optimizer=optimizer, scheduler=scheduler, criterion=criterion)\n",
    "    train(epoch,net, te_loader, device, msg=\"testing \", criterion=criterion)\n",
    "\n",
    "    if (epoch % opt.snapshot_interval == 0) and (epoch > 0):\n",
    "        path = \"{}/model_epoch_{}\".format(opt.model_folder,epoch)\n",
    "        print(\"snapshot of model saved as {}\".format(path))\n",
    "        save(net, path=path)\n",
    "\n",
    "\n",
    "if opt.epochs > 0:\n",
    "    path = \"{}/model_epoch_{}\".format(opt.model_folder,opt.epochs)\n",
    "    print(\"snapshot of model saved as {}\".format(path))\n",
    "    save(net, path=path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YGoY4bgKkOL-"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "NLP_VDCNN_final_project.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}