notebooks/gru.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis - NLU\n",
    "\n",
    "## Version: **GRU - Gated Recurrent Unit**\n",
    "Student: Francesco Laiti\n",
    "\n",
    "---"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains the source code to build, train and evaluate a GRU-based sentiment analysis model using the PyTorch library."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre requirements"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the requirements to run correctly the notebook and load properly the datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "import nltk\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "from nltk.corpus import movie_reviews\n",
    "from nltk.corpus import subjectivity\n",
    "\n",
    "import numpy \n",
    "from tqdm import tqdm\n",
    "import yaml\n",
    "import os\n",
    "from collections import Counter\n",
    "import random\n",
    "\n",
    "import wandb\n",
    "wandb.login()\n",
    "\n",
    "nltk.download('movie_reviews')\n",
    "nltk.download('subjectivity')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Declare the global constants used in this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset\n",
    "BATCH_SIZE_SUBJ = 4096\n",
    "BATCH_SIZE_POL = 512\n",
    "PAD_TOKEN = 0\n",
    "REMOVE_STOPWORDS = False\n",
    "N_SPLIT = 5 # default value of Stratified K-Fold\n",
    "K_FOLD = 0 # random.randint(0,4) to pick up a random fold of Stratified K-Fold\n",
    "\n",
    "# pre-trained word embedding\n",
    "URL_GLOVE_TOKENS = '6B' # 42B , 840B, twitter.27B \n",
    "\n",
    "# GRU configuration\n",
    "OUT_SIZE = 2\n",
    "HID_SIZE = 128\n",
    "EMB_SIZE = 300\n",
    "N_LAYER = 2\n",
    "DROPOUT = 0.5\n",
    "\n",
    "# training\n",
    "EPOCHS = 30\n",
    "OPTIMIZER = 'Adam' \n",
    "LR = 0.001\n",
    "LR_EMBEDDING = 0.01\n",
    "PATIENCE = 7\n",
    "N_AFTER_COMMA = 3\n",
    "\n",
    "# env\n",
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# wandb settings\n",
    "PROJECT_NAME = \"sentiment-analysis-nlu\"\n",
    "ENTITY = \"laitifrancesco\"\n",
    "MODE_WANDB = \"disabled\" # \"online\"\n",
    "\n",
    "# paths\n",
    "WEIGHTS_PATH = 'weights/rnn/'\n",
    "HEATMAP_PATH = 'heatmap/'\n",
    "\n",
    "# for reproducibility\n",
    "RANDOM_STATE = 42\n",
    "g = torch.Generator().manual_seed(0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_complete_saved_path(weights_path, config):\n",
    "    weights_path += config.dataset\n",
    "    if not os.path.exists(weights_path):\n",
    "        os.makedirs(weights_path, exist_ok=True)\n",
    "    weights_path += '/' + config.model.lower() + '.pt'\n",
    "    return weights_path "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset & Dataloader"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The datasets used in this notebook are ``subjectivity`` and ``movie_reviews``, both imported from NLTK library.\n",
    "\n",
    "We create a stratified k-fold validator. To access to different fold (in this case a 5-fold), simply change the ``K_FOLD`` parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "def make_stratified_k_fold(corpus, labels):\n",
    "    cv = StratifiedKFold(n_splits=N_SPLIT, shuffle=True, random_state=RANDOM_STATE)\n",
    "    fold = list(cv.split(corpus, labels))\n",
    "\n",
    "    return fold\n",
    "\n",
    "def pick_k_fold(data, labels, fold, k_fold = 0):\n",
    "    fold = fold[k_fold]\n",
    "    dataset = numpy.array(data, dtype=numpy.object0)\n",
    "    labels = numpy.array(labels)\n",
    "\n",
    "    X_train = dataset[fold[0]]\n",
    "    y_train = labels[fold[0]]\n",
    "\n",
    "    X_test = dataset[fold[1]]\n",
    "    y_test = labels[fold[1]]\n",
    "\n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.vocab import GloVe\n",
    "\n",
    "class Tokenizer():\n",
    "    def __init__(self, corpus, word2id = None, name_dataset = '', build_pretrained = False):\n",
    "        self.name_dataset = name_dataset\n",
    "        self.corpus2id, self.word2id = self._tokenizer(corpus, word2id)\n",
    "        self.pretrained_word2id = None\n",
    "        if build_pretrained:\n",
    "            self.pretrained_word2id = self._build_vocab()\n",
    "\n",
    "    # Adapted code from the lab\n",
    "    # Map sequences to number\n",
    "    def _mapping_list_of_word(self, data, mapper): \n",
    "        res = []\n",
    "        for seq in data:\n",
    "            tmp_seq = []\n",
    "            for x in seq:\n",
    "                if x in mapper:\n",
    "                    tmp_seq.append(mapper[x])\n",
    "                else:\n",
    "                    tmp_seq.append(mapper['unk'])\n",
    "            res.append(tmp_seq)\n",
    "        return res\n",
    "\n",
    "    def _mapping_list_of_list(self, data, mapper):\n",
    "        res = []\n",
    "        for doc in data:\n",
    "            tmp_seq = []\n",
    "            for sent in doc:\n",
    "                for x in sent:\n",
    "                    if x in mapper:\n",
    "                        tmp_seq.append(mapper[x])\n",
    "                    else:\n",
    "                        tmp_seq.append(mapper['unk'])\n",
    "            res.append(tmp_seq)\n",
    "        return res\n",
    "\n",
    "    # Adapted code from the lab\n",
    "    def _w2id(self, elements, unk=True):\n",
    "        vocab = {'pad': PAD_TOKEN}\n",
    "        if unk:\n",
    "            vocab['unk'] = len(vocab)\n",
    "        count = Counter(elements)\n",
    "        for k, v in count.items():\n",
    "            vocab[k] = len(vocab)\n",
    "        return vocab\n",
    "\n",
    "    def _tokenizer(self, data, word2id):\n",
    "        if self.name_dataset == 'subjectivity':\n",
    "            if word2id is None: word2id = self._w2id([w for sent in data for w in sent])\n",
    "            corpus2id = self._mapping_list_of_word(data, word2id)\n",
    "        elif self.name_dataset == 'polarity': \n",
    "            if word2id is None: word2id = self._w2id([w for doc in data for sent in doc for w in sent])\n",
    "            corpus2id = self._mapping_list_of_list(data, word2id)\n",
    "        else:\n",
    "            raise NameError(f'Name of the dataset {self.name_dataset} not valid. Please, choose between subjectivity or polarity.')\n",
    "\n",
    "        return corpus2id, word2id\n",
    "    \n",
    "    # Build matrix for pre-trained word embedding \n",
    "    # Reference: https://medium.com/@martinpella/how-to-use-pre-trained-word-embeddings-in-pytorch-71ca59249f76\n",
    "    def _build_vocab(self, emb_dim=EMB_SIZE): \n",
    "        global_vectors = GloVe(name=URL_GLOVE_TOKENS, dim=emb_dim)\n",
    "        matrix_len = len(self.word2id)\n",
    "        weights_matrix = numpy.zeros((matrix_len, emb_dim))\n",
    "\n",
    "        for i, word in enumerate(self.word2id):\n",
    "            weights_matrix[i] = global_vectors.get_vecs_by_tokens(word)\n",
    "            \n",
    "        return torch.from_numpy(weights_matrix).float() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SA_Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = tokens\n",
    "        self.labels = labels    \n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        item = {}\n",
    "        item['input_ids'] = torch.tensor(self.tokens[idx])\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference: https://www.codefull.org/2018/11/use-pytorchs-dataloader-with-variable-length-sequences-for-lstm-gru/\n",
    "def collate_fn(batch):\n",
    "    sorted_batch = sorted(batch, key=lambda x: x['input_ids'].shape[0], reverse=True) # Sort the batch (a dictionary with 2 keys: 'input_ids' and 'labels')in the descending order\n",
    "    sequences = [x['input_ids'] for x in sorted_batch] # Get each sequence and pad it\n",
    "   \n",
    "    sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=True, padding_value=PAD_TOKEN) # Also need to store the length of each sequence (each tensor). This is later needed in order to unpad the sequences\n",
    "    \n",
    "    lengths = torch.LongTensor([len(x) for x in sequences])\n",
    "    labels = torch.LongTensor([x['labels'] for x in sorted_batch]) # Don't forget to grab the labels of the *sorted* batch\n",
    "\n",
    "    return sequences_padded, lengths, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def prepare_polarity_data(k_fold = 0):\n",
    "    neg = movie_reviews.paras(categories='neg')\n",
    "    pos = movie_reviews.paras(categories='pos')\n",
    "\n",
    "    corpus = neg + pos\n",
    "    labels = numpy.array([0] * len(neg) + [1] * len(pos)) # \"0\": \"NEGATIVE\", \"1\": \"POSITIVE\"\n",
    "    \n",
    "    if REMOVE_STOPWORDS:\n",
    "        res = []\n",
    "        for doc in corpus:\n",
    "            tmp_sent = []\n",
    "            for sent in doc:\n",
    "                tmp_seq = []\n",
    "                for x in sent:\n",
    "                    if x not in stop_words:\n",
    "                        tmp_seq.append(x)\n",
    "                tmp_sent.append(tmp_seq)\n",
    "            res.append(tmp_sent)\n",
    "        corpus = res\n",
    "        del res\n",
    "    \n",
    "    fold_pols = make_stratified_k_fold(corpus, labels)\n",
    "\n",
    "    X_train, X_test, y_train, y_test = pick_k_fold(corpus, labels, fold_pols, k_fold)\n",
    "    X_train = Tokenizer(X_train, build_pretrained=True, name_dataset='polarity')\n",
    "    X_test = Tokenizer(X_test, X_train.word2id, name_dataset= 'polarity')\n",
    "\n",
    "    train_data = SA_Dataset(X_train.corpus2id, y_train)\n",
    "    test_data = SA_Dataset(X_test.corpus2id, y_test)\n",
    "    \n",
    "    train_dataloader = torch.utils.data.DataLoader(train_data, BATCH_SIZE_POL, collate_fn=collate_fn, shuffle=True, generator=g)\n",
    "    test_dataloader = torch.utils.data.DataLoader(test_data, BATCH_SIZE_POL, collate_fn=collate_fn, shuffle=True, generator=g)\n",
    "\n",
    "    print(f'Polarity [K_FOLD = {k_fold}] data loaded. Remove stop word: {REMOVE_STOPWORDS}')\n",
    "    return train_dataloader, test_dataloader, X_train\n",
    "\n",
    "def prepare_subjectivity_data(k_fold = 0):\n",
    "    subj = subjectivity.sents(categories='subj')\n",
    "    obj = subjectivity.sents(categories='obj')\n",
    "\n",
    "    corpus = subj + obj\n",
    "    labels = numpy.array([0] * len(subj) + [1] * len(obj))  # \"0\": \"SUBJECTIVE\", \"1\": \"NEUTRAL\"\n",
    "\n",
    "    if REMOVE_STOPWORDS:\n",
    "        res = []\n",
    "        for sent in corpus:\n",
    "            tmp_seq = []\n",
    "            for x in sent:\n",
    "                if x not in stop_words:\n",
    "                    tmp_seq.append(x)\n",
    "            res.append(tmp_seq)\n",
    "        corpus = res\n",
    "        del res\n",
    "    \n",
    "    fold_subj = make_stratified_k_fold(corpus, labels)\n",
    "\n",
    "    X_train, X_test, y_train, y_test = pick_k_fold(corpus, labels, fold_subj, k_fold)\n",
    "    X_train = Tokenizer(X_train, build_pretrained=True, name_dataset='subjectivity')\n",
    "    X_test = Tokenizer(X_test, X_train.word2id, name_dataset= 'subjectivity')\n",
    "\n",
    "    train_data = SA_Dataset(X_train.corpus2id, y_train)\n",
    "    test_data = SA_Dataset(X_test.corpus2id, y_test)\n",
    "\n",
    "    train_dataloader = torch.utils.data.DataLoader(train_data, BATCH_SIZE_SUBJ, collate_fn=collate_fn, shuffle=True, generator=g)\n",
    "    test_dataloader = torch.utils.data.DataLoader(test_data, BATCH_SIZE_SUBJ, collate_fn=collate_fn, shuffle=True, generator=g)\n",
    "\n",
    "    print(f'Subjectivity [K_FOLD = {k_fold}] data loaded. Remove stop word: {REMOVE_STOPWORDS}')\n",
    "    return train_dataloader, test_dataloader, X_train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train & Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiGRU(nn.Module):\n",
    "    def __init__(self, out_size, hid_size, emb_size, no_layers, vocab_len, dropout_prob, attention = True, weights_mx_embed = None, use_pretrained_embed = False, pad_index = PAD_TOKEN):\n",
    "        super(BiGRU, self).__init__()\n",
    "        \n",
    "        # embedding\n",
    "        self.embedding = nn.Embedding(vocab_len, emb_size, padding_idx=pad_index)\n",
    "        if use_pretrained_embed:\n",
    "            if weights_mx_embed is not None:\n",
    "                self.embedding.weight = nn.Parameter(weights_mx_embed)\n",
    "            else:\n",
    "                raise Exception('Weights matrix for the embedding layer is not valid.')\n",
    "            if LR_EMBEDDING == 0.:\n",
    "                self.embedding.weight.requires_grad = False\n",
    "\n",
    "        # GRU\n",
    "        self.gru = nn.GRU(input_size=emb_size,hidden_size=hid_size,num_layers=no_layers, batch_first=True, bidirectional=True)\n",
    "\n",
    "        # attention\n",
    "        self.attention = None\n",
    "        if attention:\n",
    "            self.attention = nn.Linear(hid_size*2, 1)\n",
    "\n",
    "        # droput layer\n",
    "        self.dropout = nn.Dropout(dropout_prob)\n",
    "\n",
    "        # linear layer\n",
    "        self.fc = nn.Linear(hid_size*2, out_size) # x2 because we are dealing with a bidirectional model\n",
    "    \n",
    "    def forward(self, x, x_lengths):\n",
    "        # embedding \n",
    "        embeds = self.embedding(x) # shape: Batch x Sequence x Feature since batch = True\n",
    "        packed_input = pack_padded_sequence(embeds, x_lengths.cpu().numpy(), batch_first=True)\n",
    "\n",
    "        # gru\n",
    "        packed_out, last_hidden = self.gru(packed_input)\n",
    "        alpha, input_sizes = pad_packed_sequence(packed_out, batch_first=True)\n",
    "\n",
    "        # attention\n",
    "        if self.attention is not None:\n",
    "            # Compute attention weights\n",
    "            attention_weights = self.dropout(alpha)\n",
    "            attention_weights = torch.softmax(self.attention(attention_weights), dim=1)\n",
    "            alpha = alpha * attention_weights\n",
    "\n",
    "        # dropout and fully connected layer\n",
    "        context_vector = alpha.sum(dim=1)\n",
    "        out = self.dropout(context_vector)\n",
    "        out = self.fc(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "    def _attention_heatmap(self, x, x_lengths):\n",
    "        embeds = self.embedding(x)\n",
    "        packed_input = pack_padded_sequence(embeds, x_lengths.cpu().numpy(), batch_first=True)\n",
    "        packed_out, last_hidden = self.gru(packed_input)\n",
    "        pack_encoded, input_sizes = pad_packed_sequence(packed_out, batch_first=True)\n",
    "        attention_weights = self.dropout(pack_encoded)\n",
    "        attention_weights = self.attention(attention_weights)\n",
    "\n",
    "        return attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adapted code from the lab\n",
    "def init_weights(mat):\n",
    "    for m in mat.modules():\n",
    "        if type(m) in [nn.GRU, nn.LSTM, nn.RNN]:\n",
    "            for name, param in m.named_parameters():\n",
    "                if 'weight_ih' in name:\n",
    "                    for idx in range(4):\n",
    "                        mul = param.shape[0]//4\n",
    "                        torch.nn.init.xavier_uniform_(param[idx*mul:(idx+1)*mul])\n",
    "                elif 'weight_hh' in name:\n",
    "                    for idx in range(4):\n",
    "                        mul = param.shape[0]//4\n",
    "                        torch.nn.init.orthogonal_(param[idx*mul:(idx+1)*mul])\n",
    "                elif 'bias' in name:\n",
    "                    param.data.fill_(0)\n",
    "        else:\n",
    "            if type(m) in [nn.Linear]:\n",
    "                torch.nn.init.uniform_(m.weight, -0.01, 0.01)\n",
    "                if m.bias != None:\n",
    "                    m.bias.data.fill_(0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimizer(model, lr_embedding = None):\n",
    "    if lr_embedding is not None:\n",
    "        embedding_weights = []\n",
    "        rest_of_the_net_weights = []\n",
    "        # iterate through the layers of the network\n",
    "        for name, param in model.named_parameters():\n",
    "            # separate embedding layers to the other layers\n",
    "            if name.startswith('embedding'):\n",
    "                embedding_weights.append(param)\n",
    "            else:\n",
    "                rest_of_the_net_weights.append(param)\n",
    "        lr_specs = [{'params': rest_of_the_net_weights},\n",
    "                    {'params': embedding_weights, 'lr': lr_embedding}]\n",
    "\n",
    "        opt = torch.optim.Adam(lr_specs, lr=LR)\n",
    "    else:\n",
    "        opt = torch.optim.Adam(model.parameters(), lr=LR)\n",
    "    return opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loss():\n",
    "    return torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_step(model, optimizer, train_loader):\n",
    "    model.train()\n",
    "\n",
    "    cumulative_loss = 0.\n",
    "    cumulative_accuracy = 0.\n",
    "    cumulative_f1 = 0.\n",
    "\n",
    "    cost_function = get_loss()\n",
    "\n",
    "    for batch in train_loader:\n",
    "        x, x_lengths, labels = batch\n",
    "        x = x.to(DEVICE)\n",
    "        x_lengths = x_lengths.to(DEVICE)\n",
    "        labels = labels.to(DEVICE)\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(x, x_lengths)\n",
    "        \n",
    "        loss = cost_function(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        predictions = torch.nn.functional.softmax(outputs, dim=1)\n",
    "        pred_labels = torch.argmax(predictions, dim=1)\n",
    "\n",
    "        cumulative_loss += loss.item()\n",
    "        cumulative_accuracy += accuracy_score(labels.cpu().detach().numpy(), pred_labels.cpu().detach().numpy())\n",
    "        cumulative_f1 += f1_score(labels.cpu().detach().numpy(), pred_labels.cpu().detach().numpy())\n",
    "\n",
    "    return {\"train/train_acc\":(cumulative_accuracy/len(train_loader)), \n",
    "            \"train/train_loss\": cumulative_loss/len(train_loader),\n",
    "            \"train/train_f1\": cumulative_f1/len(train_loader)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluating_step(model, test_loader):\n",
    "    model.eval()\n",
    "    \n",
    "    cumulative_loss = 0.\n",
    "    cumulative_accuracy = 0.\n",
    "    cumulative_f1 = 0.\n",
    "\n",
    "    cost_function = get_loss()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in test_loader:\n",
    "            x, x_lengths, labels = batch\n",
    "            x = x.to(DEVICE)\n",
    "            x_lengths = x_lengths.to(DEVICE)\n",
    "            labels = labels.to(DEVICE)\n",
    "\n",
    "            outputs = model(x, x_lengths)\n",
    "\n",
    "            loss = cost_function(outputs, labels)\n",
    "            \n",
    "            predictions = torch.nn.functional.softmax(outputs, dim=1)\n",
    "            pred_labels = torch.argmax(predictions, dim=1)\n",
    "\n",
    "            cumulative_loss += loss.item()\n",
    "            cumulative_accuracy += accuracy_score(labels.cpu().detach().numpy(), pred_labels.cpu().detach().numpy())\n",
    "            cumulative_f1 += f1_score(labels.cpu().detach().numpy(), pred_labels.cpu().detach().numpy())\n",
    "    \n",
    "    return {\"test/test_acc\": (cumulative_accuracy/len(test_loader)), \n",
    "            \"test/test_loss\": cumulative_loss/len(test_loader), \n",
    "            \"test/test_f1\": cumulative_f1/len(test_loader) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(wandb_run, train_data, test_data, vocab_len, weights_mx_embed):\n",
    "\n",
    "    config = wandb_run.config\n",
    "    print('CONFIGS\\n', yaml.dump(config._items, default_flow_style=False)) # pretty print of configs used\n",
    "\n",
    "    run_epochs = 0\n",
    "    best_acc = 0.\n",
    "    best_loss = 0.\n",
    "    best_f1 = 0.\n",
    "    patience = PATIENCE\n",
    "\n",
    "    model = BiGRU(OUT_SIZE, config.hidden_size, config.embedding_size, config.n_layers, vocab_len, config.dropout_prob, config.attention, weights_mx_embed, config.pretrained_embedding)\n",
    "    model.apply(init_weights)\n",
    "    optimizer = get_optimizer(model, lr_embedding=config.lr_embedding)\n",
    "    weights_path = get_complete_saved_path(WEIGHTS_PATH, config)\n",
    "    model.to(DEVICE)\n",
    "\n",
    "    for e in range(config.epochs):\n",
    "        print(f'-- Epoch [{e+1}/{config.epochs}] --')\n",
    "        train_metrics = training_step(model, optimizer, train_data)\n",
    "        test_metrics = evaluating_step(model, test_data)\n",
    "        wandb.log({**train_metrics, **test_metrics})\n",
    "        print(f'Train -> \\tLoss:{train_metrics[\"train/train_loss\"]:.5f} \\tAccuracy: {train_metrics[\"train/train_acc\"]:.2f} \\tF1-Score: {train_metrics[\"train/train_f1\"]:.2f}')\n",
    "        print(f'Test -> \\tLoss:{test_metrics[\"test/test_loss\"]:.5f} \\tAccuracy: {test_metrics[\"test/test_acc\"]:.2f} \\tF1-Score: {test_metrics[\"test/test_f1\"]:.2f}')\n",
    "\n",
    "        if (best_acc < test_metrics[\"test/test_acc\"]):\n",
    "            torch.save(model.state_dict(), weights_path)\n",
    "            best_acc = test_metrics[\"test/test_acc\"]\n",
    "            best_loss = test_metrics[\"test/test_loss\"]\n",
    "            best_f1 = test_metrics[\"test/test_f1\"]\n",
    "        else: patience -= 1\n",
    "        \n",
    "        if patience < 0: # Early stopping with patience\n",
    "            run_epochs = e+1\n",
    "            print(f'Early stopping with PATIENCE = {PATIENCE}. Model trained for {run_epochs}/{config.epochs} epochs')\n",
    "            break # Not nice but it keeps the code clean\n",
    "            \n",
    "    print('Model saved in location ', weights_path)\n",
    "    wandb.summary[\"test_best_acc\"] = best_acc\n",
    "    wandb.summary[\"test_best_loss\"] = best_loss\n",
    "    wandb.summary[\"test_best_f1\"] = best_f1\n",
    "    wandb.summary[\"real_run_epochs\"] = run_epochs\n",
    "    wandb.finish()\n",
    "\n",
    "    return best_acc, best_f1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train subjectivity classifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now train and evaluate a subjectivity detector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "        \"dataset\": \"subjectivity\",\n",
    "        \"model\": \"GRU\",\n",
    "        \"batch_size\": BATCH_SIZE_SUBJ,\n",
    "        \"epochs\": EPOCHS,\n",
    "        \"lr\": LR,\n",
    "        \"lr_embedding\": LR_EMBEDDING,\n",
    "        \"optimizer\": OPTIMIZER,\n",
    "        \"hidden_size\": HID_SIZE,\n",
    "        \"embedding_size\": EMB_SIZE,\n",
    "        \"n_layers\": N_LAYER,\n",
    "        \"dropout_prob\": DROPOUT,\n",
    "        \"attention\": True,\n",
    "        \"pretrained_embedding\": True,\n",
    "        \"remove_stopwords\": REMOVE_STOPWORDS\n",
    "    }\n",
    "\n",
    "NAME_RUN = \"subj-GRU-\"\n",
    "\n",
    "train_subj, test_subj, train_info_subj = prepare_subjectivity_data(k_fold=K_FOLD)\n",
    "run = wandb.init(project=PROJECT_NAME, entity=ENTITY, name=NAME_RUN + str(K_FOLD), config=config, mode=MODE_WANDB)\n",
    "training_loop(run, train_subj, test_subj, len(train_info_subj.word2id), train_info_subj.pretrained_word2id) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train no-filter sents polarity classifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now train and evaluate a polarity classifier **without** removing subjective sentences from movie reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "        \"dataset\": \"polarity\",\n",
    "        \"version\": \"no-filter\",\n",
    "        \"model\": \"GRU-no-filter\",\n",
    "        \"batch_size\": BATCH_SIZE_POL,\n",
    "        \"epochs\": EPOCHS,\n",
    "        \"lr\": LR,\n",
    "        \"lr_embedding\": LR_EMBEDDING,\n",
    "        \"optimizer\": OPTIMIZER,\n",
    "        \"hidden_size\": HID_SIZE,\n",
    "        \"embedding_size\": EMB_SIZE,\n",
    "        \"n_layers\": N_LAYER,\n",
    "        \"dropout_prob\": DROPOUT,\n",
    "        \"attention\": True,\n",
    "        \"pretrained_embedding\": True,\n",
    "        \"remove_stopwords\": REMOVE_STOPWORDS\n",
    "    }\n",
    "\n",
    "NAME_RUN = \"pol-no-filter-GRU-pretrained-\"\n",
    "\n",
    "train_pols, test_pols, train_info_pols = prepare_polarity_data(k_fold=K_FOLD)\n",
    "run = wandb.init(project=PROJECT_NAME, entity=ENTITY, name=NAME_RUN + str(K_FOLD), config=config , mode=MODE_WANDB)\n",
    "training_loop(run, train_pols, test_pols, len(train_info_pols.word2id), train_info_pols.pretrained_word2id)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter sentences using subjectivity classifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now use the subjectivity detector to remove the subjective sentences from the movie reviews dataset to hopefully obtain better quality data to process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "def remove_subj_sents(data, label, classifier):\n",
    "    filtered = []\n",
    "\n",
    "    len_doc = []\n",
    "    len_subj = []\n",
    "    len_sent_removed = []\n",
    "\n",
    "    for doc in tqdm(data):\n",
    "        pol_tokens = Tokenizer(doc, train_info_subj.word2id, name_dataset='subjectivity') # trated as a subjectivity dataset because at the end has small number word per sentences\n",
    "        polarity_data = SA_Dataset(pol_tokens.corpus2id, numpy.array([label]*len(doc)))\n",
    "        polarity_dl = torch.utils.data.DataLoader(polarity_data, BATCH_SIZE_SUBJ, collate_fn=collate_fn, shuffle=False, generator=g)\n",
    "\n",
    "        classify_labels = filter_step(classifier, polarity_dl)\n",
    "        sents = [d for d, estimate in zip(doc, classify_labels) if estimate == 0] # 0 = subj\n",
    "\n",
    "        if len(sents) > 0:\n",
    "            filtered.append(sents)\n",
    "\n",
    "        len_doc.append(len(doc))\n",
    "        len_subj.append(len(sents))\n",
    "        len_sent_removed.append(len(doc)-len(sents))\n",
    "\n",
    "    df = pandas.DataFrame({'doc':len_doc, 'subj-sents':len_subj, 'sents-removed':len_sent_removed})\n",
    "    df.to_csv(f'{random.randint(0,1000)}.csv', index=False)\n",
    "\n",
    "    return filtered\n",
    "\n",
    "def filter_step(model, filtered_loader):\n",
    "    model.eval()\n",
    "\n",
    "    sbj_labels = []\n",
    "    with torch.no_grad():\n",
    "        for batch in filtered_loader:\n",
    "            x, x_lengths, labels = batch\n",
    "            x = x.to(DEVICE)\n",
    "            x_lengths = x_lengths.to(DEVICE)\n",
    "\n",
    "            outputs = model(x, x_lengths)\n",
    "            \n",
    "            predictions = torch.nn.functional.softmax(outputs, dim=1)\n",
    "            pred_labels = torch.argmax(predictions, dim=1)\n",
    "\n",
    "            sbj_labels += pred_labels\n",
    "    \n",
    "    return sbj_labels\n",
    "\n",
    "def filter_sentences():\n",
    "    classifier = BiGRU(OUT_SIZE, HID_SIZE, EMB_SIZE, N_LAYER, len(train_info_subj.word2id), DROPOUT)\n",
    "    classifier.load_state_dict(torch.load(WEIGHTS_PATH + 'subjectivity/' + 'gru.pt'))\n",
    "    classifier.to(DEVICE)\n",
    "\n",
    "    neg = movie_reviews.paras(categories='neg')\n",
    "    pos = movie_reviews.paras(categories='pos')\n",
    "    \n",
    "    neg_sents = remove_subj_sents(neg, 0, classifier)\n",
    "    pos_sents = remove_subj_sents(pos, 1, classifier)\n",
    "    \n",
    "    return neg_sents, pos_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_polarity_filtered_data(filter_dict, fold, k_fold):\n",
    "    corpus = filter_dict['corpus']\n",
    "    labels = filter_dict['labels']\n",
    "\n",
    "    if REMOVE_STOPWORDS:\n",
    "        res = []\n",
    "        for doc in corpus:\n",
    "            tmp_sent = []\n",
    "            for sent in doc:\n",
    "                tmp_seq = []\n",
    "                for x in sent:\n",
    "                    if x not in stop_words:\n",
    "                        tmp_seq.append(x)\n",
    "                tmp_sent.append(tmp_seq)\n",
    "            res.append(tmp_sent)\n",
    "        corpus = res\n",
    "        del res\n",
    "        \n",
    "    X_train, X_test, y_train, y_test = pick_k_fold(corpus, labels, fold, k_fold)\n",
    "    X_train = Tokenizer(X_train, build_pretrained=True, name_dataset='polarity')\n",
    "    X_test = Tokenizer(X_test, X_train.word2id, name_dataset= 'polarity')\n",
    "\n",
    "    train_data = SA_Dataset(X_train.corpus2id, y_train)\n",
    "    test_data = SA_Dataset(X_test.corpus2id, y_test)\n",
    "    \n",
    "    train_dataloader = torch.utils.data.DataLoader(train_data, BATCH_SIZE_POL, collate_fn=collate_fn, shuffle=True, generator=g)\n",
    "    test_dataloader = torch.utils.data.DataLoader(test_data, BATCH_SIZE_POL, collate_fn=collate_fn, shuffle=False, generator=g)\n",
    "\n",
    "    print(f'Filtered polarity [K_FOLD = {k_fold}] data loaded. Remove stop word: {REMOVE_STOPWORDS}')\n",
    "    return train_dataloader, test_dataloader, X_train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We filter out the subjective sentences from the ```movie_reviews``` dataset and save them in a ```.pkl``` file for an easy access."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "filtered_saved_path = WEIGHTS_PATH + 'filtered_polarity_sents.pkl'\n",
    "dict_pols_filtered = {}\n",
    "\n",
    "if not os.path.exists(filtered_saved_path):\n",
    "    print('Creating .pkl with filtered sentences')\n",
    "\n",
    "    neg_filtered, pos_filtered = filter_sentences()\n",
    "    dict_pols_filtered = {'corpus': neg_filtered + pos_filtered, 'labels': numpy.array([0] * len(neg_filtered) + [1] * len(pos_filtered))}\n",
    "\n",
    "    with open(filtered_saved_path, 'wb') as f:\n",
    "        pickle.dump(dict_pols_filtered, f)\n",
    "        print('Saved at location ', filtered_saved_path)\n",
    "\n",
    "else:\n",
    "    print('Using .pkl with filtered sentences from ', filtered_saved_path)\n",
    "    with open(filtered_saved_path, 'rb') as f:\n",
    "        dict_pols_filtered = pickle.load(f)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train filter sents polarity classifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now train and evaluate a polarity classifier **with** removed subjective sentences from movie reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "        \"dataset\": \"polarity\",\n",
    "        \"version\": \"filter\",\n",
    "        \"model\": \"GRU-filter\",\n",
    "        \"batch_size\": BATCH_SIZE_POL,\n",
    "        \"epochs\": EPOCHS,\n",
    "        \"lr\": LR,\n",
    "        \"lr_embedding\": LR_EMBEDDING,\n",
    "        \"optimizer\": OPTIMIZER,\n",
    "        \"hidden_size\": HID_SIZE,\n",
    "        \"embedding_size\": EMB_SIZE,\n",
    "        \"n_layers\": N_LAYER,\n",
    "        \"dropout_prob\": DROPOUT,\n",
    "        \"attention\": True,\n",
    "        \"pretrained_embedding\": True,\n",
    "        \"remove_stopwords\": REMOVE_STOPWORDS\n",
    "    }\n",
    "\n",
    "NAME_RUN = \"pol-filter-GRU-pretrained-\"\n",
    "\n",
    "fold_filtered_pols = make_stratified_k_fold(dict_pols_filtered['corpus'], dict_pols_filtered['labels'])\n",
    "\n",
    "train_filtered, test_filtered, train_info_filtered = prepare_polarity_filtered_data(dict_pols_filtered, fold_filtered_pols, K_FOLD)\n",
    "run = wandb.init(project=PROJECT_NAME, entity=ENTITY, name=NAME_RUN + str(K_FOLD), config=config, mode=MODE_WANDB)\n",
    "training_loop(run, train_filtered, test_filtered, len(train_info_filtered.word2id), train_info_filtered.pretrained_word2id)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text attention heat map generator & visualization"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create and visualize the text attention heat map using two different methods:\n",
    "- generate a heat map using ```seaborn``` library;\n",
    "- generate Latex code to visualize the attention based text (adapted the source code from the following GitHub repository: https://github.com/jiesutd/Text-Attention-Heatmap-Visualization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Git command: \n",
    "# !git clone https://github.com/jiesutd/Text-Attention-Heatmap-Visualization.git\n",
    "\n",
    "# @Author: Jie Yang\n",
    "# @Date:   2019-03-29 16:10:23\n",
    "\n",
    "## convert the text/attention list to latex code, which will further generates the text heatmap based on attention weights.\n",
    "import numpy as np\n",
    "\n",
    "latex_special_token = [\"!@#$%^&*()\"]\n",
    "\n",
    "def generate(text_list, attention_list, latex_file, color='red', rescale_value = False):\n",
    "\tif rescale_value:\n",
    "\t\tattention_list = rescale(attention_list)\n",
    "\tword_num = len(text_list)\n",
    "\ttext_list = clean_word(text_list)\n",
    "\twith open(latex_file,'w') as f:\n",
    "\t\tf.write(r'''\\documentclass[varwidth]{standalone}\n",
    "\t\t\t\t\\special{papersize=210mm,297mm}\n",
    "\t\t\t\t\\usepackage{color}\n",
    "\t\t\t\t\\usepackage{tcolorbox}\n",
    "\t\t\t\t\\usepackage{CJK}\n",
    "\t\t\t\t\\usepackage{adjustbox}\n",
    "\t\t\t\t\\tcbset{width=0.9\\textwidth,boxrule=0pt,colback=red,arc=0pt,auto outer arc,left=0pt,right=0pt,boxsep=5pt}\n",
    "\t\t\t\t\\begin{document}\n",
    "\t\t\t\t\\begin{CJK*}{UTF8}{gbsn}'''+'\\n')\n",
    "\t\tstring = r'''{\\setlength{\\fboxsep}{0pt}\\colorbox{white!0}{\\parbox{0.9\\textwidth}{'''+\"\\n\"\n",
    "\t\tfor idx in range(word_num):\n",
    "\t\t\tstring += \"\\\\colorbox{%s!%s}{\"%(color, attention_list[idx])+\"\\\\strut \" + text_list[idx]+\"} \"\n",
    "\t\tstring += \"\\n}}}\"\n",
    "\t\tf.write(string+'\\n')\n",
    "\t\tf.write(r'''\\end{CJK*}\n",
    "\t\t\t\t\\end{document}''')\n",
    "\n",
    "def rescale(input_list):\n",
    "\tthe_array = np.asarray(input_list)\n",
    "\tthe_max = np.max(the_array)\n",
    "\tthe_min = np.min(the_array)\n",
    "\trescale = (the_array - the_min)/(the_max-the_min)*100\n",
    "\treturn rescale.tolist()\n",
    "\n",
    "def clean_word(word_list):\n",
    "\tnew_word_list = []\n",
    "\tfor word in word_list:\n",
    "\t\tfor latex_sensitive in [\"\\\\\", \"%\", \"&\", \"^\", \"#\", \"_\",  \"{\", \"}\"]:\n",
    "\t\t\tif latex_sensitive in word:\n",
    "\t\t\t\tword = word.replace(latex_sensitive, '\\\\'+latex_sensitive)\n",
    "\t\tnew_word_list.append(word)\n",
    "\treturn new_word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "def subj_generate_heat_map(sent, normalize = True, tex = True, png = True):\n",
    "    print('SUBJECTIVITY HEATMAP')\n",
    "    classifier = BiGRU(OUT_SIZE, HID_SIZE, EMB_SIZE, N_LAYER, len(train_info_subj.word2id), DROPOUT)\n",
    "    classifier.load_state_dict(torch.load(WEIGHTS_PATH + 'subjectivity/' + 'gru.pt'))\n",
    "    classifier.to(DEVICE)\n",
    "    classifier.eval()\n",
    "\n",
    "    ds = Tokenizer([sent], train_info_subj.word2id, build_pretrained=False, name_dataset='subjectivity')\n",
    "    cd = SA_Dataset(ds.corpus2id, [0])\n",
    "    dl = torch.utils.data.DataLoader(cd, collate_fn=collate_fn)\n",
    "\n",
    "    it = dl.__iter__()\n",
    "    x, x_length, _ = it.__next__()\n",
    "    x = x.to(DEVICE)\n",
    "\n",
    "    att = classifier._attention_heatmap(x, x_length)\n",
    "    att = att.detach().cpu().numpy()\n",
    "    att = att.squeeze(2).squeeze(0)\n",
    "\n",
    "    assert len(sent) == len(att)\n",
    "    subj_hp_path = HEATMAP_PATH + '/subjectivity'\n",
    "\n",
    "    if not os.path.exists(subj_hp_path):\n",
    "        os.makedirs(subj_hp_path, exist_ok=True)\n",
    "\n",
    "    print('Input sent:')\n",
    "    print(\" \".join([w for w in sent]))\n",
    "\n",
    "    min_val = np.min(att)\n",
    "    max_val = np.max(att)\n",
    "    normalized_vector = (att - min_val) / (max_val - min_val)\n",
    "    \n",
    "    if tex:\n",
    "        if normalize:\n",
    "            generate(sent, normalized_vector*100, subj_hp_path + '/att_visualization_normalize.tex')\n",
    "        else:\n",
    "            generate(sent, att*100, subj_hp_path + '/att_visualization.tex')\n",
    "        print('\\nSaved .tex file located in ', subj_hp_path)\n",
    "    if png:\n",
    "        if normalize:\n",
    "            df_data = {'attention weights': att}\n",
    "            df = pd.DataFrame(df_data, index = sent)\n",
    "            sns.heatmap(df, annot=False, vmin=min(att), vmax=max(att), square = True)\n",
    "            plt.show()\n",
    "        else:\n",
    "            df_data = {'attention weights': att}\n",
    "            df = pd.DataFrame(df_data, index = sent)\n",
    "            sns.heatmap(df, annot=False, vmin=0, vmax=1, square = True)\n",
    "            plt.show()\n",
    "            \n",
    "def pol_generate_heat_map(sent, filter = False, normalize = True, tex = True, png = True):\n",
    "    print('POLARITY HEATMAP')\n",
    "    if filter: gru_type = 'gru-filter.pt' \n",
    "    else: gru_type = 'gru-no-filter.pt'\n",
    "\n",
    "    classifier = BiGRU(OUT_SIZE, HID_SIZE, EMB_SIZE, N_LAYER, len(train_info_filtered.word2id), DROPOUT)\n",
    "    classifier.load_state_dict(torch.load(WEIGHTS_PATH + 'polarity/' + gru_type))\n",
    "    classifier.to(DEVICE)\n",
    "    classifier.eval()\n",
    "\n",
    "    ds = Tokenizer([sent], train_info_filtered.word2id, build_pretrained=False, name_dataset='polarity') #[sent]\n",
    "    cd = SA_Dataset(ds.corpus2id, [0])\n",
    "    dl = torch.utils.data.DataLoader(cd, collate_fn=collate_fn)\n",
    "\n",
    "    iter = dl.__iter__()\n",
    "    x, x_length, _ = iter.__next__()\n",
    "    x = x.to(DEVICE)\n",
    "\n",
    "    att = classifier._attention_heatmap(x, x_length)\n",
    "    att = att.detach().cpu().numpy()\n",
    "    att = att.squeeze(2).squeeze(0)\n",
    "\n",
    "    sent = [w for d in sent for w in d]\n",
    "\n",
    "    assert len(sent) == len(att)\n",
    "    pols_hp_path = HEATMAP_PATH + 'polarity'\n",
    "\n",
    "    if not os.path.exists(pols_hp_path):\n",
    "        os.makedirs(pols_hp_path, exist_ok=True)\n",
    "\n",
    "    print('Input sent:')\n",
    "    print(\" \".join([w for w in sent]))\n",
    "\n",
    "    # calculate quartiles\n",
    "    q1, q3 = np.percentile(att, [25, 75])\n",
    "    # calculate IQR\n",
    "    iqr = q3 - q1\n",
    "    # calculate lower and upper bounds\n",
    "    lower_bound = q1 - (1.5 * iqr)\n",
    "    upper_bound = q3 + (1.5 * iqr)\n",
    "    # remove outliers\n",
    "    att[(att < lower_bound)] = lower_bound\n",
    "    att[(att > upper_bound)] = upper_bound\n",
    "    # normalize vector\n",
    "    min_val = np.min(att)\n",
    "    max_val = np.max(att)\n",
    "    normalized_vector = (att - min_val) / (max_val - min_val)\n",
    "\n",
    "    if tex:\n",
    "        if normalize:\n",
    "            generate(sent, normalized_vector*100, pols_hp_path + '/att_visualization_normalize.tex')\n",
    "        else:\n",
    "            generate(sent, att*100, pols_hp_path + '/att_visualization.tex')\n",
    "        print('\\nSaved .tex file located in ', pols_hp_path)\n",
    "    if png:\n",
    "        if normalize:\n",
    "            df_data = {'attention weights': normalized_vector}\n",
    "            df = pd.DataFrame(df_data, index = sent)\n",
    "            sns.heatmap(df, annot=False, vmin=min_val, vmax=max_val, square = True)\n",
    "            plt.show()\n",
    "        else:\n",
    "            df_data = {'attention weights': att}\n",
    "            df = pd.DataFrame(df_data, index = sent)\n",
    "            sns.heatmap(df, annot=False, vmin=0, vmax=1, square = True)\n",
    "            plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the project report I used as example:\n",
    "- in Figure 1 the subjective sentence from ``subj[32]``;\n",
    "- in Figure 2 some non filtered sentences extracted from ``neg[1]``, where we have objective and subjective sentences. "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choose a sentence from the datasets and run it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj = subjectivity.sents(categories='obj')\n",
    "subj = subjectivity.sents(categories='subj')\n",
    "\n",
    "neg = movie_reviews.paras(categories='neg')\n",
    "pos = movie_reviews.paras(categories='pos')\n",
    "\n",
    "subj_generate_heat_map(subj[32], normalize=True, tex=False)\n",
    "pol_generate_heat_map(neg[1], filter=True, normalize=True, tex=True) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "40d3a090f54c6569ab1632332b64b2c03c39dcf918b08424e98f38b5ae0af88f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}