Chatbot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EcVd5Z4B9eQL"
   },
   "source": [
    "# Conversation Modeling and Decoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ak-qK8bkRJIq"
   },
   "source": [
    "## Support code to load pretrained model for chatbot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 496
    },
    "colab_type": "code",
    "id": "6LaiBshdRHWd",
    "outputId": "a1a2c13a-05e3-4355-8f2a-04762fcda926"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 131438/131438 [00:14<00:00, 9175.40it/s]\n",
      "100%|██████████| 7801/7801 [00:00<00:00, 9131.56it/s]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from collections import Counter\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "import editdistance\n",
    "\n",
    "RETOK = re.compile(r'\\w+|[^\\w\\s]|\\n', re.UNICODE)\n",
    "\n",
    "class ChatDictionary(object):\n",
    "    \"\"\"\n",
    "    Simple dict loader\n",
    "    \"\"\"\n",
    "    def __init__(self, dict_file_path):\n",
    "        self.word2ind = {}  # word:index\n",
    "        self.ind2word = {}  # index:word\n",
    "        self.counts = {}  # word:count\n",
    "\n",
    "        dict_raw = open(dict_file_path, 'r').readlines()\n",
    "        \n",
    "        for i, w in enumerate(dict_raw):\n",
    "            _word, _count = w.strip().split('\\t')\n",
    "            if _word == '\\\\n':\n",
    "                _word = '\\n'\n",
    "            self.word2ind[_word] = i\n",
    "            self.ind2word[i] = _word\n",
    "            self.counts[_word] = _count\n",
    "            \n",
    "    def t2v(self, tokenized_text):\n",
    "        return [self.word2ind[w] if w in self.counts else self.word2ind['__unk__'] for w in tokenized_text]\n",
    "\n",
    "    def v2t(self, list_ids):\n",
    "        return ' '.join([self.ind2word[i] for i in list_ids])\n",
    "    \n",
    "    def pred2text(self, tensor):\n",
    "        result = []\n",
    "        for i in range(tensor.size(0)):\n",
    "            if tensor[i].item() == '__end__'  or tensor[i].item() == '__null__':  # null is pad\n",
    "                break\n",
    "            else:\n",
    "                result.append(self.ind2word[tensor[i].item()])\n",
    "        return ' '.join(result)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.counts)\n",
    "\n",
    "\n",
    "class ChatDataset(Dataset):\n",
    "    \"\"\"\n",
    "    Json dataset wrapper\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, dataset_file_path, dictionary, dt='train'):\n",
    "        super().__init__()\n",
    "        \n",
    "        json_text = open(dataset_file_path, 'r').readlines()\n",
    "        self.samples = []\n",
    "        \n",
    "        for sample in tqdm(json_text):\n",
    "            sample = sample.rstrip()\n",
    "            sample = json.loads(sample)\n",
    "            _inp_toked = RETOK.findall(sample['text'])\n",
    "            _inp_toked_id = dictionary.t2v(_inp_toked)\n",
    "\n",
    "            sample['text_vec'] = torch.tensor(_inp_toked_id, dtype=torch.long)\n",
    "            \n",
    "            # train and valid have different key names for target\n",
    "            if dt == 'train':\n",
    "                _tar_toked = RETOK.findall(sample['labels'][0]) + ['__end__']\n",
    "            elif dt == 'valid':\n",
    "                _tar_toked = RETOK.findall(sample['eval_labels'][0]) + ['__end__']\n",
    "                \n",
    "            _tar_toked_id = dictionary.t2v(_tar_toked)\n",
    "            \n",
    "            sample['target_vec'] = torch.tensor(_tar_toked_id, dtype=torch.long)\n",
    "            \n",
    "            self.samples.append(sample)\n",
    "            \n",
    "    def __getitem__(self, i):\n",
    "        return self.samples[i]['text_vec'], self.samples[i]['target_vec']\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.samples)\n",
    "\n",
    "\n",
    "def pad_tensor(tensors, sort=True, pad_token=0):\n",
    "    rows = len(tensors)\n",
    "    lengths = [len(i) for i in tensors]\n",
    "    max_t = max(lengths)\n",
    "        \n",
    "    output = tensors[0].new(rows, max_t)\n",
    "    output.fill_(pad_token)  # 0 is a pad token here\n",
    "    \n",
    "    for i, (tensor, length) in enumerate(zip(tensors, lengths)):\n",
    "        output[i,:length] = tensor\n",
    "\n",
    "    return output, lengths\n",
    "\n",
    "def argsort(keys, *lists, descending=False):\n",
    "    \"\"\"Reorder each list in lists by the (descending) sorted order of keys.\n",
    "    :param iter keys: Keys to order by.\n",
    "    :param list[list] lists: Lists to reordered by keys's order.\n",
    "                             Correctly handles lists and 1-D tensors.\n",
    "    :param bool descending: Use descending order if true.\n",
    "    :returns: The reordered items.\n",
    "    \"\"\"\n",
    "    ind_sorted = sorted(range(len(keys)), key=lambda k: keys[k])\n",
    "    if descending:\n",
    "        ind_sorted = list(reversed(ind_sorted))\n",
    "    output = []\n",
    "    for lst in lists:\n",
    "        if isinstance(lst, torch.Tensor):\n",
    "            output.append(lst[ind_sorted])\n",
    "        else:\n",
    "            output.append([lst[i] for i in ind_sorted])\n",
    "    return output\n",
    "\n",
    "def batchify(batch):\n",
    "    inputs = [i[0] for i in batch]\n",
    "    labels = [i[1] for i in batch]\n",
    "    \n",
    "    input_vecs, input_lens = pad_tensor(inputs)\n",
    "    label_vecs, label_lens = pad_tensor(labels)\n",
    "    \n",
    "    # sort only wrt inputs here for encoder packinng\n",
    "    input_vecs, input_lens, label_vecs, label_lens = argsort(input_lens, input_vecs, input_lens, label_vecs, label_lens, descending=True)\n",
    "\n",
    "    return {\n",
    "        \"text_vecs\": input_vecs,\n",
    "        \"text_lens\": input_lens,\n",
    "        \"target_vecs\": label_vecs,\n",
    "        \"target_lens\": label_lens,\n",
    "        'use_packed': True\n",
    "    }\n",
    "\n",
    "\n",
    "# loading datasets and dictionary\n",
    "\n",
    "# downloading pretrained models and data\n",
    "\n",
    "### DOWNLOADING THE FILES\n",
    "import os\n",
    "\n",
    "### persona chat dataset\n",
    "if not os.path.exists('./dict'):\n",
    "    !wget \"https://nyu.box.com/shared/static/sj9f87tofpicll89xbc154pmbztu5q4h\" -O './dict'\n",
    "if not os.path.exists('./train.jsonl'):\n",
    "    !wget \"https://nyu.box.com/shared/static/aqp0jyjaixjmukm5asasivq2bcfze075.jsonl\" -O './train.jsonl'\n",
    "if not os.path.exists('./valid.jsonl'):\n",
    "    !wget \"https://nyu.box.com/shared/static/eg4ivddtqib2hkf1k8rkxnmzmo0cq27p.jsonl\" -O './valid.jsonl'\n",
    "\n",
    "if not os.path.exists('./chat_model_best_22.pt'):\n",
    "    !wget \"https://nyu.box.com/shared/static/24zsynuks8nzg7530tgakzh8o62id9xa.pt\" -O './chat_model_best_22.pt'\n",
    "\n",
    "chat_dict = ChatDictionary('./dict')\n",
    "train_dataset = ChatDataset('./train.jsonl', chat_dict)\n",
    "valid_dataset = ChatDataset('./valid.jsonl', chat_dict, 'valid')\n",
    "\n",
    "train_loader = DataLoader(train_dataset, shuffle=True, collate_fn=batchify, batch_size=256)\n",
    "valid_loader = DataLoader(valid_dataset, shuffle=False, collate_fn=batchify, batch_size=256)\n",
    "\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "\n",
    "class EncoderRNN(nn.Module):\n",
    "    \"\"\"Encodes the input context.\"\"\"\n",
    "\n",
    "    def __init__(self, vocab_size, embed_size, hidden_size, num_layers, pad_idx=0, dropout=0, shared_lt=None):\n",
    "        super().__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embed_size = embed_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        self.pad_idx = pad_idx\n",
    "        \n",
    "        if shared_lt is None:\n",
    "            self.embedding = nn.Embedding(self.vocab_size, self.embed_size, pad_idx)\n",
    "        else:\n",
    "            self.embedding = shared_lt\n",
    "            \n",
    "        self.gru = nn.GRU(\n",
    "            self.embed_size, self.hidden_size, num_layers=self.num_layers, batch_first=True, dropout=dropout if num_layers > 1 else 0,\n",
    "        )\n",
    "        \n",
    "        \n",
    "    def forward(self, text_vec, text_lens, hidden=None, use_packed=True):\n",
    "        embedded = self.embedding(text_vec)\n",
    "        attention_mask = text_vec.ne(self.pad_idx)\n",
    "\n",
    "        embedded = self.dropout(embedded)\n",
    "        if use_packed is True:\n",
    "            embedded = pack_padded_sequence(embedded, text_lens, batch_first=True)\n",
    "        output, hidden = self.gru(embedded, hidden)\n",
    "        if use_packed is True:\n",
    "            output, output_lens = pad_packed_sequence(output, batch_first=True)\n",
    "        \n",
    "        return output, hidden, attention_mask\n",
    "\n",
    "    \n",
    "class DecoderRNN(nn.Module):\n",
    "    \"\"\"Generates a sequence of tokens in response to context.\"\"\"\n",
    "\n",
    "    def __init__(self, vocab_size, embed_size, hidden_size, num_layers, dropout=0):\n",
    "        super().__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embed_size = embed_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "        self.embedding = nn.Embedding(self.vocab_size, self.embed_size, 0)\n",
    "        \n",
    "        self.gru = nn.GRU(\n",
    "            self.embed_size, self.hidden_size, num_layers=self.num_layers, batch_first=True, dropout=dropout if num_layers > 1 else 0,\n",
    "        )\n",
    "        \n",
    "        self.attention = AttentionLayer(self.hidden_size, self.embed_size)\n",
    "\n",
    "        self.out = nn.Linear(self.hidden_size, self.vocab_size)\n",
    "        self.longest_label = 100\n",
    "\n",
    "    def forward(self, text_vec, decoder_hidden, encoder_states):\n",
    "        emb = self.embedding(text_vec)\n",
    "        emb = self.dropout(emb)\n",
    "        seqlen = text_vec.size(1)\n",
    "        encoder_output, encoder_hidden, attention_mask = encoder_states\n",
    "        \n",
    "        decoder_hidden = decoder_hidden\n",
    "        output = []\n",
    "        attn_w_log = []\n",
    "\n",
    "        for i in range(seqlen):\n",
    "            decoder_output, decoder_hidden = self.gru(emb[:,i,:].unsqueeze(1), decoder_hidden)\n",
    "            \n",
    "            # compute attention at each time step\n",
    "            decoder_output_attended, attn_weights = self.attention(decoder_output, decoder_hidden, encoder_output, attention_mask)\n",
    "            output.append(decoder_output_attended)\n",
    "            attn_w_log.append(attn_weights)\n",
    "            \n",
    "        output = torch.cat(output, dim=1).to(text_vec.device)\n",
    "        scores = self.out(output)\n",
    "        \n",
    "        return scores, decoder_hidden, attn_w_log\n",
    "    \n",
    "    def decode_forced(self, ys, encoder_states, xs_lens):\n",
    "        encoder_output, encoder_hidden, attention_mask = encoder_states\n",
    "        \n",
    "        batch_size = ys.size(0)\n",
    "        target_length = ys.size(1)\n",
    "        longest_label = max(target_length, self.longest_label)\n",
    "        \n",
    "        starts = torch.Tensor([1]).long().to(self.embedding.weight.device).expand(batch_size, 1).long()  # expand to batch size\n",
    "        \n",
    "        # Teacher forcing: Feed the target as the next input\n",
    "        y_in = ys.narrow(1, 0, ys.size(1) - 1)\n",
    "        decoder_input = torch.cat([starts, y_in], 1)\n",
    "        decoder_output, decoder_hidden, attn_w_log = self.forward(decoder_input, encoder_hidden, encoder_states)\n",
    "        _, preds = decoder_output.max(dim=2)\n",
    "        \n",
    "        return decoder_output, preds, attn_w_log\n",
    "    \n",
    "    \n",
    "class AttentionLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, hidden_size, embedding_size):\n",
    "        super().__init__()\n",
    "        input_dim = hidden_size\n",
    "\n",
    "        self.linear_out = nn.Linear(hidden_size+input_dim, input_dim, bias=False)\n",
    "        self.softmax = nn.Softmax(dim=-1)\n",
    "        self.tanh = nn.Tanh()\n",
    "\n",
    "    def forward(self, decoder_output, decoder_hidden, encoder_output, attention_mask):\n",
    "\n",
    "        batch_size, seq_length, hidden_size = encoder_output.size()\n",
    "\n",
    "        encoder_output_t = encoder_output.transpose(1,2)\n",
    "        \n",
    "        attention_scores = torch.bmm(decoder_output, encoder_output_t).squeeze(1)\n",
    "\n",
    "        attention_scores.masked_fill_((~attention_mask), -10e5)\n",
    "        attention_weights = self.softmax(attention_scores)\n",
    "\n",
    "        mix = torch.bmm(attention_weights.unsqueeze(1), encoder_output)\n",
    "\n",
    "        combined = torch.cat((decoder_output.squeeze(1), mix.squeeze(1)), dim=1)\n",
    "\n",
    "        output = self.linear_out(combined).unsqueeze(1)\n",
    "        output = self.tanh(output)\n",
    "\n",
    "        return output, attention_weights\n",
    "    \n",
    "    \n",
    "class seq2seq(nn.Module):\n",
    "    \"\"\"\n",
    "    Generic seq2seq model with attention mechanism.\n",
    "    \"\"\"\n",
    "    def __init__(self, opts):\n",
    "\n",
    "        super().__init__()\n",
    "        self.opts = opts\n",
    "        \n",
    "        self.decoder = DecoderRNN(\n",
    "                                    vocab_size=self.opts['vocab_size'],\n",
    "                                    embed_size=self.opts['embedding_size'],\n",
    "                                    hidden_size=self.opts['hidden_size'],\n",
    "                                    num_layers=self.opts['num_layers_dec'],\n",
    "                                    dropout=self.opts['dropout'],\n",
    "                                )\n",
    "        \n",
    "        self.encoder = EncoderRNN(\n",
    "                                    vocab_size=self.opts['vocab_size'],\n",
    "                                    embed_size=self.opts['embedding_size'],\n",
    "                                    hidden_size=self.opts['hidden_size'],\n",
    "                                    num_layers=self.opts['num_layers_enc'],\n",
    "                                    dropout=self.opts['dropout'],\n",
    "                                    shared_lt=self.decoder.embedding\n",
    "        )\n",
    "        \n",
    "    def train(self):\n",
    "        self.encoder.train()\n",
    "        self.decoder.train()\n",
    "        \n",
    "    def eval(self):\n",
    "        self.encoder.eval()\n",
    "        self.decoder.eval()\n",
    "\n",
    "\n",
    "num_gpus = torch.cuda.device_count()\n",
    "if num_gpus > 0:\n",
    "    current_device = 'cuda'\n",
    "else:\n",
    "    current_device = 'cpu'\n",
    "\n",
    "\n",
    "if current_device == 'cuda':\n",
    "    model_pt = torch.load('./chat_model_best_22.pt')\n",
    "else:\n",
    "    model_pt = torch.load('./chat_model_best_22.pt', map_location=torch.device('cpu'))\n",
    "opts = model_pt['opts']\n",
    "\n",
    "model = seq2seq(opts)\n",
    "model.load_state_dict(model_pt['state_dict'])\n",
    "model.to(current_device)\n",
    "\n",
    "\n",
    "def greedy_search(model, batch, batch_size):\n",
    "    model.eval()\n",
    "        \n",
    "    text_vecs = batch['text_vecs'].to(current_device)\n",
    "\n",
    "    encoded = model.encoder(text_vecs, batch['text_lens'], use_packed=batch['use_packed'])\n",
    "    \n",
    "    encoder_output, encoder_hidden, attention_mask = encoded\n",
    "        \n",
    "    # 1 is __start__\n",
    "    starts = torch.Tensor([1]).long().to(model.decoder.embedding.weight.device).expand(batch_size, 1).long()  # expand to batch size\n",
    "    decoder_hidden = encoder_hidden\n",
    "\n",
    "    # greedy decoding here        \n",
    "    preds = [starts]\n",
    "    scores = []\n",
    "\n",
    "    # track if each sample in the mini batch is finished\n",
    "    # if all finished, stop predicting\n",
    "    finish_mask = torch.Tensor([0]*batch_size).byte().to(model.decoder.embedding.weight.device)\n",
    "    xs = starts\n",
    "    _attn_w_log = []\n",
    "\n",
    "    for ts in range(100):\n",
    "        decoder_output, decoder_hidden, attn_w_log = model.decoder(xs, decoder_hidden, encoded)  # decoder_output: [batch, time, vocab]\n",
    "        \n",
    "        _scores, _preds = torch.log_softmax(decoder_output, dim=-1).max(dim=-1)\n",
    "        \n",
    "        preds.append(_preds)\n",
    "        _attn_w_log.append(attn_w_log)\n",
    "        scores.append(_scores.view(-1)*(finish_mask == 0).float())\n",
    "\n",
    "        finish_mask += (_preds == 2).byte().view(-1)\n",
    "        \n",
    "        if not (torch.any(~finish_mask.bool())):\n",
    "            break\n",
    "        \n",
    "        xs = _preds\n",
    "    \n",
    "    preds = torch.cat(preds, dim=-1)\n",
    "        \n",
    "    return preds\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1kPRBxoMQjze"
   },
   "source": [
    "## Decoding with N-Gram blocking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a_OdGdFQTFy-"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "from operator import attrgetter\n",
    "import itertools\n",
    "\n",
    "class _HypothesisTail(object):\n",
    "    \"\"\"Hold some bookkeeping about a hypothesis.\"\"\"\n",
    "\n",
    "    # use slots because we don't want dynamic attributes here\n",
    "    __slots__ = ['timestep', 'hypid', 'score', 'tokenid']\n",
    "\n",
    "    def __init__(self, timestep, hypid, score, tokenid):\n",
    "        self.timestep = timestep\n",
    "        self.hypid = hypid\n",
    "        self.score = score\n",
    "        self.tokenid = tokenid\n",
    "\n",
    "class Beam(object):\n",
    "    \"\"\"\n",
    "    This class serves to keep info about partial hypothesis and perform the beam step\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(\n",
    "        self,\n",
    "        beam_size,\n",
    "        padding_token=0,\n",
    "        bos_token=1,\n",
    "        eos_token=2,\n",
    "        min_length=3,\n",
    "        min_n_best=3,\n",
    "        device='cpu',\n",
    "        # for iterbeam below\n",
    "        similarity_metric='hamming',\n",
    "        similarity_threshold=0,\n",
    "        # For n-gram ban\n",
    "        ngram_ban_n=0,\n",
    "        recent_hyps=[],\n",
    "    ):\n",
    "        \n",
    "        self.beam_size = beam_size\n",
    "        self.min_length = min_length\n",
    "        self.eos = eos_token\n",
    "        self.bos = bos_token\n",
    "        self.pad = padding_token\n",
    "        self.device = device\n",
    "        # recent score for each hypo in the beam\n",
    "        self.scores = None\n",
    "        # self.scores values per each time step\n",
    "        self.all_scores = [torch.Tensor([0.0] * beam_size).to(self.device)]\n",
    "        # backtracking id to hypothesis at previous time step\n",
    "        self.bookkeep = []\n",
    "        # output tokens at each time step\n",
    "        self.outputs = [\n",
    "            torch.Tensor(self.beam_size).long().fill_(self.bos).to(self.device)\n",
    "        ]\n",
    "        # keeps tuples (score, time_step, hyp_id)\n",
    "        self.finished = []\n",
    "        self.eos_top = False\n",
    "        self.eos_top_ts = None\n",
    "        self.n_best_counter = 0\n",
    "        self.min_n_best = min_n_best\n",
    "        self.partial_hyps = [[self.bos] for i in range(beam_size)]\n",
    "\n",
    "        # iterbeam related below\n",
    "        self.history_hyps = []\n",
    "        self.similarity_metric = similarity_metric\n",
    "        self.similarity_threshold = similarity_threshold\n",
    "        self.banned_tokens = set()\n",
    "\n",
    "        self.ngram_ban_n = ngram_ban_n\n",
    "        self.recent_hyps = recent_hyps\n",
    "        \n",
    "    def get_output_from_current_step(self):\n",
    "        \"\"\"Get the output at the current step.\"\"\"\n",
    "        return self.outputs[-1]\n",
    "\n",
    "    def get_backtrack_from_current_step(self):\n",
    "        \"\"\"Get the backtrack at the current step.\"\"\"\n",
    "        return self.bookkeep[-1]\n",
    "    \n",
    "    ##################### ITER-BEAM BLOCKING PART START #####################\n",
    "\n",
    "    def hamming_distance(self, t1, t2):\n",
    "        dist = 0\n",
    "        for tok1, tok2 in zip(t1,t2):\n",
    "            if tok1 != tok2:\n",
    "                dist += 1\n",
    "        return dist\n",
    "    \n",
    "    def edit_distance(self, t1, t2):\n",
    "        import editdistance\n",
    "        dist = editdistance.eval(t1, t2)\n",
    "        return dist\n",
    "                \n",
    "    def similarity_check(self, active_hyp, previous_hyps, metric='hamming', threshold=0):\n",
    "        banned_tokens = []\n",
    "        active_len = len(active_hyp)\n",
    "        for observed_hyp, _banned_tokens in previous_hyps.items():\n",
    "            if len(observed_hyp) != active_len:\n",
    "                continue\n",
    "            if metric == 'hamming':\n",
    "                dist = self.hamming_distance(observed_hyp, active_hyp)\n",
    "            if metric == 'edit':\n",
    "                dist = self.edit_distance(observed_hyp, active_hyp)\n",
    "            if dist <= threshold:\n",
    "                banned_tokens.extend(_banned_tokens)\n",
    "                    \n",
    "        return list(set(banned_tokens))\n",
    "    \n",
    "    ##################### ITER-BEAM BLOCKING PART END ########################\n",
    "\n",
    "    ## N-GRAM BAN\n",
    "\n",
    "    def ngram_ban(self, active_hyp, all_hyps, n):\n",
    "        if n==1:\n",
    "            # If 1-gram ban: return all previous tokens\n",
    "            return list(itertools.chain(*all_hyps)) + list(itertools.chain(*self.recent_hyps))\n",
    "        banned = []\n",
    "        history = tuple(active_hyp[-(n-1):])\n",
    "        for hyp in all_hyps + self.recent_hyps:\n",
    "            for ngram in zip(*[hyp[i:] for i in range(n)]):\n",
    "                if ngram[:-1] == history:\n",
    "                    banned.append(ngram[-1])\n",
    "        # print(active_hyp, banned)\n",
    "        return banned\n",
    "    \n",
    "    def select_paths(self, logprobs, prior_scores, previous_hyps):\n",
    "        \"\"\"Select the next vocabulary item in these beams.\"\"\"\n",
    "        # beam search actually looks over all hypotheses together so we flatten\n",
    "        beam_scores = logprobs + prior_scores.unsqueeze(1).expand_as(logprobs)\n",
    "        \n",
    "        # iterbeam blocking part\n",
    "        current_length = len(self.all_scores)\n",
    "        if len(previous_hyps) > 0 and current_length > 0:\n",
    "            for hyp_id in range(beam_scores.size(0)):\n",
    "                active_hyp = tuple(self.partial_hyps[hyp_id])\n",
    "                banned_tokens = self.similarity_check(active_hyp, previous_hyps, metric=self.similarity_metric, threshold=self.similarity_threshold)\n",
    "\n",
    "        # Add n-gram banned tokens.\n",
    "        if self.ngram_ban_n > 0:\n",
    "            for hyp_id in range(beam_scores.size(0)):\n",
    "                active_hyp = tuple(self.partial_hyps[hyp_id])\n",
    "                banned_tokens = self.ngram_ban(active_hyp, self.partial_hyps, self.ngram_ban_n)\n",
    "                if len(banned_tokens) > 0:\n",
    "                    beam_scores[hyp_id, banned_tokens] = -10e5\n",
    "            \n",
    "        flat_beam_scores = beam_scores.view(-1)\n",
    "        best_scores, best_idxs = torch.topk(flat_beam_scores, self.beam_size, dim=-1)\n",
    "        voc_size = logprobs.size(-1)\n",
    "\n",
    "        # get the backtracking hypothesis id as a multiple of full voc_sizes\n",
    "        hyp_ids = best_idxs / voc_size\n",
    "        # get the actual word id from residual of the same division\n",
    "        tok_ids = best_idxs % voc_size\n",
    "        \n",
    "        return (hyp_ids, tok_ids, best_scores)\n",
    "    \n",
    "    def advance(self, logprobs, previous_hyps):\n",
    "        \"\"\"Advance the beam one step.\"\"\"\n",
    "        current_length = len(self.all_scores) - 1\n",
    "        if current_length < self.min_length:\n",
    "            # penalize all eos probs to make it decode longer\n",
    "            for hyp_id in range(logprobs.size(0)):\n",
    "                logprobs[hyp_id][self.eos] = -10e5\n",
    "\n",
    "        if self.scores is None:\n",
    "            logprobs = logprobs[0:1]  # we use only the first hyp now, since they are all same\n",
    "            self.scores = torch.zeros(1).type_as(logprobs).to(logprobs.device)\n",
    "            \n",
    "        hyp_ids, tok_ids, self.scores = self.select_paths(logprobs, self.scores, previous_hyps)\n",
    "        \n",
    "        # clone scores here to avoid referencing penalized EOS in the future!\n",
    "        self.all_scores.append(self.scores.clone())\n",
    "\n",
    "        self.outputs.append(tok_ids)\n",
    "        self.bookkeep.append(hyp_ids)\n",
    "        self.partial_hyps = [\n",
    "            self.partial_hyps[hyp_ids[i]] + [tok_ids[i].item()]\n",
    "            for i in range(self.beam_size)\n",
    "        ]\n",
    "        self.history_hyps.extend(self.partial_hyps)\n",
    "\n",
    "        #  check new hypos for eos label, if we have some, add to finished\n",
    "        for hypid in range(self.beam_size):\n",
    "            if self.outputs[-1][hypid] == self.eos:\n",
    "                self.scores[hypid] = -10e5\n",
    "                #  this is finished hypo, adding to finished\n",
    "                eostail = _HypothesisTail(\n",
    "                    timestep=len(self.outputs) - 1,\n",
    "                    hypid=hypid,\n",
    "                    score=self.all_scores[-1][hypid],\n",
    "                    tokenid=self.eos,\n",
    "                )\n",
    "                self.finished.append(eostail)\n",
    "                self.n_best_counter += 1\n",
    "\n",
    "        if self.outputs[-1][0] == self.eos:\n",
    "            self.eos_top = True\n",
    "            if self.eos_top_ts is None:\n",
    "                self.eos_top_ts = len(self.outputs) - 1\n",
    "    \n",
    "    def is_done(self):\n",
    "        \"\"\"Return whether beam search is complete.\"\"\"\n",
    "        return self.eos_top and self.n_best_counter >= self.min_n_best\n",
    "\n",
    "    def get_top_hyp(self):\n",
    "        \"\"\"\n",
    "        Get single best hypothesis.\n",
    "        :return: hypothesis sequence and the final score\n",
    "        \"\"\"\n",
    "        return self._get_rescored_finished(n_best=1)[0]\n",
    "\n",
    "    def _get_hyp_from_finished(self, hypothesis_tail):\n",
    "        \"\"\"\n",
    "        Extract hypothesis ending with EOS at timestep with hyp_id.\n",
    "        :param timestep:\n",
    "            timestep with range up to len(self.outputs) - 1\n",
    "        :param hyp_id:\n",
    "            id with range up to beam_size - 1\n",
    "        :return:\n",
    "            hypothesis sequence\n",
    "        \"\"\"\n",
    "        hyp_idx = []\n",
    "        endback = hypothesis_tail.hypid\n",
    "        for i in range(hypothesis_tail.timestep, -1, -1):\n",
    "            hyp_idx.append(\n",
    "                _HypothesisTail(\n",
    "                    timestep=i,\n",
    "                    hypid=endback,\n",
    "                    score=self.all_scores[i][endback],\n",
    "                    tokenid=self.outputs[i][endback],\n",
    "                )\n",
    "            )\n",
    "            endback = self.bookkeep[i - 1][endback]\n",
    "\n",
    "        return hyp_idx\n",
    "\n",
    "    def _get_pretty_hypothesis(self, list_of_hypotails):\n",
    "        \"\"\"Return hypothesis as a tensor of token ids.\"\"\"\n",
    "        return torch.stack([ht.tokenid for ht in reversed(list_of_hypotails)])\n",
    "\n",
    "    def _get_rescored_finished(self, n_best=None, add_length_penalty=False):\n",
    "        \"\"\"\n",
    "        Return finished hypotheses according to adjusted scores.\n",
    "        Score adjustment is done according to the Google NMT paper, which\n",
    "        penalizes long utterances.\n",
    "        :param n_best:\n",
    "            number of finalized hypotheses to return\n",
    "        :return:\n",
    "            list of (tokens, score) pairs, in sorted order, where:\n",
    "              - tokens is a tensor of token ids\n",
    "              - score is the adjusted log probability of the entire utterance\n",
    "        \"\"\"\n",
    "        # if we never actually finished, force one\n",
    "        if not self.finished:\n",
    "            self.finished.append(\n",
    "                _HypothesisTail(\n",
    "                    timestep=len(self.outputs) - 1,\n",
    "                    hypid=0,\n",
    "                    score=self.all_scores[-1][0],\n",
    "                    tokenid=self.eos,\n",
    "                )\n",
    "            )\n",
    "\n",
    "        rescored_finished = []\n",
    "        for finished_item in self.finished:\n",
    "            if add_length_penalty:\n",
    "                current_length = finished_item.timestep + 1\n",
    "                # these weights are from Google NMT paper\n",
    "                length_penalty = math.pow((1 + current_length) / 6, 0.65)\n",
    "            else:\n",
    "                length_penalty = 1\n",
    "            rescored_finished.append(\n",
    "                _HypothesisTail(\n",
    "                    timestep=finished_item.timestep,\n",
    "                    hypid=finished_item.hypid,\n",
    "                    score=finished_item.score / length_penalty,\n",
    "                    tokenid=finished_item.tokenid,\n",
    "                )\n",
    "            )\n",
    "\n",
    "        # Note: beam size is almost always pretty small, so sorting is cheap enough\n",
    "        srted = sorted(rescored_finished, key=attrgetter('score'), reverse=True)\n",
    "\n",
    "        if n_best is not None:\n",
    "            srted = srted[:n_best]\n",
    "\n",
    "        return [\n",
    "            (self._get_pretty_hypothesis(self._get_hyp_from_finished(hyp)), hyp.score)\n",
    "            for hyp in srted\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cdZRmRu_TG5J"
   },
   "outputs": [],
   "source": [
    "def reorder_encoder_states(encoder_states, indices):\n",
    "        \"\"\"Reorder encoder states according to a new set of indices.\"\"\"\n",
    "        enc_out, hidden, attention_mask = encoder_states\n",
    "\n",
    "        # LSTM or GRU/RNN hidden state?\n",
    "        if isinstance(hidden, torch.Tensor):\n",
    "            hid, cell = hidden, None\n",
    "        else:\n",
    "            hid, cell = hidden\n",
    "\n",
    "        if not torch.is_tensor(indices):\n",
    "            # cast indices to a tensor if needed\n",
    "            indices = torch.LongTensor(indices).to(hid.device)\n",
    "\n",
    "        hid = hid.index_select(1, indices)\n",
    "        if cell is None:\n",
    "            hidden = hid\n",
    "        else:\n",
    "            cell = cell.index_select(1, indices)\n",
    "            hidden = (hid, cell)\n",
    "\n",
    "        enc_out = enc_out.index_select(0, indices)\n",
    "        attention_mask = attention_mask.index_select(0, indices)\n",
    "\n",
    "        return enc_out, hidden, attention_mask\n",
    "    \n",
    "    \n",
    "def reorder_decoder_incremental_state(incremental_state, inds):\n",
    "    if torch.is_tensor(incremental_state):\n",
    "        # gru or lstm\n",
    "        return torch.index_select(incremental_state, 1, inds).contiguous()\n",
    "    elif isinstance(incremental_state, tuple):\n",
    "        return tuple(\n",
    "            self.reorder_decoder_incremental_state(x, inds)\n",
    "            for x in incremental_state)\n",
    "\n",
    "def get_nbest_list_from_beam(beam, dictionary, n_best=None, add_length_penalty=False):\n",
    "    if n_best is None:\n",
    "        n_best = beam.min_n_best\n",
    "    nbest_list = beam._get_rescored_finished(n_best=n_best, add_length_penalty=add_length_penalty)\n",
    "    \n",
    "    nbest_list_text = [(dictionary.v2t(i[0].cpu().tolist()), i[1].item()) for i in nbest_list]\n",
    "    \n",
    "    return nbest_list_text\n",
    "\n",
    "def generate_with_beam(beam_size, min_n_best, model, batch, batch_size, \n",
    "                       previous_hyps=None, similarity_metric='hamming', \n",
    "                       similarity_threshold=0, verbose=False, ngram_ban_n=0, \n",
    "                       recent_hyps=[]):\n",
    "    \"\"\"\n",
    "    This function takes a model, batch, beam settings and performs decoding with a beam\n",
    "    \"\"\"\n",
    "    beams = [Beam(beam_size, min_n_best=min_n_best, \n",
    "                  eos_token=chat_dict.word2ind['__end__'], \n",
    "                  padding_token=chat_dict.word2ind['__null__'], \n",
    "                  bos_token=chat_dict.word2ind['__start__'], \n",
    "                  device=current_device, similarity_metric=similarity_metric, \n",
    "                  similarity_threshold=similarity_threshold, ngram_ban_n=ngram_ban_n, recent_hyps=recent_hyps) for _ in range(batch_size)]\n",
    "    repeated_inds = torch.arange(batch_size).to(current_device).unsqueeze(1).repeat(1, beam_size).view(-1)\n",
    "    \n",
    "    text_vecs = batch['text_vecs'].to(current_device)\n",
    "\n",
    "    encoder_states = model.encoder(text_vecs, batch['text_lens'], use_packed=batch['use_packed'])\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    encoder_states = reorder_encoder_states(encoder_states, repeated_inds)  # no actual reordering here, but repeating beam size times each sample in the minibatch\n",
    "    encoder_output, encoder_hidden, attention_mask = encoder_states\n",
    "    \n",
    "    incr_state = encoder_hidden  # we init decoder hidden with last encoder_hidden\n",
    "    \n",
    "    # 1 is a start token id\n",
    "    starts = torch.Tensor([1]).long().to(model.decoder.embedding.weight.device).expand(batch_size*beam_size, 1).long()  # expand to batch_size * beam_size\n",
    "    decoder_input = starts\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for ts in range(100):\n",
    "            if all((b.is_done() for b in beams)):\n",
    "                break\n",
    "            score, incr_state, attn_w_log = model.decoder(decoder_input, incr_state, encoder_states)\n",
    "            score = score[:, -1:, :]  # take last time step and eliminate the dimension\n",
    "            score = score.view(batch_size, beam_size, -1)\n",
    "            score = torch.log_softmax(score, dim=-1)\n",
    "         \n",
    "            for i, b in enumerate(beams):\n",
    "                if not b.is_done():\n",
    "                    # make mock previous_hyps if not used #\n",
    "                    if previous_hyps is None:\n",
    "                        previous_hyps = [{} for i in range(batch_size)]\n",
    "\n",
    "                    b.advance(score[i], previous_hyps[i])\n",
    "\n",
    "            incr_state_inds = torch.cat([beam_size * i + b.get_backtrack_from_current_step() for i, b in enumerate(beams)])\n",
    "            incr_state = reorder_decoder_incremental_state(incr_state, incr_state_inds)\n",
    "            selection = torch.cat([b.get_output_from_current_step() for b in beams]).unsqueeze(-1)\n",
    "            decoder_input = selection\n",
    "\n",
    "    beam_preds_scores = [list(b.get_top_hyp()) for b in beams]\n",
    "\n",
    "    if verbose:\n",
    "        for bi in range(batch_size):\n",
    "            print(f'batch {bi}')\n",
    "            for i in get_nbest_list_from_beam(beams[bi], chat_dict, n_best=min_n_best):\n",
    "                print(i)\n",
    "    \n",
    "    return beam_preds_scores, beams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5LlIPdYUQjzi"
   },
   "source": [
    "## You present here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 638
    },
    "colab_type": "code",
    "id": "DFeGrL7tQjzk",
    "outputId": "117dfd92-eede-4070-e6aa-24180dd84ff3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n-gram ban: 1\n",
      "batch 0\n",
      "('__start__ hi how are you today __end__', -6.499421119689941)\n",
      "(\"__start__ i ' m good thanks for asking __end__\", -7.303680896759033)\n",
      "('__start__ hi how are you tonight __end__', -8.7974853515625)\n",
      "('__start__ hi how are you ? __end__', -10.198396682739258)\n",
      "('__start__ just got back from the gym ! __end__', -12.563788414001465)\n",
      "('__start__ i am working on a new job __end__', -12.893826484680176)\n",
      "('__start__ just got back from a long walk __end__', -13.656246185302734)\n",
      "('__start__ i am working on a new project __end__', -13.694055557250977)\n",
      "('__start__ i am going to work for google __end__', -13.83371353149414)\n",
      "('__start__ just got back from a run ! __end__', -13.835736274719238)\n",
      "n-gram ban: 2\n",
      "batch 0\n",
      "('__start__ i am good how are you __end__', -6.686244487762451)\n",
      "(\"__start__ i ' m good thanks for asking __end__\", -6.9844536781311035)\n",
      "('__start__ i am good how are you ? __end__', -7.428712844848633)\n",
      "(\"__start__ i ' m good and you ? __end__\", -7.656593322753906)\n",
      "(\"__start__ i ' m good and you __end__\", -8.07814884185791)\n",
      "('__start__ i am good how are you today __end__', -10.262678146362305)\n",
      "(\"__start__ i ' m good thanks for asking . __end__\", -10.735372543334961)\n",
      "(\"__start__ i ' m good thanks for asking ! __end__\", -11.5530366897583)\n",
      "('__start__ i am good . just got home from work __end__', -12.570554733276367)\n",
      "('__start__ i am good . just got back from work __end__', -13.484786987304688)\n",
      "n-gram ban: 3\n",
      "batch 0\n",
      "(\"__start__ i ' m good thanks for asking __end__\", -6.9844536781311035)\n",
      "(\"__start__ i ' m good how are you __end__\", -7.29628849029541)\n",
      "(\"__start__ i ' m doing well and you ? __end__\", -7.599836826324463)\n",
      "(\"__start__ i ' m good how are you ? __end__\", -7.99958610534668)\n",
      "('__start__ i am doing well , how about yourself ? __end__', -8.518004417419434)\n",
      "(\"__start__ i ' m well , how about yourself ? __end__\", -9.44328498840332)\n",
      "(\"__start__ i ' m well . how about yourself ? __end__\", -9.737001419067383)\n",
      "(\"__start__ i ' m great . how about yourself ? __end__\", -9.845620155334473)\n",
      "(\"__start__ i ' m good thanks for asking . __end__\", -10.735372543334961)\n",
      "(\"__start__ i ' m doing very well , thank you for asking __end__\", -13.598217010498047)\n",
      "[tensor([  1,   5,  16,  26,  41, 247,  33, 623,   2]), tensor(-6.9845)]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1\n",
    "beam_size = 20\n",
    "beam_n_best = 10\n",
    "\n",
    "valid_loader_single = DataLoader(valid_dataset, shuffle=False, collate_fn=batchify, batch_size=batch_size)\n",
    "\n",
    "valid_sample = next(iter(valid_loader_single))\n",
    "\n",
    "print(\"n-gram ban: 1\")\n",
    "beam_preds_scores, beams = generate_with_beam(beam_size, beam_n_best, model, valid_sample, batch_size=batch_size, \n",
    "                                              verbose=True, \n",
    "                                              ngram_ban_n=1)\n",
    "\n",
    "print(\"n-gram ban: 2\")\n",
    "beam_preds_scores, beams = generate_with_beam(beam_size, beam_n_best, model, valid_sample, batch_size=batch_size, \n",
    "                                              verbose=True, \n",
    "                                              ngram_ban_n=2)\n",
    "\n",
    "print(\"n-gram ban: 3\")\n",
    "beam_preds_scores, beams = generate_with_beam(beam_size, beam_n_best, model, valid_sample, batch_size=batch_size, \n",
    "                                              verbose=True, \n",
    "                                              ngram_ban_n=3)\n",
    "\n",
    "print(beam_preds_scores[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0p4C-_XlQjzm"
   },
   "source": [
    "## Interactive chatbot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1atcw1PiaHWL"
   },
   "outputs": [],
   "source": [
    "def fill_prefixes(prefix_dict, history_hyps):\n",
    "    for hyp in history_hyps:\n",
    "        for j in range(len(hyp)):\n",
    "            _prefix = tuple(hyp[:j])\n",
    "            if _prefix in prefix_dict:\n",
    "                if hyp[j] in prefix_dict[_prefix]:\n",
    "                    continue\n",
    "                else:\n",
    "                    prefix_dict[_prefix].append(hyp[j])\n",
    "            else:\n",
    "                prefix_dict[_prefix] = [hyp[j]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LMAwdM-2Qjzo"
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    filename='input.log', #doesn't seem to save expli\n",
    "                    filemode='w') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cH94XJLd3hPH"
   },
   "source": [
    "## Live session: run the following cell and enjoy your chat with our curious bot :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "_4XQpD3xQjzq",
    "outputId": "a2f17f86-4651-4ede-abc6-c4c439ab785f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "your persona: i live in Hong Kong.\n",
      "your persona: I love computer game.\n",
      "good morning\n",
      ">> hi how are you today\n",
      "I am good. you?\n",
      ">> i ' m good thanks for asking\n",
      "what are you up to?\n",
      ">> do you have any hobbies ?\n",
      "I love coding\n",
      ">> what is your favorite food\n",
      "Any thing green and healthy\n",
      ">> that sounds like a lot of fun\n",
      "bye\n"
     ]
    }
   ],
   "source": [
    "#chat logic\n",
    "#global history variable\n",
    "history = [\"your persona: i live in Hong Kong.\\nyour persona: I love computer game.\"] \n",
    "\n",
    "#BEAM settings\n",
    "beam_size = 10\n",
    "beam_n_best = 10\n",
    "\n",
    "flag=True\n",
    "for h in history: print(h)\n",
    "prefix_dict={}\n",
    "last_hyps=[[]]\n",
    "logging_entries=[]\n",
    "while(flag==True):\n",
    "    input_sentence = input()\n",
    "    if input_sentence != 'bye':\n",
    "        # history.append(f'__start__ {input_sentence} __end__') \n",
    "        history.append(f'{input_sentence} \\n')\n",
    "        logging.info(input_sentence)\n",
    "        inputs = [RETOK.findall(sentence) for sentence in history]\n",
    "        inputs = [word for sentence in inputs for word in sentence] #flatten list\n",
    "        # print(inputs)\n",
    "        test_batch = {\n",
    "            'text_vecs': torch.tensor([chat_dict.t2v(inputs)], dtype=torch.long, device=model.decoder.embedding.weight.device),\n",
    "            'text_lens': torch.tensor([len(inputs)], dtype=torch.long),\n",
    "            'use_packed': True,\n",
    "        }\n",
    "        # output = greedy_search(model, test_batch, 1)\n",
    "        logging_entity=dict()\n",
    "        logging_entity['history'] = copy.deepcopy(history)\n",
    "        logging_entity['input_sentence'] = copy.deepcopy(input_sentence)\n",
    "        logging_entity['recent_hyps'] = copy.deepcopy(last_hyps)\n",
    "        _, beam_output = generate_with_beam(beam_size, beam_n_best, model, test_batch, 1, \n",
    "                                            similarity_metric='edit', similarity_threshold=3,\n",
    "                                            previous_hyps=[prefix_dict],\n",
    "                                            recent_hyps=last_hyps,\n",
    "                                            # verbose=True,\n",
    "                                            ngram_ban_n=2)\n",
    "        output=[]\n",
    "        for bo in beam_output[0]._get_rescored_finished(beam_n_best):\n",
    "            if tuple(bo[0]) not in last_hyps:\n",
    "                output=bo[0].tolist()\n",
    "                break\n",
    "        response = chat_dict.v2t(output[1:-1]) # Stripping __start__ and __end__\n",
    "        logging_entity['response'] = copy.deepcopy(response)\n",
    "        print(f'>> {response}')\n",
    "        history.append(response)\n",
    "        last_hyps.append(tuple(output))\n",
    "        # only keep last 5\n",
    "        last_hyps = last_hyps[-5:]\n",
    "        fill_prefixes(prefix_dict, beam_output[0].history_hyps)\n",
    "        logging_entries.append(logging_entity)\n",
    "    else:\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Chatbot.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "NLP",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}