actual.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import create_vocab_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "d, v = create_vocab_dictionary('..', 'all_tokens.pickle', False, 4500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4997"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ebmalen(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# import pandas as pd\n",
    "# import numpy as np\n",
    "\n",
    "# x = np.loadtxt('Retrieve-and-Edit-Framework/datasets/word_vectors/github.txt', delimiter=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.read_csv('Retrieve-and-Edit-Framework/datasets/word_vectors/github.txt', delimiter = ' ', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0               .\n",
       "1               )\n",
       "2               (\n",
       "3               §\n",
       "4               ,\n",
       "          ...    \n",
       "3853       Lookup\n",
       "3854    getPlugin\n",
       "3855         sess\n",
       "3856       Engine\n",
       "3857          std\n",
       "Name: 0, Length: 3858, dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import ConcatDataset, DataLoader\n",
    "from transformer.bart import BartModel\n",
    "from transformer.configuration_bart import BartConfig\n",
    "from DataClass.torchData import *\n",
    "\n",
    "model_config = BartConfig(\n",
    "        vocab_size=len(word2idx), pad_token_id=0,\n",
    "        eos_token_id=2, d_model=128,\n",
    "        encoder_ffn_dim=512,\n",
    "        encoder_layers=6,\n",
    "        encoder_attention_heads=8,\n",
    "        decoder_ffn_dim=512,\n",
    "        decoder_layers=6,\n",
    "        decoder_attention_heads=8,\n",
    "        dropout=0.1,\n",
    "        max_encoder_position_embeddings=386,\n",
    "        max_decoder_position_embeddings=128)\n",
    "\n",
    "model = BartModel(model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object Module.parameters at 0x145b8bde0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.decoder.layers.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load(\"checkpoint-ruleseq2seq.pth\", map_location=torch.device('cpu'))\n",
    "hey = checkpoint['model']\n",
    "\n",
    "a, b = zip(*hey.items())\n",
    "\n",
    "for k, v in zip(a,b):\n",
    "    hey[k[7:]] = v\n",
    "    del hey[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(hey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = \"def send_css(path):\"\n",
    "actual_y = \"return send_from_directory('static/css', path)\"\n",
    "retx = \"def send_js(path):\"\n",
    "rety = \"return  send_from_directory('static/js', path)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import preprocess_tokens, tokenize_fine_grained\n",
    "from DataClass.torchData import word2idx, UNKNOWN_IDX, idx2word\n",
    "SEP_CONTEXT_WORD = '<SEP_CONTEXT>'\n",
    "SEP_PAIR_WORD = '<SEP_PAIR>'\n",
    "SEP_RET_WORD = '<SEP_RET>'\n",
    "\n",
    "# line = 'if tool_name == \"mllr\":'#\"domain = config.get('instance', 'domain')\"\n",
    "context_tokens = preprocess_tokens(tokenize_fine_grained(line), 128) + [SEP_CONTEXT_WORD]\n",
    "# retx = 'elif tool_name == \"vtln\":'#'domain = config.get(\"instance\", \"domain\")'\n",
    "retx_tokens = preprocess_tokens(tokenize_fine_grained(retx), 128)\n",
    "# rety = 'command.extend([\"-v\", \"vtln\"])'#'headers  = {\"Authorization\" : \"Bearer %s\" % token}'\n",
    "rety_tokens = preprocess_tokens(tokenize_fine_grained(rety), 128)\n",
    "context_tokens += retx_tokens\n",
    "context_tokens += [SEP_RET_WORD]\n",
    "context_tokens += rety_tokens\n",
    "in_ = torch.LongTensor([word2idx.get(token, UNKNOWN_IDX) for token in context_tokens]).contiguous().view(1, 386)\n",
    "\n",
    "dec = \"\"\n",
    "y_tokens = preprocess_tokens(tokenize_fine_grained(dec), 128)\n",
    "y_tokens = y_tokens[:y_tokens.index(\"<EOS>\")]\n",
    "deco = [word2idx.get(token, UNKNOWN_IDX) for token in y_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"<BOS>returnsend_from_directory('static/js',path)<EOS>\""
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# deco = y_tokens\n",
    "dec_ = torch.LongTensor(deco).contiguous().view(1, len(deco))\n",
    "for i in range(127 - len(y_tokens)):\n",
    "    hey = model(input_ids = in_, decoder_input_ids=dec_)[0, -1, :]\n",
    "    new_char = int(hey.max(0)[1])\n",
    "    if new_char == UNKNOWN_IDX:\n",
    "        new_char = context_tokens[i+1]\n",
    "    deco += [new_char]\n",
    "    if int(hey.max(0)[1]) == 2: break\n",
    "#     y_tokens = preprocess_tokens(tokenize_fine_grained(deco), 128)[:i]\n",
    "    dec_ = torch.LongTensor(deco).contiguous().view(1, len(deco))\n",
    "    \n",
    "''.join([idx2word[idx] for idx in deco])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"<BOS>returnsend_from_directory('static/css',path)<EOS><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD><PAD>\""
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# actual_y = \"headers = {'Authorization': 'Bearer %s' % token}\"\n",
    "''.join(preprocess_tokens(tokenize_fine_grained(actual_y), 128))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import ConcatDataset, DataLoader\n",
    "from transformer.bart import BartModel\n",
    "from transformer.configuration_bart import BartConfig\n",
    "from DataClass.torchData import *\n",
    "\n",
    "model_config = BartConfig(\n",
    "        vocab_size=len(word2idx), pad_token_id=0,\n",
    "        eos_token_id=2, d_model=128,\n",
    "        encoder_ffn_dim=512,\n",
    "        encoder_layers=6,\n",
    "        encoder_attention_heads=8,\n",
    "        decoder_ffn_dim=512,\n",
    "        decoder_layers=6,\n",
    "        decoder_attention_heads=8,\n",
    "        dropout=0.1,\n",
    "        max_encoder_position_embeddings=128,\n",
    "        max_decoder_position_embeddings=128)\n",
    "\n",
    "model = BartModel(model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [-0.0221, -0.0118, -0.0161,  ..., -0.0460,  0.0221,  0.0192],\n",
       "        [-0.0207, -0.0227,  0.0015,  ...,  0.0046,  0.0218,  0.0117],\n",
       "        ...,\n",
       "        [-0.0132,  0.0028, -0.0170,  ...,  0.0057, -0.0064, -0.0052],\n",
       "        [ 0.0198,  0.0163, -0.0217,  ..., -0.0164,  0.0257,  0.0467],\n",
       "        [ 0.0345,  0.0384, -0.0264,  ..., -0.0031, -0.0233,  0.0296]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.shared.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load(\"checkpoint-120000.pth\", map_location=torch.device('cpu'))\n",
    "hey = checkpoint['model']\n",
    "\n",
    "a, b = zip(*hey.items())\n",
    "\n",
    "for k, v in zip(a,b):\n",
    "    hey[k[7:]] = v\n",
    "    del hey[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(hey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0046, -0.0076, -0.0695,  ..., -0.0314, -0.0249, -0.0206],\n",
       "        [-0.0230, -0.0604, -0.1208,  ..., -0.0293, -0.0233, -0.0037],\n",
       "        [ 0.0137, -0.0524, -0.0123,  ...,  0.0865, -0.0476, -0.1183],\n",
       "        ...,\n",
       "        [ 0.2625, -0.0448,  0.2158,  ...,  0.0953,  0.1499,  0.0506],\n",
       "        [ 0.1097, -0.0928,  0.0395,  ..., -0.0195,  0.1533, -0.2183],\n",
       "        [ 0.0278, -0.1492, -0.1252,  ...,  0.1920, -0.1664, -0.1495]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.shared.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import preprocess_tokens, tokenize_fine_grained\n",
    "from DataClass.torchData import word2idx, UNKNOWN_IDX, idx2word\n",
    "\n",
    "line = \"if x == 5: break\"\n",
    "context_tokens = preprocess_tokens(tokenize_fine_grained(line), 128)\n",
    "in_ = torch.LongTensor([word2idx.get(token, UNKNOWN_IDX) for token in context_tokens]).contiguous().view(1, 128)\n",
    "\n",
    "dec = \"\"\n",
    "y_tokens = preprocess_tokens(tokenize_fine_grained(dec), 128)\n",
    "y_tokens = y_tokens[:y_tokens.index(\"<EOS>\")]\n",
    "deco = [word2idx.get(token, UNKNOWN_IDX) for token in y_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<BOS>',\n",
       " 'if',\n",
       " 'x',\n",
       " '=',\n",
       " '=',\n",
       " '5',\n",
       " ':',\n",
       " 'break',\n",
       " '<EOS>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>',\n",
       " '<PAD>']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dec_ = torch.LongTensor(deco).contiguous().view(1, len(deco))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deco = y_tokens\n",
    "for i in range(127 - len(y_tokens)):\n",
    "    hey = model(input_ids = in_, decoder_input_ids=dec_)[0, -1, :]\n",
    "    deco += [int(hey.max(0)[1])]\n",
    "    if int(hey.max(0)[1]) == 2: break\n",
    "#     y_tokens = preprocess_tokens(tokenize_fine_grained(deco), 128)[:i]\n",
    "    dec_ = torch.LongTensor(deco).contiguous().view(1, len(deco))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<BOS>ifx==5:continue<EOS>'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([idx2word[idx] for idx in deco])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'_self._done=False<EOS>'"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(translate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "params argument given to the optimizer should be an iterable of Tensors or dicts, but got torch.FloatTensor",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-2456975cdc66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, params, lr, betas, eps, weight_decay, amsgrad)\u001b[0m\n\u001b[1;32m     40\u001b[0m         defaults = dict(lr=lr, betas=betas, eps=eps,\n\u001b[1;32m     41\u001b[0m                         weight_decay=weight_decay, amsgrad=amsgrad)\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, params, defaults)\u001b[0m\n\u001b[1;32m     37\u001b[0m             raise TypeError(\"params argument given to the optimizer should be \"\n\u001b[1;32m     38\u001b[0m                             \u001b[0;34m\"an iterable of Tensors or dicts, but got \"\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m                             torch.typename(params))\n\u001b[0m\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefaultdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: params argument given to the optimizer should be an iterable of Tensors or dicts, but got torch.FloatTensor"
     ]
    }
   ],
   "source": [
    "optim.Adam(x[0], lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    eps: 1e-08\n",
       "    lr: 0.01\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim.Adam(d, lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = (1, 2)\n",
    "b = (4,)\n",
    "new = b+x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 1, 2)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.MetaTorchData import MetaRepo, MetaRetrieved\n",
    "from torch.utils.data import DataLoader\n",
    "ds = MetaRepo('repo_files/beaker_line_pairs.csv', False, k_shot=3)\n",
    "d = DataLoader(ds, shuffle=True)\n",
    "loader = iter(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax, bx, cx, dx = next(loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 128])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ax.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer.Models import Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Transformer(100, 100, 0, 0)\n",
    "\n",
    "temp_params = list(model.parameters())\n",
    "\n",
    "src_emb_params = list(model.encoder.src_word_emb.parameters())\n",
    "trg_emb_params = list(model.decoder.trg_word_emb.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "The size of tensor a (512) must match the size of tensor b (100) at non-singleton dimension 0",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-60-67dddbfa56af>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtemp_params\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc_emb_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# temp_params.remove(trg_emb_params)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# trg_emb_params[0]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (512) must match the size of tensor b (100) at non-singleton dimension 0"
     ]
    }
   ],
   "source": [
    "temp_params.remove(src_emb_params[0])\n",
    "# temp_params.remove(trg_emb_params)\n",
    "# trg_emb_params[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(d.encoder.src_word_emb.parameters())[0][0]\n",
    "hey = list(d.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "eq() received an invalid combination of arguments - got (list), but expected one of:\n * (Tensor other)\n      didn't match because some of the arguments have invalid types: (\u001b[31;1mlist\u001b[0m)\n * (Number other)\n      didn't match because some of the arguments have invalid types: (\u001b[31;1mlist\u001b[0m)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-494b794627d2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msrc_word_emb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrg_word_emb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: eq() received an invalid combination of arguments - got (list), but expected one of:\n * (Tensor other)\n      didn't match because some of the arguments have invalid types: (\u001b[31;1mlist\u001b[0m)\n * (Number other)\n      didn't match because some of the arguments have invalid types: (\u001b[31;1mlist\u001b[0m)\n"
     ]
    }
   ],
   "source": [
    "hey.remove([list(d.encoder.src_word_emb.parameters())[0]), list(d.decoder.trg_word_emb.parameters())[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "185"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(list(d.parameters()))\n",
    "len(hey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for -: 'list' and 'list'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-2f7d459a8015>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msrc_word_emb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for -: 'list' and 'list'"
     ]
    }
   ],
   "source": [
    "len(list(d.parameters()) - list(d.encoder.src_word_emb.parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "from DataClass.Constants import NO_CONTEXT_WORD, UNKNOWN_IDX\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'repo_files/lantern_line_pairs.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import fix_quote_strings, tokenize_fine_grained, preprocess_tokens\n",
    "idx = 100\n",
    "try:\n",
    "    x = next(pd.read_csv(filename,\n",
    "                                skiprows=idx * 1+1,\n",
    "                                chunksize=1, header=None, dtype=str)).fillna(NO_CONTEXT_WORD).values\n",
    "except:\n",
    "        x = next(pd.read_csv(filename,\n",
    "                            skiprows=idx * 1+1,\n",
    "                            chunksize=1, header=None,\n",
    "                            sep=',\\s+', quoting=csv.QUOTE_ALL, dtype=str)).fillna(NO_CONTEXT_WORD).values\n",
    "        x = np.array(fix_quote_strings(x[0, 0]))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['        self._p_transmit = self._p.getCharacteristics(uuid=self.TRANSMIT_UUID)[0]',\n",
       "        '    def disconnect(self):']], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import create_vocab_dictionary\n",
    "\n",
    "word2idx, idx2ord = create_vocab_dictionary(path='..', file='all_tokens.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "all_tokens = pickle.load(open('../all_tokens.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_tokens['self']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'self'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-0d5564ffea4a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mword2idx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'self'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m: 'self'"
     ]
    }
   ],
   "source": [
    "word2idx['self']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = tokenize_fine_grained(x[0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['self',\n",
       " '.',\n",
       " '_',\n",
       " 'p',\n",
       " '_',\n",
       " 'transmit',\n",
       " '=',\n",
       " 'self',\n",
       " '.',\n",
       " '_',\n",
       " 'p',\n",
       " '.',\n",
       " 'get',\n",
       " 'Characteristics',\n",
       " '(',\n",
       " 'uuid',\n",
       " '=',\n",
       " 'self',\n",
       " '.',\n",
       " 'TRANSMIT',\n",
       " '_',\n",
       " 'UUID',\n",
       " ')',\n",
       " '[',\n",
       " '0',\n",
       " ']']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 344,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 1280,\n",
       " 4,\n",
       " 4,\n",
       " 4,\n",
       " 4]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[word2idx.get(token, UNKNOWN_IDX) for token in d]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['self',\n",
       " '.',\n",
       " '_',\n",
       " 'p',\n",
       " '_',\n",
       " 'transmit',\n",
       " '=',\n",
       " 'self',\n",
       " '.',\n",
       " '_',\n",
       " 'p',\n",
       " '.',\n",
       " 'get',\n",
       " 'Characteristics',\n",
       " '(',\n",
       " 'uuid',\n",
       " '=',\n",
       " 'self',\n",
       " '.',\n",
       " 'TRANSMIT',\n",
       " '_',\n",
       " 'UUID',\n",
       " ')',\n",
       " '[',\n",
       " '0',\n",
       " ']']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.MetaTorchData import MetaRepo, MetaRetrieved\n",
    "from torch.utils.data import DataLoader\n",
    "ds = MetaRepo('repo_files/beaker_line_pairs.csv', False)\n",
    "d = DataLoader(ds, shuffle=True)\n",
    "loader = iter(d)\n",
    "ax, bx, cx, dx = next(loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('./repo_files/beaker_line_pairs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.concat([df, df, df, df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.to_csv('lets_check.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = MetaRepo('repo_files/beaker_line_pairs.csv', False)\n",
    "d = DataLoader(ds, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = iter(d)\n",
    "ax, bx, cx, dx = next(loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5, 128])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ax.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ds = MetaRepo('./repo_files/beaker_line_pairs.csv', False)\n",
    "ds = MetaRepo('lets_check.csv', True, 3)\n",
    "d = DataLoader(ds, shuffle=True, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = iter(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "support_x, support_y, query_x, query_y = next(loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[  1, 505,  21,  ...,   0,   0,   0],\n",
       "         [  1,   4,  19,  ...,   0,   0,   0],\n",
       "         [  1,  13,   4,  ...,   0,   0,   0],\n",
       "         [  1, 220, 619,  ...,   0,   0,   0],\n",
       "         [  1,  36, 147,  ...,   0,   0,   0]]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "support_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5, 128])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "support_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 902])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 128])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>line</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ahe sf  sdlkgh ag s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Idk  flah fgq e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>idk if you should be doing this anyways ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         line\n",
       "0                         ahe sf  sdlkgh ag s\n",
       "1                          Idk  flah fgq e   \n",
       "2  idk if you should be doing this anyways ok"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = y[(y['line'].str.lstrip().str.rstrip().str.len() > 0) & (~y['line'].str.contains('import'))].reset_index(drop=True)\n",
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     0\n",
       "1    19\n",
       "2    18\n",
       "3    42\n",
       "4    10\n",
       "Name: line, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "y = pd.DataFrame([['    '], [' ahe sf  sdlkgh ag s'], ['Idk  flah fgq e   '], ['idk if you should be doing this anyways ok'], ['import sys']])\n",
    "y.columns = ['line']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([' ahe sf  sdlkgh ag s'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h.values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, csv\n",
    "from torch.utils.data import ConcatDataset, DataLoader\n",
    "from DataClass.torchData import PairDataset, batch_collate_fn\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = list(filter(lambda x: True if x.endswith('.csv') else False, next(os.walk('./repo_files'))[2]))\n",
    "data_loader = DataLoader(ConcatDataset([PairDataset('./repo_files' + '/' + dataset) for dataset in datasets]), \n",
    "                        batch_size=16, \n",
    "                        shuffle=True, \n",
    "                        collate_fn=batch_collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 872/2095149 [05:04<170:10:21,  3.42it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/Documents/Masters/Winter/CS224n/Neural_Code_Generator/DataClass/torchData.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     33\u001b[0m                                 \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0midx\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m                                 chunksize=self.chunksize, header=None, dtype=str)).fillna(NO_CONTEXT_WORD).values\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    447\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1113\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1114\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1115\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1891\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1892\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-264a22e2ca48>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtqdm\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mbatch_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mbatch_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1102\u001b[0m                 fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m   1103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1104\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1105\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1106\u001b[0m             \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m             \u001b[0msample_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0midx\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcumulative_sizes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdataset_idx\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 207\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdataset_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msample_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/Masters/Winter/CS224n/Neural_Code_Generator/DataClass/torchData.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     45\u001b[0m                                 \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0midx\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m                                 \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m                                 sep=',\\s+', quoting=csv.QUOTE_ALL, dtype=str)).fillna(NO_CONTEXT_WORD).values\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfix_quote_strings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    674\u001b[0m         )\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    878\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1124\u001b[0m                     \u001b[0;34m'\"python-fwf\")'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1125\u001b[0m                 )\n\u001b[0;32m-> 1126\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mklass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1128\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, **kwds)\u001b[0m\n\u001b[1;32m   2284\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_original_columns\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2285\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2286\u001b[0;31m         ) = self._infer_columns()\n\u001b[0m\u001b[1;32m   2287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2288\u001b[0m         \u001b[0;31m# Now self.columns has the set of columns that we will process.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_infer_columns\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2657\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2658\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2659\u001b[0;31m                 \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffered_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2661\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_buffered_line\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2732\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2733\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2734\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2735\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2736\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_check_for_bom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfirst_row\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_next_line\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2825\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2826\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mskipfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2827\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2828\u001b[0m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2829\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for x, y in tqdm(data_loader):\n",
    "    batch_x = x\n",
    "    batch_y = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = './repo_files/flaskjinja_line_pairs.csv'\n",
    "x = next(pd.read_csv(filename,\n",
    "                                skiprows=3456 * 1+1,\n",
    "                                chunksize=1, header=None, dtype=str)).fillna('OSOFo').values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([    1,   510,    10,    13,   173,    53,   256,    53, 30466,    53,\n",
       "         1363,    16,    18,    11,    11, 34486,    53,  1363,     2,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix(v):\n",
    "    idx = v.find('\",\"')\n",
    "    x, y = v[1:idx], v[idx+3:-1]\n",
    "    if \"\\'\" in x:\n",
    "        x = x.replace('\"', \"'\").replace(\"\\t\", ' ')\n",
    "    else:\n",
    "        x = x.replace('\"\"', '\"').replace(\"'\", '\"').replace('\\t', ' ')\n",
    "\n",
    "    if \"\\'\" in y:\n",
    "        y = y.replace('\"', \"'\").replace(\"\\t\", ' ')\n",
    "    else:\n",
    "        y = y.replace('\"\"', '\"').replace(\"'\", '\"').replace('\\t', ' ')\n",
    "    \n",
    "    return [[x, y]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = next(pd.read_csv('./repo_files/Milis-Yukleyici_line_pairs.csv', skiprows=1+32, chunksize=1, header=None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_collate_fn(data):\n",
    "        data = list(filter(lambda z : z is not None, data))\n",
    "        x, y = zip(*data)\n",
    "        return x, y\n",
    "#         x = pd.DataFrame(x)\n",
    "#         y = pd.DataFrame(y)\n",
    "        \n",
    "#         x = np.where(x.isin(word2idx.keys()), x.replace(word2idx), UNKNOWN_IDX).astype('int64')\n",
    "#         y = np.where(y.isin(word2idx.keys()), y.replace(word2idx), UNKNOWN_IDX).astype('int64')\n",
    "\n",
    "#         batch_xs = torch.LongTensor(x.values)\n",
    "#         batch_ys = torch.LongTensor(y.values)\n",
    "        return batch_xs, batch_ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pickle.load(open('pickle_files/10DaysOfStatistics_line_pairsdataset_edit.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "lantern = pd.read_csv('./repo_files/lantern_line_pairs.csv')\n",
    "electron = pd.read_csv('./repo_files/electron_line_pairs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "electron = pd.concat([electron, electron, electron], axis=1)\n",
    "lantern = pd.concat([lantern, lantern, lantern], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch, os, pickle\n",
    "from torch.utils.data import Dataset, Dataset, ConcatDataset, DataLoader\n",
    "from DataClass.data_utils import tokenize_fine_grained, create_vocab_dictionary\n",
    "import numpy as np\n",
    "from DataClass.Constants import NO_CONTEXT_WORD, UNKNOWN_WORD, PAD_WORD\n",
    "\n",
    "tokens_file = './repo_files/all_tokens.pickle'\n",
    "tokens_dict = pickle.load(open(tokens_file, 'rb'))\n",
    "tokens_dict, _ = create_vocab_dictionary(tokens_dict)\n",
    "UNKNOWN_IDX = tokens_dict[UNKNOWN_WORD]\n",
    "MAX_DIMENSION = 10\n",
    "\n",
    "def preprocess_tokens(tokens, max_dim):\n",
    "    tokens = [START_WORD] + tokens\n",
    "    n = len(tokens) + 1\n",
    "    # minus one since end word needs to go on too\n",
    "    tokens = tokens[:min(n, max_dim-1)] + [END_WORD] + [PAD_WORD]*max(0, max_dim-n)\n",
    "    return tokens\n",
    "\n",
    "def preprocess_context(context, n, max_dim):\n",
    "    context_tokens = preprocess_tokens(tokenize_fine_grained(context[0, 0]), max_dim)\n",
    "    context_tokens += [SEP_CONTEXT_WORD]\n",
    "    \n",
    "    for idx in range(n-1):\n",
    "        context_tokens += preprocess_tokens(tokenize_fine_grained(context[0, idx*2-1]), max_dim) + [SEP_PAIR_WORD]\n",
    "        context_tokens += preprocess_tokens(tokenize_fine_grained(context[0, idx*2]), max_dim) + [SEP_RET_WORD]\n",
    "    \n",
    "    context_tokens += preprocess_tokens(tokenize_fine_grained(context[0, -2]), max_dim) + [SEP_PAIR_WORD]\n",
    "    context_tokens += preprocess_tokens(tokenize_fine_grained(context[0, -1]), max_dim)\n",
    "    return context_tokens\n",
    "\n",
    "\n",
    "class MetaRetrieveDataset(Dataset):\n",
    "\n",
    "    def __init__(self, filename, chunksize, n_retrieved, k_shot, meta_search_range=100):\n",
    "        \n",
    "        self.filename = filename\n",
    "        self.chunksize = 1 # more than this and requires a lot more changes in collate fn\n",
    "        temp = next(pd.read_csv(self.filename, skiprows = 0, chunksize=1, header=None))\n",
    "        self.n_retrieved = n_retrieved\n",
    "        self.max_dim = MAX_DIMENSION\n",
    "        self.meta_search_range = meta_search_range\n",
    "        self.k_shot = k_shot\n",
    "        self.len = int(temp.values[0][0] / self.chunksize)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        x = next(pd.read_csv(self.filename,\n",
    "                            skiprows=idx * self.chunksize+1,\n",
    "                            chunksize=self.chunksize, header=None)).fillna(NO_CONTEXT_WORD).values\n",
    "\n",
    "        support_sample_idxs = np.random.randint(0, self.len, (self.k_shot))\n",
    "        while(np.where(np.logical_end(support_sample_idxs >= idx-meta_search_range, support_sample_idxs <= idx+meta_search_range))):\n",
    "            support_sample_idxs = np.random.randint(0, self.len, (self.k_shot))\n",
    "        \n",
    "        support_context_tokens = []\n",
    "        support_y_tokens = []\n",
    "        for support_idx in support_sample_idxs:\n",
    "            support_temp = next(pd.read_csv(self.filename,\n",
    "                            skiprows=support_idx * self.chunksize+1,\n",
    "                            chunksize=self.chunksize, header=None)).fillna(NO_CONTEXT_WORD).values\n",
    "            \n",
    "            support_context_tokens.append(preprocess_context(support_temp[:, :-1], self.n_retrieved, self.max_dim))\n",
    "            support_y_tokens.append(preprocess_tokens(tokenize_fine_grained(support_temp[0, -1]), self.max_dim))\n",
    "        \n",
    "        context_tokens = preprocess_context(x[:, :-1], self.n_retrieved, self.max_dim)\n",
    "        y_tokens = preprocess_tokens(tokenize_fine_grained(x[0, -1]), self.max_dim)\n",
    "        return (support_context_tokens, support_y_tokens, context_tokens, y_tokens)\n",
    "    \n",
    "def batch_collate_fn(data):\n",
    "#         print(data)\n",
    "        x, y = zip(*data)\n",
    "        x = pd.DataFrame(support_c)\n",
    "#         x = pd.DataFrame(x)\n",
    "#         y = pd.DataFrame(y)\n",
    "        x = np.where(x.isin(tokens_dict.keys()), x.replace(tokens_dict), UNKNOWN_IDX).astype('int64')\n",
    "        y = np.where(y.isin(tokens_dict.keys()), y.replace(tokens_dict), UNKNOWN_IDX).astype('int64')\n",
    "\n",
    "        batch_xs = torch.LongTensor(x)\n",
    "        batch_ys = torch.LongTensor(y)\n",
    "        return batch_xs, batch_ys\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "# def createDataLoaderAllFiles(dataset_dir, dataset_class=PairDatasetLazy, shuffle=True, batch_size=128):\n",
    "#     datasets = list(filter(lambda x: True if x.endswith('.csv') else False, next(os.walk(dataset_dir))[2]))\n",
    "#     datasets = list(filter(lambda x: True if x.startswith('check') else False, next(os.walk(dataset_dir))[2]))\n",
    "#     return DataLoader(ConcatDataset([dataset_class(dataset_dir + '/' + dataset, chunksize=batch_size, n_retrieved=RETRIEVED_EXAMPLES) for dataset in datasets]), \n",
    "#                         batch_size=batch_size, \n",
    "#                         shuffle=shuffle, \n",
    "#                         collate_fn=batch_collate_fn)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# d = createDataLoaderAllFiles('./repo_files', RetrieveDataset, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['<BOS>', 'return', 'os', '.', 'path', '.', 'basename', '(', 'binary', '<EOS>', '<SEP_CONTEXT>', '<BOS>', 'return', 'os', '.', 'path', '.', 'basename', '(', 'binary', '<EOS>', '<SEP_PAIR>', '<BOS>', 'return', 'os', '.', 'path', '.', 'basename', '(', 'binary', '<EOS>', '<SEP_RET>', '<BOS>', 'def', 'main', '(', ')', ':', '<EOS>', '<PAD>', '<PAD>', '<PAD>', '<SEP_PAIR>', '<BOS>', 'return', 'os', '.', 'path', '.', 'basename', '(', 'binary', '<EOS>'], ['<BOS>', 'upload', '_', 'options', '.', 'server', '=', 'server', '<EOS>', '<PAD>', '<SEP_CONTEXT>', '<BOS>', 'upload', '_', 'options', '.', 'server', '=', 'server', '<EOS>', '<PAD>', '<SEP_PAIR>', '<BOS>', 'upload', '_', 'options', '.', 'server', '=', 'server', '<EOS>', '<PAD>', '<SEP_RET>', '<BOS>', 'upload', '_', 'options', '.', 'save', '_', 'cookies', '=', '<EOS>', '<SEP_PAIR>', '<BOS>', 'upload', '_', 'options', '.', 'server', '=', 'server', '<EOS>', '<PAD>'])\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'support_c' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-105-9cc9679ca134>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-101-7314fefd5879>\u001b[0m in \u001b[0;36mbatch_collate_fn\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msupport_c\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m \u001b[0;31m#         x = pd.DataFrame(x)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;31m#         y = pd.DataFrame(y)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'support_c' is not defined"
     ]
    }
   ],
   "source": [
    "for x, y in d:\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([12, 128])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "hey = next(pd.read_csv('./repo_files/check_lantern_retrieve.csv', skiprows=1, chunksize=1, header=None)).fillna(NO_CONTEXT_WORD).values[:, :-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['OSOFo']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize_fine_grained(hey[0, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "lantern.columns = [lantern.columns[0]] + [0]*(len(lantern.columns)-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "electron.columns = [electron.columns[0]] + [0]*(len(electron.columns)-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "lantern.to_csv('./repo_files/check_lantern_retrieve.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 0\n",
    "\n",
    "x = next(pd.read_csv('./repo_files/check_lantern_retrieve.csv',\n",
    "                            skiprows=idx * 1+1,\n",
    "                            chunksize=1, header=None)).fillna(NO_CONTEXT_WORD).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['OSOFo',\n",
       "        'from mercurial import cmdutil, commands, hg, util, error, match',\n",
       "        'OSOFo',\n",
       "        'from mercurial import cmdutil, commands, hg, util, error, match',\n",
       "        'OSOFo',\n",
       "        'from mercurial import cmdutil, commands, hg, util, error, match']],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2579</th>\n",
       "      <th>0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>import os</td>\n",
       "      <td>NaN</td>\n",
       "      <td>import os</td>\n",
       "      <td>NaN</td>\n",
       "      <td>import os</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>import os</td>\n",
       "      <td>import subprocess</td>\n",
       "      <td>import os</td>\n",
       "      <td>import subprocess</td>\n",
       "      <td>import os</td>\n",
       "      <td>import subprocess</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>import subprocess</td>\n",
       "      <td>import sys</td>\n",
       "      <td>import subprocess</td>\n",
       "      <td>import sys</td>\n",
       "      <td>import subprocess</td>\n",
       "      <td>import sys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>import sys</td>\n",
       "      <td>TEMPLATE =</td>\n",
       "      <td>import sys</td>\n",
       "      <td>TEMPLATE =</td>\n",
       "      <td>import sys</td>\n",
       "      <td>TEMPLATE =</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEMPLATE =</td>\n",
       "      <td>def main():</td>\n",
       "      <td>TEMPLATE =</td>\n",
       "      <td>def main():</td>\n",
       "      <td>TEMPLATE =</td>\n",
       "      <td>def main():</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2574</th>\n",
       "      <td>for dirname in args:</td>\n",
       "      <td>try:</td>\n",
       "      <td>for dirname in args:</td>\n",
       "      <td>try:</td>\n",
       "      <td>for dirname in args:</td>\n",
       "      <td>try:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2575</th>\n",
       "      <td>try:</td>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "      <td>try:</td>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "      <td>try:</td>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2576</th>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "      <td>except OSError as e:</td>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "      <td>except OSError as e:</td>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "      <td>except OSError as e:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2577</th>\n",
       "      <td>except OSError as e:</td>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "      <td>except OSError as e:</td>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "      <td>except OSError as e:</td>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2578</th>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "      <td>main(sys.argv[1:])</td>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "      <td>main(sys.argv[1:])</td>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "      <td>main(sys.argv[1:])</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2579 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      2579  \\\n",
       "0                                      NaN   \n",
       "1                                import os   \n",
       "2                        import subprocess   \n",
       "3                               import sys   \n",
       "4                              TEMPLATE =    \n",
       "...                                    ...   \n",
       "2574                  for dirname in args:   \n",
       "2575                                  try:   \n",
       "2576                  os.makedirs(dirname)   \n",
       "2577                  except OSError as e:   \n",
       "2578        if e.errno == os.errno.EEXIST:   \n",
       "\n",
       "                                         0  \\\n",
       "0                                import os   \n",
       "1                        import subprocess   \n",
       "2                               import sys   \n",
       "3                              TEMPLATE =    \n",
       "4                              def main():   \n",
       "...                                    ...   \n",
       "2574                                  try:   \n",
       "2575                  os.makedirs(dirname)   \n",
       "2576                  except OSError as e:   \n",
       "2577        if e.errno == os.errno.EEXIST:   \n",
       "2578                    main(sys.argv[1:])   \n",
       "\n",
       "                                       0.1  \\\n",
       "0                                      NaN   \n",
       "1                                import os   \n",
       "2                        import subprocess   \n",
       "3                               import sys   \n",
       "4                              TEMPLATE =    \n",
       "...                                    ...   \n",
       "2574                  for dirname in args:   \n",
       "2575                                  try:   \n",
       "2576                  os.makedirs(dirname)   \n",
       "2577                  except OSError as e:   \n",
       "2578        if e.errno == os.errno.EEXIST:   \n",
       "\n",
       "                                       0.2  \\\n",
       "0                                import os   \n",
       "1                        import subprocess   \n",
       "2                               import sys   \n",
       "3                              TEMPLATE =    \n",
       "4                              def main():   \n",
       "...                                    ...   \n",
       "2574                                  try:   \n",
       "2575                  os.makedirs(dirname)   \n",
       "2576                  except OSError as e:   \n",
       "2577        if e.errno == os.errno.EEXIST:   \n",
       "2578                    main(sys.argv[1:])   \n",
       "\n",
       "                                       0.3  \\\n",
       "0                                      NaN   \n",
       "1                                import os   \n",
       "2                        import subprocess   \n",
       "3                               import sys   \n",
       "4                              TEMPLATE =    \n",
       "...                                    ...   \n",
       "2574                  for dirname in args:   \n",
       "2575                                  try:   \n",
       "2576                  os.makedirs(dirname)   \n",
       "2577                  except OSError as e:   \n",
       "2578        if e.errno == os.errno.EEXIST:   \n",
       "\n",
       "                                       0.4  \n",
       "0                                import os  \n",
       "1                        import subprocess  \n",
       "2                               import sys  \n",
       "3                              TEMPLATE =   \n",
       "4                              def main():  \n",
       "...                                    ...  \n",
       "2574                                  try:  \n",
       "2575                  os.makedirs(dirname)  \n",
       "2576                  except OSError as e:  \n",
       "2577        if e.errno == os.errno.EEXIST:  \n",
       "2578                    main(sys.argv[1:])  \n",
       "\n",
       "[2579 rows x 6 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('./repo_files/check_electron_retrieve.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch, os, pickle\n",
    "from torch.utils.data import Dataset, Dataset, ConcatDataset, DataLoader\n",
    "from DataClass.data_utils import tokenize_fine_grained, create_vocab_dictionary\n",
    "import numpy as np\n",
    "from DataClass.Constants import PAD_WORD, START_WORD, END_WORD, PAD_IDX, START_IDX, END_IDX, NO_CONTEXT_IDX, NO_CONTEXT_WORD, UNKNOWN_IDX, UNKNOWN_WORD\n",
    "\n",
    "tokens_file = './repo_files/all_tokens.pickle'\n",
    "tokens_dict = pickle.load(open(tokens_file, 'rb'))\n",
    "tokens_dict, _ = create_vocab_dictionary(tokens_dict)\n",
    "UNKNOWN_IDX = tokens_dict[UNKNOWN_WORD]\n",
    "MAX_DIMENSION = 128\n",
    "\n",
    "class PairDatasetLazy(Dataset):\n",
    "\n",
    "    def __init__(self, filename, chunksize):\n",
    "        \n",
    "        self.filename = filename\n",
    "        self.chunksize = 1 # more than this and requires a lot more changes in collate fn\n",
    "        temp = next(pd.read_csv(self.filename, skiprows = 0, chunksize=1, header=None))\n",
    "        self.max_dim = MAX_DIMENSION-2\n",
    "        self.len = int(temp.values[0][0] / self.chunksize)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        x = next(pd.read_csv(self.filename,\n",
    "                            skiprows=idx * self.chunksize+1,\n",
    "                            chunksize=self.chunksize, header=None, dtype=str)).fillna(NO_CONTEXT_WORD).values\n",
    "\n",
    "        x_tokens = tokenize_fine_grained(x[0, 0])\n",
    "        y_tokens = tokenize_fine_grained(x[0, 1])\n",
    "\n",
    "        x_n = len(x_tokens); y_n = len(y_tokens)\n",
    "\n",
    "        x_tokens = [START_WORD] + x_tokens[:(self.max_dim) if self.max_dim < x_n else x_n] + [PAD_WORD]*(max(0, self.max_dim-x_n)) + [END_WORD]\n",
    "        y_tokens = [START_WORD] + y_tokens[:(self.max_dim) if self.max_dim < y_n else y_n] + [PAD_WORD]*(max(0, self.max_dim-y_n)) + [END_WORD]\n",
    "\n",
    "        return x_tokens, y_tokens\n",
    "        # return np.array(x_tokens).reshape(-1, 1), np.array(y_tokens).reshape(-1, 1)\n",
    "    \n",
    "def batch_collate_fn(data):\n",
    "        x, y = zip(*data)\n",
    "        \n",
    "        x = pd.DataFrame(x)#.fillna(PAD_WORD)\n",
    "        y = pd.DataFrame(y)#.fillna(PAD_WORD)\n",
    "        x = np.where(x.isin(tokens_dict.keys()), x.replace(tokens_dict), UNKNOWN_IDX).astype('int64')\n",
    "        y = np.where(y.isin(tokens_dict.keys()), y.replace(tokens_dict), UNKNOWN_IDX).astype('int64')\n",
    "\n",
    "        batch_xs = torch.LongTensor(x)\n",
    "        batch_ys = torch.LongTensor(y)\n",
    "        # x = pd.DataFrame(x).replace(tokens_dict).fillna(PAD_IDX)\n",
    "        # y = pd.DataFrame(y).replace(tokens_dict).fillna(PAD_IDX)\n",
    "\n",
    "        # batch_xs = torch.LongTensor(x.values)\n",
    "        # batch_ys = torch.LongTensor(y.values)\n",
    "        return batch_xs, batch_ys\n",
    "    \n",
    "    \n",
    "\n",
    "def createDataLoaderAllFiles(dataset_dir, dataset_class=PairDatasetLazy, shuffle=True, batch_size=128):\n",
    "    datasets = list(filter(lambda x: True if x.endswith('.csv') else False, next(os.walk(dataset_dir))[2]))\n",
    "    return DataLoader(ConcatDataset([dataset_class(dataset_dir + '/' + dataset, chunksize=batch_size) for dataset in datasets]), \n",
    "                        batch_size=batch_size, \n",
    "                        shuffle=shuffle, \n",
    "                        collate_fn=batch_collate_fn)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import glob\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import distance\n",
    "import collections\n",
    "import configargparse\n",
    "from tqdm import tqdm\n",
    "from  more_itertools import unique_everseen\n",
    "\n",
    "# import opts\n",
    "\n",
    "from flask import Flask, render_template, url_for, redirect, jsonify, request  # noqa\n",
    "from flask_cors import CORS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On repo [1/4]\n",
      "On repo [2/4]\n",
      "On repo [3/4]\n",
      "On repo [4/4]\n"
     ]
    }
   ],
   "source": [
    "from DataClass.Crawler import Crawler\n",
    "c = Crawler()\n",
    "c.tokenize_from_files('./repo_files')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.torchData import createDataLoaderAllFiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = createDataLoaderAllFiles('./repo_files')\n",
    "# for i, (x, y) in "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(['<BOS>', 'return', 'retcode', '<EOS>'], ['<BOS>', 'if', '_', '_', 'name', '_', '_', '=', '=', \"'\", '_', '_', 'main', '_', '_', \"'\", ':', '<EOS>']), (['<BOS>', 'return', 'len', '(', 'broken', 'Links', ')', '<EOS>'], ['<BOS>', 'def', 'check', 'Sections', '(', 'sections', ',', 'lines', ')', ':', '<EOS>']), (['<BOS>', 'raise', 'hg', '_', 'util', '.', 'Abort', '(', 'codereview', '_', 'disabled', ')', '<EOS>'], ['<BOS>', 'dirty', '=', '{', '}', '<EOS>']), (['<BOS>', 'root', '=', \"'\", \"'\", '<EOS>'], ['<BOS>', 'try', ':', '<EOS>']), (['<BOS>', \"'\", 'libffmpeg', '.', 'so', \"'\", ',', '<EOS>'], ['<BOS>', \"'\", 'lib', 'GLESv', '2', '.', 'so', \"'\", ',', '<EOS>']), (['<BOS>', 'line', '=', 'line', '.', 'strip', '(', ')', '<EOS>'], ['<BOS>', 'if', 'line', '=', '=', \"'\", \"'\", ':', '<EOS>']), (['<BOS>', 'return', 'newpatch', ',', '\"', '\"', '<EOS>'], ['<BOS>', 'if', 'not', 'm', ':', '<EOS>']), (['<BOS>', \"'\", 'You', 'have', 'to', 'pass', '-', '-', 'overwrite', 'to', 'overwrite', 'a', 'published', 'release', \"'\", '<EOS>'], ['<BOS>', 'electron', '_', 'zip', '=', 'os', '.', 'path', '.', 'join', '(', 'OUT', '_', 'DIR', ',', 'DIST', '_', 'NAME', ')', '<EOS>']), (['<BOS>', 'content', '_', 'type', '=', '\"', 'application', '/', 'octet', '-', 'stream', '\"', ',', '<EOS>'], ['<BOS>', 'timeout', '=', 'None', ',', 'force', '_', 'auth', '=', 'True', ',', '<EOS>']), (['<BOS>', 'import', 'subprocess', '<EOS>'], ['<BOS>', 'import', 'sys', '<EOS>']), (['<BOS>', 'raise', 'hg', '_', 'util', '.', 'Abort', '(', '\"', 'cannot', 'use', '\"', '+', 'flag', '+', '\"', 'with', 'file', 'patterns', '\"', ')', '<EOS>'], ['<BOS>', 'if', 'opts', '[', '\"', 'stdin', '\"', ']', 'or', 'opts', '[', '\"', 'stdout', '\"', ']', ':', '<EOS>']), (['<BOS>', 'print', '\"', 'Uploading', '%', 's', 'file', 'for', '%', 's', '\"', '%', '(', 'type', ',', 'filename', ')', '<EOS>'], ['<BOS>', 'url', '=', '\"', '/', '%', 'd', '/', 'upload', '_', 'content', '/', '%', 'd', '/', '%', 'd', '\"', '%', '(', 'int', '(', 'issue', ')', ',', 'int', '(', 'patchset', ')', ',', 'file', '_', 'id', ')', '<EOS>']), (['<BOS>', 'action', '=', \"'\", 'store', '_', 'true', \"'\", ')', '<EOS>'], ['<BOS>', 'parser', '.', 'add', '_', 'argument', '(', \"'\", '-', 's', \"'\", ',', \"'\", '-', '-', 'upload', '_', 'to', '_', 's', '3', \"'\", ',', '<EOS>']), (['<BOS>', 'sys', '.', 'stdout', '.', 'writelines', '(', 'diff', '_', 'lines', ')', '<EOS>'], ['<BOS>', 'def', 'print', '_', 'trouble', '(', 'prog', ',', 'message', ',', 'use', '_', 'colors', ')', ':', '<EOS>']), (['<BOS>', 'else', ':', '<EOS>'], ['<BOS>', 'err', '=', 'Edit', 'CL', '(', 'ui', ',', 'repo', ',', 'cl', ')', '<EOS>']), (['<BOS>', 'import', 'sys', '<EOS>'], ['<BOS>', 'sys', '.', 'path', '.', 'append', '(', '<EOS>']), (['<BOS>', 'if', 'get', '_', 'target', '_', 'arch', '(', ')', '=', '=', '\"', 'mips', '64', 'el', '\"', ':', '<EOS>'], ['<BOS>', 'SOURCE', '_', 'ROOT', '=', 'os', '.', 'path', '.', 'abspath', '(', 'os', '.', 'path', '.', 'dirname', '(', '<EOS>']), (['<BOS>', 'type', '=', '\"', 'current', '\"', '<EOS>'], ['<BOS>', 'if', 'len', '(', 'content', ')', '>', 'MAX', '_', 'UPLOAD', '_', 'SIZE', ':', '<EOS>']), (['<BOS>', 'else', ':', '<EOS>'], ['<BOS>', 'copy', '_', 'debug', '_', 'from', '_', 'binaries', '(', 'args', '.', 'directory', ',', 'args', '.', 'out', '_', 'dir', ',', 'args', '.', 'target', '_', 'cpu', ',', '<EOS>']), (['<BOS>', 'with', 'open', '(', 'os', '.', 'path', '.', 'join', '(', 'app', '_', 'path', ',', \"'\", 'mksnapshot', '_', 'args', \"'\", ')', ')', 'as', 'f', ':', '<EOS>'], ['<BOS>', 'mkargs', '=', 'f', '.', 'read', '(', ')', '.', 'splitlines', '(', ')', '<EOS>']), (['<BOS>', 'if', 'ws', '=', '=', 'None', ':', '<EOS>'], ['<BOS>', 'return', 'text', '<EOS>']), (['<BOS>', 'if', 'downloaded', '_', 'sha', '!', '=', 'sha', ':', '<EOS>'], ['<BOS>', 'raise', 'Exception', '(', '\"', 'SHA', 'for', 'external', 'binary', 'file', '{', '}', 'does', 'not', 'match', 'expected', \"'\", '{', '}', \"'\", '!', '=', \"'\", '{', '}', \"'\", '\"', '.', 'format', '(', 'file', '_', 'path', ',', 'downloaded', '_', 'sha', ',', 'sha', ')', ')', '<EOS>']), (['<BOS>', 'continue', '<EOS>'], ['<BOS>', 'if', 'line', '=', '=', \"'\", \"'\", 'or', 'line', '[', '0', ']', '=', '=', \"'\", \"'\", 'or', 'line', '[', '0', ']', '=', '=', \"'\", '\\\\', 't', \"'\", ':', '<EOS>']), (['<BOS>', 'content', '_', 'type', '=', '\"', 'application', '/', 'octet', '-', 'stream', '\"', ',', '<EOS>'], ['<BOS>', 'timeout', '=', 'None', ',', '<EOS>']), (['<BOS>', 'Set', 'Environment', 'And', 'Get', 'Runtime', 'Dll', 'Dirs', ',', '\\\\', '<EOS>'], ['<BOS>', 'Set', 'Environment', 'And', 'Get', 'SDKDir', ',', '\\\\', '<EOS>']), (['<BOS>', 'cl', '.', 'files', '=', 'clx', '.', 'files', '<EOS>'], ['<BOS>', 'cl', '.', 'private', '=', 'clx', '.', 'private', '<EOS>']), (['<BOS>', 'help', '=', \"'\", 'path', 'to', 'a', 'tests', 'config', \"'\", ')', '<EOS>'], ['<BOS>', 'parser', '.', 'add', '_', 'argument', '(', \"'\", '-', 't', \"'\", ',', \"'\", '-', '-', 'tests', '-', 'dir', \"'\", ',', 'required', '=', 'False', ',', '<EOS>']), (['<BOS>', 'continue', '<EOS>'], ['<BOS>', 'ui', '.', 'warn', '(', '\"', 'error', ':', '%', 's', 'does', 'not', 'exist', ';', 'omitting', '\\\\', 'n', '\"', '%', '(', 'f', ',', ')', ')', '<EOS>']), (['<BOS>', 'return', 'False', '<EOS>'], ['<BOS>', 'return', 'mimetype', '.', 'startswith', '(', '\"', 'image', '/', '\"', ')', '<EOS>']), (['<BOS>', 'global', 'match', '_', 'repo', '<EOS>'], ['<BOS>', 'global', 'match', '_', 'ui', '<EOS>']), (['<BOS>', 'prompt', '+', '=', '\"', '[', '%', 's', ']', '\"', '%', 'last', '_', 'email', '<EOS>'], ['<BOS>', 'except', 'IOError', ',', 'e', ':', '<EOS>']), (['<BOS>', 'return', 'False', '<EOS>'], ['<BOS>', 'def', 'Download', 'CL', '(', 'ui', ',', 'repo', ',', 'clname', ')', ':', '<EOS>']), (['<BOS>', 'patchid', '=', 'diffs', '[', '-', '1', ']', '<EOS>'], ['<BOS>', 'patchset', '=', 'JSONGet', '(', 'ui', ',', '\"', '/', 'api', '/', '\"', '+', 'clname', '+', '\"', '/', '\"', '+', 'str', '(', 'patchid', ')', ')', '<EOS>']), (['<BOS>', 'if', 'not', 'cl', '.', 'files', ':', '<EOS>'], ['<BOS>', 'if', 'not', 'cl', '.', 'copied', '_', 'from', ':', '<EOS>']), (['<BOS>', 'disabled', '_', 'tests', '_', 'policy', ')', '<EOS>'], ['<BOS>', '@', 'staticmethod', '<EOS>']), (['<BOS>', 'if', 'not', 'rev', ':', '<EOS>'], ['<BOS>', 'return', '\"', 'unknown', 'revision', '%', 's', '\"', '%', 'clname', '<EOS>']), (['<BOS>', 'line', '=', 'line', '.', 'rstrip', '(', ')', '<EOS>'], ['<BOS>', 'if', 'line', '=', '=', \"'\", \"'\", ':', '<EOS>']), (['<BOS>', 'noise', '=', '[', '<EOS>'], ['<BOS>', '\"', '\"', ',', '<EOS>']), (['<BOS>', 's', '=', '\"', 'code', 'review', '%', 's', ':', '%', 's', '\"', '%', '(', 'self', '.', 'name', ',', 's', ')', '<EOS>'], ['<BOS>', 'typecheck', '(', 's', ',', 'str', ')', '<EOS>']), (['<BOS>', 'id', '=', \"'\", '+', \"'\", '.', 'join', '(', '[', 'hg', '_', 'node', '.', 'short', '(', 'p', '.', 'node', '(', ')', ')', 'for', 'p', 'in', 'parents', ']', ')', '<EOS>'], ['<BOS>', 'if', 'vers', '!', '=', '\"', '\"', 'and', 'id', '!', '=', 'vers', ':', '<EOS>']), (['<BOS>', \"'\", '.', '/', 'lib', 'Vk', 'ICD', '_', 'mock', '_', \"'\", ',', 'Skipping', 'because', 'these', 'are', 'outputs', 'that', 'we', 'don', \"'\", 't', 'need', '<EOS>'], ['<BOS>', \"'\", '.', '/', 'Vk', 'ICD', '_', 'mock', '_', \"'\", ',', 'Skipping', 'because', 'these', 'are', 'outputs', 'that', 'we', 'don', \"'\", 't', 'need', '<EOS>']), (['<BOS>', 'from', 'struct', 'import', 'Struct', '<EOS>'], ['<BOS>', 'import', 'sys', '<EOS>']), (['<BOS>', 'if', 'not', 'ui', '.', 'quiet', ':', '<EOS>'], ['<BOS>', 'ui', '.', 'status', '(', '\"', 'hg', 'file', '%', 's', '%', 's', '\\\\', 'n', '\"', '%', '(', 'ocl', '.', 'name', ',', 'f', ')', ')', '<EOS>']), (['<BOS>', 'if', 'not', 'lines', 'or', 'lines', '[', '0', ']', '!', '=', '\"', 'OK', '\"', ':', '<EOS>'], ['<BOS>', 'Status', 'Update', '(', '\"', '-', '-', '>', '%', 's', '\"', '%', 'response', '_', 'body', ')', '<EOS>']), (['<BOS>', 'initial', '_', 'app', '_', 'path', '=', 'os', '.', 'path', '.', 'join', '(', 'source', '_', 'root', ',', 'args', '.', 'build', '_', 'dir', ')', '<EOS>'], ['<BOS>', 'app', '_', 'path', '=', 'create', '_', 'app', '_', 'copy', '(', 'initial', '_', 'app', '_', 'path', ')', '<EOS>']), (['<BOS>', 'data', '=', '{', '}', '<EOS>'], ['<BOS>', 'for', 'argpath', 'in', 'argpaths', ':', '<EOS>']), (['<BOS>', 'typecheck', '(', 'about', ',', 'str', ')', '<EOS>'], ['<BOS>', 'if', 'not', 'cl', '.', 'mailed', 'and', 'not', 'cl', '.', 'copied', '_', 'from', ':', '<EOS>']), (['<BOS>', 'st', ',', 'fn', '=', 'line', '.', 'split', '(', '\"', '\"', ',', '1', ')', '<EOS>'], ['<BOS>', 'if', 'st', '=', '=', '\"', '?', '\"', ':', '<EOS>']), (['<BOS>', 'except', ':', '<EOS>'], ['<BOS>', 'ui', '.', 'warn', '(', '\"', 'JSONGet', '%', 's', ':', '%', 's', '\\\\', 'n', '\"', '%', '(', 'path', ',', 'Exception', 'Detail', '(', ')', ')', ')', '<EOS>']), (['<BOS>', 'files', '=', '[', 'Relative', 'Path', '(', 'repo', '.', 'root', '+', \"'\", '/', \"'\", '+', 'f', ',', 'cwd', ')', 'for', 'f', 'in', 'files', ']', '<EOS>'], ['<BOS>', 'files', '=', '[', 'f', 'for', 'f', 'in', 'files', 'if', 'os', '.', 'access', '(', 'f', ',', '0', ')', ']', '<EOS>']), (['<BOS>', 'self', '.', 'ui', '=', 'ui', '<EOS>'], ['<BOS>', 'self', '.', 'repo', '=', 'repo', '<EOS>']), (['<BOS>', 'return', 'err', '<EOS>'], ['<BOS>', 'if', 'not', 'cl', '.', 'local', ':', '<EOS>']), (['<BOS>', 't', '+', '=', 'line', '+', \"'\", '\\\\', 'n', \"'\", '<EOS>'], ['<BOS>', 'while', 'len', '(', 't', ')', '>', '=', '2', 'and', 't', '[', '-', '2', ':', ']', '=', '=', \"'\", '\\\\', 'n', '\\\\', 'n', \"'\", ':', '<EOS>']), (['<BOS>', 'line', '=', 'line', '.', 'rstrip', '(', ')', '<EOS>'], ['<BOS>', 'if', 'line', '!', '=', \"'\", \"'\", 'and', 'line', '[', '0', ']', '=', '=', \"'\", \"'\", ':', '<EOS>']), (['<BOS>', 'proc', '_', 'stderr', '=', 'codecs', '.', 'getreader', '(', 'encoding', ')', '(', 'proc', '_', 'stderr', ')', '<EOS>'], ['<BOS>', 'proc', '.', 'returncode', ',', 'file', '_', 'name', ')', ',', 'errs', ')', '<EOS>']), (['<BOS>', 'cl', '.', 'copied', '_', 'from', '=', 'sections', '[', \"'\", 'Author', \"'\", ']', '<EOS>'], ['<BOS>', 'cl', '.', 'desc', '=', 'sections', '[', \"'\", 'Description', \"'\", ']', '<EOS>']), (['<BOS>', 'def', 'main', '(', ')', ':', '<EOS>'], ['<BOS>', 'parser', '=', 'argparse', '.', 'Argument', 'Parser', '(', 'description', '=', '_', '_', 'doc', '_', '_', ')', '<EOS>']), (['<BOS>', 'set', '_', 'status', '(', '\"', 'uploading', 'CL', 'metadata', '+', 'diffs', '\"', ')', '<EOS>'], ['<BOS>', 'os', '.', 'chdir', '(', 'repo', '.', 'root', ')', '<EOS>']), (['<BOS>', 'item', '[', \"'\", 'vendor', \"'\", ']', '=', 'obj', 'Item', '.', 'Vendor', '<EOS>'], ['<BOS>', 'if', 'obj', 'Item', '.', 'Version', ':', '<EOS>']), (['<BOS>', 'directory', '=', 'tempfile', '.', 'mkdtemp', '(', 'prefix', '=', \"'\", 'electron', '-', 'tmp', \"'\", ')', '<EOS>'], ['<BOS>', 'result', '=', '[', ']', '<EOS>']), (['<BOS>', \"'\", '-', '-', 'author', \"'\", ',', 'author', ',', '<EOS>'], ['<BOS>', \"'\", '-', '-', 'message', \"'\", ',', 'message', '<EOS>']), (['<BOS>', '@', 'staticmethod', '<EOS>'], ['<BOS>', 'def', '_', '_', 'get', '_', 'config', '_', 'data', '(', 'config', '_', 'path', ')', ':', '<EOS>']), (['<BOS>', 'class', 'Abstract', 'Rpc', 'Server', '(', 'object', ')', ':', '<EOS>'], ['<BOS>', 'def', '_', '_', 'init', '_', '_', '(', 'self', ',', 'host', ',', 'auth', '_', 'function', ',', 'host', '_', 'override', '=', 'None', ',', 'extra', '_', 'headers', '=', '{', '}', ',', 'save', '_', 'cookies', '=', 'False', ')', ':', '<EOS>']), (['<BOS>', 'parser', '.', 'add', '_', 'argument', '(', \"'\", '-', 'v', \"'\", ',', \"'\", '-', '-', 'verbose', \"'\", ',', '<EOS>'], ['<BOS>', 'action', '=', \"'\", 'store', '_', 'true', \"'\", ',', '<EOS>']), (['<BOS>', 'return', 'ret', '<EOS>'], ['<BOS>', 'def', 'hg', '_', 'heads', '(', 'ui', ',', 'repo', ')', ':', '<EOS>']), (['<BOS>', '(', '\"', 'content', '_', 'upload', '\"', ',', '\"', '1', '\"', ')', ',', '<EOS>'], ['<BOS>', '(', '\"', 'reviewers', '\"', ',', 'Join', 'Comma', '(', 'self', '.', 'reviewer', ')', ')', ',', '<EOS>']), (['<BOS>', 'continue', '<EOS>'], ['<BOS>', 'modified', ',', 'added', ',', 'removed', ',', 'deleted', ',', 'unknown', ',', 'ignored', ',', 'clean', '=', 'matchpats', '(', 'ui', ',', 'repo', ',', 'pats', ',', '{', '}', ')', '<EOS>']), (['<BOS>', 'with', 'open', '(', 'config', '_', 'hash', '_', 'path', ',', \"'\", 'r', \"'\", ')', 'as', 'f', ':', '<EOS>'], ['<BOS>', 'existing', '_', 'hash', '=', 'f', '.', 'readline', '(', ')', '.', 'strip', '(', ')', '<EOS>']), (['<BOS>', 'elif', 'opts', '[', '\"', 'pending', '\"', ']', ':', '<EOS>'], ['<BOS>', 'ui', '.', 'write', '(', 'cl', '.', 'Pending', 'Text', '(', ')', ')', '<EOS>']), (['<BOS>', 'if', 'PLATFORM', '=', '=', \"'\", 'linux', \"'\", 'and', '(', 'target', '_', 'cpu', '=', '=', \"'\", 'x', '86', \"'\", 'or', 'target', '_', 'cpu', '=', '=', \"'\", 'arm', \"'\", 'or', '<EOS>'], ['<BOS>', 'target', '_', 'cpu', '=', '=', \"'\", 'arm', '64', \"'\", ')', ':', '<EOS>']), (['<BOS>', 'relative', 'Link', '=', 'match', 'Links', '.', 'group', '(', \"'\", 'links', \"'\", ')', '<EOS>'], ['<BOS>', 'if', 'not', 'str', '(', 'relative', 'Link', ')', '.', 'startswith', '(', \"'\", 'http', \"'\", ')', ':', '<EOS>']), (['<BOS>', 'if', 'ret', 'and', 'ret', '!', '=', '1', ':', '<EOS>'], ['<BOS>', 'raise', 'hg', '_', 'util', '.', 'Abort', '(', 'ret', ')', '<EOS>']), (['<BOS>', 'if', 'len', '(', 'lines', ')', '>', '=', '2', ':', '<EOS>'], ['<BOS>', 'msg', '=', 'lines', '[', '0', ']', '<EOS>']), (['<BOS>', 'id', '=', \"'\", '+', \"'\", '.', 'join', '(', '[', 'short', '(', 'p', '.', 'node', '(', ')', ')', 'for', 'p', 'in', 'parents', ']', ')', '<EOS>'], ['<BOS>', 'if', 'vers', '!', '=', '\"', '\"', 'and', 'id', '!', '=', 'vers', ':', '<EOS>']), (['<BOS>', 'last', '_', 'email', '=', 'last', '_', 'email', '_', 'file', '.', 'readline', '(', ')', '.', 'strip', '(', '\"', '\\\\', 'n', '\"', ')', '<EOS>'], ['<BOS>', 'last', '_', 'email', '_', 'file', '.', 'close', '(', ')', '<EOS>']), (['<BOS>', 'zero', '_', 'zip', '_', 'date', '_', 'time', '(', 'file', '_', 'path', ')', '<EOS>'], ['<BOS>', 'except', 'Non', 'Zip', 'File', 'Error', ':', '<EOS>']), (['<BOS>', 'for', 'line', 'in', 'f', ':', '<EOS>'], ['<BOS>', 'if', 'line', '.', 'startswith', '(', \"'\", \"'\", ')', ':', '<EOS>']), (['<BOS>', 'codereview', '_', 'init', '=', 'True', '<EOS>'], ['<BOS>', 'start', '_', 'status', '_', 'thread', '(', ')', '<EOS>']), (['<BOS>', 'self', '.', 'old', 'Quiet', '=', 'ui', '.', 'quiet', '<EOS>'], ['<BOS>', 'ui', '.', 'quiet', '=', 'True', '<EOS>']), (['<BOS>', 'upload', '_', 'options', '.', 'send', '_', 'mail', '=', 'False', '<EOS>'], ['<BOS>', 'upload', '_', 'options', '.', 'vcs', '=', 'None', '<EOS>']), (['<BOS>', 'DIST', '_', 'URL', '=', \"'\", 'https', ':', '/', '/', 'electronjs', '.', 'org', '/', 'headers', '/', \"'\", '<EOS>'], ['<BOS>', 'def', 'main', '(', ')', ':', '<EOS>']), (['<BOS>', 'if', 'filename', 'and', 'diff', ':', '<EOS>'], ['<BOS>', 'patches', '.', 'append', '(', '(', 'filename', ',', \"'\", \"'\", '.', 'join', '(', 'diff', ')', ')', ')', '<EOS>']), (['<BOS>', 'line', '=', \"'\", 'mv', \"'\", '+', 'line', '[', 'len', '(', \"'\", 'moving', \"'\", ')', ':', ']', '<EOS>'], ['<BOS>', 'if', 'line', '.', 'startswith', '(', \"'\", 'getting', \"'\", ')', 'and', 'line', '.', 'find', '(', \"'\", 'to', \"'\", ')', '>', '=', '0', ':', '<EOS>']), (['<BOS>', 'else', ':', '<EOS>'], ['<BOS>', 'ext', '=', 'os', '.', 'path', '.', 'splitext', '(', 'f', ')', '[', '1', ']', '[', '1', ':', ']', '<EOS>']), (['<BOS>', 'ui', '.', 'warn', '(', '\"', 'warning', ':', '%', 's', 'is', 'listed', 'in', 'the', 'CL', 'but', 'unchanged', '\\\\', 'n', '\"', '%', '(', 'f', ',', ')', ')', '<EOS>'], ['<BOS>', 'files', '.', 'append', '(', 'f', ')', '<EOS>']), (['<BOS>', '*', '*', 'kwargs', ')', ':', '<EOS>'], ['<BOS>', 'try', ':', '<EOS>']), (['<BOS>', 'line', '=', \"'\", '-', \"'\", '+', 'line', '[', 'len', '(', \"'\", 'removing', \"'\", ')', ':', ']', '<EOS>'], ['<BOS>', 'ui', '.', 'write', '(', 'line', '+', \"'\", '\\\\', 'n', \"'\", ')', '<EOS>']), (['<BOS>', 'raise', 'Unexpected', 'Error', '(', \"'\", '{', '}', ':', '{', '}', ':', '{', '}', \"'\", '.', 'format', '(', '<EOS>'], ['<BOS>', 'file', '_', 'name', ',', 'e', '.', '_', '_', 'class', '_', '_', '.', '_', '_', 'name', '_', '_', ',', 'e', ')', ',', 'e', ')', '<EOS>']), (['<BOS>', 'print', '(', '\"', 'An', 'error', 'occurred', 'while', 'running', \"'\", '{', '}', \"'\", ':', '\"', '.', 'format', '(', 'self', '.', 'binary', '_', 'path', ')', ',', '<EOS>'], ['<BOS>', \"'\", '\\\\', 'n', \"'\", ',', 'exception', ',', 'file', '=', 'sys', '.', 'stderr', ')', '<EOS>']), (['<BOS>', 'return', '\"', 'no', 'such', 'modified', 'files', '\"', '<EOS>'], ['<BOS>', 'files', '=', 'Sub', '(', 'files', ',', 'cl', '.', 'files', ')', '<EOS>']), (['<BOS>', 'try', ':', '<EOS>'], ['<BOS>', 'basestring', '<EOS>']), (['<BOS>', 'offset', '=', '0', '<EOS>'], ['<BOS>', 'mm', '=', 'mmap', '.', 'mmap', '(', 'zip', '_', '.', 'fileno', '(', ')', ',', '0', ')', '<EOS>']), (['<BOS>', 's', '+', '=', '\"', 'Description', ':', '\\\\', 'n', '\"', '<EOS>'], ['<BOS>', 'if', 'cl', '.', 'desc', '=', '=', \"'\", \"'\", ':', '<EOS>']), (['<BOS>', 'committer', '_', 'name', '=', '\"', 'Electron', 'Scripts', '\"', ',', 'committer', '_', 'email', '=', '\"', 'scripts', '@', 'electron', '\"', ')', '<EOS>'], ['<BOS>', 'def', 'parse', '_', 'args', '(', ')', ':', '<EOS>']), (['<BOS>', 'import', 'os', '<EOS>'], ['<BOS>', 'import', 'platform', '<EOS>']), (['<BOS>', 'if', 'get', '_', 'target', '_', 'arch', '(', ')', '!', '=', \"'\", 'mips', '64', 'el', \"'\", ':', '<EOS>'], ['<BOS>', 'symbols', '_', 'zip', '=', 'os', '.', 'path', '.', 'join', '(', 'OUT', '_', 'DIR', ',', 'SYMBOLS', '_', 'NAME', ')', '<EOS>']), (['<BOS>', 'def', 'matchpats', '(', 'ui', ',', 'repo', ',', 'pats', ',', 'opts', ')', ':', '<EOS>'], ['<BOS>', 'matcher', '=', 'scmutil', '.', 'match', '(', 'repo', ',', 'pats', ',', 'opts', ')', '<EOS>']), (['<BOS>', 'if', 'subject', 'is', 'not', 'None', ':', '<EOS>'], ['<BOS>', 'form', '_', 'fields', '[', \"'\", 'subject', \"'\", ']', '=', 'subject', '<EOS>']), (['<BOS>', 'print', '(', '\"', '\\\\', 'n', 'To', 'patch', 'these', 'files', ',', 'run', ':', '\\\\', 'n', '$', 'git', 'apply', '{', '}', '\\\\', 'n', '\"', '<EOS>'], ['<BOS>', '.', 'format', '(', 'patch', '_', 'file', '.', 'name', ')', ')', '<EOS>']), (['<BOS>', 'raise', '<EOS>'], ['<BOS>', 'elif', 'e', '.', 'code', '=', '=', '401', ':', '<EOS>']), (['<BOS>', 'import', 'sys', '<EOS>'], ['<BOS>', 'sys', '.', 'path', '.', 'append', '(', 'os', '.', 'path', '.', 'dirname', '(', 'os', '.', 'path', '.', 'abspath', '(', '_', '_', 'file', '_', '_', ')', ')', '+', '\"', '/', '.', '.', '\"', ')', '<EOS>']), (['<BOS>', 'import', 'sys', '<EOS>'], ['<BOS>', 'stdout', ',', '_', '_', 'stdout', '_', '_', '=', 'sys', '.', 'stdout', ',', 'sys', '.', '_', '_', 'stdout', '_', '_', '<EOS>']), (['<BOS>', 'def', 'Check', 'Contributor', '(', 'ui', ',', 'repo', ',', 'user', '=', 'None', ')', ':', '<EOS>'], ['<BOS>', 'set', '_', 'status', '(', '\"', 'checking', 'CONTRIBUTORS', 'file', '\"', ')', '<EOS>']), (['<BOS>', 'patches', '=', 'Split', 'Patch', '(', 'data', ')', '<EOS>'], ['<BOS>', 'rv', '=', '[', ']', '<EOS>']), (['<BOS>', 'if', 'os', '.', 'sys', '.', 'platform', '=', '=', \"'\", 'plan', '9', \"'\", ':', '<EOS>'], ['<BOS>', 'try', ':', '<EOS>']), (['<BOS>', 'else', ':', '<EOS>'], ['<BOS>', 'ui', '.', 'status', '(', 'msg', '+', '\"', '\\\\', 'n', '\"', ')', '<EOS>']), (['<BOS>', 'raise', 'hg', '_', 'util', '.', 'Abort', '(', '\"', 'no', 'reviewers', 'listed', 'in', 'CL', '\"', ')', '<EOS>'], ['<BOS>', 'if', 'not', 'cl', '.', 'local', ':', '<EOS>']), (['<BOS>', 'repourl', '=', 'getremote', '(', 'ui', ',', 'repo', ',', '{', '}', ')', '.', 'path', '<EOS>'], ['<BOS>', 'if', 'not', 'self', '.', 'mailed', ':', '<EOS>']), (['<BOS>', 's', '=', 's', '[', '0', ':', '55', ']', '+', '\"', '.', '.', '.', '\"', '<EOS>'], ['<BOS>', 'if', 'self', '.', 'name', '!', '=', '\"', 'new', '\"', ':', '<EOS>']), (['<BOS>', 'if', 'closed', ':', '<EOS>'], ['<BOS>', 'form', '_', 'fields', '[', \"'\", 'closed', \"'\", ']', '=', '\"', 'checked', '\"', '<EOS>']), (['<BOS>', 'if', 'sections', '[', \"'\", 'Private', \"'\", ']', 'in', '(', \"'\", 'True', \"'\", ',', \"'\", 'true', \"'\", ',', \"'\", 'Yes', \"'\", ',', \"'\", 'yes', \"'\", ')', ':', '<EOS>'], ['<BOS>', 'cl', '.', 'private', '=', 'True', '<EOS>']), (['<BOS>', 'patch', '=', 'patch', '1', '<EOS>'], ['<BOS>', 'argv', '=', '[', '\"', 'hgapplydiff', '\"', ']', '<EOS>']), (['<BOS>', 'for', 'line', 'in', 'sections', '[', \"'\", 'Files', \"'\", ']', '.', 'split', '(', \"'\", '\\\\', 'n', \"'\", ')', ':', '<EOS>'], ['<BOS>', 'i', '=', 'line', '.', 'find', '(', \"'\", \"'\", ')', '<EOS>']), (['<BOS>', 'ui', '.', 'write', '(', 'line', '+', \"'\", '\\\\', 'n', \"'\", ')', '<EOS>'], ['<BOS>', 'return', 'err', '<EOS>']), (['<BOS>', 'def', 'JSONGet', '(', 'ui', ',', 'path', ')', ':', '<EOS>'], ['<BOS>', 'try', ':', '<EOS>']), (['<BOS>', 'dsyms', '=', 'glob', '.', 'glob', '(', \"'\", '*', '.', 'd', 'SYM', \"'\", ')', '<EOS>'], ['<BOS>', 'dsym', '_', 'zip', '_', 'file', '=', 'os', '.', 'path', '.', 'join', '(', 'args', '.', 'build', '_', 'dir', ',', 'dsym', '_', 'name', ')', '<EOS>']), (['<BOS>', 'if', 'is', '_', 'base', ':', '<EOS>'], ['<BOS>', 'type', '=', '\"', 'base', '\"', '<EOS>']), (['<BOS>', 'SOURCE', '_', 'ROOT', '=', 'os', '.', 'path', '.', 'dirname', '(', 'os', '.', 'path', '.', 'dirname', '(', '_', '_', 'file', '_', '_', ')', ')', '<EOS>'], ['<BOS>', 'def', 'main', '(', ')', ':', '<EOS>']), (['<BOS>', 'if', 'e', '.', 'message', '.', 'find', '(', '\"', 'push', 'creates', 'new', 'heads', '\"', ')', '>', '=', '0', ':', '<EOS>'], ['<BOS>', 'need', '_', 'sync', '(', ')', '<EOS>']), (['<BOS>', '[', 'patches', '.', 'setdefault', '(', 'v', ',', 'k', ')', 'for', 'k', ',', 'v', 'in', 'patch', '_', 'list', ']', '<EOS>'], ['<BOS>', 'for', 'filename', 'in', 'patches', '.', 'keys', '(', ')', ':', '<EOS>']), (['<BOS>', 'print', '(', \"'\", 'ok', 'Chrome', 'Driver', 'is', 'able', 'to', 'be', 'initialized', '.', \"'\", ')', '<EOS>'], ['<BOS>', 'return', 'returncode', '<EOS>']), (['<BOS>', 'add', '_', 'exec', '_', 'bit', '_', 'to', '_', 'sccache', '_', 'binary', '(', 'output', '_', 'dir', ')', '<EOS>'], ['<BOS>', 'with', 'open', '(', 'config', '_', 'hash', '_', 'path', ',', \"'\", 'w', \"'\", ')', 'as', 'f', ':', '<EOS>']), (['<BOS>', 'env', '=', 'os', '.', 'environ', '<EOS>'], ['<BOS>', 'if', 'is', '_', 'verbose', '_', 'mode', '(', ')', ':', '<EOS>']), (['<BOS>', '(', \"'\", 'D', \"'\", ',', \"'\", 'deletelocal', \"'\", ',', 'None', ',', \"'\", 'delete', 'locally', ',', 'but', 'do', 'not', 'change', 'CL', 'on', 'server', \"'\", ')', ',', '<EOS>'], ['<BOS>', '(', \"'\", 'i', \"'\", ',', \"'\", 'stdin', \"'\", ',', 'None', ',', \"'\", 'read', 'change', 'list', 'from', 'standard', 'input', \"'\", ')', ',', '<EOS>']), (['<BOS>', 'required', '=', 'False', ')', '<EOS>'], ['<BOS>', 'return', 'parser', '.', 'parse', '_', 'args', '(', ')', '<EOS>']), (['<BOS>', 'prefix', '=', 'to', '_', 'slash', '(', 'os', '.', 'path', '.', 'realpath', '(', 'repo', '.', 'root', ')', ')', '+', \"'\", '/', \"'\", '<EOS>'], ['<BOS>', 'for', 'line', 'in', 'text', '.', 'split', '(', \"'\", '\\\\', 'n', \"'\", ')', ':', '<EOS>']), (['<BOS>', 'cmd', '=', 'subprocess', '.', 'Popen', '(', '[', '\"', 'gofmt', '\"', ',', '\"', '-', 'l', '\"', ']', '+', 'files', ',', 'shell', '=', 'False', ',', 'stdin', '=', 'subprocess', '.', 'PIPE', ',', 'stdout', '=', 'subprocess', '.', 'PIPE', ',', 'stderr', '=', 'subprocess', '.', 'PIPE', ',', 'close', '_', 'fds', '=', 'sys', '.', 'platform', '!', '=', '\"', 'win', '32', '\"', ')', '<EOS>'], ['<BOS>', 'cmd', '.', 'stdin', '.', 'close', '(', ')', '<EOS>']), (['<BOS>', 'import', 'cookielib', '<EOS>'], ['<BOS>', 'import', 'getpass', '<EOS>'])]\n",
      "        0         1          2           3       4      5      6   \\\n",
      "0    <BOS>    return    retcode       <EOS>   <PAD>  <PAD>  <PAD>   \n",
      "1    <BOS>    return        len           (  broken  Links      )   \n",
      "2    <BOS>     raise         hg           _    util      .  Abort   \n",
      "3    <BOS>      root          =           '       '  <EOS>  <PAD>   \n",
      "4    <BOS>         '  libffmpeg           .      so      '      ,   \n",
      "..     ...       ...        ...         ...     ...    ...    ...   \n",
      "123  <BOS>         (          '           D       '      ,      '   \n",
      "124  <BOS>  required          =       False       )  <EOS>  <PAD>   \n",
      "125  <BOS>    prefix          =          to       _  slash      (   \n",
      "126  <BOS>       cmd          =  subprocess       .  Popen      (   \n",
      "127  <BOS>    import  cookielib       <EOS>   <PAD>  <PAD>  <PAD>   \n",
      "\n",
      "              7           8      9   ...     47        48     49     50  \\\n",
      "0          <PAD>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "1          <EOS>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "2              (  codereview      _  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "3          <PAD>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "4          <EOS>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "..           ...         ...    ...  ...    ...       ...    ...    ...   \n",
      "123  deletelocal           '      ,  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "124        <PAD>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "125           os           .   path  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "126            [           \"  gofmt  ...      .  platform      !      =   \n",
      "127        <PAD>       <PAD>  <PAD>  ...  <PAD>     <PAD>  <PAD>  <PAD>   \n",
      "\n",
      "        51     52     53     54     55     56  \n",
      "0    <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "1    <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "2    <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "3    <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "4    <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "..     ...    ...    ...    ...    ...    ...  \n",
      "123  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "124  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "125  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "126      \"    win     32      \"      )  <EOS>  \n",
      "127  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  <PAD>  \n",
      "\n",
      "[128 rows x 57 columns]\n"
     ]
    }
   ],
   "source": [
    "for i, (x, y) in enumerate(z):\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   1,  110,   12,  ...,    0,    0,    0],\n",
       "        [   1,   42,    6,  ...,    0,    0,    0],\n",
       "        [   1,  729,  266,  ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [   1, 1152,    9,  ...,    0,    0,    0],\n",
       "        [   1,  100,   22,  ...,    0,    0,    0],\n",
       "        [   1,  146,    9,  ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   1, 1028,   20,   57,   20,  117,   15,   72,   16,    5,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (torchData.py, line 12)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[0;36m(most recent call last)\u001b[0m:\n",
      "  File \u001b[1;32m\"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\"\u001b[0m, line \u001b[1;32m3326\u001b[0m, in \u001b[1;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \u001b[1;32m\"<ipython-input-2-ce45c5093a23>\"\u001b[0m, line \u001b[1;32m1\u001b[0m, in \u001b[1;35m<module>\u001b[0m\n    from EditorNoRet import EditorNoRetrieval\n",
      "\u001b[0;36m  File \u001b[0;32m\"/Users/DanielSalz/Documents/Masters/Winter/CS224n/Neural_Code_Generator/EditorNoRet.py\"\u001b[0;36m, line \u001b[0;32m4\u001b[0;36m, in \u001b[0;35m<module>\u001b[0;36m\u001b[0m\n\u001b[0;31m    from DataClass.torchData import tokens_dict, createDataLoaderAllFiles, PAD_IDX\u001b[0m\n",
      "\u001b[0;36m  File \u001b[0;32m\"/Users/DanielSalz/Documents/Masters/Winter/CS224n/Neural_Code_Generator/DataClass/torchData.py\"\u001b[0;36m, line \u001b[0;32m12\u001b[0m\n\u001b[0;31m    UNKNOWN_WORD = = '<UNK>'\u001b[0m\n\u001b[0m                   ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from EditorNoRet import EditorNoRetrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'EditorNoRetrieval' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-4b5f74a79ddb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEditorNoRetrieval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_layers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_heads\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'EditorNoRetrieval' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "hey = EditorNoRetrieval(num_layers=1, num_heads=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'hey' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e3cc1511f85b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mhey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'hey' is not defined"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "optimizer = optim.Adam(hey.model.parameters(), lr=1e-2)\n",
    "hey.train(optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'hey' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-199d9694af2e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'hey' is not defined"
     ]
    }
   ],
   "source": [
    "for i, (x, y) in enumerate(hey.data_loader):\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic Flask setup\n",
    "# app = Flask(__name__)\n",
    "# CORS(app)\n",
    "\n",
    "# app.config['SECRET_KEY'] = 'fe093b0354f9ba0b1237a5e36f58caf55cc9f5682c2627b7463c30f0bbd97672'  # noqa\n",
    "\n",
    "\n",
    "# Regex\n",
    "newline_ptr = re.compile(r'(?:\"[^\"]*\"|.)+')  # \\n outside of quotes (https://stackoverflow.com/questions/24018577/parsing-a-string-in-python-how-to-split-newlines-while-ignoring-newline-inside)\n",
    "comment_ptr = re.compile(r'\"\"\"(.*?)\"\"\"|\\'\\'\\'(.*?)\\'\\'\\'|#[^\"\\']*?(?=\\n)|#.*?(\".*?\"|\\'.*?\\').*?(?=\\n)', re.DOTALL|re.MULTILINE)\n",
    "literal_ptr = re.compile(r'\".*?\"|\\'.*?\\'|[-+]?\\d*\\.\\d+|\\d+')\n",
    "camelcase_ptr = re.compile(r\"(?<=[a-z])([A-Z]+[a-z]*)\")\n",
    "number_ptr = re.compile(r'(?<=[^a-zA-Z])([-+]?\\d*\\.\\d+|\\d+)')\n",
    "number_with_alpha_ptr = re.compile(r'(?<=[a-zA-Z])([-+]?\\d*\\.\\d+|\\d+)')  # split numbers from alpha\n",
    "string_ptr = re.compile(r'\".*?\"|\\'.*?\\'')\n",
    "code_ptr = re.compile(r\"([^a-zA-Z0-9])\")\n",
    "whitespace_ptr = re.compile(r\"(\\s+)\")\n",
    "\n",
    "\n",
    "# Read default options\n",
    "# parser = configargparse.ArgumentParser(description=\"index.py\")\n",
    "# opts.system_opts(parser)\n",
    "# opt = parser.parse_args()\n",
    "\n",
    "results = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (torchData.py, line 12)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[0;36m(most recent call last)\u001b[0m:\n",
      "  File \u001b[1;32m\"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\"\u001b[0m, line \u001b[1;32m3326\u001b[0m, in \u001b[1;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-7-af2170a0ee29>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0;36m, in \u001b[0;35m<module>\u001b[0;36m\u001b[0m\n\u001b[0;31m    from DataClass.torchData import createDataLoaderAllFiles\u001b[0m\n",
      "\u001b[0;36m  File \u001b[0;32m\"/Users/DanielSalz/Documents/Masters/Winter/CS224n/Neural_Code_Generator/DataClass/torchData.py\"\u001b[0;36m, line \u001b[0;32m12\u001b[0m\n\u001b[0;31m    UNKNOWN_WORD = = '<UNK>'\u001b[0m\n\u001b[0m                   ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from DataClass.torchData import createDataLoaderAllFiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-8-9db71dc3d418>, line 4)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-8-9db71dc3d418>\"\u001b[0;36m, line \u001b[0;32m4\u001b[0m\n\u001b[0;31m    dl =\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# path = \"scrape_github/electron\"\n",
    "\n",
    "\n",
    "dl = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_newlines(s):\n",
    "    \"\"\"\n",
    "    Split based on new lines (\\n) outside of quotes.\n",
    "\n",
    "    Note that this coalesces several newlines into one,\n",
    "    as blank lines are ignored. To avoid that, give a null case:\n",
    "\n",
    "    (?:\"[^\"]*\"|.)+|(?!\\Z)\n",
    "    \"\"\"\n",
    "    return re.findall(newline_ptr, s)\n",
    "\n",
    "def remove_comments(source):\n",
    "    return re.sub(r\"\\n\\n+\", \"\\n\\n\", re.sub(comment_ptr, \"\", source))\n",
    "\n",
    "\n",
    "def get_lines_from_source(source):\n",
    "    \"\"\"\n",
    "    Remove comments and empty lines from source.\n",
    "    Return a list of lines\n",
    "    \"\"\"\n",
    "    source = remove_comments(source)\n",
    "\n",
    "    lines = split_newlines(source)\n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_examples_line_by_line(source):  # If line, no need to filter based on length\n",
    "    examples = []\n",
    "\n",
    "    lines = get_lines_from_source(source)\n",
    "    if not lines:\n",
    "        return []\n",
    "    \n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'token_dict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-069e1f245017>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'token_dict.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'token_dict' is not defined"
     ]
    }
   ],
   "source": [
    "pickle.dump(token_dict, open('token_dict.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from DataClass.data_utils import tokenize_fine_grained\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_source(source):\n",
    "    \"\"\"\n",
    "    Remove # in strings for comment_ptr.\n",
    "\n",
    "    TODO. Later on, fix regex instead.\n",
    "    \"\"\"\n",
    "    for string in re.findall(r'\".*?#.*?\"|\\'.*?#.*?\\'', source):\n",
    "        if '#' in string:\n",
    "            modified = string.replace('#', '')\n",
    "            source = source.replace(string, modified)\n",
    "    return source\n",
    "\n",
    "\n",
    "def read_data(path):\n",
    "    if not os.path.exists(path):\n",
    "        print(\"[!] Data does not exist\")\n",
    "    elif os.path.isfile(path):\n",
    "        return read_file(path)\n",
    "    else:\n",
    "        return read_dir(path)\n",
    "\n",
    "\n",
    "def read_file(path):\n",
    "    num_file = 1\n",
    "    sources = []\n",
    "    try:\n",
    "        with open(path, 'r') as f:\n",
    "            source.append(preprocess_source(f.read()))\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    return sources, num_file\n",
    "\n",
    "\n",
    "def read_dir(path):\n",
    "    num_file = 0\n",
    "    sources = []\n",
    "    for filename in glob.iglob(os.path.join(path, \"**/*.py\"), recursive=True):\n",
    "        if not filename.endswith('.py'): continue\n",
    "        num_file += 1\n",
    "        with open(filename, \"r\") as f:\n",
    "            sources.append(preprocess_source(f.read()))\n",
    "    return sources, num_file\n",
    "\n",
    "\n",
    "token_dict = {}\n",
    "import numpy as np\n",
    "\n",
    "path = 'scrape_github/lantern'\n",
    "sources, num_file = read_data(path)\n",
    "all_lines = None\n",
    "for source in sources:\n",
    "    lines = extract_examples_line_by_line(source)\n",
    "    if lines == []: continue\n",
    "    for line in lines:\n",
    "        for token in tokenize_fine_grained(line):\n",
    "            token_dict[token] = token_dict.get(token, len(token_dict))\n",
    "    y = pd.DataFrame(lines)\n",
    "    y.columns = ['line']\n",
    "    y = y[y['line'].apply(lambda x: len(str(x).strip()) > 0)].reset_index(drop=True)    \n",
    "    x = pd.concat([pd.DataFrame([\"\"]), y['line'][:-1]]).reset_index(drop=True)\n",
    "    pair = pd.concat([x, y], axis=1)\n",
    "    all_lines = pd.concat([all_lines, pair], axis=0)# if all_lines is not None else pair\n",
    "\n",
    "all_lines = pd.concat([pd.DataFrame(np.array([all_lines.shape[0], None]).reshape(1, -1), columns=all_lines.columns), all_lines], axis=0)\n",
    "all_lines.to_csv(path[len(path) - path[::-1].find('/'):] + '_line_pairs.csv', header=None, index=None)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File b'electron_line_pairs.csv' does not exist: b'electron_line_pairs.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-ff14a7ae6868>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mdatasets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mMyDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'electron_line_pairs.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMyDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'lantern_line_pairs.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mConcatDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatasets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollate_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_collate_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;31m# for i, (x, y) in enumerate(z):\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-ff14a7ae6868>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename, chunksize)\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunksize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;31m#chunksize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mskiprows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    683\u001b[0m         )\n\u001b[1;32m    684\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 685\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    686\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1135\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1136\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1915\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1917\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1918\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File b'electron_line_pairs.csv' does not exist: b'electron_line_pairs.csv'"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, Dataset, ConcatDataset, DataLoader\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "class MyDataset(Dataset):\n",
    "\n",
    "    def __init__(self, filename, chunksize):\n",
    "        \n",
    "        self.filename = filename\n",
    "        self.chunksize = 2#chunksize\n",
    "        temp = next(pd.read_csv(self.filename, skiprows = 0, chunksize=1, header=None))\n",
    "        self.len = int(temp.values[0][0] / self.chunksize)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        x = next(pd.read_csv(self.filename,\n",
    "                            skiprows=idx * self.chunksize+1,\n",
    "                            chunksize=self.chunksize, header=None, dtype=str)).fillna('OSOFo').values\n",
    "        x_tokens = ['BOS'] + tokenize_fine_grained(x[0, 0]) + ['EOS']# for idx in range(self.chunksize)]\n",
    "        y_tokens = ['BOS'] + tokenize_fine_grained(x[0, 1]) + ['EOS']# for idx in range(self.chunksize)]\n",
    "\n",
    "        return x_tokens, y_tokens\n",
    "    \n",
    "    \n",
    "def batch_collate_fn(data):\n",
    "        x, y = zip(*data)#pd.DataFrame(zip(*h)).T\n",
    "        return x, y\n",
    "#         print(x)\n",
    "        # off by one in tokenize dictionary\n",
    "        x = pd.DataFrame(x).replace(token_dict).fillna(-1)+1\n",
    "        y = pd.DataFrame(y).replace(token_dict).fillna(-1)+1\n",
    "        x.insert(0, 'start', 1)\n",
    "        x.insert(x.shape[1], 'end', 2)\n",
    "        y.insert(0, 'start', 1)\n",
    "        y.insert(y.shape[1], 'end', 2)\n",
    "        batch_xs = torch.LongTensor(x.values)\n",
    "        batch_ys = torch.LongTensor(y.values)\n",
    "        return batch_xs, batch_ys\n",
    "    \n",
    "    \n",
    "batch_size = 4\n",
    "datasets = [MyDataset('electron_line_pairs.csv', batch_size), MyDataset('lantern_line_pairs.csv', batch_size)]\n",
    "z = DataLoader(ConcatDataset(datasets), batch_size=batch_size, shuffle=True, collate_fn=batch_collate_fn)\n",
    "# for i, (x, y) in enumerate(z):\n",
    "# #     continue\n",
    "# # #     print(i)\n",
    "#     print(\"the x: %s and y: %s\" % (x, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "count = 0\n",
    "for i, (x, y) in enumerate(z):\n",
    "    h = x\n",
    "    aa = y\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['BOS',\n",
       "  'unknown',\n",
       "  '_',\n",
       "  'files',\n",
       "  '=',\n",
       "  'self',\n",
       "  '.',\n",
       "  'Get',\n",
       "  'Unknown',\n",
       "  'Files',\n",
       "  '(',\n",
       "  ')',\n",
       "  'EOS'],\n",
       " ['BOS', 'if', 'not', 'files', ':', 'EOS'],\n",
       " ['BOS', 'def', 'effective', '_', 'revpair', '(', 'repo', ')', ':', 'EOS'],\n",
       " ['BOS', 'promptadd', '(', 'ui', ',', 'repo', ',', 'f', ')', 'EOS'])"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.insert(len(x.columns), 'dprde', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 543,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>36</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>36</td>\n",
       "      <td>24</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16\n",
       "0  24   2   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
       "1  25  25  12  26  26  26   0   0   0   0   0   0   0   0   0   0   0\n",
       "2  34  10  11  25  12  36   0   0   0   0   0   0   0   0   0   0   0\n",
       "3  13  36  15  16  25  17  36  36  25  20  36  24  22  23  24  36  36"
      ]
     },
     "execution_count": 424,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pd.DataFrame(x).replace(tokens_dict)+1).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'import': 23,\n",
       " 'mimetypes': 1,\n",
       " 'self': 24,\n",
       " '.': 24,\n",
       " 'reviewer': 11,\n",
       " '=': 25,\n",
       " '[': 25,\n",
       " ']': 25,\n",
       " 'if': 33,\n",
       " 'not': 9,\n",
       " 'cl': 10,\n",
       " ':': 35,\n",
       " 'for': 12,\n",
       " 'line': 35,\n",
       " 'in': 14,\n",
       " 'w': 15,\n",
       " 'output': 16,\n",
       " '(': 35,\n",
       " ')': 35,\n",
       " 'split': 19,\n",
       " \"'\": 23,\n",
       " '\\\\': 21,\n",
       " 'n': 22,\n",
       " 'optparse': 23,\n",
       " 'cc': 24,\n",
       " 'return': 25,\n",
       " 'pending': 26,\n",
       " 'ui': 27,\n",
       " ',': 32,\n",
       " 'repo': 29,\n",
       " '*': 32,\n",
       " 'pats': 31,\n",
       " 'opts': 32,\n",
       " 'is': 33,\n",
       " 'Noise': 34}"
      ]
     },
     "execution_count": 419,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['import', 'os']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from DataClass.data_utils import read_data, tokenize_fine_grained, get_urls_from_csv\n",
    "\n",
    "tokenize_fine_grained(x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MyDataset('electron_line_pairs.csv', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the x: tensor([nan], dtype=torch.float64) and y: ('import os',)\n"
     ]
    }
   ],
   "source": [
    "# z = DataLoader(ConcatDataset(datasets), batch_size=1, shuffle=False)\n",
    "for i, (x, y) in enumerate(z):\n",
    "#     continue\n",
    "# #     print(i)\n",
    "    print(\"the x: %s and y: %s\" % (x, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "unexpected EOF while parsing (<ipython-input-338-96a581c0152e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-338-96a581c0152e>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    datasets = [MyDataset('electron_line_pairs.csv', 1, 2), MyDataset('lantern_line_pairs.csv', 1,\u001b[0m\n\u001b[0m                                                                                                   ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unexpected EOF while parsing\n"
     ]
    }
   ],
   "source": [
    "datasets = [MyDataset('electron_line_pairs.csv', 1, 2), MyDataset('lantern_line_pairs.csv', 1, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_lines.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # dataset, dataset_edit = generate_datasets(opt)\n",
    "# sources, num_file = read_data(opt.path)\n",
    "# dataset = construct_dataset(sources)#generate_datasets(opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>import os</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>import sys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>def main(args):</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>for dirname in args:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>try:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>except OSError as e:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0\n",
       "0                                      \n",
       "1                             import os\n",
       "2                            import sys\n",
       "3                       def main(args):\n",
       "4                  for dirname in args:\n",
       "5                                  try:\n",
       "6                  os.makedirs(dirname)\n",
       "7                  except OSError as e:\n",
       "8        if e.errno == os.errno.EEXIST:"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>line</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>import os</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>import sys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>def main(args):</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for dirname in args:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>try:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>except OSError as e:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>main(sys.argv[1:])</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   line\n",
       "0                             import os\n",
       "1                            import sys\n",
       "2                       def main(args):\n",
       "3                  for dirname in args:\n",
       "4                                  try:\n",
       "5                  os.makedirs(dirname)\n",
       "6                  except OSError as e:\n",
       "7        if e.errno == os.errno.EEXIST:\n",
       "9                    main(sys.argv[1:])"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>line</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>import os</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>import sys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>def main(args):</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for dirname in args:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>try:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>os.makedirs(dirname)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>except OSError as e:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>if e.errno == os.errno.EEXIST:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>main(sys.argv[1:])</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   line\n",
       "0                             import os\n",
       "1                            import sys\n",
       "2                       def main(args):\n",
       "3                  for dirname in args:\n",
       "4                                  try:\n",
       "5                  os.makedirs(dirname)\n",
       "6                  except OSError as e:\n",
       "7        if e.errno == os.errno.EEXIST:\n",
       "9                    main(sys.argv[1:])"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[y['line'].apply(lambda x: len(str(x).replace(' ', '')) > 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'        '"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y['line'][8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "True",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: True",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-267-ef9675bdba12>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# y = y[y.apply(lambda x: len(x) > 0)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'liney'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# df[\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# df['names'].apply(lambda x: len(x)>1) &\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2993\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2994\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2995\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2996\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2997\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2897\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: True"
     ]
    }
   ],
   "source": [
    "# y = y[y.apply(lambda x: len(x) > 0)]\n",
    "y[(len(str(y['liney'])) > 0)]\n",
    "\n",
    "# df[\n",
    "# df['names'].apply(lambda x: len(x)>1) &\n",
    "# df['cars'].apply(lambda x: \"i\" in x) &\n",
    "# df['age'].apply(lambda x: int(x)<2)\n",
    "#   ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_examples_line_by_line(source,\n",
    "                                  n=opt.n_leftmost_tokens,\n",
    "                                  min_len=opt.min_len):  # If line, no need to filter based on length\n",
    "    examples = []\n",
    "\n",
    "    lines = get_lines_from_source(source,\n",
    "                                  remove_comments_from_source=True,\n",
    "                                  remove_empty_lines_from_source=True)\n",
    "    if not lines:\n",
    "        return []\n",
    "    \n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "hey = extract_examples_line_by_line(dataset)\n",
    "\n",
    "y = pd.DataFrame(hey)\n",
    "x = pd.concat([pd.DataFrame([\"\"]), y[0][:-1]]).reset_index(drop=True)\n",
    "pair = pd.concat([x, y], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = {\"debug\": True, \"save\": True, \"overwrite\": True, \"verbose\": True, \"unit\": \"line\", \"min_len\": 3, \"distance_metric_x\": \"nc\",\n",
    "          \"distance_threshold_x\": 0.3, \"distance_metric_y\": \"c\", \"distance_threshold_y\": 2, \"n_leftmost_tokens\": 1, \"max_num_candidates\": 1,\n",
    "      \"check_exact_match_suffix\": True}\n",
    "from collections import namedtuple\n",
    "MyStruct = namedtuple('MyStruct', 'path debug save overwrite verbose unit min_len distance_metric_x distance_threshold_x distance_metric_y distance_threshold_y n_leftmost_tokens max_num_candidates check_exact_match_suffix')\n",
    "\n",
    "opt = MyStruct(path = path, debug = True, save = True, overwrite = True, verbose = True, unit = \"line\", min_len = 10, distance_metric_x = \"nc\",\n",
    "          distance_threshold_x = 0.3, distance_metric_y = \"c\", distance_threshold_y = 2, n_leftmost_tokens = 1, max_num_candidates = 1, \n",
    "               check_exact_match_suffix = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-104-7a02113e77f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhey\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1315\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "hey[1315]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Read data\n",
    "###############################################################################\n",
    "\n",
    "def preprocess_source(source):\n",
    "    \"\"\"\n",
    "    Remove # in strings for comment_ptr.\n",
    "\n",
    "    TODO. Later on, fix regex instead.\n",
    "    \"\"\"\n",
    "    for string in re.findall(r'\".*?#.*?\"|\\'.*?#.*?\\'', source):\n",
    "        if '#' in string:\n",
    "            modified = string.replace('#', '')\n",
    "            source = source.replace(string, modified)\n",
    "    return source\n",
    "\n",
    "\n",
    "def read_data(path):\n",
    "    if not os.path.exists(path):\n",
    "        print(\"[!] Data does not exist\")\n",
    "    elif os.path.isfile(path):\n",
    "        return read_file(path)\n",
    "    else:\n",
    "        return read_dir(path)\n",
    "\n",
    "\n",
    "def read_file(path):\n",
    "    num_file = 1\n",
    "    sources = []\n",
    "    try:\n",
    "        with open(path, 'r') as f:\n",
    "            source.append(preprocess_source(f.read()))\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    return sources, num_file\n",
    "\n",
    "\n",
    "def read_dir(path):\n",
    "    num_file = 0\n",
    "    sources = []\n",
    "    for filename in glob.iglob(os.path.join(path, \"**/*.py\"), recursive=True):\n",
    "#         print(filename)\n",
    "        num_file += 1\n",
    "        with open(filename, \"r\") as f:\n",
    "            sources.append(preprocess_source(f.read()))\n",
    "    return sources, num_file\n",
    "\n",
    "\n",
    "def read_from_pickle(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        data = pickle.load(f)\n",
    "    return data\n",
    "\n",
    "\n",
    "def save_as_pickle(path, data):\n",
    "    with open(path, 'wb') as f:\n",
    "        pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def construct_dataset(sources):\n",
    "#     \"\"\"\n",
    "#     From a project, extract all (x, y) pairs\n",
    "#     \"\"\"\n",
    "#     dataset = []\n",
    "#     num_lines = 0\n",
    "#     for source in sources:\n",
    "\n",
    "#         num_lines += len(get_lines_from_source(source,\n",
    "#                                                remove_comments_from_source=True,\n",
    "#                                                remove_empty_lines_from_source=True))\n",
    "#         return source\n",
    "#         examples = extract_examples_line_by_line(source)\n",
    "# #         print(examples)\n",
    "#         dataset.extend(examples)\n",
    "\n",
    "#     print(f\"\\n[construct_dataset] Number of lines in the project: {num_lines}\")\n",
    "#     print(f\"[construct_dataset] Number of examples: {len(dataset)} ({len(set(dataset))} unique examples)\")\n",
    "#     return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_datasets(opt):\n",
    "    \"\"\"\n",
    "    Check if processed datasets exist.\n",
    "\n",
    "    If so, read. Otherwise, generate and save to the path.\n",
    "    \"\"\"\n",
    "    print(f\"\\n[generate_datasets] Processing {opt.path}\")\n",
    "    options = to_string_opt(opt)\n",
    "    path_dataset = os.path.join(opt.path, f\"dataset_{options}.p\")\n",
    "    path_dataset_edit = os.path.join(opt.path, f\"dataset_edit_{options}.p\")\n",
    "\n",
    "    if os.path.exists(path_dataset) and os.path.exists(path_dataset_edit) and not opt.overwrite:\n",
    "        print(\"[generate_datasets] Read from pickled files\")\n",
    "        dataset = read_from_pickle(path_dataset)\n",
    "        dataset_edit = read_from_pickle(path_dataset_edit)\n",
    "    else:\n",
    "        sources, num_file = read_data(opt.path)\n",
    "\n",
    "        dataset = construct_dataset(sources)  # D_proj = {(x, y)}\n",
    "        if opt.save:\n",
    "            save_as_pickle(path_dataset, dataset)\n",
    "            save_as_pickle(path_dataset_edit, dataset_edit)\n",
    "\n",
    "    print(f\"\\n[generate_datasets] Number of examples: {len(dataset)} ({len(set(dataset))} unique examples)\")\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Utils\n",
    "###############################################################################\n",
    "\n",
    "def rand_select(list, k):\n",
    "    print(f\"Randomly selected {k} examples from {len(list)} examples:\")\n",
    "    return random.choices(list, k=k)\n",
    "\n",
    "\n",
    "def strip_empty_lines(s):\n",
    "    \"\"\"\n",
    "    Remove empty lines at first and last.\n",
    "    \"\"\"\n",
    "    lines = s.splitlines()\n",
    "    while lines and not lines[0].strip():\n",
    "        lines.pop(0)\n",
    "    while lines and not lines[-1].strip():\n",
    "        lines.pop()\n",
    "    return '\\n'.join(lines)\n",
    "\n",
    "\n",
    "def split_newlines(s):\n",
    "    \"\"\"\n",
    "    Split based on new lines (\\n) outside of quotes.\n",
    "\n",
    "    Note that this coalesces several newlines into one,\n",
    "    as blank lines are ignored. To avoid that, give a null case:\n",
    "\n",
    "    (?:\"[^\"]*\"|.)+|(?!\\Z)\n",
    "    \"\"\"\n",
    "    return re.findall(newline_ptr, s)\n",
    "\n",
    "\n",
    "def remove_null(l):\n",
    "    return list(filter(None, l))\n",
    "\n",
    "\n",
    "def remove_comments(source):\n",
    "    return re.sub(r\"\\n\\n+\", \"\\n\\n\", re.sub(comment_ptr, \"\", source))\n",
    "\n",
    "\n",
    "def remove_redundant_indentation(code):\n",
    "    lines = split_newlines(code)\n",
    "    redundant_indentation = min([len(line) - len(line.lstrip())\n",
    "                                 for line in lines\n",
    "                                 if len(line.strip()) > 0])\n",
    "    lines = [line[redundant_indentation:] for line in lines]\n",
    "    return lines\n",
    "\n",
    "\n",
    "def to_string_opt(opt):\n",
    "    \"\"\"\n",
    "    Generate a string for filename that contains current options\n",
    "\n",
    "    E.g. u_line__d_metric_n__d_thre_0.5__n_1__max_can_5\n",
    "    \"\"\"\n",
    "    s = []\n",
    "    s.append(f'u_{opt.unit}')\n",
    "    s.append(f'd_metric_x_{opt.distance_metric_x}')\n",
    "    s.append(f'd_thre_x_{opt.distance_threshold_x}')\n",
    "    s.append(f'd_metric_y_{opt.distance_metric_y}')\n",
    "    s.append(f'd_thre_y_{opt.distance_threshold_y}')\n",
    "    s.append(f'n_{opt.n_leftmost_tokens}')\n",
    "    s.append(f'max_can_{opt.max_num_candidates}')\n",
    "    return '__'.join(s)\n",
    "\n",
    "\n",
    "def get_lines_from_source(source,\n",
    "                          remove_comments_from_source,\n",
    "                          remove_empty_lines_from_source):\n",
    "    \"\"\"\n",
    "    Remove comments and empty lines from source.\n",
    "    Return a list of lines\n",
    "    \"\"\"\n",
    "    if remove_comments_from_source:\n",
    "        source = remove_comments(source)\n",
    "\n",
    "    lines = split_newlines(source)\n",
    "\n",
    "    if remove_empty_lines_from_source:\n",
    "        lines = [line for line in lines if len(line.strip()) > 0]\n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Tokenize\n",
    "###############################################################################\n",
    "\n",
    "def tokenize(s,\n",
    "             split_camelcase,\n",
    "             split_number_from_alpha,\n",
    "             keep_literal,\n",
    "             keep_whitespace,\n",
    "             verbose=False):\n",
    "\n",
    "    numbers, strings, delimiters = [], [], []\n",
    "\n",
    "    if keep_literal:\n",
    "        numbers = remove_null(set(re.findall(number_ptr, s)))\n",
    "        strings = remove_null(set(re.findall(string_ptr, s)))\n",
    "\n",
    "        literals = [f\"(?<=[^a-zA-Z0-9]){re.escape(l)}|^{re.escape(l)}\" for l in numbers]  # Add negative look ahead to exclude cases like fc1\n",
    "        literals.extend([re.escape(l) for l in strings])\n",
    "\n",
    "        delimiters = sorted(literals, key=len, reverse=True)\n",
    "\n",
    "    # Basic tokenization based on non alphanumeric tokens\n",
    "    delimiters.append(\"[^a-zA-Z0-9]\")  # Be careful of the order\n",
    "    delimiters = remove_null(delimiters)\n",
    "\n",
    "    tmp_code_ptr = \"({})\".format(\"|\".join(delimiters))\n",
    "    tokens = remove_null(re.split(tmp_code_ptr, s))\n",
    "    if verbose:\n",
    "        print('[tokenize] Basic:', tokens)\n",
    "\n",
    "    if split_camelcase:\n",
    "        before = tokens\n",
    "        tokens = []\n",
    "        for token in before:\n",
    "            if not token:\n",
    "                continue\n",
    "            elif token in numbers or token in strings:\n",
    "                tokens.append(token)\n",
    "            else:\n",
    "                tokens.extend(re.split(camelcase_ptr, token))\n",
    "        tokens = remove_null(tokens)\n",
    "        if verbose:\n",
    "            print('[tokenize] Split camel cases:', tokens)\n",
    "\n",
    "    if split_number_from_alpha:\n",
    "        before = tokens\n",
    "        tokens = []\n",
    "        for token in before:\n",
    "            if not token:\n",
    "                continue\n",
    "            elif token in numbers or token in strings:\n",
    "                tokens.append(token)\n",
    "            else:\n",
    "                tokens.extend(re.split(number_with_alpha_ptr, token))\n",
    "        tokens = remove_null(tokens)\n",
    "        if verbose:\n",
    "            print('[tokenize] Split numbers from alpha:', tokens)\n",
    "\n",
    "    if not keep_whitespace:\n",
    "        tokens = [token for token in tokens if len(token.strip()) > 0]\n",
    "        if verbose:\n",
    "            print('[tokenize] Remove whitespace:', tokens)\n",
    "\n",
    "    return tokens\n",
    "\n",
    "\n",
    "def tokenize_fine_grained(s, keep_whitespace=False):\n",
    "    \"\"\"\n",
    "    Tokenize as much as possible. Used when calculating edit distance.\n",
    "\n",
    "    E.g., camelCase45 = \"hi there\" -> camel, Case, 45, \", hi, there, \"\n",
    "    \"\"\"\n",
    "    return tokenize(s,\n",
    "                    split_camelcase=True,\n",
    "                    split_number_from_alpha=True,\n",
    "                    keep_literal=False,\n",
    "                    keep_whitespace=keep_whitespace)\n",
    "\n",
    "\n",
    "def tokenize_keywords(s):\n",
    "    \"\"\"\n",
    "    Tokenize as much as human-preferable. Used when generating keywords.\n",
    "\n",
    "    Do not tokenize based on literals.\n",
    "    By default, whitespace is entirely removed.\n",
    "    \"\"\"\n",
    "    return tokenize(s,\n",
    "                    split_camelcase=True,\n",
    "                    split_number_from_alpha=True,\n",
    "                    keep_literal=True,\n",
    "                    keep_whitespace=False)\n",
    "\n",
    "\n",
    "def get_prefix(y,\n",
    "               n=1):#opt.n_leftmost_tokens):\n",
    "    return tuple(tokenize_keywords(y)[:n])\n",
    "\n",
    "\n",
    "def get_suffix(x,\n",
    "               n=1):#opt.n_leftmost_tokens):\n",
    "    return tuple(tokenize_keywords(x)[-n:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Print stuff in color\n",
    "###############################################################################\n",
    "\n",
    "def print_example(example):\n",
    "    x, y = example\n",
    "    print(colored(x, 'red') + colored(y, 'blue'))\n",
    "\n",
    "\n",
    "def print_examples(examples):\n",
    "    print(\"-------------------------------\")\n",
    "    for example in examples:\n",
    "        print_example(example)\n",
    "        print(\"-------------------------------\")\n",
    "\n",
    "\n",
    "def print_candidates(example, candidates):\n",
    "    print('[x] -------------------------------')\n",
    "    print(colored(example[0], 'red') + colored(example[1], 'blue'))\n",
    "    for i, (edit_distance, x, y) in enumerate(candidates, 1):\n",
    "        print(f\"[{i}] {edit_distance:.2f} --------------------------\")\n",
    "        print_example((x, y))\n",
    "\n",
    "\n",
    "def print_example_edit(example_edit):\n",
    "    x, y_abs, x_prime, y_prime = example_edit\n",
    "    print_example((x_prime, y_prime))\n",
    "    print(colored('-------------------------------', 'white'))\n",
    "    print_example((x, y_abs))\n",
    "\n",
    "\n",
    "def print_dataset_edit(dataset_edit):\n",
    "    for i, example_edit in enumerate(dataset_edit, 1):\n",
    "        print(f\"[{i}] -------------------------------\")\n",
    "        print_example_edit(example_edit)\n",
    "\n",
    "\n",
    "def plot_histogram(array, title=\"\", xlabel=\"\", ylabel=\"\"):\n",
    "    n, bins, patches = plt.hist(array, bins=30, facecolor='g', alpha=0.75)\n",
    "    plt.grid(True)\n",
    "\n",
    "    if title:\n",
    "        plt.title(title)\n",
    "    if xlabel:\n",
    "        plt.xlabel(xlabel)\n",
    "    if ylabel:\n",
    "        plt.ylabel(ylabel)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Extract (x, y) pairs\n",
    "###############################################################################\n",
    "\n",
    "def process_example(x, y, n=opt.n_leftmost_tokens):\n",
    "    \"\"\"\n",
    "    Cut and paste the prefix of y at the end of x\n",
    "    \"\"\"\n",
    "    prefix_tokens = get_prefix(y, n=n)\n",
    "    tmp_prefix_ptr = '\\s*' + '\\s*'.join([re.escape(token) for token in prefix_tokens])\n",
    "    prefix_index = re.search(tmp_prefix_ptr, y).end()\n",
    "    x += '\\n' + y[:prefix_index]  # Valid only if each unit is separated by lines\n",
    "    y = y[prefix_index:]\n",
    "    return x, y\n",
    "\n",
    "\n",
    "def reverse_process_example(x, y, n=opt.n_leftmost_tokens):\n",
    "    \"\"\"\n",
    "    Cut and paste the suffix of x at the beginning of y\n",
    "    \"\"\"\n",
    "    suffix_tokens = get_suffix(x, n=n)\n",
    "    tmp_suffix_ptr = '\\s*' + '\\s*'.join([re.escape(token) for token in suffix_tokens]) + '$'\n",
    "    prefix_index = re.search(tmp_suffix_ptr, x).start()\n",
    "    y = strip_empty_lines(x[prefix_index:] + y)\n",
    "    x = x[:prefix_index]\n",
    "    return x, y\n",
    "\n",
    "\n",
    "def extract_examples_line_by_line(source,\n",
    "                                  n=opt.n_leftmost_tokens,\n",
    "                                  min_len=opt.min_len):  # If line, no need to filter based on length\n",
    "    examples = []\n",
    "    def add_example(x, y):\n",
    "        x, y = process_example(x, y, n=n)\n",
    "        examples.append((x, y))\n",
    "\n",
    "    lines = get_lines_from_source(source,\n",
    "                                  remove_comments_from_source=True,\n",
    "                                  remove_empty_lines_from_source=True)\n",
    "    if not lines:\n",
    "        return []\n",
    "\n",
    "    add_example(x=\"\", y=lines[0])  # First example doesn't have context\n",
    "    for i in range(1, len(lines) - 1):\n",
    "        add_example(x=lines[i], y=lines[i + 1])\n",
    "\n",
    "    # Filter out examples that are too short\n",
    "    examples = [(x, y) for (x, y) in examples if len(x.strip()) > min_len and len(y.strip()) > min_len]\n",
    "\n",
    "    # Filter out examples that are import stmts\n",
    "    examples = [(x, y) for (x, y) in examples if 'import' not in x and 'import' not in y]\n",
    "    return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Calculate distance\n",
    "###############################################################################\n",
    "\n",
    "def abstract(s):\n",
    "    return re.sub(literal_ptr, ' ', s)\n",
    "\n",
    "\n",
    "def has_alpha(tokens):\n",
    "    return any([token.isalpha() for token in tokens])\n",
    "\n",
    "\n",
    "def iterative_levenshtein(s, t, costs=(1, 1, 1)):\n",
    "    \"\"\"\n",
    "        iterative_levenshtein(s, t) -> ldist\n",
    "        ldist is the Levenshtein distance between the strings\n",
    "        s and t.\n",
    "        For all i and j, dist[i,j] will contain the Levenshtein\n",
    "        distance between the first i characters of s and the\n",
    "        first j characters of t\n",
    "\n",
    "        costs: a tuple or a list with three integers (d, i, s)\n",
    "               where d defines the costs for a deletion\n",
    "                     i defines the costs for an insertion and\n",
    "                     s defines the costs for a substitution\n",
    "    \"\"\"\n",
    "    rows = len(s)+1\n",
    "    cols = len(t)+1\n",
    "    deletes, inserts, substitutes = costs\n",
    "\n",
    "    dist = [[0 for x in range(cols)] for x in range(rows)]\n",
    "    # source prefixes can be transformed into empty strings\n",
    "    # by deletions:\n",
    "    for row in range(1, rows):\n",
    "        dist[row][0] = row * deletes\n",
    "    # target prefixes can be created from an empty source string\n",
    "    # by inserting the characters\n",
    "    for col in range(1, cols):\n",
    "        dist[0][col] = col * inserts\n",
    "\n",
    "    for col in range(1, cols):\n",
    "        for row in range(1, rows):\n",
    "            if s[row-1] == t[col-1]:\n",
    "                cost = 0\n",
    "            else:\n",
    "                cost = substitutes\n",
    "            dist[row][col] = min(dist[row-1][col] + deletes,\n",
    "                                 dist[row][col-1] + inserts,\n",
    "                                 dist[row-1][col-1] + cost) # substitution\n",
    "    return dist\n",
    "\n",
    "\n",
    "def backtrack_levenshtein(tokens1, tokens2, dist, verbose=False):\n",
    "    \"\"\"\n",
    "    Edit tokens2 to tokens1\n",
    "    \"\"\"\n",
    "    i = len(tokens1)\n",
    "    j = len(tokens2)\n",
    "\n",
    "    replaced_pairs = []  # list of (token2, token1) pairs\n",
    "    replaced_indices = [] # list of (index2, index1) pairs\n",
    "\n",
    "#     if verbose:\n",
    "#         for row in dist:\n",
    "#             print(row)\n",
    "#         print(tokens1)\n",
    "#         print(tokens2)\n",
    "\n",
    "    while i > 0 or j > 0:\n",
    "        if j <= 0:\n",
    "            if verbose:\n",
    "                print(\"Insert\", tokens1[i - 1])\n",
    "            i -= 1\n",
    "            continue\n",
    "        if i <= 0:\n",
    "            if verbose:\n",
    "                print(\"Delete\", tokens2[j - 1])\n",
    "            j -= 1\n",
    "            continue\n",
    "\n",
    "        if tokens1[i - 1] == tokens2[j - 1]:\n",
    "            if verbose:\n",
    "                print(f\"Same {tokens1[i - 1]} (i={i - 1}, j={j - 1})\")\n",
    "            i -= 1\n",
    "            j -= 1\n",
    "        elif j > 0 and dist[i][j] == dist[i][j - 1] + 1:\n",
    "            if verbose:\n",
    "                print(\"Delete\", tokens2[j - 1])\n",
    "            j -= 1\n",
    "        elif i > 0 and j > 0 and dist[i][j] == dist[i - 1][j - 1] + 1:\n",
    "            if verbose:\n",
    "                print(f\"Replace {tokens2[j - 1]} with {tokens1[i - 1]}\")\n",
    "            replaced_pairs.append((tokens2[j - 1], tokens1[i - 1]))\n",
    "            replaced_indices.append((j - 1, i - 1))\n",
    "            i -= 1\n",
    "            j -= 1\n",
    "        elif i > 0 and dist[i][j] == dist[i - 1][j] + 1:\n",
    "            if verbose:\n",
    "                print(\"Insert\", tokens1[i - 1])\n",
    "            i -= 1\n",
    "        else:\n",
    "            if verbose:\n",
    "                print(f\"Error: i={i}, j={j}\")\n",
    "            break\n",
    "\n",
    "\n",
    "    return replaced_pairs, replaced_indices\n",
    "\n",
    "\n",
    "def collapse_edit_distance(tokens1, tokens2, verbose=False):\n",
    "    dist = iterative_levenshtein(tokens1, tokens2)\n",
    "    replaced_pairs, replaced_indices = backtrack_levenshtein(tokens1, tokens2, dist, verbose=verbose)\n",
    "    if verbose:\n",
    "        print(replaced_pairs)\n",
    "\n",
    "    edit_distance = dist[-1][-1]\n",
    "    collapse = len(replaced_pairs) - len(set(replaced_pairs))  # Do not count same replacement multiple times\n",
    "    return edit_distance - collapse\n",
    "\n",
    "\n",
    "def calculate_edit_distance(code_block1,\n",
    "                            code_block2,\n",
    "                            ignore_literals,\n",
    "                            distance_metric,\n",
    "                            verbose=False):\n",
    "    if ignore_literals:  # Todo. Just ignore difference in strings if they are substentially different\n",
    "        block1 = abstract(code_block1)\n",
    "        block2 = abstract(code_block2)\n",
    "        if verbose:\n",
    "            print(\"[.] Abstracted code blocks:\")\n",
    "            print(block1.strip())\n",
    "            print(block2.strip())\n",
    "\n",
    "    else:\n",
    "        block1 = code_block1\n",
    "        block2 = code_block2\n",
    "\n",
    "    # Tokenize\n",
    "    tokens1 = tokenize_fine_grained(block1, keep_whitespace=False)\n",
    "    tokens2 = tokenize_fine_grained(block2, keep_whitespace=False)\n",
    "\n",
    "    if not tokens1 or not tokens2:\n",
    "        return float('inf')\n",
    "\n",
    "    if not has_alpha(tokens1) or not has_alpha(tokens2):\n",
    "        return float('inf')\n",
    "\n",
    "    if verbose:\n",
    "        print(tokens1)\n",
    "        print(tokens2)\n",
    "\n",
    "    # https://github.com/doukremt/distance\n",
    "    if distance_metric == \"j\":\n",
    "        return distance.jaccard(tokens1, tokens2)\n",
    "    elif distance_metric == \"l\":\n",
    "        return distance.levenshtein(tokens1, tokens2)\n",
    "    elif distance_metric == \"h\":\n",
    "        return distance.hamming(tokens1, tokens2)\n",
    "    elif distance_metric == \"s\":\n",
    "        return distance.sorensen(tokens1, tokens2)\n",
    "    elif distance_metric == \"n\":  # Normalized Levenshtein\n",
    "        return distance.nlevenshtein(tokens1, tokens2)\n",
    "    elif distance_metric == \"c\":  # Collapsed Levenshtein edit distance\n",
    "        return collapse_edit_distance(tokens1, tokens2, verbose=verbose)\n",
    "    elif distance_metric == \"nc\":  # Normalized collapsed Levenshtein edit distance\n",
    "        collapsed = collapse_edit_distance(tokens1, tokens2, verbose=verbose)\n",
    "        return collapsed / max(len(tokens1), len(tokens2))\n",
    "\n",
    "\n",
    "def replace_diff_with_placeholders(string1, string2):\n",
    "    \"\"\"\n",
    "    Replace string2-specific tokens with placeholders. Keep original values.\n",
    "\n",
    "    E.g. string1 = 'self.fc7 = (self.relu7, 4096, 4096, \"fc7\")'\n",
    "         string2 = 'self.fc8 = (self.relu8, 4096, 1000, \"fc8\")'\n",
    "\n",
    "         return 'self.fc[[8]] = (self.relu[[8]], 4096, [[1000]], \"fc[[8]]\")'\n",
    "\n",
    "    Parameters:\n",
    "        string1 (str): y\n",
    "        string2 (str): y'; to be abstracted to be consistent with y\n",
    "    \"\"\"\n",
    "    tokens1 = tokenize_fine_grained(string1, keep_whitespace=True)  # Need to keep whitespace\n",
    "    tokens2 = tokenize_fine_grained(string2, keep_whitespace=True)  # Need to keep whitespace\n",
    "\n",
    "    dist = iterative_levenshtein(tokens1, tokens2)\n",
    "    replaced_pairs, replaced_indices = backtrack_levenshtein(tokens1, tokens2, dist)\n",
    "\n",
    "    replaced = tokens2\n",
    "    for index2, index1 in replaced_indices:\n",
    "        replaced[index2] = f'[[{replaced[index2]}]]'\n",
    "    return ''.join(replaced)\n",
    "\n",
    "\n",
    "def rank_based_on_distance(opt,\n",
    "                           example,\n",
    "                           examples,\n",
    "                           include_unsatisfying_examples,\n",
    "                           exclude_same_context,\n",
    "                           exclude_same_example,\n",
    "                           n=opt.n_leftmost_tokens,\n",
    "                           check_exact_match_suffix=opt.check_exact_match_suffix,\n",
    "                           verbose=False):\n",
    "    \"\"\"\n",
    "    Parameters:\n",
    "        check_exact_match_suffix (bool): make sure that the suffix of x\n",
    "            (n_leftmost_tokens tokens) exactly matches\n",
    "        exclude_unsatisfying_examples (bool): filter out examples whose\n",
    "            edit distance is larger than distance_threshold\n",
    "        exclude_same_context (bool): exclude examples whose context exactly matches x\n",
    "    \"\"\"\n",
    "    ranked_examples = []\n",
    "    x, y = example\n",
    "    if verbose:\n",
    "        print(\"[..] x:\", x)\n",
    "\n",
    "    candidate_examples = examples\n",
    "    \n",
    "    # Select examples whose suffix of x exactly matches with that of example\n",
    "    if check_exact_match_suffix:\n",
    "        candidate_examples = []\n",
    "        x_suffix = get_suffix(x, n)\n",
    "        if verbose:\n",
    "            print(f\"[.] Enforce to have same {n} tokens as suffix: {x_suffix}\")\n",
    "        for x_prime, y_prime in examples:\n",
    "            x_prime_suffix = get_suffix(x_prime, n)\n",
    "            if verbose:\n",
    "                print(\"[..] x':\", x_prime)\n",
    "                print(\"[..] Rightmost tokens:\", x_prime_suffix, \"\\n\")\n",
    "            if x_suffix == x_prime_suffix:\n",
    "                candidate_examples.append((x_prime, y_prime))\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"[.] Ranking {len(candidate_examples)} examples\")\n",
    "    for x_prime, y_prime in candidate_examples:\n",
    "        edit_distance_x = calculate_edit_distance(x,\n",
    "                                                  x_prime,\n",
    "                                                  distance_metric=opt.distance_metric_x,\n",
    "                                                  ignore_literals=False)\n",
    "        edit_distance_y = calculate_edit_distance(y,\n",
    "                                                  y_prime,\n",
    "                                                  distance_metric=opt.distance_metric_y,\n",
    "                                                  ignore_literals=False)\n",
    "\n",
    "        if include_unsatisfying_examples:  # Include all examples\n",
    "            ranked_examples.append((edit_distance, x_prime, y_prime))\n",
    "        elif edit_distance_x <= opt.distance_threshold_x and edit_distance_y <= opt.distance_threshold_y:\n",
    "            if exclude_same_context and edit_distance_x == 0:\n",
    "                continue\n",
    "            if exclude_same_example and edit_distance_x == 0 and edit_distance_y == 0:\n",
    "                continue\n",
    "            ranked_examples.append((edit_distance_x, edit_distance_y, x_prime, y_prime))\n",
    "\n",
    "    return sorted(set(ranked_examples), key=lambda x:(x[1], x[0]))  # Remove duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Construct datasets\n",
    "###############################################################################\n",
    "\n",
    "def construct_examples_with_suffix(dataset):\n",
    "    \"\"\"\n",
    "    To speed up, construct clusters of examples based on their suffix\n",
    "    \"\"\"\n",
    "    examples_with_suffix = collections.defaultdict(list)\n",
    "    for example in dataset:\n",
    "        x = example[0]\n",
    "        x_suffix = get_suffix(x)\n",
    "        examples_with_suffix[x_suffix].append(example)\n",
    "    return examples_with_suffix\n",
    "\n",
    "\n",
    "def construct_dataset(sources):\n",
    "    \"\"\"\n",
    "    From a project, extract all (x, y) pairs\n",
    "    \"\"\"\n",
    "    dataset = []\n",
    "    num_lines = 0\n",
    "    for source in sources:\n",
    "        num_lines += len(get_lines_from_source(source,\n",
    "                                               remove_comments_from_source=True,\n",
    "                                               remove_empty_lines_from_source=True))\n",
    "        examples = extract_examples_line_by_line(source)\n",
    "        dataset.extend(examples)\n",
    "\n",
    "    print(f\"\\n[construct_dataset] Number of lines in the project: {num_lines}\")\n",
    "    print(f\"[construct_dataset] Number of examples: {len(dataset)} ({len(set(dataset))} unique examples)\")\n",
    "    return dataset\n",
    "\n",
    "\n",
    "def construct_dataset_edit(opt,\n",
    "                           dataset,\n",
    "                           verbose=False):\n",
    "    \"\"\"\n",
    "    Given (x, y) pairs, generate (x, y_abs, x', y')\n",
    "    \"\"\"\n",
    "    dataset_edit = []\n",
    "    num_examples_with_candidates = 0\n",
    "    examples_with_suffix = construct_examples_with_suffix(dataset)\n",
    "\n",
    "    print(\"\\n[construct_dataset_edit] Start generating dataset for edit\")\n",
    "    for example in tqdm(dataset):\n",
    "        x, y = example\n",
    "        x_suffix = get_suffix(x)\n",
    "        candidates = rank_based_on_distance(opt,\n",
    "                                            example,\n",
    "                                            examples_with_suffix[x_suffix],\n",
    "                                            check_exact_match_suffix=False,  # No need to check if passing examples_with_suffix\n",
    "                                            include_unsatisfying_examples=False,  # Difference: filter out\n",
    "                                            exclude_same_context=False,\n",
    "                                            exclude_same_example=True,\n",
    "                                            verbose=verbose)\n",
    "        if not candidates:\n",
    "            pass\n",
    "        else:\n",
    "            num_examples_with_candidates += 1\n",
    "            for candidate in candidates[:opt.max_num_candidates]:\n",
    "                edit_distance_x, edit_distance_y, x_prime, y_prime = candidate\n",
    "                y_abs = y  # TODO\n",
    "                dataset_edit.append((x, y_abs, x_prime, y_prime))\n",
    "    print(f\"[construct_dataset_edit] Number of examples covered for edit: {num_examples_with_candidates}/{len(dataset)} ({num_examples_with_candidates/len(dataset)*100:.2f}%)\")\n",
    "    print(f\"[construct_dataset_edit] Number of examples for edit: {len(dataset_edit)} ({len(set(dataset_edit))} unique examples for edit)\")\n",
    "    return dataset_edit\n",
    "\n",
    "\n",
    "def generate_datasets(opt):\n",
    "    \"\"\"\n",
    "    Check if processed datasets exist.\n",
    "\n",
    "    If so, read. Otherwise, generate and save to the path.\n",
    "    \"\"\"\n",
    "    print(f\"\\n[generate_datasets] Processing {opt.path}\")\n",
    "    options = to_string_opt(opt)\n",
    "    path_dataset = os.path.join(opt.path, f\"dataset_{options}.p\")\n",
    "    path_dataset_edit = os.path.join(opt.path, f\"dataset_edit_{options}.p\")\n",
    "\n",
    "    if os.path.exists(path_dataset) and os.path.exists(path_dataset_edit) and not opt.overwrite:\n",
    "        print(\"[generate_datasets] Read from pickled files\")\n",
    "        dataset = read_from_pickle(path_dataset)\n",
    "        dataset_edit = read_from_pickle(path_dataset_edit)\n",
    "    else:\n",
    "        sources, num_file = read_data(opt.path)\n",
    "\n",
    "        dataset = construct_dataset(sources)  # D_proj = {(x, y)}\n",
    "        return dataset\n",
    "        if len(dataset) > 10000:\n",
    "            print(f\"[generate_datasets] Skipping too large project (|dataset| = {len(dataset)})\")\n",
    "            dataset_edit = []\n",
    "        else:\n",
    "            dataset_edit = construct_dataset_edit(opt, dataset)  # D_edit = {(x, y, x', y')}\n",
    "\n",
    "            # Save datasets for future use\n",
    "            if opt.save:\n",
    "                save_as_pickle(path_dataset, dataset)\n",
    "                save_as_pickle(path_dataset_edit, dataset_edit)\n",
    "\n",
    "    print(f\"\\n[generate_datasets] Number of examples: {len(dataset)} ({len(set(dataset))} unique examples)\")\n",
    "    print(f\"[generate_datasets] Number of examples for edit: {len(dataset_edit)} ({len(set(dataset_edit))} unique examples)\\n\")\n",
    "\n",
    "    return dataset, dataset_edit\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Render html\n",
    "###############################################################################\n",
    "\n",
    "ignorable = '(\\s*\"\"\".*?\"\"\"\\s*|\\s*\\'\\'\\'.*?\\'\\'\\'\\s*|\\s)*'\n",
    "\n",
    "\n",
    "def generate_html(sources):\n",
    "    html = \"\"\n",
    "    for source in sources:\n",
    "        if len(source.strip()) == 0:\n",
    "            continue\n",
    "        html += f\"#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#\\n\\n{source}\\n\\n\"\n",
    "    return html\n",
    "\n",
    "\n",
    "def collect_code_with_edits(dataset_edit, html):\n",
    "    \"\"\"\n",
    "    Collect all exact code (~ x + y) that have example edits in dataset_edit.\n",
    "    \"\"\"\n",
    "    # TODO Ignore one liner comments starting with #\n",
    "    code_with_edits = set()\n",
    "    cnt_not_found, cnt_found = 0, 0\n",
    "    for x, y, x_prime, y_prime in dataset_edit:  # NOTE y is not abstracted\n",
    "        x, y = reverse_process_example(x, y)  # Cut and paste the suffix of x to the beginning of y\n",
    "        code_to_find_ptr = re.escape(x.strip()) + ignorable + re.escape(y.strip())  # Add ignorable\n",
    "        code_to_find_ptr = re.compile(code_to_find_ptr, re.MULTILINE|re.DOTALL)\n",
    "\n",
    "        # Find actual code from html\n",
    "        found = re.search(code_to_find_ptr, html)\n",
    "        if not found:\n",
    "            print(\"[!] code block not found in source code\")\n",
    "            print(x)\n",
    "            print(y)\n",
    "            cnt_not_found += 1\n",
    "        else:\n",
    "            code = found.group(0).strip()\n",
    "            code_with_edits.add(code)\n",
    "            cnt_found += 1\n",
    "    print(\"\\nMark code with edits in html\")\n",
    "    print(\"Found:\", cnt_found)\n",
    "    print(\"Not found:\", cnt_not_found)\n",
    "    return sorted(code_with_edits, key=len, reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[main] Executing main function with options\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'path'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-38898812aceb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;31m# Execute main function with default options\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-41-38898812aceb>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(opt)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"[main] Executing main function with options\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0msources\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_files\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset_edit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_datasets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'path'"
     ]
    }
   ],
   "source": [
    "###############################################################################\n",
    "# Main\n",
    "###############################################################################\n",
    "\n",
    "def main(opt):\n",
    "    print(\"[main] Executing main function with options\")\n",
    "    sources, num_files = read_data(opt.path)\n",
    "    dataset, dataset_edit = generate_datasets(opt)\n",
    "\n",
    "    html = generate_html(sources)  # For rendering\n",
    "    code_with_edits = collect_code_with_edits(dataset_edit, html)  # For rendering\n",
    "    examples_with_suffix = construct_examples_with_suffix(dataset)  # For metadata\n",
    "\n",
    "    print(\"\\nTotal number of files processed:\", num_files)\n",
    "\n",
    "    print(\"\\nDistance metric (x):\", opt.distance_metric_x)\n",
    "    print(\"Distance threshold (x):\", opt.distance_threshold_x)\n",
    "    print(\"Distance metric (y):\", opt.distance_metric_y)\n",
    "    print(\"Distance threshold (y):\", opt.distance_threshold_y)\n",
    "\n",
    "    # print(\"\\nNumber of leftmost tokens for keywords (n):\", opt.n_leftmost_tokens)\n",
    "    # print(\"Maximum number of candidates to generate example edits (k):\", opt.max_num_candidates)\n",
    "    # print(\"Enforce exact match of the suffix of x and x':\", opt.check_exact_match_suffix)\n",
    "\n",
    "    print(f\"\\nTotal number of examples: {len(dataset)}\")\n",
    "    print(f\"Total number of example edits: {len(dataset_edit)} ({len(dataset_edit) / len(dataset) * 100:.2f}%)\")\n",
    "    print(f\"Total number of code with edits: {len(code_with_edits)}\", \"\\n\")\n",
    "\n",
    "    return {\n",
    "        'sources': sources,\n",
    "        'num_files': num_files,\n",
    "        'dataset': dataset,\n",
    "        'dataset_edit': dataset_edit,\n",
    "        'html': html,\n",
    "        'code_with_edits': code_with_edits,\n",
    "        'examples_with_suffix': examples_with_suffix\n",
    "    }\n",
    "\n",
    "\n",
    "# Execute main function with default options\n",
    "results = main(opt)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'app' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-b9ae3311c9a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m###############################################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0mapp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/apply_options\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethods\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"POST\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mapply_options\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0moptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'app' is not defined"
     ]
    }
   ],
   "source": [
    "###############################################################################\n",
    "# Flask\n",
    "###############################################################################\n",
    "\n",
    "@app.route(\"/apply_options\", methods=[\"POST\"])\n",
    "def apply_options():\n",
    "    options = request.get_json(force=True)\n",
    "    print(\"\\n[apply_options] Applying requested options:\", options)\n",
    "\n",
    "    # Update opt values\n",
    "    global opt\n",
    "    opt.distance_metric_x = options['distance_metric_x']\n",
    "    opt.distance_threshold_x = float(options['distance_threshold_x'])\n",
    "    opt.distance_metric_y = options['distance_metric_y']\n",
    "    opt.distance_threshold_y = float(options['distance_threshold_y'])\n",
    "\n",
    "    # Update processed data for new options\n",
    "    global results\n",
    "    results = main(opt)\n",
    "    print('HERERERERE')\n",
    "    print(results)\n",
    "    return render_template(\"index.html\",\n",
    "                           html=results['html'],\n",
    "                           dataset_size=len(results['dataset']),\n",
    "                           dataset_edit_size=len(results['dataset_edit']),\n",
    "                           coverage=f\"{len(results['dataset_edit']) / len(results['dataset']) * 100:.2f}\",\n",
    "                           distance_metric_x=opt.distance_metric_x,\n",
    "                           distance_threshold_x=opt.distance_threshold_x,\n",
    "                           distance_metric_y=opt.distance_metric_y,\n",
    "                           distance_threshold_y=opt.distance_threshold_y)\n",
    "\n",
    "\n",
    "\n",
    "@app.route(\"/get_code_with_edits\", methods=[\"POST\"])\n",
    "def get_code_with_edits():\n",
    "    return jsonify(results['code_with_edits'])\n",
    "\n",
    "\n",
    "@app.route(\"/get_metadata\", methods=[\"POST\"])\n",
    "def get_metadata():\n",
    "    example = request.get_json(force=True)\n",
    "    print(example)\n",
    "    # Process example\n",
    "\n",
    "    x = example['x'].replace(\"&lt;\", \"<\").replace(\"&gt;\", \">\").replace(\"&amp;\", \"&\")\n",
    "    y = example['y'].replace(\"&lt;\", \"<\").replace(\"&gt;\", \">\").replace(\"&amp;\", \"&\")\n",
    "    # print(\"Pre POrcexx examples ??????\")\n",
    "    # print(x)\n",
    "    # print(y)\n",
    "    x, y = process_example(x, y)\n",
    "    # print(\"POrcexx examples ??????\")\n",
    "    # print(x)\n",
    "    # print(y)\n",
    "\n",
    "    # Get metadata\n",
    "    x_suffix = get_suffix(x)\n",
    "    print(\"about to process example edits\")\n",
    "    example_edits = rank_based_on_distance(opt,\n",
    "                                           (x, y),\n",
    "                                           results['examples_with_suffix'][x_suffix],\n",
    "                                           check_exact_match_suffix=False,  # No need to check if passing examples_with_suffix\n",
    "                                           include_unsatisfying_examples=False,\n",
    "                                           exclude_same_context=False,\n",
    "                                           exclude_same_example=True)\n",
    "    print(example_edits)\n",
    "    print(\"in between example edits\")\n",
    "    example_edits = [{\n",
    "                        'edit_distance_x': f'{edit_distance_x:.2f}',\n",
    "                        'edit_distance_y': f'{edit_distance_y:.2f}',\n",
    "                        'x': x,\n",
    "                        'y': y,\n",
    "                    }\n",
    "                    for (edit_distance_x, edit_distance_y, x, y) in example_edits]\n",
    "\n",
    "    data = {\n",
    "        'x': x,\n",
    "        'y': y,\n",
    "        'example_edits': example_edits,\n",
    "    }\n",
    "    return jsonify(data)\n",
    "\n",
    "\n",
    "@app.route(\"/\")\n",
    "def intro():\n",
    "    return render_template(\"index.html\",\n",
    "                           html=results['html'],\n",
    "                           dataset_size=len(results['dataset']),\n",
    "                           dataset_edit_size=len(results['dataset_edit']),\n",
    "                           coverage=f\"{len(results['dataset_edit']) / len(results['dataset']) * 100:.2f}\",\n",
    "                           distance_metric_x=opt.distance_metric_x,\n",
    "                           distance_threshold_x=opt.distance_threshold_x,\n",
    "                           distance_metric_y=opt.distance_metric_y,\n",
    "                           distance_threshold_y=opt.distance_threshold_y,\n",
    "                           min_len=opt.min_len)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    app.run(host=\"0.0.0.0\",  # Public\n",
    "            debug=opt.debug)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}