BERT_all_tag.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT NE recognition LOC/ORG/FACILITY"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1)  Preprocess / format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import codecs\n",
    "import glob\n",
    "import logging\n",
    "import multiprocessing\n",
    "import os\n",
    "import pprint\n",
    "import re\n",
    "import keras\n",
    "import nltk\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import ne_chunk, pos_tag\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize, PunktSentenceTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "import gensim.models.word2vec as w2v\n",
    "from gensim import logging\n",
    "import sklearn.manifold\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as  pd\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import json\n",
    "from nltk.tokenize import WhitespaceTokenizer\n",
    "from nltk.tokenize import MWETokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/chantana/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) load keyword: LOC/ORG from text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Petpimarn Boutique Resort',\n",
       " 'Airport Suite Bangkok (Don Muang Airport)',\n",
       " 'The Riche Residence',\n",
       " 'Regent Home1 at Donmuang',\n",
       " 'Charoenpong Apartment']"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_files = ['./keywords/list_hotel.txt']\n",
    "word_list_name = []\n",
    "for word_file in word_files:\n",
    "    with open(word_file) as f:\n",
    "        lines = f.readlines()\n",
    "    word_list2 = [x.strip() for x in lines] \n",
    "    \n",
    "    word_list_name.extend(word_list2)\n",
    "word_list_name[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['24-hour Receptionist',\n",
       " 'Laundry service',\n",
       " 'Luggage storage',\n",
       " 'Tours',\n",
       " 'Parking']"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_files = ['./keywords/facility.txt']\n",
    "word_list_fac = []\n",
    "for word_file in word_files:\n",
    "    with open(word_file) as f:\n",
    "        lines = f.readlines()\n",
    "    word_list2 = [x.strip() for x in lines] \n",
    "    \n",
    "    word_list_fac.extend(word_list2)\n",
    "word_list_fac[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['99/89 Moo 6',\n",
       " 'VIPHAVADEE RANDSIT RD',\n",
       " '111/1 Soi Ngam Wong Wan 47 Yaek 1',\n",
       " '310/72 Soi Phaholyothin 67/1',\n",
       " '3/2899 Soi Yak']"
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_hotels = ['./keywords/list_location.txt']\n",
    "lists_location = []\n",
    "for word_hotel in word_hotels:\n",
    "    with open(word_hotel) as f:\n",
    "        lines = f.readlines()\n",
    "    lists_l= [x.strip() for x in lines] \n",
    "    \n",
    "    lists_location.extend(lists_l)\n",
    "lists_location[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove duplicates\n",
    "def ordered_set(in_list):\n",
    "    out_list = []\n",
    "    added = set()\n",
    "    for val in in_list:\n",
    "        if not val in added:\n",
    "            out_list.append(val)\n",
    "            added.add(val)\n",
    "    return out_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list_name = ordered_set(word_list_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list_fac = ordered_set(word_list_fac)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_location= ordered_set(lists_location)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "word_locates = ['../keywords/hotel_location.txt']\n",
    "word_hotel_location = []\n",
    "for word_locate in word_locates:\n",
    "    with open(word_locate) as f:\n",
    "        lines = f.readlines()\n",
    "    word_location2 = [x.strip() for x in lines] \n",
    "    \n",
    "    word_hotel_location.extend(word_location2)\n",
    "word_hotel_location[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Petpimarn', 'Boutique', 'Resort'),\n",
       " ('Airport', 'Suite', 'Bangkok', '(Don', 'Muang', 'Airport)'),\n",
       " ('The', 'Riche', 'Residence'),\n",
       " ('Regent', 'Home1', 'at', 'Donmuang'),\n",
       " ('Charoenpong', 'Apartment')]"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mw_list = []\n",
    "for x in word_list_name:\n",
    "    ws = x.split() \n",
    "    if len(ws) > 1:\n",
    "        tws = tuple(ws)\n",
    "        mw_list.append(tws)\n",
    "        \n",
    "\n",
    "for x in word_location:\n",
    "    ws = x.split() \n",
    "    if len(ws) > 1:\n",
    "        tws = tuple(ws)\n",
    "        mw_list.append(tws)\n",
    "        \n",
    "\n",
    "        \n",
    "for x in word_list_fac:\n",
    "    ws = x.split() \n",
    "    if len(ws) > 1:\n",
    "        tws = tuple(ws)\n",
    "        mw_list.append(tws)\n",
    "         \n",
    "            \n",
    "mw_list = ordered_set(mw_list)\n",
    "mw_list[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35968\n"
     ]
    }
   ],
   "source": [
    "my_idx = {}\n",
    "for i,w in enumerate(mw_list):\n",
    "    my_idx[\" \".join(w)] = i\n",
    "print(len(my_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found\n",
      "found\n"
     ]
    }
   ],
   "source": [
    "for i in mw_list:\n",
    "    if i == ('M&A','Guesthouse'):\n",
    "        print('found')\n",
    "    if i == ('Central','Pattaya'):\n",
    "        print('found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "mwe = MWETokenizer(mw_list,separator=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews = pd.read_csv(\"./keywords/all_loc_fac.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26541, 1)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Staying at @ Home Executive Apartment is a goo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Staying at @ Oasis Resort is a good choice whe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Staying at @ T Boutique Hotel is a good choice...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Staying at AC Resort is a good choice when you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Staying at Acqua Condominium is a good choice ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            location\n",
       "0  Staying at @ Home Executive Apartment is a goo...\n",
       "1  Staying at @ Oasis Resort is a good choice whe...\n",
       "2  Staying at @ T Boutique Hotel is a good choice...\n",
       "3  Staying at AC Resort is a good choice when you...\n",
       "4  Staying at Acqua Condominium is a good choice ..."
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews = reviews.dropna()\n",
    "reviews = reviews.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spans(txt):\n",
    "    tokens=mwe.tokenize(word_tokenize(txt))\n",
    "    offset = 0\n",
    "    for token in tokens:\n",
    "        offset = txt.find(token, offset)\n",
    "        yield token, offset, offset+len(token)\n",
    "        offset += len(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f550c8f0a7846da8a9c24d8dba2972f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=26541), HTML(value=u'')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "all_item = []\n",
    "\n",
    "for i in tqdm(range(len(reviews))):\n",
    "    word_ls = []\n",
    "    #print(\"sentence:\",i+1)\n",
    "    \n",
    "    for token in spans(reviews.location[i]):\n",
    "        #print(token)\n",
    "        #assert token[0]==reviews.location[i][token[1]:token[2]]\n",
    "        my_tuple = token[0]\n",
    "        #print(\"###\",token)\n",
    "        \n",
    "        #my_tuples = ' , '.join(map(str, my_tuple))\n",
    "        if token[0] in word_location:\n",
    "            #word_ls.append(my_tuple)\n",
    "            subwords = token[0].split()\n",
    "            pos_list = [nltk.pos_tag([w]) for w in subwords]\n",
    "            tag_list = ['I-LOC']*len(pos_list)\n",
    "            tag_list[0] = 'B-LOC' \n",
    "            \n",
    "            \n",
    "            for s,p,t in zip(subwords,pos_list,tag_list):           \n",
    "                if type(p) == list:\n",
    "                    p = p[0][1]\n",
    "                    #print('list')\n",
    "                new_item = dict({'Sentence #': i+1, 'Tag' : t, 'Word': s,'POS': p})\n",
    "                all_item.append(new_item)\n",
    "                \n",
    "                \n",
    "            #lis_lo = nltk.pos_tag(word_ls),LOC\n",
    "            #print(' , '.join(map(str, lis_lo)))\n",
    " \n",
    "        elif token[0] in word_list_name:\n",
    "            \n",
    "            subwords = token[0].split()\n",
    "            pos_list = [nltk.pos_tag([w]) for w in subwords]\n",
    "            tag_list = ['I-ORG']*len(pos_list)\n",
    "            tag_list[0] = 'B-ORG' \n",
    "            \n",
    "            #print(subwords)\n",
    "            for s,p,t in zip(subwords,pos_list,tag_list):           \n",
    "                new_item = dict({'Sentence #': i+1, 'Tag' : t, 'Word': s,'POS': p[0][1]})\n",
    "                all_item.append(new_item)\n",
    "                #print(new_item)\n",
    "            #word_ls.append(my_tuple)\n",
    "            #print(\"found LOC\")\n",
    "            #lis_lo = nltk.pos_tag(word_ls),\"LOC\"\n",
    "            #print(' , '.join(map(str, lis_lo)))\n",
    "             \n",
    "        elif token[0] in word_list_fac:\n",
    "            #word_ls.append(my_tuple)\n",
    "            #lis_lo = nltk.pos_tag(word_ls),\"FACILITY\"\n",
    "            #print(' , '.join(map(str, lis_lo)))\n",
    "            \n",
    "            #my_pos = nltk.pos_tag([my_tuple])[0][1]\n",
    "            #new_item =  dict({'Sentence #': i+1, 'Tag' : 'FACILITY', 'Word': my_tuple,'POS': my_pos})\n",
    "            \n",
    "            subwords = token[0].split()\n",
    "            pos_list = [nltk.pos_tag([w]) for w in subwords]\n",
    "            tag_list = ['I-FAC']*len(pos_list)\n",
    "            tag_list[0] = 'B-FAC' \n",
    "            \n",
    "             \n",
    "            for s,p,t in zip(subwords,pos_list,tag_list):   \n",
    "                if type(p) == list:\n",
    "                    p = p[0][1]\n",
    "                new_item = dict({'Sentence #': i+1, 'Tag' : t, 'Word': s,'POS': p})\n",
    "                all_item.append(new_item)\n",
    "                \n",
    "\n",
    "             \n",
    "        else:\n",
    "            #print(type(my_tuple))\n",
    "            my_pos = nltk.pos_tag([my_tuple])[0][1]\n",
    "            new_item = dict({'Sentence #': i+1, 'Tag' : 'O', 'Word': my_tuple.lower(),'POS': my_pos})\n",
    "            all_item.append(new_item)\n",
    "            \n",
    "            #if i+1 == 5177:\n",
    "                #print(new_item)\n",
    "             \n",
    "            #print(nltk.pos_tag([my_tuple]),',','O')\n",
    "        #print(new_item)\n",
    "        #if not(my_pos == '.' or  my_pos == ',' or my_pos == ':' or my_pos == '(' or my_pos == ')') :\n",
    "            \n",
    "            #all_item.append(new_item) \n",
    "\n",
    "        \n",
    "\n",
    "        \n",
    "    \n",
    "\n",
    "          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3) save preproc data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_csv = 'data-loc-fac-bert.csv'\n",
    "with open(file_csv, 'w') as csv_file:\n",
    "    csv_file.write('Sentence #*Word*POS*Tag\\n')\n",
    "    for item in all_item:\n",
    "        \n",
    "        #print(item['POS'])    \n",
    "        csv_file.write(str(item['Sentence #'])+'*'+item['Word']+'*'+item['POS']+'*'+item['Tag']+'\\n')\n",
    "csv_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4) Load preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence #</th>\n",
       "      <th>Word</th>\n",
       "      <th>POS</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>staying</td>\n",
       "      <td>VBG</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>at</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>@</td>\n",
       "      <td>NN</td>\n",
       "      <td>B-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Home</td>\n",
       "      <td>NN</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Executive</td>\n",
       "      <td>NN</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>NN</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>is</td>\n",
       "      <td>VBZ</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>a</td>\n",
       "      <td>DT</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>good</td>\n",
       "      <td>JJ</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>choice</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sentence #       Word  POS    Tag\n",
       "0           1    staying  VBG      O\n",
       "1           1         at   IN      O\n",
       "2           1          @   NN  B-ORG\n",
       "3           1       Home   NN  I-ORG\n",
       "4           1  Executive   NN  I-ORG\n",
       "5           1  Apartment   NN  I-ORG\n",
       "6           1         is  VBZ      O\n",
       "7           1          a   DT      O\n",
       "8           1       good   JJ      O\n",
       "9           1     choice   NN      O"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "#load data\n",
    "# explore data\n",
    "file_csv = 'data-loc-fac-bert.csv'\n",
    "data = pd.read_csv(file_csv, sep='*',encoding=\"utf-8\").fillna(method=\"ffill\")\n",
    "\n",
    "data.head(10)\n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for i in range(100):\n",
    "    locate_lis = []\n",
    "    print(\"sentence:\",i+1)\n",
    "    for token in spans(reviews.location[i]):\n",
    "        #print(token)\n",
    "        #assert token[0]==reviews.location[i][token[1]:token[2]]\n",
    "        my_tuple = token[0]\n",
    "        print(token)\n",
    "        #my_tuples = ' , '.join(map(str, my_tuple))\n",
    "        if token[0] in word_location:\n",
    "            locate_lis.append(my_tuple)\n",
    "            lis_lo1 = \"\\n\".join(map(str, (nltk.pos_tag(locate_lis))))\n",
    "            print(lis_lo1,',',\"LOC\")\n",
    "            continue\n",
    "        print(nltk.pos_tag([my_tuple]),',','O')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load word2vec representation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_50 =\"trained/hotel2vec-desc-50.w2v\"\n",
    "filename_g300 =\"trained/hotel2vec-glove-desc-300.w2v\"\n",
    "filename_num =\"trained/hotel2vec-num-desc-300.w2v\"\n",
    "filename_g50 =\"trained/hotel2vec-glove-desc-50.w2v\"\n",
    "filename_gg =\"trained/hotel2vec-gg-desc-300.w2v\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_num = './model/numberbatch-en-17.06.txt' \n",
    "file_gg = 'GoogleNews-vectors-negative300.bin'\n",
    "file_glove_50 = 'glove.6B.300d.txt'\n",
    "file_glove_300 = 'glove.6B.300d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-03-16 13:42:41,552 : INFO : loading Word2Vec object from trained/hotel2vec-glove-desc-50.w2v\n",
      "2019-03-16 13:42:41,564 : INFO : loading vocabulary recursively from trained/hotel2vec-glove-desc-50.w2v.vocabulary.* with mmap=None\n",
      "2019-03-16 13:42:41,567 : INFO : loading wv recursively from trained/hotel2vec-glove-desc-50.w2v.wv.* with mmap=None\n",
      "2019-03-16 13:42:41,569 : INFO : setting ignored attribute vectors_norm to None\n",
      "2019-03-16 13:42:41,576 : INFO : loading trainables recursively from trained/hotel2vec-glove-desc-50.w2v.trainables.* with mmap=None\n",
      "2019-03-16 13:42:41,581 : INFO : setting ignored attribute cum_table to None\n",
      "2019-03-16 13:42:41,584 : INFO : loaded trained/hotel2vec-glove-desc-50.w2v\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word embeddings: 400000\n",
      "Word embeddings: 401831\n"
     ]
    }
   ],
   "source": [
    "#load based model\n",
    "filename = file_glove_50\n",
    "\n",
    "embeddings_index = {}\n",
    "seq = 0\n",
    "with open(filename) as f:\n",
    "#with open('./model/numberbatch-en-17.06.txt', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split(' ')\n",
    "        #print(values)\n",
    "        word = values[0].decode('utf-8')\n",
    "        embedding = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = (seq,embedding)\n",
    "        seq += 1\n",
    "\n",
    "print('Word embeddings:', len(embeddings_index))\n",
    "\n",
    "#load desc model\n",
    "filename = filename_g50\n",
    "\n",
    "\n",
    "\n",
    "model_d =  w2v.Word2Vec.load(filename)\n",
    "word_vectors = model_d.wv\n",
    "\n",
    "vocab_gg = word_vectors.vocab\n",
    " \n",
    "for v in vocab_gg:\n",
    "     \n",
    "    word = v \n",
    "    embedding = np.asarray(model_d.wv[v], dtype='float32')\n",
    "    embeddings_index[word] = (seq,embedding)\n",
    "    seq +=1\n",
    "\n",
    "    \n",
    "print('Word embeddings:', len(embeddings_index))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5) BERT trained-formating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentenceGetter(object):\n",
    "    \n",
    "    def __init__(self, data):\n",
    "        self.n_sent = 1\n",
    "        self.data = data\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"Word\"].values.tolist(),\n",
    "                                                           s[\"POS\"].values.tolist(),\n",
    "                                                           s[\"Tag\"].values.tolist())]\n",
    "        self.grouped = self.data.groupby(\"Sentence #\").apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "    \n",
    "    def get_next(self):\n",
    "        try:\n",
    "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    " \n",
    "#concat sentence\n",
    "getter = SentenceGetter(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list = [ [s[0] for s in sent] for sent in getter.sentences] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'staying at @ Home Executive Apartment is a good choice when you are visiting Central Pattaya .'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = [\" \".join([s[0] for s in sent]) for sent in getter.sentences]\n",
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26541\n"
     ]
    }
   ],
   "source": [
    "print(len(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "labels = [[s[2] for s in sent] for sent in getter.sentences]\n",
    "print(len(labels[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "print(len(word_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26541\n"
     ]
    }
   ],
   "source": [
    "print(len(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_vals = list(set(data[\"Tag\"].values))\n",
    "tag2idx = {t: i for i, t in enumerate(tags_vals)}\n",
    "idx2tag = {i: t for i, t in enumerate(tags_vals) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_vals_save = tags_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'B-ORG'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx2tag[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_word2idx (text_list):\n",
    "    \n",
    "    values = []\n",
    "    #print(text_list)\n",
    "     \n",
    "    for word in text_list:\n",
    "        if word in my_idx:\n",
    "            values.append(my_idx[word])\n",
    "        else:\n",
    "            if word.lower() in embeddings_index:\n",
    "                v = len(my_idx)+embeddings_index[word.lower()][0] \n",
    "            else:\n",
    "                val = list(embeddings_index.keys())[0]\n",
    "                embeddings_index[word.lower()] = (len(embeddings_index),[0.5]*len(val[1]))\n",
    "                v = len(my_idx) +len(embeddings_index)\n",
    "            values.append(v)\n",
    "            \n",
    "    return values\n",
    "\n",
    "def convert_idx2word (idx_list):\n",
    "    words_list = []\n",
    "    for idx in idx_list:\n",
    "        found = False\n",
    "        for key, value in my_idx.items(): \n",
    "            if idx == value:\n",
    "                words_list.append(key)\n",
    "                found = True\n",
    "                break\n",
    "        if not found:\n",
    "            for key, value in embeddings_index.items(): \n",
    "                if idx == value[0]+len(my_idx):\n",
    "                    words_list.append(key)\n",
    "\n",
    "                    break\n",
    "\n",
    "            \n",
    "         \n",
    "        \n",
    "    return words_list\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'TITAN Xp'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_LEN = 120\n",
    "bs = 8\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.get_device_name(0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-03-18 13:42:57,328 : INFO : loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/chantana/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mwe = MWETokenizer(mw_list,separator=' ')\n",
    "\n",
    "tokenized_texts = [mwe.tokenize(word_tokenize(sent)) for sent in sentences]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "print(tokenized_texts[0],len(tokenized_texts[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'staying', u'at', u'@', u'home', u'executive', u'apartment', u'is', u'a', u'good', u'choice', u'when', u'you', u'are', u'visiting', u'central', u'pat', u'##ta', u'##ya', u'.']\n"
     ]
    }
   ],
   "source": [
    "tokenized_texts = [ tokenizer.tokenize(sent)   for sent in sentences]\n",
    "print(tokenized_texts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'staying at @ Home Executive Apartment is a good choice when you are visiting Central Pattaya .'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'staying', u'at', u'@', u'home', u'executive', u'apartment', u'is', u'a', u'good', u'choice', u'when', u'you', u'are', u'visiting', u'central', u'pat', u'##ta', u'##ya', u'.']\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = PorterStemmer() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_word2idx (text_list):\n",
    "    \n",
    "    values = []\n",
    "    #print(text_list)\n",
    "    start = len(tokenizer.vocab)\n",
    "    for word in text_list:\n",
    "        \n",
    "        if tokenizer.vocab.has_key(word):\n",
    "            values.append(tokenizer.vocab[word])\n",
    "        else:\n",
    "            #print(word,\"not found\")\n",
    "            tokenizer.vocab[word] = start\n",
    "            values.append(tokenizer.vocab[word])\n",
    "            start = start + 1\n",
    "        \n",
    "            \n",
    "    return values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30522"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_save =  tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([(u'[PAD]', 0),\n",
       "             (u'[unused0]', 1),\n",
       "             (u'[unused1]', 2),\n",
       "             (u'[unused2]', 3),\n",
       "             (u'[unused3]', 4),\n",
       "             (u'[unused4]', 5),\n",
       "             (u'[unused5]', 6),\n",
       "             (u'[unused6]', 7),\n",
       "             (u'[unused7]', 8),\n",
       "             (u'[unused8]', 9),\n",
       "             (u'[unused9]', 10),\n",
       "             (u'[unused10]', 11),\n",
       "             (u'[unused11]', 12),\n",
       "             (u'[unused12]', 13),\n",
       "             (u'[unused13]', 14),\n",
       "             (u'[unused14]', 15),\n",
       "             (u'[unused15]', 16),\n",
       "             (u'[unused16]', 17),\n",
       "             (u'[unused17]', 18),\n",
       "             (u'[unused18]', 19),\n",
       "             (u'[unused19]', 20),\n",
       "             (u'[unused20]', 21),\n",
       "             (u'[unused21]', 22),\n",
       "             (u'[unused22]', 23),\n",
       "             (u'[unused23]', 24),\n",
       "             (u'[unused24]', 25),\n",
       "             (u'[unused25]', 26),\n",
       "             (u'[unused26]', 27),\n",
       "             (u'[unused27]', 28),\n",
       "             (u'[unused28]', 29),\n",
       "             (u'[unused29]', 30),\n",
       "             (u'[unused30]', 31),\n",
       "             (u'[unused31]', 32),\n",
       "             (u'[unused32]', 33),\n",
       "             (u'[unused33]', 34),\n",
       "             (u'[unused34]', 35),\n",
       "             (u'[unused35]', 36),\n",
       "             (u'[unused36]', 37),\n",
       "             (u'[unused37]', 38),\n",
       "             (u'[unused38]', 39),\n",
       "             (u'[unused39]', 40),\n",
       "             (u'[unused40]', 41),\n",
       "             (u'[unused41]', 42),\n",
       "             (u'[unused42]', 43),\n",
       "             (u'[unused43]', 44),\n",
       "             (u'[unused44]', 45),\n",
       "             (u'[unused45]', 46),\n",
       "             (u'[unused46]', 47),\n",
       "             (u'[unused47]', 48),\n",
       "             (u'[unused48]', 49),\n",
       "             (u'[unused49]', 50),\n",
       "             (u'[unused50]', 51),\n",
       "             (u'[unused51]', 52),\n",
       "             (u'[unused52]', 53),\n",
       "             (u'[unused53]', 54),\n",
       "             (u'[unused54]', 55),\n",
       "             (u'[unused55]', 56),\n",
       "             (u'[unused56]', 57),\n",
       "             (u'[unused57]', 58),\n",
       "             (u'[unused58]', 59),\n",
       "             (u'[unused59]', 60),\n",
       "             (u'[unused60]', 61),\n",
       "             (u'[unused61]', 62),\n",
       "             (u'[unused62]', 63),\n",
       "             (u'[unused63]', 64),\n",
       "             (u'[unused64]', 65),\n",
       "             (u'[unused65]', 66),\n",
       "             (u'[unused66]', 67),\n",
       "             (u'[unused67]', 68),\n",
       "             (u'[unused68]', 69),\n",
       "             (u'[unused69]', 70),\n",
       "             (u'[unused70]', 71),\n",
       "             (u'[unused71]', 72),\n",
       "             (u'[unused72]', 73),\n",
       "             (u'[unused73]', 74),\n",
       "             (u'[unused74]', 75),\n",
       "             (u'[unused75]', 76),\n",
       "             (u'[unused76]', 77),\n",
       "             (u'[unused77]', 78),\n",
       "             (u'[unused78]', 79),\n",
       "             (u'[unused79]', 80),\n",
       "             (u'[unused80]', 81),\n",
       "             (u'[unused81]', 82),\n",
       "             (u'[unused82]', 83),\n",
       "             (u'[unused83]', 84),\n",
       "             (u'[unused84]', 85),\n",
       "             (u'[unused85]', 86),\n",
       "             (u'[unused86]', 87),\n",
       "             (u'[unused87]', 88),\n",
       "             (u'[unused88]', 89),\n",
       "             (u'[unused89]', 90),\n",
       "             (u'[unused90]', 91),\n",
       "             (u'[unused91]', 92),\n",
       "             (u'[unused92]', 93),\n",
       "             (u'[unused93]', 94),\n",
       "             (u'[unused94]', 95),\n",
       "             (u'[unused95]', 96),\n",
       "             (u'[unused96]', 97),\n",
       "             (u'[unused97]', 98),\n",
       "             (u'[unused98]', 99),\n",
       "             (u'[UNK]', 100),\n",
       "             (u'[CLS]', 101),\n",
       "             (u'[SEP]', 102),\n",
       "             (u'[MASK]', 103),\n",
       "             (u'[unused99]', 104),\n",
       "             (u'[unused100]', 105),\n",
       "             (u'[unused101]', 106),\n",
       "             (u'[unused102]', 107),\n",
       "             (u'[unused103]', 108),\n",
       "             (u'[unused104]', 109),\n",
       "             (u'[unused105]', 110),\n",
       "             (u'[unused106]', 111),\n",
       "             (u'[unused107]', 112),\n",
       "             (u'[unused108]', 113),\n",
       "             (u'[unused109]', 114),\n",
       "             (u'[unused110]', 115),\n",
       "             (u'[unused111]', 116),\n",
       "             (u'[unused112]', 117),\n",
       "             (u'[unused113]', 118),\n",
       "             (u'[unused114]', 119),\n",
       "             (u'[unused115]', 120),\n",
       "             (u'[unused116]', 121),\n",
       "             (u'[unused117]', 122),\n",
       "             (u'[unused118]', 123),\n",
       "             (u'[unused119]', 124),\n",
       "             (u'[unused120]', 125),\n",
       "             (u'[unused121]', 126),\n",
       "             (u'[unused122]', 127),\n",
       "             (u'[unused123]', 128),\n",
       "             (u'[unused124]', 129),\n",
       "             (u'[unused125]', 130),\n",
       "             (u'[unused126]', 131),\n",
       "             (u'[unused127]', 132),\n",
       "             (u'[unused128]', 133),\n",
       "             (u'[unused129]', 134),\n",
       "             (u'[unused130]', 135),\n",
       "             (u'[unused131]', 136),\n",
       "             (u'[unused132]', 137),\n",
       "             (u'[unused133]', 138),\n",
       "             (u'[unused134]', 139),\n",
       "             (u'[unused135]', 140),\n",
       "             (u'[unused136]', 141),\n",
       "             (u'[unused137]', 142),\n",
       "             (u'[unused138]', 143),\n",
       "             (u'[unused139]', 144),\n",
       "             (u'[unused140]', 145),\n",
       "             (u'[unused141]', 146),\n",
       "             (u'[unused142]', 147),\n",
       "             (u'[unused143]', 148),\n",
       "             (u'[unused144]', 149),\n",
       "             (u'[unused145]', 150),\n",
       "             (u'[unused146]', 151),\n",
       "             (u'[unused147]', 152),\n",
       "             (u'[unused148]', 153),\n",
       "             (u'[unused149]', 154),\n",
       "             (u'[unused150]', 155),\n",
       "             (u'[unused151]', 156),\n",
       "             (u'[unused152]', 157),\n",
       "             (u'[unused153]', 158),\n",
       "             (u'[unused154]', 159),\n",
       "             (u'[unused155]', 160),\n",
       "             (u'[unused156]', 161),\n",
       "             (u'[unused157]', 162),\n",
       "             (u'[unused158]', 163),\n",
       "             (u'[unused159]', 164),\n",
       "             (u'[unused160]', 165),\n",
       "             (u'[unused161]', 166),\n",
       "             (u'[unused162]', 167),\n",
       "             (u'[unused163]', 168),\n",
       "             (u'[unused164]', 169),\n",
       "             (u'[unused165]', 170),\n",
       "             (u'[unused166]', 171),\n",
       "             (u'[unused167]', 172),\n",
       "             (u'[unused168]', 173),\n",
       "             (u'[unused169]', 174),\n",
       "             (u'[unused170]', 175),\n",
       "             (u'[unused171]', 176),\n",
       "             (u'[unused172]', 177),\n",
       "             (u'[unused173]', 178),\n",
       "             (u'[unused174]', 179),\n",
       "             (u'[unused175]', 180),\n",
       "             (u'[unused176]', 181),\n",
       "             (u'[unused177]', 182),\n",
       "             (u'[unused178]', 183),\n",
       "             (u'[unused179]', 184),\n",
       "             (u'[unused180]', 185),\n",
       "             (u'[unused181]', 186),\n",
       "             (u'[unused182]', 187),\n",
       "             (u'[unused183]', 188),\n",
       "             (u'[unused184]', 189),\n",
       "             (u'[unused185]', 190),\n",
       "             (u'[unused186]', 191),\n",
       "             (u'[unused187]', 192),\n",
       "             (u'[unused188]', 193),\n",
       "             (u'[unused189]', 194),\n",
       "             (u'[unused190]', 195),\n",
       "             (u'[unused191]', 196),\n",
       "             (u'[unused192]', 197),\n",
       "             (u'[unused193]', 198),\n",
       "             (u'[unused194]', 199),\n",
       "             (u'[unused195]', 200),\n",
       "             (u'[unused196]', 201),\n",
       "             (u'[unused197]', 202),\n",
       "             (u'[unused198]', 203),\n",
       "             (u'[unused199]', 204),\n",
       "             (u'[unused200]', 205),\n",
       "             (u'[unused201]', 206),\n",
       "             (u'[unused202]', 207),\n",
       "             (u'[unused203]', 208),\n",
       "             (u'[unused204]', 209),\n",
       "             (u'[unused205]', 210),\n",
       "             (u'[unused206]', 211),\n",
       "             (u'[unused207]', 212),\n",
       "             (u'[unused208]', 213),\n",
       "             (u'[unused209]', 214),\n",
       "             (u'[unused210]', 215),\n",
       "             (u'[unused211]', 216),\n",
       "             (u'[unused212]', 217),\n",
       "             (u'[unused213]', 218),\n",
       "             (u'[unused214]', 219),\n",
       "             (u'[unused215]', 220),\n",
       "             (u'[unused216]', 221),\n",
       "             (u'[unused217]', 222),\n",
       "             (u'[unused218]', 223),\n",
       "             (u'[unused219]', 224),\n",
       "             (u'[unused220]', 225),\n",
       "             (u'[unused221]', 226),\n",
       "             (u'[unused222]', 227),\n",
       "             (u'[unused223]', 228),\n",
       "             (u'[unused224]', 229),\n",
       "             (u'[unused225]', 230),\n",
       "             (u'[unused226]', 231),\n",
       "             (u'[unused227]', 232),\n",
       "             (u'[unused228]', 233),\n",
       "             (u'[unused229]', 234),\n",
       "             (u'[unused230]', 235),\n",
       "             (u'[unused231]', 236),\n",
       "             (u'[unused232]', 237),\n",
       "             (u'[unused233]', 238),\n",
       "             (u'[unused234]', 239),\n",
       "             (u'[unused235]', 240),\n",
       "             (u'[unused236]', 241),\n",
       "             (u'[unused237]', 242),\n",
       "             (u'[unused238]', 243),\n",
       "             (u'[unused239]', 244),\n",
       "             (u'[unused240]', 245),\n",
       "             (u'[unused241]', 246),\n",
       "             (u'[unused242]', 247),\n",
       "             (u'[unused243]', 248),\n",
       "             (u'[unused244]', 249),\n",
       "             (u'[unused245]', 250),\n",
       "             (u'[unused246]', 251),\n",
       "             (u'[unused247]', 252),\n",
       "             (u'[unused248]', 253),\n",
       "             (u'[unused249]', 254),\n",
       "             (u'[unused250]', 255),\n",
       "             (u'[unused251]', 256),\n",
       "             (u'[unused252]', 257),\n",
       "             (u'[unused253]', 258),\n",
       "             (u'[unused254]', 259),\n",
       "             (u'[unused255]', 260),\n",
       "             (u'[unused256]', 261),\n",
       "             (u'[unused257]', 262),\n",
       "             (u'[unused258]', 263),\n",
       "             (u'[unused259]', 264),\n",
       "             (u'[unused260]', 265),\n",
       "             (u'[unused261]', 266),\n",
       "             (u'[unused262]', 267),\n",
       "             (u'[unused263]', 268),\n",
       "             (u'[unused264]', 269),\n",
       "             (u'[unused265]', 270),\n",
       "             (u'[unused266]', 271),\n",
       "             (u'[unused267]', 272),\n",
       "             (u'[unused268]', 273),\n",
       "             (u'[unused269]', 274),\n",
       "             (u'[unused270]', 275),\n",
       "             (u'[unused271]', 276),\n",
       "             (u'[unused272]', 277),\n",
       "             (u'[unused273]', 278),\n",
       "             (u'[unused274]', 279),\n",
       "             (u'[unused275]', 280),\n",
       "             (u'[unused276]', 281),\n",
       "             (u'[unused277]', 282),\n",
       "             (u'[unused278]', 283),\n",
       "             (u'[unused279]', 284),\n",
       "             (u'[unused280]', 285),\n",
       "             (u'[unused281]', 286),\n",
       "             (u'[unused282]', 287),\n",
       "             (u'[unused283]', 288),\n",
       "             (u'[unused284]', 289),\n",
       "             (u'[unused285]', 290),\n",
       "             (u'[unused286]', 291),\n",
       "             (u'[unused287]', 292),\n",
       "             (u'[unused288]', 293),\n",
       "             (u'[unused289]', 294),\n",
       "             (u'[unused290]', 295),\n",
       "             (u'[unused291]', 296),\n",
       "             (u'[unused292]', 297),\n",
       "             (u'[unused293]', 298),\n",
       "             (u'[unused294]', 299),\n",
       "             (u'[unused295]', 300),\n",
       "             (u'[unused296]', 301),\n",
       "             (u'[unused297]', 302),\n",
       "             (u'[unused298]', 303),\n",
       "             (u'[unused299]', 304),\n",
       "             (u'[unused300]', 305),\n",
       "             (u'[unused301]', 306),\n",
       "             (u'[unused302]', 307),\n",
       "             (u'[unused303]', 308),\n",
       "             (u'[unused304]', 309),\n",
       "             (u'[unused305]', 310),\n",
       "             (u'[unused306]', 311),\n",
       "             (u'[unused307]', 312),\n",
       "             (u'[unused308]', 313),\n",
       "             (u'[unused309]', 314),\n",
       "             (u'[unused310]', 315),\n",
       "             (u'[unused311]', 316),\n",
       "             (u'[unused312]', 317),\n",
       "             (u'[unused313]', 318),\n",
       "             (u'[unused314]', 319),\n",
       "             (u'[unused315]', 320),\n",
       "             (u'[unused316]', 321),\n",
       "             (u'[unused317]', 322),\n",
       "             (u'[unused318]', 323),\n",
       "             (u'[unused319]', 324),\n",
       "             (u'[unused320]', 325),\n",
       "             (u'[unused321]', 326),\n",
       "             (u'[unused322]', 327),\n",
       "             (u'[unused323]', 328),\n",
       "             (u'[unused324]', 329),\n",
       "             (u'[unused325]', 330),\n",
       "             (u'[unused326]', 331),\n",
       "             (u'[unused327]', 332),\n",
       "             (u'[unused328]', 333),\n",
       "             (u'[unused329]', 334),\n",
       "             (u'[unused330]', 335),\n",
       "             (u'[unused331]', 336),\n",
       "             (u'[unused332]', 337),\n",
       "             (u'[unused333]', 338),\n",
       "             (u'[unused334]', 339),\n",
       "             (u'[unused335]', 340),\n",
       "             (u'[unused336]', 341),\n",
       "             (u'[unused337]', 342),\n",
       "             (u'[unused338]', 343),\n",
       "             (u'[unused339]', 344),\n",
       "             (u'[unused340]', 345),\n",
       "             (u'[unused341]', 346),\n",
       "             (u'[unused342]', 347),\n",
       "             (u'[unused343]', 348),\n",
       "             (u'[unused344]', 349),\n",
       "             (u'[unused345]', 350),\n",
       "             (u'[unused346]', 351),\n",
       "             (u'[unused347]', 352),\n",
       "             (u'[unused348]', 353),\n",
       "             (u'[unused349]', 354),\n",
       "             (u'[unused350]', 355),\n",
       "             (u'[unused351]', 356),\n",
       "             (u'[unused352]', 357),\n",
       "             (u'[unused353]', 358),\n",
       "             (u'[unused354]', 359),\n",
       "             (u'[unused355]', 360),\n",
       "             (u'[unused356]', 361),\n",
       "             (u'[unused357]', 362),\n",
       "             (u'[unused358]', 363),\n",
       "             (u'[unused359]', 364),\n",
       "             (u'[unused360]', 365),\n",
       "             (u'[unused361]', 366),\n",
       "             (u'[unused362]', 367),\n",
       "             (u'[unused363]', 368),\n",
       "             (u'[unused364]', 369),\n",
       "             (u'[unused365]', 370),\n",
       "             (u'[unused366]', 371),\n",
       "             (u'[unused367]', 372),\n",
       "             (u'[unused368]', 373),\n",
       "             (u'[unused369]', 374),\n",
       "             (u'[unused370]', 375),\n",
       "             (u'[unused371]', 376),\n",
       "             (u'[unused372]', 377),\n",
       "             (u'[unused373]', 378),\n",
       "             (u'[unused374]', 379),\n",
       "             (u'[unused375]', 380),\n",
       "             (u'[unused376]', 381),\n",
       "             (u'[unused377]', 382),\n",
       "             (u'[unused378]', 383),\n",
       "             (u'[unused379]', 384),\n",
       "             (u'[unused380]', 385),\n",
       "             (u'[unused381]', 386),\n",
       "             (u'[unused382]', 387),\n",
       "             (u'[unused383]', 388),\n",
       "             (u'[unused384]', 389),\n",
       "             (u'[unused385]', 390),\n",
       "             (u'[unused386]', 391),\n",
       "             (u'[unused387]', 392),\n",
       "             (u'[unused388]', 393),\n",
       "             (u'[unused389]', 394),\n",
       "             (u'[unused390]', 395),\n",
       "             (u'[unused391]', 396),\n",
       "             (u'[unused392]', 397),\n",
       "             (u'[unused393]', 398),\n",
       "             (u'[unused394]', 399),\n",
       "             (u'[unused395]', 400),\n",
       "             (u'[unused396]', 401),\n",
       "             (u'[unused397]', 402),\n",
       "             (u'[unused398]', 403),\n",
       "             (u'[unused399]', 404),\n",
       "             (u'[unused400]', 405),\n",
       "             (u'[unused401]', 406),\n",
       "             (u'[unused402]', 407),\n",
       "             (u'[unused403]', 408),\n",
       "             (u'[unused404]', 409),\n",
       "             (u'[unused405]', 410),\n",
       "             (u'[unused406]', 411),\n",
       "             (u'[unused407]', 412),\n",
       "             (u'[unused408]', 413),\n",
       "             (u'[unused409]', 414),\n",
       "             (u'[unused410]', 415),\n",
       "             (u'[unused411]', 416),\n",
       "             (u'[unused412]', 417),\n",
       "             (u'[unused413]', 418),\n",
       "             (u'[unused414]', 419),\n",
       "             (u'[unused415]', 420),\n",
       "             (u'[unused416]', 421),\n",
       "             (u'[unused417]', 422),\n",
       "             (u'[unused418]', 423),\n",
       "             (u'[unused419]', 424),\n",
       "             (u'[unused420]', 425),\n",
       "             (u'[unused421]', 426),\n",
       "             (u'[unused422]', 427),\n",
       "             (u'[unused423]', 428),\n",
       "             (u'[unused424]', 429),\n",
       "             (u'[unused425]', 430),\n",
       "             (u'[unused426]', 431),\n",
       "             (u'[unused427]', 432),\n",
       "             (u'[unused428]', 433),\n",
       "             (u'[unused429]', 434),\n",
       "             (u'[unused430]', 435),\n",
       "             (u'[unused431]', 436),\n",
       "             (u'[unused432]', 437),\n",
       "             (u'[unused433]', 438),\n",
       "             (u'[unused434]', 439),\n",
       "             (u'[unused435]', 440),\n",
       "             (u'[unused436]', 441),\n",
       "             (u'[unused437]', 442),\n",
       "             (u'[unused438]', 443),\n",
       "             (u'[unused439]', 444),\n",
       "             (u'[unused440]', 445),\n",
       "             (u'[unused441]', 446),\n",
       "             (u'[unused442]', 447),\n",
       "             (u'[unused443]', 448),\n",
       "             (u'[unused444]', 449),\n",
       "             (u'[unused445]', 450),\n",
       "             (u'[unused446]', 451),\n",
       "             (u'[unused447]', 452),\n",
       "             (u'[unused448]', 453),\n",
       "             (u'[unused449]', 454),\n",
       "             (u'[unused450]', 455),\n",
       "             (u'[unused451]', 456),\n",
       "             (u'[unused452]', 457),\n",
       "             (u'[unused453]', 458),\n",
       "             (u'[unused454]', 459),\n",
       "             (u'[unused455]', 460),\n",
       "             (u'[unused456]', 461),\n",
       "             (u'[unused457]', 462),\n",
       "             (u'[unused458]', 463),\n",
       "             (u'[unused459]', 464),\n",
       "             (u'[unused460]', 465),\n",
       "             (u'[unused461]', 466),\n",
       "             (u'[unused462]', 467),\n",
       "             (u'[unused463]', 468),\n",
       "             (u'[unused464]', 469),\n",
       "             (u'[unused465]', 470),\n",
       "             (u'[unused466]', 471),\n",
       "             (u'[unused467]', 472),\n",
       "             (u'[unused468]', 473),\n",
       "             (u'[unused469]', 474),\n",
       "             (u'[unused470]', 475),\n",
       "             (u'[unused471]', 476),\n",
       "             (u'[unused472]', 477),\n",
       "             (u'[unused473]', 478),\n",
       "             (u'[unused474]', 479),\n",
       "             (u'[unused475]', 480),\n",
       "             (u'[unused476]', 481),\n",
       "             (u'[unused477]', 482),\n",
       "             (u'[unused478]', 483),\n",
       "             (u'[unused479]', 484),\n",
       "             (u'[unused480]', 485),\n",
       "             (u'[unused481]', 486),\n",
       "             (u'[unused482]', 487),\n",
       "             (u'[unused483]', 488),\n",
       "             (u'[unused484]', 489),\n",
       "             (u'[unused485]', 490),\n",
       "             (u'[unused486]', 491),\n",
       "             (u'[unused487]', 492),\n",
       "             (u'[unused488]', 493),\n",
       "             (u'[unused489]', 494),\n",
       "             (u'[unused490]', 495),\n",
       "             (u'[unused491]', 496),\n",
       "             (u'[unused492]', 497),\n",
       "             (u'[unused493]', 498),\n",
       "             (u'[unused494]', 499),\n",
       "             (u'[unused495]', 500),\n",
       "             (u'[unused496]', 501),\n",
       "             (u'[unused497]', 502),\n",
       "             (u'[unused498]', 503),\n",
       "             (u'[unused499]', 504),\n",
       "             (u'[unused500]', 505),\n",
       "             (u'[unused501]', 506),\n",
       "             (u'[unused502]', 507),\n",
       "             (u'[unused503]', 508),\n",
       "             (u'[unused504]', 509),\n",
       "             (u'[unused505]', 510),\n",
       "             (u'[unused506]', 511),\n",
       "             (u'[unused507]', 512),\n",
       "             (u'[unused508]', 513),\n",
       "             (u'[unused509]', 514),\n",
       "             (u'[unused510]', 515),\n",
       "             (u'[unused511]', 516),\n",
       "             (u'[unused512]', 517),\n",
       "             (u'[unused513]', 518),\n",
       "             (u'[unused514]', 519),\n",
       "             (u'[unused515]', 520),\n",
       "             (u'[unused516]', 521),\n",
       "             (u'[unused517]', 522),\n",
       "             (u'[unused518]', 523),\n",
       "             (u'[unused519]', 524),\n",
       "             (u'[unused520]', 525),\n",
       "             (u'[unused521]', 526),\n",
       "             (u'[unused522]', 527),\n",
       "             (u'[unused523]', 528),\n",
       "             (u'[unused524]', 529),\n",
       "             (u'[unused525]', 530),\n",
       "             (u'[unused526]', 531),\n",
       "             (u'[unused527]', 532),\n",
       "             (u'[unused528]', 533),\n",
       "             (u'[unused529]', 534),\n",
       "             (u'[unused530]', 535),\n",
       "             (u'[unused531]', 536),\n",
       "             (u'[unused532]', 537),\n",
       "             (u'[unused533]', 538),\n",
       "             (u'[unused534]', 539),\n",
       "             (u'[unused535]', 540),\n",
       "             (u'[unused536]', 541),\n",
       "             (u'[unused537]', 542),\n",
       "             (u'[unused538]', 543),\n",
       "             (u'[unused539]', 544),\n",
       "             (u'[unused540]', 545),\n",
       "             (u'[unused541]', 546),\n",
       "             (u'[unused542]', 547),\n",
       "             (u'[unused543]', 548),\n",
       "             (u'[unused544]', 549),\n",
       "             (u'[unused545]', 550),\n",
       "             (u'[unused546]', 551),\n",
       "             (u'[unused547]', 552),\n",
       "             (u'[unused548]', 553),\n",
       "             (u'[unused549]', 554),\n",
       "             (u'[unused550]', 555),\n",
       "             (u'[unused551]', 556),\n",
       "             (u'[unused552]', 557),\n",
       "             (u'[unused553]', 558),\n",
       "             (u'[unused554]', 559),\n",
       "             (u'[unused555]', 560),\n",
       "             (u'[unused556]', 561),\n",
       "             (u'[unused557]', 562),\n",
       "             (u'[unused558]', 563),\n",
       "             (u'[unused559]', 564),\n",
       "             (u'[unused560]', 565),\n",
       "             (u'[unused561]', 566),\n",
       "             (u'[unused562]', 567),\n",
       "             (u'[unused563]', 568),\n",
       "             (u'[unused564]', 569),\n",
       "             (u'[unused565]', 570),\n",
       "             (u'[unused566]', 571),\n",
       "             (u'[unused567]', 572),\n",
       "             (u'[unused568]', 573),\n",
       "             (u'[unused569]', 574),\n",
       "             (u'[unused570]', 575),\n",
       "             (u'[unused571]', 576),\n",
       "             (u'[unused572]', 577),\n",
       "             (u'[unused573]', 578),\n",
       "             (u'[unused574]', 579),\n",
       "             (u'[unused575]', 580),\n",
       "             (u'[unused576]', 581),\n",
       "             (u'[unused577]', 582),\n",
       "             (u'[unused578]', 583),\n",
       "             (u'[unused579]', 584),\n",
       "             (u'[unused580]', 585),\n",
       "             (u'[unused581]', 586),\n",
       "             (u'[unused582]', 587),\n",
       "             (u'[unused583]', 588),\n",
       "             (u'[unused584]', 589),\n",
       "             (u'[unused585]', 590),\n",
       "             (u'[unused586]', 591),\n",
       "             (u'[unused587]', 592),\n",
       "             (u'[unused588]', 593),\n",
       "             (u'[unused589]', 594),\n",
       "             (u'[unused590]', 595),\n",
       "             (u'[unused591]', 596),\n",
       "             (u'[unused592]', 597),\n",
       "             (u'[unused593]', 598),\n",
       "             (u'[unused594]', 599),\n",
       "             (u'[unused595]', 600),\n",
       "             (u'[unused596]', 601),\n",
       "             (u'[unused597]', 602),\n",
       "             (u'[unused598]', 603),\n",
       "             (u'[unused599]', 604),\n",
       "             (u'[unused600]', 605),\n",
       "             (u'[unused601]', 606),\n",
       "             (u'[unused602]', 607),\n",
       "             (u'[unused603]', 608),\n",
       "             (u'[unused604]', 609),\n",
       "             (u'[unused605]', 610),\n",
       "             (u'[unused606]', 611),\n",
       "             (u'[unused607]', 612),\n",
       "             (u'[unused608]', 613),\n",
       "             (u'[unused609]', 614),\n",
       "             (u'[unused610]', 615),\n",
       "             (u'[unused611]', 616),\n",
       "             (u'[unused612]', 617),\n",
       "             (u'[unused613]', 618),\n",
       "             (u'[unused614]', 619),\n",
       "             (u'[unused615]', 620),\n",
       "             (u'[unused616]', 621),\n",
       "             (u'[unused617]', 622),\n",
       "             (u'[unused618]', 623),\n",
       "             (u'[unused619]', 624),\n",
       "             (u'[unused620]', 625),\n",
       "             (u'[unused621]', 626),\n",
       "             (u'[unused622]', 627),\n",
       "             (u'[unused623]', 628),\n",
       "             (u'[unused624]', 629),\n",
       "             (u'[unused625]', 630),\n",
       "             (u'[unused626]', 631),\n",
       "             (u'[unused627]', 632),\n",
       "             (u'[unused628]', 633),\n",
       "             (u'[unused629]', 634),\n",
       "             (u'[unused630]', 635),\n",
       "             (u'[unused631]', 636),\n",
       "             (u'[unused632]', 637),\n",
       "             (u'[unused633]', 638),\n",
       "             (u'[unused634]', 639),\n",
       "             (u'[unused635]', 640),\n",
       "             (u'[unused636]', 641),\n",
       "             (u'[unused637]', 642),\n",
       "             (u'[unused638]', 643),\n",
       "             (u'[unused639]', 644),\n",
       "             (u'[unused640]', 645),\n",
       "             (u'[unused641]', 646),\n",
       "             (u'[unused642]', 647),\n",
       "             (u'[unused643]', 648),\n",
       "             (u'[unused644]', 649),\n",
       "             (u'[unused645]', 650),\n",
       "             (u'[unused646]', 651),\n",
       "             (u'[unused647]', 652),\n",
       "             (u'[unused648]', 653),\n",
       "             (u'[unused649]', 654),\n",
       "             (u'[unused650]', 655),\n",
       "             (u'[unused651]', 656),\n",
       "             (u'[unused652]', 657),\n",
       "             (u'[unused653]', 658),\n",
       "             (u'[unused654]', 659),\n",
       "             (u'[unused655]', 660),\n",
       "             (u'[unused656]', 661),\n",
       "             (u'[unused657]', 662),\n",
       "             (u'[unused658]', 663),\n",
       "             (u'[unused659]', 664),\n",
       "             (u'[unused660]', 665),\n",
       "             (u'[unused661]', 666),\n",
       "             (u'[unused662]', 667),\n",
       "             (u'[unused663]', 668),\n",
       "             (u'[unused664]', 669),\n",
       "             (u'[unused665]', 670),\n",
       "             (u'[unused666]', 671),\n",
       "             (u'[unused667]', 672),\n",
       "             (u'[unused668]', 673),\n",
       "             (u'[unused669]', 674),\n",
       "             (u'[unused670]', 675),\n",
       "             (u'[unused671]', 676),\n",
       "             (u'[unused672]', 677),\n",
       "             (u'[unused673]', 678),\n",
       "             (u'[unused674]', 679),\n",
       "             (u'[unused675]', 680),\n",
       "             (u'[unused676]', 681),\n",
       "             (u'[unused677]', 682),\n",
       "             (u'[unused678]', 683),\n",
       "             (u'[unused679]', 684),\n",
       "             (u'[unused680]', 685),\n",
       "             (u'[unused681]', 686),\n",
       "             (u'[unused682]', 687),\n",
       "             (u'[unused683]', 688),\n",
       "             (u'[unused684]', 689),\n",
       "             (u'[unused685]', 690),\n",
       "             (u'[unused686]', 691),\n",
       "             (u'[unused687]', 692),\n",
       "             (u'[unused688]', 693),\n",
       "             (u'[unused689]', 694),\n",
       "             (u'[unused690]', 695),\n",
       "             (u'[unused691]', 696),\n",
       "             (u'[unused692]', 697),\n",
       "             (u'[unused693]', 698),\n",
       "             (u'[unused694]', 699),\n",
       "             (u'[unused695]', 700),\n",
       "             (u'[unused696]', 701),\n",
       "             (u'[unused697]', 702),\n",
       "             (u'[unused698]', 703),\n",
       "             (u'[unused699]', 704),\n",
       "             (u'[unused700]', 705),\n",
       "             (u'[unused701]', 706),\n",
       "             (u'[unused702]', 707),\n",
       "             (u'[unused703]', 708),\n",
       "             (u'[unused704]', 709),\n",
       "             (u'[unused705]', 710),\n",
       "             (u'[unused706]', 711),\n",
       "             (u'[unused707]', 712),\n",
       "             (u'[unused708]', 713),\n",
       "             (u'[unused709]', 714),\n",
       "             (u'[unused710]', 715),\n",
       "             (u'[unused711]', 716),\n",
       "             (u'[unused712]', 717),\n",
       "             (u'[unused713]', 718),\n",
       "             (u'[unused714]', 719),\n",
       "             (u'[unused715]', 720),\n",
       "             (u'[unused716]', 721),\n",
       "             (u'[unused717]', 722),\n",
       "             (u'[unused718]', 723),\n",
       "             (u'[unused719]', 724),\n",
       "             (u'[unused720]', 725),\n",
       "             (u'[unused721]', 726),\n",
       "             (u'[unused722]', 727),\n",
       "             (u'[unused723]', 728),\n",
       "             (u'[unused724]', 729),\n",
       "             (u'[unused725]', 730),\n",
       "             (u'[unused726]', 731),\n",
       "             (u'[unused727]', 732),\n",
       "             (u'[unused728]', 733),\n",
       "             (u'[unused729]', 734),\n",
       "             (u'[unused730]', 735),\n",
       "             (u'[unused731]', 736),\n",
       "             (u'[unused732]', 737),\n",
       "             (u'[unused733]', 738),\n",
       "             (u'[unused734]', 739),\n",
       "             (u'[unused735]', 740),\n",
       "             (u'[unused736]', 741),\n",
       "             (u'[unused737]', 742),\n",
       "             (u'[unused738]', 743),\n",
       "             (u'[unused739]', 744),\n",
       "             (u'[unused740]', 745),\n",
       "             (u'[unused741]', 746),\n",
       "             (u'[unused742]', 747),\n",
       "             (u'[unused743]', 748),\n",
       "             (u'[unused744]', 749),\n",
       "             (u'[unused745]', 750),\n",
       "             (u'[unused746]', 751),\n",
       "             (u'[unused747]', 752),\n",
       "             (u'[unused748]', 753),\n",
       "             (u'[unused749]', 754),\n",
       "             (u'[unused750]', 755),\n",
       "             (u'[unused751]', 756),\n",
       "             (u'[unused752]', 757),\n",
       "             (u'[unused753]', 758),\n",
       "             (u'[unused754]', 759),\n",
       "             (u'[unused755]', 760),\n",
       "             (u'[unused756]', 761),\n",
       "             (u'[unused757]', 762),\n",
       "             (u'[unused758]', 763),\n",
       "             (u'[unused759]', 764),\n",
       "             (u'[unused760]', 765),\n",
       "             (u'[unused761]', 766),\n",
       "             (u'[unused762]', 767),\n",
       "             (u'[unused763]', 768),\n",
       "             (u'[unused764]', 769),\n",
       "             (u'[unused765]', 770),\n",
       "             (u'[unused766]', 771),\n",
       "             (u'[unused767]', 772),\n",
       "             (u'[unused768]', 773),\n",
       "             (u'[unused769]', 774),\n",
       "             (u'[unused770]', 775),\n",
       "             (u'[unused771]', 776),\n",
       "             (u'[unused772]', 777),\n",
       "             (u'[unused773]', 778),\n",
       "             (u'[unused774]', 779),\n",
       "             (u'[unused775]', 780),\n",
       "             (u'[unused776]', 781),\n",
       "             (u'[unused777]', 782),\n",
       "             (u'[unused778]', 783),\n",
       "             (u'[unused779]', 784),\n",
       "             (u'[unused780]', 785),\n",
       "             (u'[unused781]', 786),\n",
       "             (u'[unused782]', 787),\n",
       "             (u'[unused783]', 788),\n",
       "             (u'[unused784]', 789),\n",
       "             (u'[unused785]', 790),\n",
       "             (u'[unused786]', 791),\n",
       "             (u'[unused787]', 792),\n",
       "             (u'[unused788]', 793),\n",
       "             (u'[unused789]', 794),\n",
       "             (u'[unused790]', 795),\n",
       "             (u'[unused791]', 796),\n",
       "             (u'[unused792]', 797),\n",
       "             (u'[unused793]', 798),\n",
       "             (u'[unused794]', 799),\n",
       "             (u'[unused795]', 800),\n",
       "             (u'[unused796]', 801),\n",
       "             (u'[unused797]', 802),\n",
       "             (u'[unused798]', 803),\n",
       "             (u'[unused799]', 804),\n",
       "             (u'[unused800]', 805),\n",
       "             (u'[unused801]', 806),\n",
       "             (u'[unused802]', 807),\n",
       "             (u'[unused803]', 808),\n",
       "             (u'[unused804]', 809),\n",
       "             (u'[unused805]', 810),\n",
       "             (u'[unused806]', 811),\n",
       "             (u'[unused807]', 812),\n",
       "             (u'[unused808]', 813),\n",
       "             (u'[unused809]', 814),\n",
       "             (u'[unused810]', 815),\n",
       "             (u'[unused811]', 816),\n",
       "             (u'[unused812]', 817),\n",
       "             (u'[unused813]', 818),\n",
       "             (u'[unused814]', 819),\n",
       "             (u'[unused815]', 820),\n",
       "             (u'[unused816]', 821),\n",
       "             (u'[unused817]', 822),\n",
       "             (u'[unused818]', 823),\n",
       "             (u'[unused819]', 824),\n",
       "             (u'[unused820]', 825),\n",
       "             (u'[unused821]', 826),\n",
       "             (u'[unused822]', 827),\n",
       "             (u'[unused823]', 828),\n",
       "             (u'[unused824]', 829),\n",
       "             (u'[unused825]', 830),\n",
       "             (u'[unused826]', 831),\n",
       "             (u'[unused827]', 832),\n",
       "             (u'[unused828]', 833),\n",
       "             (u'[unused829]', 834),\n",
       "             (u'[unused830]', 835),\n",
       "             (u'[unused831]', 836),\n",
       "             (u'[unused832]', 837),\n",
       "             (u'[unused833]', 838),\n",
       "             (u'[unused834]', 839),\n",
       "             (u'[unused835]', 840),\n",
       "             (u'[unused836]', 841),\n",
       "             (u'[unused837]', 842),\n",
       "             (u'[unused838]', 843),\n",
       "             (u'[unused839]', 844),\n",
       "             (u'[unused840]', 845),\n",
       "             (u'[unused841]', 846),\n",
       "             (u'[unused842]', 847),\n",
       "             (u'[unused843]', 848),\n",
       "             (u'[unused844]', 849),\n",
       "             (u'[unused845]', 850),\n",
       "             (u'[unused846]', 851),\n",
       "             (u'[unused847]', 852),\n",
       "             (u'[unused848]', 853),\n",
       "             (u'[unused849]', 854),\n",
       "             (u'[unused850]', 855),\n",
       "             (u'[unused851]', 856),\n",
       "             (u'[unused852]', 857),\n",
       "             (u'[unused853]', 858),\n",
       "             (u'[unused854]', 859),\n",
       "             (u'[unused855]', 860),\n",
       "             (u'[unused856]', 861),\n",
       "             (u'[unused857]', 862),\n",
       "             (u'[unused858]', 863),\n",
       "             (u'[unused859]', 864),\n",
       "             (u'[unused860]', 865),\n",
       "             (u'[unused861]', 866),\n",
       "             (u'[unused862]', 867),\n",
       "             (u'[unused863]', 868),\n",
       "             (u'[unused864]', 869),\n",
       "             (u'[unused865]', 870),\n",
       "             (u'[unused866]', 871),\n",
       "             (u'[unused867]', 872),\n",
       "             (u'[unused868]', 873),\n",
       "             (u'[unused869]', 874),\n",
       "             (u'[unused870]', 875),\n",
       "             (u'[unused871]', 876),\n",
       "             (u'[unused872]', 877),\n",
       "             (u'[unused873]', 878),\n",
       "             (u'[unused874]', 879),\n",
       "             (u'[unused875]', 880),\n",
       "             (u'[unused876]', 881),\n",
       "             (u'[unused877]', 882),\n",
       "             (u'[unused878]', 883),\n",
       "             (u'[unused879]', 884),\n",
       "             (u'[unused880]', 885),\n",
       "             (u'[unused881]', 886),\n",
       "             (u'[unused882]', 887),\n",
       "             (u'[unused883]', 888),\n",
       "             (u'[unused884]', 889),\n",
       "             (u'[unused885]', 890),\n",
       "             (u'[unused886]', 891),\n",
       "             (u'[unused887]', 892),\n",
       "             (u'[unused888]', 893),\n",
       "             (u'[unused889]', 894),\n",
       "             (u'[unused890]', 895),\n",
       "             (u'[unused891]', 896),\n",
       "             (u'[unused892]', 897),\n",
       "             (u'[unused893]', 898),\n",
       "             (u'[unused894]', 899),\n",
       "             (u'[unused895]', 900),\n",
       "             (u'[unused896]', 901),\n",
       "             (u'[unused897]', 902),\n",
       "             (u'[unused898]', 903),\n",
       "             (u'[unused899]', 904),\n",
       "             (u'[unused900]', 905),\n",
       "             (u'[unused901]', 906),\n",
       "             (u'[unused902]', 907),\n",
       "             (u'[unused903]', 908),\n",
       "             (u'[unused904]', 909),\n",
       "             (u'[unused905]', 910),\n",
       "             (u'[unused906]', 911),\n",
       "             (u'[unused907]', 912),\n",
       "             (u'[unused908]', 913),\n",
       "             (u'[unused909]', 914),\n",
       "             (u'[unused910]', 915),\n",
       "             (u'[unused911]', 916),\n",
       "             (u'[unused912]', 917),\n",
       "             (u'[unused913]', 918),\n",
       "             (u'[unused914]', 919),\n",
       "             (u'[unused915]', 920),\n",
       "             (u'[unused916]', 921),\n",
       "             (u'[unused917]', 922),\n",
       "             (u'[unused918]', 923),\n",
       "             (u'[unused919]', 924),\n",
       "             (u'[unused920]', 925),\n",
       "             (u'[unused921]', 926),\n",
       "             (u'[unused922]', 927),\n",
       "             (u'[unused923]', 928),\n",
       "             (u'[unused924]', 929),\n",
       "             (u'[unused925]', 930),\n",
       "             (u'[unused926]', 931),\n",
       "             (u'[unused927]', 932),\n",
       "             (u'[unused928]', 933),\n",
       "             (u'[unused929]', 934),\n",
       "             (u'[unused930]', 935),\n",
       "             (u'[unused931]', 936),\n",
       "             (u'[unused932]', 937),\n",
       "             (u'[unused933]', 938),\n",
       "             (u'[unused934]', 939),\n",
       "             (u'[unused935]', 940),\n",
       "             (u'[unused936]', 941),\n",
       "             (u'[unused937]', 942),\n",
       "             (u'[unused938]', 943),\n",
       "             (u'[unused939]', 944),\n",
       "             (u'[unused940]', 945),\n",
       "             (u'[unused941]', 946),\n",
       "             (u'[unused942]', 947),\n",
       "             (u'[unused943]', 948),\n",
       "             (u'[unused944]', 949),\n",
       "             (u'[unused945]', 950),\n",
       "             (u'[unused946]', 951),\n",
       "             (u'[unused947]', 952),\n",
       "             (u'[unused948]', 953),\n",
       "             (u'[unused949]', 954),\n",
       "             (u'[unused950]', 955),\n",
       "             (u'[unused951]', 956),\n",
       "             (u'[unused952]', 957),\n",
       "             (u'[unused953]', 958),\n",
       "             (u'[unused954]', 959),\n",
       "             (u'[unused955]', 960),\n",
       "             (u'[unused956]', 961),\n",
       "             (u'[unused957]', 962),\n",
       "             (u'[unused958]', 963),\n",
       "             (u'[unused959]', 964),\n",
       "             (u'[unused960]', 965),\n",
       "             (u'[unused961]', 966),\n",
       "             (u'[unused962]', 967),\n",
       "             (u'[unused963]', 968),\n",
       "             (u'[unused964]', 969),\n",
       "             (u'[unused965]', 970),\n",
       "             (u'[unused966]', 971),\n",
       "             (u'[unused967]', 972),\n",
       "             (u'[unused968]', 973),\n",
       "             (u'[unused969]', 974),\n",
       "             (u'[unused970]', 975),\n",
       "             (u'[unused971]', 976),\n",
       "             (u'[unused972]', 977),\n",
       "             (u'[unused973]', 978),\n",
       "             (u'[unused974]', 979),\n",
       "             (u'[unused975]', 980),\n",
       "             (u'[unused976]', 981),\n",
       "             (u'[unused977]', 982),\n",
       "             (u'[unused978]', 983),\n",
       "             (u'[unused979]', 984),\n",
       "             (u'[unused980]', 985),\n",
       "             (u'[unused981]', 986),\n",
       "             (u'[unused982]', 987),\n",
       "             (u'[unused983]', 988),\n",
       "             (u'[unused984]', 989),\n",
       "             (u'[unused985]', 990),\n",
       "             (u'[unused986]', 991),\n",
       "             (u'[unused987]', 992),\n",
       "             (u'[unused988]', 993),\n",
       "             (u'[unused989]', 994),\n",
       "             (u'[unused990]', 995),\n",
       "             (u'[unused991]', 996),\n",
       "             (u'[unused992]', 997),\n",
       "             (u'[unused993]', 998),\n",
       "             (u'!', 999),\n",
       "             ...])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.vocab "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'mw_list' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-81-87c5a3ac2d46>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmw_list\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmw_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'@ Home Executive Apartment'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'found'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'mw_list' is not defined"
     ]
    }
   ],
   "source": [
    "print(mw_list[:5])\n",
    "for i in mw_list:\n",
    "    if \" \".join(i) == '@ Home Executive Apartment':\n",
    "        print('found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_vocab(vocab_tuple_list):\n",
    "    start = len(tokenizer.vocab)\n",
    "    \n",
    "    for v in vocab_tuple_list:\n",
    "        ww = \" \".join(v)\n",
    "        if ww not in tokenizer.vocab:\n",
    "            #print(v)\n",
    "            tokenizer.vocab[ww] = start\n",
    "            print(ww)\n",
    "            start += 1\n",
    "    return tokenizer.vocab\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_ids = [  tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41276"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.vocab['Pattya']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 17\n"
     ]
    }
   ],
   "source": [
    " print(len(tokens_ids[0]),len(labels[0]))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "#input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "input_ids = pad_sequences(tokens_ids,\n",
    "                          maxlen=MAX_LEN, dtype=\"int64\", truncating=\"post\", padding=\"post\")\n",
    "#input_ids = pad_sequences([convert_word2idx(txt) for txt in tokenized_texts],\n",
    "                          #maxlen=MAX_LEN, dtype=\"int64\", truncating=\"post\", padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                          maxlen=MAX_LEN, dtype=\"int64\", truncating=\"post\", padding=\"post\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120\n",
      "120\n"
     ]
    }
   ],
   "source": [
    "print(MAX_LEN)\n",
    "for i in tokens_ids:\n",
    "    if len(i) > MAX_LEN:\n",
    "        #print(tokens_ids)\n",
    "        print(\"need more\")\n",
    "        MAX_LEN = len(i)\n",
    "print(MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_list = [[tag2idx.get(l) for l in lab] for lab in labels]\n",
    "len(t_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6595 2012 1030 2188 3237 4545 2003 1037 2204 3601 2043 2017 2024 5873\n",
      " 2430 6986 2696 3148 1012    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0]\n"
     ]
    }
   ],
   "source": [
    "print(input_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels],\n",
    "                     maxlen=MAX_LEN, value=tag2idx[\"O\"], padding=\"post\",\n",
    "                     dtype=\"int64\", truncating=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_masks = [[float(i>0) for i in ii] for ii in input_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split train test\n",
    "tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(input_ids, tags, \n",
    "                                                            random_state=2018, test_size=0.3)\n",
    "tr_masks, val_masks, _, _ = train_test_split(attention_masks, input_ids,\n",
    "                                             random_state=2018, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#change to torhc tensor\n",
    "tr_inputs = torch.tensor(tr_inputs)\n",
    "val_inputs = torch.tensor(val_inputs)\n",
    "tr_tags = torch.tensor(tr_tags)\n",
    "val_tags = torch.tensor(val_tags)\n",
    "tr_masks = torch.tensor(tr_masks)\n",
    "val_masks = torch.tensor(val_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([18578, 120])\n",
      "torch.Size([18578, 120])\n",
      "torch.Size([18578, 120])\n"
     ]
    }
   ],
   "source": [
    "print(tr_inputs.shape)\n",
    "print(tr_masks.shape)\n",
    "print(tr_tags.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([7963, 120])\n",
      "torch.Size([7963, 120])\n",
      "torch.Size([7963, 120])\n"
     ]
    }
   ],
   "source": [
    "print(val_inputs.shape)\n",
    "print(val_masks.shape)\n",
    "print(val_tags.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=bs)\n",
    "\n",
    "valid_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "valid_sampler = SequentialSampler(valid_data)\n",
    "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "print(len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-03-18 13:44:06,944 : INFO : loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /home/chantana/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n",
      "2019-03-18 13:44:06,948 : INFO : extracting archive file /home/chantana/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /tmp/tmpyWtLQx\n",
      "2019-03-18 13:44:10,503 : INFO : Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1, \n",
      "  \"hidden_act\": \"gelu\", \n",
      "  \"hidden_dropout_prob\": 0.1, \n",
      "  \"hidden_size\": 768, \n",
      "  \"initializer_range\": 0.02, \n",
      "  \"intermediate_size\": 3072, \n",
      "  \"max_position_embeddings\": 512, \n",
      "  \"num_attention_heads\": 12, \n",
      "  \"num_hidden_layers\": 12, \n",
      "  \"type_vocab_size\": 2, \n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "2019-03-18 13:44:13,586 : INFO : Weights of BertForTokenClassification not initialized from pretrained model: [u'classifier.bias', u'classifier.weight']\n",
      "2019-03-18 13:44:13,587 : INFO : Weights from pretrained model not used in BertForTokenClassification: [u'cls.predictions.bias', u'cls.predictions.transform.dense.weight', u'cls.predictions.transform.dense.bias', u'cls.predictions.decoder.weight', u'cls.seq_relationship.weight', u'cls.seq_relationship.bias', u'cls.predictions.transform.LayerNorm.weight', u'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "model = BertForTokenClassification.from_pretrained(u\"bert-base-uncased\", num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.cuda();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters()) \n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fine tune BERT\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch:   0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.381081105619\n",
      "Validation loss: 0.255078543651\n",
      "Validation Accuracy: 0.800869450022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch:  20%|██        | 1/5 [08:23<33:34, 503.75s/it]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-Score: 0.267616214074\n",
      "Train loss: 0.221693574455\n",
      "Validation loss: 0.171089398483\n",
      "Validation Accuracy: 0.833422579206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch:  40%|████      | 2/5 [27:59<35:16, 705.39s/it]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-Score: 0.366658020763\n",
      "Train loss: 0.169982631978\n",
      "Validation loss: 0.155627313331\n",
      "Validation Accuracy: 0.840877398483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch:  60%|██████    | 3/5 [36:43<21:41, 650.88s/it]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-Score: 0.384366901261\n",
      "Train loss: 0.141815415096\n",
      "Validation loss: 0.139297030737\n",
      "Validation Accuracy: 0.842304635207\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch:  80%|████████  | 4/5 [44:12<09:50, 590.32s/it]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-Score: 0.391983509768\n",
      "Train loss: 0.122520411009\n",
      "Validation loss: 0.1095122506\n",
      "Validation Accuracy: 0.866085174029\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Epoch: 100%|██████████| 5/5 [50:58<00:00, 535.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-Score: 0.464652200901\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        # forward pass\n",
    "        loss = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
    "    # VALIDATION on validation set\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions , true_labels = [], []\n",
    "    for batch in valid_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                                  attention_mask=b_input_mask, labels=b_labels)\n",
    "            logits = model(b_input_ids, token_type_ids=None,\n",
    "                           attention_mask=b_input_mask)\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        true_labels.append(label_ids)\n",
    "        \n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "        \n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "        \n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss/nb_eval_steps\n",
    "    print(\"Validation loss: {}\".format(eval_loss))\n",
    "    print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))\n",
    "    pred_tags = [tags_vals[p_i] for p in predictions for p_i in p]\n",
    "    valid_tags = [tags_vals[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
    "    print(\"F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/996 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 1/996 [00:00<05:22,  3.09it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 2/996 [00:00<05:24,  3.06it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 3/996 [00:00<05:24,  3.06it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 4/996 [00:01<05:11,  3.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 5/996 [00:01<05:03,  3.27it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 6/996 [00:01<05:01,  3.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 7/996 [00:02<05:06,  3.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 8/996 [00:02<05:09,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 9/996 [00:02<04:51,  3.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 10/996 [00:03<04:56,  3.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 11/996 [00:03<04:59,  3.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|          | 12/996 [00:03<05:00,  3.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|▏         | 13/996 [00:03<04:55,  3.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  1%|▏         | 14/996 [00:04<04:55,  3.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 15/996 [00:04<04:59,  3.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 16/996 [00:04<04:57,  3.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 17/996 [00:05<04:56,  3.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 18/996 [00:05<05:05,  3.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 19/996 [00:05<05:10,  3.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 20/996 [00:06<05:13,  3.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 21/996 [00:06<05:15,  3.09it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 22/996 [00:06<05:17,  3.07it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 23/996 [00:07<05:25,  2.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  2%|▏         | 24/996 [00:07<05:30,  2.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 25/996 [00:07<05:34,  2.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 26/996 [00:08<05:25,  2.98it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 27/996 [00:08<05:20,  3.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 28/996 [00:08<05:17,  3.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 29/996 [00:09<05:09,  3.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 30/996 [00:09<05:06,  3.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 31/996 [00:09<05:05,  3.16it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 32/996 [00:10<05:05,  3.16it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 33/996 [00:10<05:02,  3.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 34/996 [00:10<04:56,  3.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▎         | 35/996 [00:11<04:55,  3.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▎         | 36/996 [00:11<04:51,  3.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▎         | 37/996 [00:11<04:55,  3.24it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 38/996 [00:11<04:58,  3.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 39/996 [00:12<04:59,  3.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 40/996 [00:12<04:59,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 41/996 [00:12<04:58,  3.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 42/996 [00:13<04:55,  3.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 43/996 [00:13<04:57,  3.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  4%|▍         | 44/996 [00:13<04:58,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▍         | 45/996 [00:14<04:59,  3.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▍         | 46/996 [00:14<05:01,  3.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▍         | 47/996 [00:14<05:03,  3.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▍         | 48/996 [00:15<05:06,  3.09it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▍         | 49/996 [00:15<04:55,  3.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▌         | 50/996 [00:15<04:55,  3.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▌         | 51/996 [00:16<04:56,  3.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▌         | 52/996 [00:16<04:52,  3.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▌         | 53/996 [00:16<04:55,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  5%|▌         | 54/996 [00:16<04:55,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 55/996 [00:17<04:55,  3.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 56/996 [00:17<04:52,  3.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 57/996 [00:17<04:45,  3.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 58/996 [00:18<04:45,  3.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 59/996 [00:18<04:49,  3.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 60/996 [00:18<04:43,  3.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▌         | 61/996 [00:18<04:06,  3.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 63/996 [00:19<03:11,  4.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  7%|▋         | 65/996 [00:19<02:32,  6.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  7%|▋         | 67/996 [00:19<02:05,  7.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  7%|▋         | 69/996 [00:19<01:45,  8.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  7%|▋         | 71/996 [00:19<01:32,  9.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  7%|▋         | 73/996 [00:19<01:23, 11.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  8%|▊         | 75/996 [00:19<01:16, 12.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  8%|▊         | 77/996 [00:20<01:11, 12.80it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  8%|▊         | 79/996 [00:20<01:08, 13.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  8%|▊         | 81/996 [00:20<01:06, 13.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  8%|▊         | 83/996 [00:20<01:04, 14.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  9%|▊         | 85/996 [00:20<01:03, 14.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  9%|▊         | 87/996 [00:20<01:02, 14.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  9%|▉         | 89/996 [00:20<01:02, 14.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  9%|▉         | 91/996 [00:20<01:01, 14.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  9%|▉         | 93/996 [00:21<01:00, 14.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 10%|▉         | 95/996 [00:21<01:00, 14.87it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 10%|▉         | 97/996 [00:21<01:00, 14.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 10%|▉         | 99/996 [00:21<01:00, 14.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 10%|█         | 101/996 [00:21<01:00, 14.72it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 10%|█         | 103/996 [00:21<01:00, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 11%|█         | 105/996 [00:21<01:01, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 11%|█         | 107/996 [00:22<01:01, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 11%|█         | 109/996 [00:22<01:00, 14.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 11%|█         | 111/996 [00:22<01:01, 14.47it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 11%|█▏        | 113/996 [00:22<01:00, 14.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▏        | 115/996 [00:22<01:00, 14.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▏        | 117/996 [00:22<00:59, 14.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▏        | 119/996 [00:22<00:59, 14.78it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▏        | 121/996 [00:23<00:58, 14.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▏        | 123/996 [00:23<00:58, 14.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 125/996 [00:23<00:59, 14.70it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 127/996 [00:23<00:59, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 129/996 [00:23<00:59, 14.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 131/996 [00:23<01:00, 14.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 133/996 [00:23<01:01, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 14%|█▎        | 135/996 [00:24<00:59, 14.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 14%|█▍        | 137/996 [00:24<01:00, 14.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 14%|█▍        | 139/996 [00:24<01:00, 14.12it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 14%|█▍        | 141/996 [00:24<01:00, 14.08it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 14%|█▍        | 143/996 [00:24<01:00, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 15%|█▍        | 145/996 [00:24<01:00, 14.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 15%|█▍        | 147/996 [00:24<01:00, 14.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 15%|█▍        | 149/996 [00:24<00:58, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 15%|█▌        | 151/996 [00:25<00:58, 14.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 15%|█▌        | 153/996 [00:25<00:58, 14.47it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▌        | 155/996 [00:25<00:58, 14.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▌        | 157/996 [00:25<00:58, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▌        | 159/996 [00:25<00:58, 14.31it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▌        | 161/996 [00:25<00:59, 14.08it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▋        | 163/996 [00:25<00:59, 14.07it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 17%|█▋        | 165/996 [00:26<00:58, 14.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 17%|█▋        | 167/996 [00:26<00:57, 14.49it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 17%|█▋        | 169/996 [00:26<00:57, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 17%|█▋        | 171/996 [00:26<00:57, 14.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 17%|█▋        | 173/996 [00:26<00:57, 14.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 18%|█▊        | 175/996 [00:26<00:57, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 18%|█▊        | 177/996 [00:26<00:57, 14.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 18%|█▊        | 179/996 [00:27<00:56, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 18%|█▊        | 181/996 [00:27<00:57, 14.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 18%|█▊        | 183/996 [00:27<00:57, 14.07it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▊        | 185/996 [00:27<00:56, 14.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 187/996 [00:27<00:55, 14.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 189/996 [00:27<00:55, 14.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 191/996 [00:27<00:54, 14.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 193/996 [00:28<00:54, 14.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 20%|█▉        | 195/996 [00:28<00:53, 14.87it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 20%|█▉        | 197/996 [00:28<00:53, 14.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 20%|█▉        | 199/996 [00:28<00:53, 14.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 20%|██        | 201/996 [00:28<00:54, 14.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 20%|██        | 203/996 [00:28<00:54, 14.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 21%|██        | 205/996 [00:28<00:54, 14.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 21%|██        | 207/996 [00:29<00:55, 14.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 21%|██        | 209/996 [00:29<00:55, 14.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 21%|██        | 211/996 [00:29<00:55, 14.09it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 21%|██▏       | 213/996 [00:29<00:54, 14.27it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 22%|██▏       | 215/996 [00:29<00:54, 14.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 22%|██▏       | 217/996 [00:29<00:54, 14.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 22%|██▏       | 219/996 [00:29<00:54, 14.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 22%|██▏       | 221/996 [00:29<00:54, 14.22it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 22%|██▏       | 223/996 [00:30<00:56, 13.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 225/996 [00:30<00:55, 13.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 227/996 [00:30<00:54, 14.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 229/996 [00:30<00:54, 14.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 231/996 [00:30<00:54, 14.07it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 233/996 [00:30<00:54, 14.02it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 24%|██▎       | 235/996 [00:30<00:54, 14.08it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 24%|██▍       | 237/996 [00:31<00:53, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 24%|██▍       | 239/996 [00:31<00:53, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 24%|██▍       | 241/996 [00:31<00:55, 13.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 24%|██▍       | 243/996 [00:31<00:55, 13.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▍       | 245/996 [00:31<00:55, 13.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▍       | 247/996 [00:31<00:54, 13.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 249/996 [00:32<00:53, 13.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 251/996 [00:32<00:52, 14.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 253/996 [00:32<00:51, 14.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▌       | 255/996 [00:32<00:52, 14.22it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▌       | 257/996 [00:32<00:51, 14.26it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▌       | 259/996 [00:32<00:51, 14.31it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▌       | 261/996 [00:32<00:51, 14.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▋       | 263/996 [00:32<00:51, 14.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 27%|██▋       | 265/996 [00:33<00:51, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 27%|██▋       | 267/996 [00:33<00:51, 14.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 27%|██▋       | 269/996 [00:33<00:51, 14.01it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 27%|██▋       | 271/996 [00:33<00:50, 14.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 27%|██▋       | 273/996 [00:33<00:50, 14.43it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 28%|██▊       | 275/996 [00:33<00:49, 14.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 28%|██▊       | 277/996 [00:33<00:49, 14.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 28%|██▊       | 279/996 [00:34<00:49, 14.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 28%|██▊       | 281/996 [00:34<00:49, 14.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 28%|██▊       | 283/996 [00:34<00:49, 14.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▊       | 285/996 [00:34<00:49, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▉       | 287/996 [00:34<00:48, 14.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▉       | 289/996 [00:34<00:48, 14.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▉       | 291/996 [00:34<00:49, 14.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▉       | 293/996 [00:35<00:48, 14.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 30%|██▉       | 295/996 [00:35<00:48, 14.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 30%|██▉       | 297/996 [00:35<00:49, 14.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 30%|███       | 299/996 [00:35<00:48, 14.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 30%|███       | 301/996 [00:35<00:48, 14.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 30%|███       | 303/996 [00:35<00:48, 14.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███       | 305/996 [00:35<00:48, 14.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███       | 307/996 [00:36<00:47, 14.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███       | 309/996 [00:36<00:46, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███       | 311/996 [00:36<00:47, 14.42it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 313/996 [00:36<00:47, 14.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 315/996 [00:36<00:47, 14.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 317/996 [00:36<00:46, 14.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 319/996 [00:36<00:46, 14.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 321/996 [00:37<00:46, 14.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 323/996 [00:37<00:46, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 33%|███▎      | 325/996 [00:37<00:46, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 33%|███▎      | 327/996 [00:37<00:46, 14.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 33%|███▎      | 329/996 [00:37<00:46, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 33%|███▎      | 331/996 [00:37<00:45, 14.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 33%|███▎      | 333/996 [00:37<00:45, 14.72it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 34%|███▎      | 335/996 [00:37<00:45, 14.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 34%|███▍      | 337/996 [00:38<00:45, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 34%|███▍      | 339/996 [00:38<00:45, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 34%|███▍      | 341/996 [00:38<00:45, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 34%|███▍      | 343/996 [00:38<00:46, 14.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▍      | 345/996 [00:38<00:45, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▍      | 347/996 [00:38<00:44, 14.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▌      | 349/996 [00:38<00:44, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▌      | 351/996 [00:39<00:44, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▌      | 353/996 [00:39<00:44, 14.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 36%|███▌      | 355/996 [00:39<00:44, 14.42it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 36%|███▌      | 357/996 [00:39<00:44, 14.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 36%|███▌      | 359/996 [00:39<00:44, 14.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 36%|███▌      | 361/996 [00:39<00:44, 14.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 36%|███▋      | 363/996 [00:39<00:44, 14.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 37%|███▋      | 365/996 [00:40<00:43, 14.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 37%|███▋      | 367/996 [00:40<00:43, 14.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 37%|███▋      | 369/996 [00:40<00:42, 14.66it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 37%|███▋      | 371/996 [00:40<00:42, 14.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 37%|███▋      | 373/996 [00:40<00:42, 14.70it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 375/996 [00:40<00:42, 14.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 377/996 [00:40<00:42, 14.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 379/996 [00:41<00:42, 14.66it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 381/996 [00:41<00:41, 14.66it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 383/996 [00:41<00:41, 14.70it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▊      | 385/996 [00:41<00:41, 14.78it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▉      | 387/996 [00:41<00:41, 14.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▉      | 389/996 [00:41<00:41, 14.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▉      | 391/996 [00:41<00:44, 13.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▉      | 393/996 [00:42<00:46, 13.08it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 40%|███▉      | 395/996 [00:42<00:46, 12.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 40%|███▉      | 397/996 [00:42<00:46, 12.96it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 40%|████      | 399/996 [00:42<00:44, 13.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 40%|████      | 401/996 [00:42<00:43, 13.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 40%|████      | 403/996 [00:42<00:42, 13.82it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 41%|████      | 405/996 [00:42<00:42, 13.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 41%|████      | 407/996 [00:43<00:41, 14.02it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 41%|████      | 409/996 [00:43<00:41, 14.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 41%|████▏     | 411/996 [00:43<00:43, 13.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 41%|████▏     | 413/996 [00:43<00:43, 13.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 415/996 [00:43<00:45, 12.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 417/996 [00:43<00:45, 12.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 419/996 [00:43<00:45, 12.73it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 421/996 [00:44<00:44, 13.01it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 423/996 [00:44<00:42, 13.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 43%|████▎     | 425/996 [00:44<00:42, 13.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 43%|████▎     | 427/996 [00:44<00:42, 13.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 43%|████▎     | 429/996 [00:44<00:42, 13.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 43%|████▎     | 431/996 [00:44<00:41, 13.49it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 43%|████▎     | 433/996 [00:45<00:41, 13.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▎     | 435/996 [00:45<00:41, 13.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 437/996 [00:45<00:41, 13.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 439/996 [00:45<00:41, 13.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 441/996 [00:45<00:40, 13.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 443/996 [00:45<00:40, 13.72it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 45%|████▍     | 445/996 [00:45<00:39, 13.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 447/996 [00:46<00:39, 14.07it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 45%|████▌     | 449/996 [00:46<00:38, 14.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 45%|████▌     | 451/996 [00:46<00:38, 14.02it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 45%|████▌     | 453/996 [00:46<00:38, 14.02it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 46%|████▌     | 455/996 [00:46<00:38, 14.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 46%|████▌     | 457/996 [00:46<00:38, 14.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 46%|████▌     | 459/996 [00:46<00:37, 14.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 46%|████▋     | 461/996 [00:47<00:37, 14.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 46%|████▋     | 463/996 [00:47<00:37, 14.20it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 47%|████▋     | 465/996 [00:47<00:36, 14.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 47%|████▋     | 467/996 [00:47<00:36, 14.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 47%|████▋     | 469/996 [00:47<00:36, 14.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 47%|████▋     | 471/996 [00:47<00:38, 13.70it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 47%|████▋     | 473/996 [00:47<00:38, 13.69it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 475/996 [00:48<00:37, 14.00it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 477/996 [00:48<00:36, 14.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 479/996 [00:48<00:35, 14.47it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 481/996 [00:48<00:35, 14.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 483/996 [00:48<00:36, 14.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 49%|████▊     | 485/996 [00:48<00:36, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 49%|████▉     | 487/996 [00:48<00:36, 13.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 49%|████▉     | 489/996 [00:49<00:36, 13.80it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 49%|████▉     | 491/996 [00:49<00:36, 13.72it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 49%|████▉     | 493/996 [00:49<00:36, 13.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|████▉     | 495/996 [00:49<00:36, 13.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|████▉     | 497/996 [00:49<00:37, 13.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 499/996 [00:49<00:36, 13.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 501/996 [00:49<00:35, 13.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 51%|█████     | 503/996 [00:50<00:35, 13.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 51%|█████     | 505/996 [00:50<00:35, 13.98it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 51%|█████     | 507/996 [00:50<00:34, 13.98it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 51%|█████     | 509/996 [00:50<00:35, 13.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 51%|█████▏    | 511/996 [00:50<00:35, 13.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 513/996 [00:50<00:34, 14.00it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 515/996 [00:50<00:33, 14.26it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 517/996 [00:51<00:33, 14.12it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 519/996 [00:51<00:34, 13.87it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 521/996 [00:51<00:34, 13.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 53%|█████▎    | 523/996 [00:51<00:34, 13.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 53%|█████▎    | 525/996 [00:51<00:34, 13.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 53%|█████▎    | 527/996 [00:51<00:36, 13.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 53%|█████▎    | 529/996 [00:51<00:35, 13.16it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 53%|█████▎    | 531/996 [00:52<00:34, 13.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 54%|█████▎    | 533/996 [00:52<00:33, 13.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 54%|█████▎    | 535/996 [00:52<00:33, 13.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 54%|█████▍    | 537/996 [00:52<00:32, 14.00it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 54%|█████▍    | 539/996 [00:52<00:33, 13.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 54%|█████▍    | 541/996 [00:52<00:34, 13.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▍    | 543/996 [00:52<00:33, 13.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▍    | 545/996 [00:53<00:33, 13.31it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▍    | 547/996 [00:53<00:34, 13.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▌    | 549/996 [00:53<00:33, 13.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▌    | 551/996 [00:53<00:32, 13.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▌    | 553/996 [00:53<00:32, 13.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▌    | 555/996 [00:53<00:32, 13.73it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▌    | 557/996 [00:53<00:31, 13.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▌    | 559/996 [00:54<00:31, 13.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 561/996 [00:54<00:31, 13.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 57%|█████▋    | 563/996 [00:54<00:31, 13.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 57%|█████▋    | 565/996 [00:54<00:31, 13.82it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 57%|█████▋    | 567/996 [00:54<00:30, 14.00it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 57%|█████▋    | 569/996 [00:54<00:30, 14.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 57%|█████▋    | 571/996 [00:54<00:29, 14.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 573/996 [00:55<00:29, 14.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 575/996 [00:55<00:29, 14.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 577/996 [00:55<00:29, 13.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 579/996 [00:55<00:29, 13.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 581/996 [00:55<00:29, 13.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 59%|█████▊    | 583/996 [00:55<00:29, 14.01it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 59%|█████▊    | 585/996 [00:55<00:28, 14.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 59%|█████▉    | 587/996 [00:56<00:29, 14.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 59%|█████▉    | 589/996 [00:56<00:29, 14.00it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 59%|█████▉    | 591/996 [00:56<00:29, 13.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 60%|█████▉    | 593/996 [00:56<00:28, 13.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 60%|█████▉    | 595/996 [00:56<00:28, 13.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 60%|█████▉    | 597/996 [00:56<00:28, 13.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 60%|██████    | 599/996 [00:56<00:28, 13.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 60%|██████    | 601/996 [00:57<00:28, 13.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████    | 603/996 [00:57<00:28, 13.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████    | 605/996 [00:57<00:28, 13.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████    | 607/996 [00:57<00:27, 13.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████    | 609/996 [00:57<00:27, 13.94it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████▏   | 611/996 [00:57<00:27, 13.80it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▏   | 613/996 [00:58<00:28, 13.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▏   | 615/996 [00:58<00:28, 13.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▏   | 617/996 [00:58<00:28, 13.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▏   | 619/996 [00:58<00:28, 13.46it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▏   | 621/996 [00:58<00:27, 13.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 63%|██████▎   | 623/996 [00:58<00:27, 13.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 63%|██████▎   | 625/996 [00:58<00:27, 13.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 63%|██████▎   | 627/996 [00:59<00:27, 13.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 63%|██████▎   | 629/996 [00:59<00:26, 13.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 63%|██████▎   | 631/996 [00:59<00:26, 13.85it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 64%|██████▎   | 633/996 [00:59<00:26, 13.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 64%|██████▍   | 635/996 [00:59<00:25, 13.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 64%|██████▍   | 637/996 [00:59<00:25, 13.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 64%|██████▍   | 639/996 [00:59<00:25, 13.96it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 64%|██████▍   | 641/996 [01:00<00:25, 13.84it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▍   | 643/996 [01:00<00:25, 13.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▍   | 645/996 [01:00<00:25, 13.69it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▍   | 647/996 [01:00<00:25, 13.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▌   | 649/996 [01:00<00:25, 13.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▌   | 651/996 [01:00<00:25, 13.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 66%|██████▌   | 653/996 [01:00<00:25, 13.31it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 66%|██████▌   | 655/996 [01:01<00:25, 13.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 66%|██████▌   | 657/996 [01:01<00:26, 12.91it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 66%|██████▌   | 659/996 [01:01<00:26, 12.85it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 66%|██████▋   | 661/996 [01:01<00:26, 12.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 67%|██████▋   | 663/996 [01:01<00:26, 12.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 67%|██████▋   | 665/996 [01:01<00:26, 12.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 67%|██████▋   | 667/996 [01:02<00:26, 12.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 67%|██████▋   | 669/996 [01:02<00:25, 12.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 67%|██████▋   | 671/996 [01:02<00:25, 12.68it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 673/996 [01:02<00:25, 12.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 675/996 [01:02<00:24, 12.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 677/996 [01:02<00:24, 12.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 679/996 [01:02<00:24, 13.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 681/996 [01:03<00:25, 12.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▊   | 683/996 [01:03<00:25, 12.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 685/996 [01:03<00:25, 12.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 687/996 [01:03<00:25, 12.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 689/996 [01:03<00:24, 12.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 691/996 [01:03<00:24, 12.43it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 70%|██████▉   | 693/996 [01:04<00:24, 12.49it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 70%|██████▉   | 695/996 [01:04<00:24, 12.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 70%|██████▉   | 697/996 [01:04<00:24, 12.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 70%|███████   | 699/996 [01:04<00:24, 12.01it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 70%|███████   | 701/996 [01:04<00:24, 12.28it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████   | 703/996 [01:04<00:23, 12.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████   | 705/996 [01:05<00:22, 12.69it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████   | 707/996 [01:05<00:22, 12.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████   | 709/996 [01:05<00:23, 12.43it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████▏  | 711/996 [01:05<00:24, 11.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 72%|███████▏  | 713/996 [01:05<00:24, 11.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 72%|███████▏  | 715/996 [01:05<00:24, 11.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 72%|███████▏  | 717/996 [01:06<00:24, 11.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 72%|███████▏  | 719/996 [01:06<00:24, 11.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 72%|███████▏  | 721/996 [01:06<00:24, 11.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 73%|███████▎  | 723/996 [01:06<00:24, 11.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 73%|███████▎  | 725/996 [01:06<00:24, 11.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 73%|███████▎  | 727/996 [01:07<00:23, 11.31it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 73%|███████▎  | 729/996 [01:07<00:23, 11.42it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 73%|███████▎  | 731/996 [01:07<00:22, 11.85it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▎  | 733/996 [01:07<00:21, 12.01it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▍  | 735/996 [01:07<00:21, 12.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▍  | 737/996 [01:07<00:20, 12.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▍  | 739/996 [01:07<00:20, 12.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▍  | 741/996 [01:08<00:19, 12.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▍  | 743/996 [01:08<00:19, 12.86it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▍  | 745/996 [01:08<00:19, 12.98it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 747/996 [01:08<00:18, 13.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 749/996 [01:08<00:18, 13.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 751/996 [01:08<00:18, 13.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 76%|███████▌  | 753/996 [01:09<00:18, 13.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 76%|███████▌  | 755/996 [01:09<00:18, 13.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 76%|███████▌  | 757/996 [01:09<00:17, 13.30it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 76%|███████▌  | 759/996 [01:09<00:17, 13.24it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 76%|███████▋  | 761/996 [01:09<00:17, 13.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 763/996 [01:09<00:17, 13.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 765/996 [01:09<00:17, 13.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 767/996 [01:10<00:17, 13.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 769/996 [01:10<00:16, 13.47it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 771/996 [01:10<00:16, 13.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 78%|███████▊  | 773/996 [01:10<00:16, 13.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 78%|███████▊  | 775/996 [01:10<00:16, 13.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 78%|███████▊  | 777/996 [01:10<00:16, 13.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 78%|███████▊  | 779/996 [01:10<00:15, 13.73it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 78%|███████▊  | 781/996 [01:11<00:15, 13.89it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 79%|███████▊  | 783/996 [01:11<00:15, 14.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 79%|███████▉  | 785/996 [01:11<00:14, 14.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 79%|███████▉  | 787/996 [01:11<00:14, 14.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 79%|███████▉  | 789/996 [01:11<00:14, 14.43it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 79%|███████▉  | 791/996 [01:11<00:14, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 80%|███████▉  | 793/996 [01:11<00:14, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 80%|███████▉  | 795/996 [01:12<00:13, 14.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 80%|████████  | 797/996 [01:12<00:13, 14.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 80%|████████  | 799/996 [01:12<00:13, 14.42it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 80%|████████  | 801/996 [01:12<00:13, 14.43it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████  | 803/996 [01:12<00:13, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████  | 805/996 [01:12<00:13, 14.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████  | 807/996 [01:12<00:12, 14.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████  | 809/996 [01:13<00:12, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 811/996 [01:13<00:12, 14.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 82%|████████▏ | 813/996 [01:13<00:12, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 82%|████████▏ | 815/996 [01:13<00:12, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 82%|████████▏ | 817/996 [01:13<00:12, 14.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 82%|████████▏ | 819/996 [01:13<00:12, 14.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 82%|████████▏ | 821/996 [01:13<00:11, 14.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 83%|████████▎ | 823/996 [01:13<00:11, 14.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 83%|████████▎ | 825/996 [01:14<00:11, 14.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 83%|████████▎ | 827/996 [01:14<00:11, 14.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 83%|████████▎ | 829/996 [01:14<00:11, 14.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 83%|████████▎ | 831/996 [01:14<00:11, 14.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▎ | 833/996 [01:14<00:11, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▍ | 835/996 [01:14<00:11, 14.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▍ | 837/996 [01:14<00:11, 14.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▍ | 839/996 [01:15<00:10, 14.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▍ | 841/996 [01:15<00:11, 13.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 85%|████████▍ | 843/996 [01:15<00:11, 13.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 85%|████████▍ | 845/996 [01:15<00:11, 13.33it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 85%|████████▌ | 847/996 [01:15<00:11, 13.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 85%|████████▌ | 849/996 [01:15<00:10, 13.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 85%|████████▌ | 851/996 [01:16<00:10, 13.70it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 86%|████████▌ | 853/996 [01:16<00:10, 13.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 86%|████████▌ | 855/996 [01:16<00:10, 14.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 86%|████████▌ | 857/996 [01:16<00:09, 14.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 86%|████████▌ | 859/996 [01:16<00:09, 13.96it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 86%|████████▋ | 861/996 [01:16<00:09, 13.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 863/996 [01:16<00:09, 13.69it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 865/996 [01:17<00:09, 13.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 867/996 [01:17<00:09, 13.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 869/996 [01:17<00:09, 13.81it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 871/996 [01:17<00:08, 14.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 873/996 [01:17<00:08, 14.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 875/996 [01:17<00:08, 13.81it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 877/996 [01:17<00:08, 13.65it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 879/996 [01:18<00:08, 13.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 881/996 [01:18<00:08, 14.13it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 89%|████████▊ | 883/996 [01:18<00:07, 14.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 89%|████████▉ | 885/996 [01:18<00:07, 14.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 89%|████████▉ | 887/996 [01:18<00:07, 14.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 89%|████████▉ | 889/996 [01:18<00:07, 14.24it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 89%|████████▉ | 891/996 [01:18<00:07, 14.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 90%|████████▉ | 893/996 [01:19<00:07, 13.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 90%|████████▉ | 895/996 [01:19<00:07, 13.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 90%|█████████ | 897/996 [01:19<00:07, 13.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 90%|█████████ | 899/996 [01:19<00:07, 13.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 90%|█████████ | 901/996 [01:19<00:07, 13.05it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 91%|█████████ | 903/996 [01:19<00:07, 12.95it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 91%|█████████ | 905/996 [01:19<00:07, 12.78it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 91%|█████████ | 907/996 [01:20<00:07, 12.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 91%|█████████▏| 909/996 [01:20<00:06, 12.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 91%|█████████▏| 911/996 [01:20<00:06, 12.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 92%|█████████▏| 913/996 [01:20<00:06, 12.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 92%|█████████▏| 915/996 [01:20<00:06, 12.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 92%|█████████▏| 917/996 [01:20<00:06, 12.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 92%|█████████▏| 919/996 [01:21<00:06, 12.11it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 92%|█████████▏| 921/996 [01:21<00:06, 11.95it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 93%|█████████▎| 923/996 [01:21<00:06, 12.04it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 93%|█████████▎| 925/996 [01:21<00:05, 11.93it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 93%|█████████▎| 927/996 [01:21<00:05, 11.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 93%|█████████▎| 929/996 [01:21<00:05, 12.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 93%|█████████▎| 931/996 [01:22<00:05, 12.26it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▎| 933/996 [01:22<00:05, 12.15it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 935/996 [01:22<00:04, 12.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 937/996 [01:22<00:04, 12.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 939/996 [01:22<00:04, 12.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 941/996 [01:22<00:04, 12.80it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 95%|█████████▍| 943/996 [01:23<00:04, 12.90it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 95%|█████████▍| 945/996 [01:23<00:03, 12.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 95%|█████████▌| 947/996 [01:23<00:03, 12.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 95%|█████████▌| 949/996 [01:23<00:03, 12.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 951/996 [01:23<00:03, 12.44it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 96%|█████████▌| 953/996 [01:23<00:03, 12.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 96%|█████████▌| 955/996 [01:24<00:03, 12.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 96%|█████████▌| 957/996 [01:24<00:03, 12.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 96%|█████████▋| 959/996 [01:24<00:02, 13.14it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 96%|█████████▋| 961/996 [01:24<00:02, 13.24it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 963/996 [01:24<00:02, 13.03it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 965/996 [01:24<00:02, 13.19it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 967/996 [01:24<00:02, 13.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 969/996 [01:25<00:01, 13.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 971/996 [01:25<00:01, 13.92it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 98%|█████████▊| 973/996 [01:25<00:01, 14.16it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 98%|█████████▊| 975/996 [01:25<00:01, 14.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 98%|█████████▊| 977/996 [01:25<00:01, 13.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 98%|█████████▊| 979/996 [01:25<00:01, 13.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 98%|█████████▊| 981/996 [01:25<00:01, 13.83it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 99%|█████████▊| 983/996 [01:26<00:00, 13.81it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 99%|█████████▉| 985/996 [01:26<00:00, 13.74it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 99%|█████████▉| 987/996 [01:26<00:00, 13.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 99%|█████████▉| 989/996 [01:26<00:00, 13.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 99%|█████████▉| 991/996 [01:26<00:00, 13.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|█████████▉| 993/996 [01:26<00:00, 13.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|█████████▉| 995/996 [01:26<00:00, 13.79it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 996/996 [01:26<00:00, 11.45it/s]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss: 0.1095122506\n",
      "Validation Accuracy: 0.866085174029\n",
      "Validation F1-Score: 0.464652200901\n"
     ]
    }
   ],
   "source": [
    "#evaluate model\n",
    "model.eval()\n",
    "predictions = []\n",
    "true_labels = []\n",
    "eval_loss, eval_accuracy = 0, 0\n",
    "nb_eval_steps, nb_eval_examples = 0, 0\n",
    "print(len(valid_dataloader))\n",
    "for batch in tqdm(valid_dataloader):\n",
    "    #print(len(batch))\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "    with torch.no_grad():\n",
    "        tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                              attention_mask=b_input_mask, labels=b_labels)\n",
    "        logits = model(b_input_ids, token_type_ids=None,\n",
    "                       attention_mask=b_input_mask)\n",
    "        \n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
    "    label_ids = b_labels.to('cpu').numpy()\n",
    "    true_labels.append(label_ids)\n",
    "    tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "    eval_loss += tmp_eval_loss.mean().item()\n",
    "    eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "    nb_eval_examples += b_input_ids.size(0)\n",
    "    nb_eval_steps += 1\n",
    "\n",
    "pred_tags = [[tags_vals[p_i] for p_i in p] for p in predictions]\n",
    "valid_tags = [[tags_vals[l_ii] for l_ii in l_i] for l in true_labels for l_i in l ]\n",
    "print(\"Validation loss: {}\".format(eval_loss/nb_eval_steps))\n",
    "print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))\n",
    "print(\"Validation F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O']]"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for lab in b_labels:\n",
    "    for l in lab:\n",
    "        print(idx2tag[int(l)])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'FACILITY',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O'],\n",
       " [u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'ORG',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O',\n",
       "  u'O']]"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ idx2tag[int(l)] for l in lab] for lab in b_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}