FineTuneRobertaNER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get books from goodreads metadata\n",
    "\n",
    "import pandas as pd\n",
    "import pickle\n",
    "metadata_goodreads = pickle.load(open( \"pickled_metadata/metadata_goodreads.p\", \"rb\" ))\n",
    "books = list(set(metadata_goodreads['clean_title']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "books = [b for b in books if len(b.split())<3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "books;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepro_jsonl_dataset(ds,augument_chance=0,book_list_to_pick=None,remove_negatives=True):\n",
    "    new_ds = []\n",
    "    \n",
    "    for d in ds:\n",
    "\n",
    "        if d['entities']:\n",
    "\n",
    "            if(np.random.rand()>1-augument_chance):\n",
    "\n",
    "                d_ = d.copy()\n",
    "                book = d_['entities'][0]['text']\n",
    "                book_to_change = np.random.choice(book_list_to_pick,1)[0]\n",
    "                temp = d_['text'].replace(book,book_to_change)\n",
    "                d_['text'] = temp\n",
    "\n",
    "\n",
    "                b = temp.find(book_to_change)\n",
    "                e = b + len(book_to_change)\n",
    "\n",
    "                #dict(start=5,end=14,text=\"Tal Perry\",label=\"Person\")\n",
    "                entities = {\n",
    "                    'start' : b,\n",
    "                    'end' : e,\n",
    "                    'text' : book_to_change,\n",
    "                    'label' : \"OBRA\",\n",
    "                    'tag' : \"OBRA\"\n",
    "                }\n",
    "\n",
    "                d_['meta']['book'] = book_to_change\n",
    "\n",
    "                d_['entities'] = [entities]\n",
    "                \n",
    "                # add original and augumented\n",
    "                new_ds.append(d_)\n",
    "            new_ds.append(d)   \n",
    "\n",
    "        else:\n",
    "            if(not remove_negatives):\n",
    "                print('opa')\n",
    "                new_ds.append(d) \n",
    "        \n",
    "\n",
    "    return new_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def load_jsonl_dataset(path = None):\n",
    "    with open(path,'r') as f:\n",
    "        return list(map(lambda x : json.loads(x),f.readlines()))\n",
    "\n",
    "ds_odissey  = load_jsonl_dataset ('annotated/odyssey_annotaded.jsonl') \n",
    "ds_calibre = load_jsonl_dataset ('annotated/dataset_annotated-fixed-done.jsonl') \n",
    "ds_crimeandpun =  load_jsonl_dataset ('annotated/crimeandpun.jsonl') \n",
    "ds_republic =  load_jsonl_dataset ('annotated/therepublic.jsonl') \n",
    "ds_goodreads = load_jsonl_dataset ('annotated/smaller_dataset_goodreads(ANNOTATED).jsonl') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_calibre = prepro_jsonl_dataset(ds_calibre,augument_chance=0.9,book_list_to_pick=books,remove_negatives=True)\n",
    "ds_odissey = prepro_jsonl_dataset(ds_odissey,augument_chance=.9,book_list_to_pick=books,remove_negatives=True)\n",
    "ds_crimeandpun = prepro_jsonl_dataset(ds_crimeandpun,augument_chance=.9,book_list_to_pick=books,remove_negatives=True)\n",
    "ds_republic = prepro_jsonl_dataset(ds_republic,augument_chance=.9,book_list_to_pick=books,remove_negatives=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Goodreads dataset needs preproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in ds_goodreads:\n",
    "    book = d['meta']['book']\n",
    "    \n",
    "    temp = d['text'].replace('[[BOOK]]',book)\n",
    "    d['text'] = temp\n",
    "    b = temp.find(book)\n",
    "    e = b + len(book)\n",
    "\n",
    "    #dict(start=5,end=14,text=\"Tal Perry\",label=\"Person\")\n",
    "    entities = {\n",
    "        'start' : b,\n",
    "        'end' : e,\n",
    "        'text' : book,\n",
    "        'label' : \"OBRA\",\n",
    "        'tag' : \"OBRA\"\n",
    "    }\n",
    "    if(d['labels']==['negative']):\n",
    "        d['entities'] = []\n",
    "    else:\n",
    "        d['entities'] = [entities]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1153\n"
     ]
    }
   ],
   "source": [
    "dsi = []\n",
    "list_ds = [ds_calibre,ds_odissey,ds_crimeandpun,ds_republic]\n",
    "list_ds = [ds_calibre]\n",
    "for d in list_ds:\n",
    "\n",
    "    dsi.extend(d)\n",
    "print(len(dsi))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "random.shuffle(dsi)\n",
    "\n",
    "dataset_train = dsi[:-50]\n",
    "dataset_test = dsi[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('ner_datasets/dataset_annotated_ner_calibre.jsonl','w') as f:\n",
    "#    for js in ds: \n",
    "#        f.write(json.dumps(js) + '\\n')\n",
    "#    f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### convert ENTITIES to Hugginface Format (tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaTokenizerFast, RobertaModel\n",
    "import torch\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing_extensions import TypedDict\n",
    "from typing import List,Any\n",
    "IntList = List[int] # A list of token_ids\n",
    "IntListList = List[IntList] # A List of List of token_ids, e.g. a Batch\n",
    "\n",
    "def align_tokens_and_annotations_bilou(tokenized, annotations):\n",
    "    tokens = tokenized.tokens\n",
    "    aligned_labels = [\"O\"] * len(\n",
    "        tokens\n",
    "    )  # Make a list to store our labels the same length as our tokens\n",
    "    for anno in annotations:\n",
    "        annotation_token_ix_set = (\n",
    "            set()\n",
    "        )  # A set that stores the token indices of the annotation\n",
    "        for char_ix in range(anno[\"start\"], anno[\"end\"]):\n",
    "\n",
    "            token_ix = tokenized.char_to_token(char_ix)\n",
    "            if token_ix is not None:\n",
    "                annotation_token_ix_set.add(token_ix)\n",
    "        if len(annotation_token_ix_set) == 1:\n",
    "            # If there is only one token\n",
    "            token_ix = annotation_token_ix_set.pop()\n",
    "            prefix = (\n",
    "                \"U\"  # This annotation spans one token so is prefixed with U for unique\n",
    "            )\n",
    "            aligned_labels[token_ix] = f\"{prefix}-{anno['label']}\"\n",
    "\n",
    "        else:\n",
    "\n",
    "            last_token_in_anno_ix = len(annotation_token_ix_set) - 1\n",
    "            for num, token_ix in enumerate(sorted(annotation_token_ix_set)):\n",
    "                if num == 0:\n",
    "                    prefix = \"B\"\n",
    "                elif num == last_token_in_anno_ix:\n",
    "                    prefix = \"L\"  # Its the last token\n",
    "                else:\n",
    "                    prefix = \"I\"  # We're inside of a multi token annotation\n",
    "                aligned_labels[token_ix] = f\"{prefix}-{anno['label']}\"\n",
    "    return aligned_labels\n",
    "\n",
    "\n",
    "import itertools\n",
    "\n",
    "\n",
    "class LabelSet:\n",
    "    def __init__(self, labels: List[str]):\n",
    "        self.labels_to_id = {}\n",
    "        self.ids_to_label = {}\n",
    "        self.labels_to_id[\"O\"] = 0\n",
    "        self.ids_to_label[0] = \"O\"\n",
    "        num = 0  # in case there are no labels\n",
    "        # Writing BILU will give us incremntal ids for the labels\n",
    "        for _num, (label, s) in enumerate(itertools.product(labels, \"BILU\")):\n",
    "            num = _num + 1  # skip 0\n",
    "            l = f\"{s}-{label}\"\n",
    "            self.labels_to_id[l] = num\n",
    "            self.ids_to_label[num] = l\n",
    "        # Add the OUTSIDE label - no label for the token\n",
    "\n",
    "    def get_aligned_label_ids_from_annotations(self, tokenized_text, annotations):\n",
    "        raw_labels = align_tokens_and_annotations_bilou(tokenized_text, annotations)    \n",
    "        return list(map(self.labels_to_id.get, raw_labels))\n",
    "from dataclasses import dataclass\n",
    "from torch.utils.data import Dataset\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "\n",
    "@dataclass\n",
    "class TrainingExample:\n",
    "    input_ids: IntList\n",
    "    attention_masks: IntList\n",
    "    labels: IntList\n",
    "\n",
    "\n",
    "class TraingDataset(Dataset):\n",
    "    def __init__(\n",
    "        self,\n",
    "        data: Any,\n",
    "        label_set: LabelSet,\n",
    "        tokenizer: PreTrainedTokenizerFast,\n",
    "        tokens_per_batch=32,\n",
    "        window_stride=None,\n",
    "    ):\n",
    "        self.label_set = label_set\n",
    "        if window_stride is None:\n",
    "            self.window_stride = tokens_per_batch\n",
    "        self.tokenizer = tokenizer\n",
    "        for example in data:\n",
    "            # changes tag key to label\n",
    "            for a in example[\"entities\"]:\n",
    "                a[\"label\"] = a[\"tag\"]\n",
    "                pass\n",
    "        self.texts = []\n",
    "        self.annotations = []\n",
    "\n",
    "        for example in data:\n",
    "            self.texts.append(example[\"text\"])\n",
    "            self.annotations.append(example[\"entities\"])\n",
    "        ###TOKENIZE All THE DATA\n",
    "        tokenized_batch = self.tokenizer(self.texts, add_special_tokens=False)\n",
    "        ###ALIGN LABELS ONE EXAMPLE AT A TIME\n",
    "        aligned_labels = []\n",
    "        for ix in range(len(tokenized_batch.encodings)):\n",
    "            encoding = tokenized_batch.encodings[ix]\n",
    "            raw_annotations = self.annotations[ix]            \n",
    "            aligned = label_set.get_aligned_label_ids_from_annotations(\n",
    "                encoding, raw_annotations\n",
    "            )\n",
    "            aligned_labels.append(aligned)\n",
    "        ###END OF LABEL ALIGNMENT\n",
    "\n",
    "        ###MAKE A LIST OF TRAINING EXAMPLES. (This is where we add padding)\n",
    "        self.training_examples: List[TrainingExample] = []\n",
    "        empty_label_id = \"O\"\n",
    "        for encoding, label in zip(tokenized_batch.encodings, aligned_labels):\n",
    "            length = len(label)  # How long is this sequence\n",
    "            for start in range(0, length, self.window_stride):\n",
    "\n",
    "                end = min(start + tokens_per_batch, length)\n",
    "\n",
    "                # How much padding do we need ?\n",
    "                padding_to_add = max(0, tokens_per_batch - end + start)\n",
    "                self.training_examples.append(\n",
    "                    TrainingExample(\n",
    "                        # Record the tokens\n",
    "                        input_ids=encoding.ids[start:end]  # The ids of the tokens\n",
    "                        + [self.tokenizer.pad_token_id]\n",
    "                        * padding_to_add,  # padding if needed\n",
    "                        labels=(\n",
    "                            label[start:end]\n",
    "                            + [-100] * padding_to_add  # padding if needed\n",
    "                        ),  # -100 is a special token for padding of labels,\n",
    "                        attention_masks=(\n",
    "                            encoding.attention_mask[start:end]\n",
    "                            + [0]\n",
    "                            * padding_to_add  # 0'd attenetion masks where we added padding\n",
    "                        ),\n",
    "                    )\n",
    "                )\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.training_examples)\n",
    "\n",
    "    def __getitem__(self, idx) -> TrainingExample:\n",
    "\n",
    "        return self.training_examples[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TraingingBatch:\n",
    "    def __getitem__(self, item):\n",
    "        return getattr(self, item)\n",
    "\n",
    "    def __init__(self, examples: List[TrainingExample]):\n",
    "        self.input_ids: torch.Tensor\n",
    "        self.attention_masks: torch.Tensor\n",
    "        self.labels: torch.Tensor\n",
    "        input_ids: IntListList = []\n",
    "        masks: IntListList = []\n",
    "        labels: IntListList = []\n",
    "        for ex in examples:\n",
    "            input_ids.append(ex.input_ids)\n",
    "            masks.append(ex.attention_masks)\n",
    "            labels.append(ex.labels)\n",
    "        self.input_ids = torch.LongTensor(input_ids)\n",
    "        self.attention_masks = torch.LongTensor(masks)\n",
    "        self.labels = torch.LongTensor(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (564 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "label_set = LabelSet(labels=[\"OBRA\"])\n",
    "tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
    "\n",
    "ds = TraingDataset(\n",
    "    data=dataset_train, tokenizer=tokenizer, label_set=label_set, tokens_per_batch=40\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing RobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForTokenClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers.optimization import get_linear_schedule_with_warmup\n",
    "from transformers import AdamW\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from transformers import RobertaForTokenClassification\n",
    "\n",
    "\n",
    "model = RobertaForTokenClassification.from_pretrained(\n",
    "    \"roberta-base\", num_labels=len(ds.label_set.ids_to_label.values())\n",
    ")\n",
    "\n",
    "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
    "nb_grad_acc_steps = 2\n",
    "warmup_steps = 1\n",
    "num_epochs = 5\n",
    "\n",
    "dataloader = DataLoader(\n",
    "    ds,\n",
    "    collate_fn=TraingingBatch,\n",
    "    batch_size=4,\n",
    "    shuffle=True,\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "nb_training_steps = len(dataloader)//nb_grad_acc_steps\n",
    "\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer,warmup_steps,num_training_steps=nb_training_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.cuda();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "def check_accuracy_model(model,dataset_raw):\n",
    "\n",
    "    outs = []\n",
    "    for d in dataset_raw:\n",
    "\n",
    "            y_pred = ''\n",
    "            if len(d['entities'])>0:\n",
    "            \n",
    "                y_true = d['entities'][0]['text']\n",
    "            else:\n",
    "                y_true = ''\n",
    "                \n",
    "            inputs = tokenizer(d['text'], return_tensors=\"pt\",truncation=True,padding=True).to(device)\n",
    "\n",
    "            logits = model(**inputs)\n",
    "\n",
    "            idx = torch.nonzero(torch.argmax(logits[0][0],axis=1))\n",
    "\n",
    "            if (len(idx)):\n",
    "                ids_labeled = inputs.input_ids[0][idx]\n",
    "                tokens = tokenizer.convert_ids_to_tokens(ids_labeled)\n",
    "                y_pred = tokenizer.convert_tokens_to_string(tokens).strip()\n",
    "\n",
    "                success = y_pred.find(y_true)!=-1\n",
    "                outs.append(success)\n",
    "            else:\n",
    "                pass\n",
    "                \n",
    "            outs.append(y_pred==y_true)\n",
    "\n",
    "                \n",
    "\n",
    "    return np.mean(outs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=0/5: 100%|██████████| 395/395 [00:23<00:00, 16.91it/s, loss=0.054]   \n",
      "Epoch=1/5:   1%|          | 2/395 [00:00<00:23, 16.65it/s, loss=0.0154]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch val loss Goodreads:0.5887096774193549 Val Set:0.71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=1/5: 100%|██████████| 395/395 [00:23<00:00, 16.87it/s, loss=0.0513]  \n",
      "Epoch=2/5:   1%|          | 2/395 [00:00<00:24, 15.87it/s, loss=0.0198] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch val loss Goodreads:0.5887096774193549 Val Set:0.7551020408163265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=2/5: 100%|██████████| 395/395 [00:23<00:00, 16.47it/s, loss=0.00391] \n",
      "Epoch=3/5:   1%|          | 2/395 [00:00<00:24, 16.07it/s, loss=0.00465]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch val loss Goodreads:0.5887096774193549 Val Set:0.7551020408163265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=3/5: 100%|██████████| 395/395 [00:23<00:00, 16.48it/s, loss=0.138]   \n",
      "Epoch=4/5:   1%|          | 2/395 [00:00<00:23, 16.74it/s, loss=0.0149]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch val loss Goodreads:0.5887096774193549 Val Set:0.7551020408163265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=4/5: 100%|██████████| 395/395 [00:23<00:00, 16.77it/s, loss=0.0129]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch val loss Goodreads:0.5887096774193549 Val Set:0.7551020408163265\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    iterator = dataloader.__iter__()\n",
    "\n",
    "    loop = tqdm(range(int(nb_training_steps)))\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    for i_train in loop:\n",
    "        \n",
    "        batch = next(iterator)\n",
    "            \n",
    "        device = 'cuda'\n",
    "        input_ids = batch.input_ids.to(device)\n",
    "        attention_masks = batch.attention_masks.to(device)\n",
    "        labels = batch.labels.to(device)\n",
    "\n",
    "        loss, logits = model(\n",
    "            input_ids=input_ids,\n",
    "            attention_mask=attention_masks,\n",
    "            labels=labels,\n",
    "            return_dict=False\n",
    "        )\n",
    "        \n",
    "        loss = loss/nb_grad_acc_steps\n",
    "        loss.backward()  \n",
    "        if (i_train+1) % nb_grad_acc_steps == 0:      \n",
    "\n",
    "            optimizer.step()\n",
    "            scheduler.step()\n",
    "            optimizer.zero_grad()   \n",
    "            \n",
    "        loop.set_description(f'Epoch={epoch}/{num_epochs}')\n",
    "        loop.set_postfix(loss=loss.item())\n",
    "        \n",
    "    #gd_val = check_accuracy_model(model,ds_goodreads)\n",
    "    val_val = check_accuracy_model(model,dataset_test)\n",
    "\n",
    "    print(f'Epoch val loss Goodreads:{gd_val} Val Set:{val_val}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.device"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "model.save_pretrained(f\"fine-tuned-model-ner-better-data-3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values(['O', 'B-OBRA', 'I-OBRA', 'L-OBRA', 'U-OBRA'])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.label_set.ids_to_label.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEST MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/lira/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaTokenizerFast, RobertaForTokenClassification\n",
    "import torch\n",
    "\n",
    "tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
    "model = RobertaForTokenClassification.from_pretrained(\"fine-tuned-model-ner-better-data-3/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import check_if_citation_ner\n",
    "model.cpu();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Prince The Republic'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,'Reading The Prince and The Republic has changed my life')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Meditations'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,'The prince reads Marcus Aurelius Meditations to relax')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Odyssey'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,\"When Odysseus returns to Ithaca in Book 13 of The Odyssey, Athena disguises him as an old beggar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Infinite Jest'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,\"But these more outré materials combine to form what is finally a thematic second tier. The foreground of Infinite Jest features three basic plot systems. At the center of one is Hal Incandenza, an adolescent tennis star attending Enfield Tennis Academy (ETA), which his family founded,\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Henry IV'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,\"the commencement of war a herald might be called upon to recite the causes of the conflict; in effect, to provide the motivation. In Shakespeare's Henry IV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Meditations'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,\"is impersonal in the Meditations agrees closely with Epictetus. Marcus Aurelius is doubtful about immortality, but says, as a Christian might: 'Since it is possible th\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Moby-Dick'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,\"8 Melville’s Moby-Dick, for instance.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([24], dtype='int64', name='id')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "metadata_calibre = pickle.load(open( \"pickled_metadata/calibre_metadata.p\", \"rb\" ))\n",
    "metadata_calibre.query('clean_title==\"Story of Philosophy\"').index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "nietz = []\n",
    "with open(f'books_raw/{186}.txt', 'r') as book_text:\n",
    "    nietz.append(book_text.readlines())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "line = \"Reading The Prince and The Republic has changed my life\"\n",
    "\n",
    "book_detected = check_if_citation_ner(model,tokenizer,line)\n",
    "\n",
    "book = 'The Machieavli'\n",
    "is_citation = False\n",
    "if (book_detected.find(book)!=-1 or book.find(book_detected)!=-1):\n",
    "    is_citation= True\n",
    "else:\n",
    "    is_citation = False\n",
    "print(is_citation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_detected.find(book)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book.find(book_detected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in nietz[0]:\n",
    "    if line.find('The New Republic')!=-1:\n",
    "        sentence = line\n",
    "        #print(check_if_citation_ner(model,tokenizer,line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = sentence+sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([512])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tokenizer(sentence, return_tensors=\"pt\",truncation=True,padding=True)\n",
    "inputs.input_ids[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The New Republic The New Republic'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_if_citation_ner(model,tokenizer,sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predicting on a Batch directly "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_if_citation_ner_multiline(model,tokenizer,sentences):\n",
    "\n",
    "    inputs = tokenizer(sentences, return_tensors=\"pt\",truncation=True,padding=True)\n",
    "\n",
    "    logits = model(**inputs)\n",
    "    idx = torch.nonzero(torch.argmax(logits[0][0],axis=1))\n",
    "\n",
    "\n",
    "\n",
    "    num_sents = logits[0].shape[0]\n",
    "    idx_sentence = torch.nonzero(torch.argmax(logits[0],axis=2)).numpy()\n",
    "\n",
    "    dict_tokens_idx = {num:[] for num in range(num_sents)}\n",
    "\n",
    "    for row,idx in idx_sentence:\n",
    "        dict_tokens_idx[row].append(idx)\n",
    "    books_found = []\n",
    "    for row,ids in dict_tokens_idx.items():\n",
    "        ids_labeled = inputs.input_ids[row][ids]\n",
    "        tokens = tokenizer.convert_ids_to_tokens(ids_labeled)\n",
    "        books_found.append(tokenizer.convert_tokens_to_string(tokens).strip())\n",
    "    \n",
    "    return books_found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### seeing outputs from model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('to_annotate/output_ner_multi.jsonl','r') as f:\n",
    "    \n",
    "    ds_goodreads = list(map(lambda x : json.loads(x),f.readlines()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': 'Resistance, Rebellion, and Death (Actuelles—a selection) 1961', 'labels': ['positive'], 'detected_ner': 'Resistance, and Death', 'meta': {'book': 'Rebellion'}}\n",
      "-1\n",
      "-1\n",
      "{'text': 'Resistance, Rebellion, and Death displays Camus’ rigorous moral intelligence addressing issues that range from colonial warfare in Algeria to the social cancer of capital punishment.', 'labels': ['positive'], 'detected_ner': 'Res The Stranger The Rebel The Myth of Sisyphus', 'meta': {'book': 'Rebellion'}}\n",
      "-1\n",
      "-1\n",
      "{'text': 'Resistance, Rebellion, and Death (Actuelles', 'labels': ['positive'], 'detected_ner': 'Res', 'meta': {'book': 'Rebellion'}}\n",
      "-1\n",
      "-1\n",
      "{'text': 'Resistance, Rebellion, and Death displays Camus’ rigorous moral intelligence addressing issues that range from colonial warfare in Algeria to the social cancer of capital punishment.', 'labels': ['positive'], 'detected_ner': 'Res The Stranger The Rebel The Myth of Sisyphus', 'meta': {'book': 'Rebellion'}}\n",
      "-1\n",
      "-1\n",
      "{'text': 'Cole, J. R. 1992. The Olympian Dreams and Youthful Rebellion of René Descartes (Champaign: University of Illinois Press).', 'labels': ['positive'], 'detected_ner': 'The Olympian Dreams and Youthfules', 'meta': {'book': 'Rebellion'}}\n",
      "-1\n",
      "-1\n"
     ]
    }
   ],
   "source": [
    "for d in ds_goodreads:\n",
    "    if(d['meta']['book']=='Rebellion'):\n",
    "        \n",
    "        if(d['detected_ner'] != ''):\n",
    "            print(d)\n",
    "            print(d['detected_ner'].find('Rebellion'))\n",
    "            print('Rebellion'.find(d['detected_ner']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([63], dtype='int64', name='id')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "metadata_calibre = pickle.load(open( \"pickled_metadata/calibre_metadata.p\", \"rb\" ))\n",
    "metadata_calibre.query('clean_title==\"Enlightenment Now\"').index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "book_memory = []\n",
    "with open(f'books_raw/{63}.txt', 'r') as book_text:\n",
    "    book_memory.append(book_text.readlines())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\t77. In analyzing history’s deadliest conflicts, Matthew White comments, “I’m amazed at how often the immediate cause of a conflict is a mistake, unfounded suspicion, or rumor.” In addition to the first two listed here he includes the First World War, Sino-Japanese War, Seven Years’ War, Second French War of Religion, An Lushan Rebellion in China, Indonesian Purge, and Russia’s Time of Troubles; White 2011, p. 537.\n",
      "\t\t\tAn Lushan Rebellion, 484n77\n"
     ]
    }
   ],
   "source": [
    "!grep Rebellion books_raw/63.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import BookProcesserFactory\n",
    "\n",
    "bkp = BookProcesserFactory(create_dataset = True,\n",
    "                    verbose = False,\n",
    "                    use_citation_model = True,\n",
    "                    metadata_to_use = 'goodreads'       \n",
    "                )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_ = 63\n",
    "\n",
    "iterator = list(zip([63],\n",
    "                    metadata_calibre['aliases'].loc[id_],\n",
    "                    [metadata_calibre['clean_title'].loc[id_]],\n",
    "                    book_memory))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Enlightenment Now'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata_calibre['clean_title'].loc[id_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "book_processer = bkp.GetProcessFunction()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = book_processer(iterator[0],model,tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in results[2]:\n",
    "    d= json.loads(d)\n",
    "    if(d['meta']['book']=='Rebellion'):\n",
    "\n",
    "        if(d['detected_ner'] != ''):\n",
    "            print(d)\n",
    "            print(d['detected_ner'].find('Rebellion'))\n",
    "            print('Rebellion'.find(d['detected_ner']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Enlightenment Now',\n",
       " [('Enlightenment Now', 'The Age of Reason'),\n",
       "  ('Enlightenment Now', 'A Clockwork Orange'),\n",
       "  ('Enlightenment Now', 'Housekeeping'),\n",
       "  ('Enlightenment Now', 'Walden'),\n",
       "  ('Enlightenment Now', 'The Road to Serfdom'),\n",
       "  ('Enlightenment Now', 'The Origin of Species'),\n",
       "  ('Enlightenment Now', 'Brave New World'),\n",
       "  ('Enlightenment Now', 'The Wealth of Nations'),\n",
       "  ('Enlightenment Now', 'Much Ado About Nothing'),\n",
       "  ('Enlightenment Now', 'Cutting Edge'),\n",
       "  ('Enlightenment Now', 'The Untouchables'),\n",
       "  ('Enlightenment Now', 'The Origin of Species'),\n",
       "  ('Enlightenment Now', 'All of Us'),\n",
       "  ('Enlightenment Now', 'Beyond Good and Evil'),\n",
       "  ('Enlightenment Now', 'The Will to Power'),\n",
       "  ('Enlightenment Now', 'The Black Book')],\n",
       " ['{\"text\": \"12-9: Lightning strike deaths, US, 1900–2015\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Lightning\"} }\\n',\n",
       "  '{\"text\": \"My gratitude goes as well to the other data scientists I pestered and to the institutions that collect and maintain their data: Karlyn Bowman, Daniel Cox (PRRI), Tamar Epner (Social Progress Index), Christopher Fariss, Chelsea Follett (HumanProgress), Andrew Gelman, Yair Ghitza, April Ingram (Science Heroes), Jill Janocha (Bureau of Labor Statistics), Gayle Kelch (US Fire Administration/FEMA), Alaina Kolosh (National Safety Council), Kalev Leetaru (Global Database of Events, Language, and Tone), Monty Marshall (Polity Project), Bruce Meyer, Branko Milanović (World Bank), Robert Muggah (Homicide Monitor), Pippa Norris (World Values Survey), Thomas Olshanski (US Fire Administration/FEMA), Amy Pearce (Science Heroes), Mark Perry, Therese Pettersson (Uppsala Conflict Data Program), Leandro Prados de la Escosura, Stephen Radelet, Auke Rijpma (OECD Clio Infra), Hannah Ritchie (Our World in Data), Seth Stephens-Davidowitz (Google Trends), James X. Sullivan, Sam Taub (Uppsala Conflict Data Program), Kyla Thomas, Jennifer Truman (Bureau of Justice Statistics), Jean Twenge, Bas van Leeuwen (OECD Clio Infra), Carlos Vilalta, Christian Welzel (World Values Survey), Justin Wolfers, and Billy Woodward (Science Heroes).\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Truman\"} }\\n',\n",
       "  '{\"text\": \"A 21st-century statement of the same idea may be found in the physicist David Deutsch’s defense of enlightenment, The Beginning of Infinity.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"The Beginning\"} }\\n',\n",
       "  '{\"text\": \"Thanks to language, ideas are not just abstracted and combined inside the head of a single thinker but can be pooled across a community of thinkers. Thomas Jefferson explained the power of language with the help of an analogy: “He who receives an idea from me, receives instruction himself without lessening mine; as he who lights his taper at mine, receives light without darkening me.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Thomas Jefferson\"} }\\n',\n",
       "  '{\"text\": \"Information about human progress, though absent from major news outlets and intellectual forums, is easy enough to find. The data are not entombed in dry reports but are displayed in gorgeous Web sites, particularly Max Roser’s Our World in Data, Marian Tupy’s HumanProgress, and Hans Rosling’s Gapminder. (Rosling learned that not even swallowing a sword during a 2007 TED talk was enough to get the world’s attention.) The case has been made in beautifully written books, some by Nobel laureates, which flaunt the news in their titles—Progress, The Progress Paradox, Infinite Progress, The Infinite Resource, The Rational Optimist, The Case for Rational Optimism, Utopia for Realists, Mass Flourishing, Abundance, The Improving State of the World, Getting Better, The End of Doom, The Moral Arc, The Big Ratchet, The Great Escape, The Great Surge, The Great Convergence.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Utopia\"} }\\n',\n",
       "  '{\"text\": \"The historian Fernand Braudel has documented that premodern Europe suffered from famines every few decades.2 Desperate peasants would harvest grain before it was ripe, eat grass or human flesh, and pour into cities to beg. Even in good times, many would get the bulk of their calories from bread or gruel, and not many at that: in The Escape from Hunger and Premature Death, 1700–2100, the economist Robert Fogel noted that “the energy value of the typical diet in France at the start of the eighteenth century was as low as that of Rwanda in 1965, the most malnourished nation for that year.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hunger\"} }\\n',\n",
       "  '{\"text\": \"It is all very well for us, sitting pretty, to think that material standards of living don’t matter all that much. It is all very well for one, as a personal choice, to reject industrialisation—do a modern Walden if you like, and if you go without much food, see most of your children die in infancy, despise the comforts of literacy, accept twenty years off your own life, then I respect you for the strength of your aesthetic revulsion.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Walden\"} }\\n',\n",
       "  '{\"text\": \"Let’s complete our tour of the history of inequality by turning to the final segment in figure 9-3, the rise of inequality in wealthy nations that began around 1980. This is the development that inspired the claim that life has gotten worse for everyone but the richest. The rebound defies the Kuznets curve, in which inequality was supposed to have settled into a low equilibrium. Many explanations have been proffered for this surprise.41 Wartime restrictions on economic competition may have been sticky, outlasting World War II, but they finally dissipated, freeing the rich to get richer from their investment income and opening up an arena of dynamic economic competition with winner-take-all payoffs. The ideological shift associated with Ronald Reagan and Margaret Thatcher slowed the movement toward greater social spending financed by taxes on the rich while eroding social norms against extravagant salaries and conspicuous wealth.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Ronald Reagan\"} }\\n',\n",
       "  '{\"text\": \"Many of the improvements can be seen with the naked eye. Cities are less often shrouded in purple-brown haze, and London no longer has the fog—actually coal smoke—that was immortalized in Impressionist paintings, gothic novels, the Gershwin song, and the brand of raincoats. Urban waterways that had been left for dead—including Puget Sound, Chesapeake Bay, Boston Harbor, Lake Erie, and the Hudson, Potomac, Chicago, Charles, Seine, Rhine, and Thames rivers (the last described by Disraeli as “a Stygian pool reeking with ineffable and intolerable horrors”)—have been recolonized by fish, birds, marine mammals, and sometimes swimmers.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Chesapeake\"} }\\n',\n",
       "  '{\"text\": \"Nordhaus and Shellenberger summarize the calculations of an increasing number of climate scientists: “There is no credible path to reducing global carbon emissions without an enormous expansion of nuclear power. It is the only low carbon technology we have today with the demonstrated capability to generate large quantities of centrally generated electric power.”83 The Deep Carbonization Pathways Project, a consortium of research teams that have worked out roadmaps for countries to reduce their emissions enough to meet the 2°C target, estimates that the United States will have to get between 30 and 60 percent of its electricity from nuclear power by 2050 (1.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"The Deep\"} }\\n',\n",
       "  '{\"text\": \"“It takes more skill to cross Broadway . . . than to cross the Atlantic in a clamboat.” . . . The engine of city mayhem was the horse. Underfed and nervous, this vital brute was often flogged to exhaustion by pitiless drivers, who exulted in pushing ahead “with utmost fury, defying law and delighting in destruction.” Runaways were common.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Runaways\"} }\\n',\n",
       "  '{\"text\": \"Figure 12-9: Lightning strike deaths, US, 1900–2015\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Lightning\"} }\\n',\n",
       "  '{\"text\": \"Sources: US Bureau of Justice Statistics, National Crime Victimization Survey, Victimization Analysis Tool, http://www.bjs.gov/index.cfm?ty=nvat, with additional data provided by Jennifer Truman of BJS.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Truman\"} }\\n',\n",
       "  '{\"text\": \"As a feminist-era husband I can truthfully use the first-person plural in celebrating this gain. But in most times and places housework is gendered, so the liberation of humankind from household labor is in practice the liberation of women from household labor. Perhaps the liberation of women in general. Arguments for the equality of women go back to Mary Astell’s 1700 treatise and are irrefutable, so why did they take centuries to catch on? In a 1912 interview in Good Housekeeping magazine, Thomas Edison prophesied one of the great social transformations of the 20th century:\" , \"labels\":[\"positive\"], \"detected_ner\":\"keeping\", \"meta\":{\"book\":\"Housekeeping\"} }\\n',\n",
       "  '{\"text\": \"Even for wealthy Western urbanites, who always had the run of the palaces of culture, access to arts and letters has expanded tremendously. When I was a student, a movie buff had to wait years for a classic film to be shown at a local repertory theater or on late-night television, if it was shown at all; today it can be streamed on demand. I can listen to any of thousands of songs while jogging, washing the dishes, or waiting in line at the Registry of Motor Vehicles. With a few keystrokes, I could lose myself in the complete works of Caravaggio, the original trailer for Rashomon, Dylan Thomas reciting “And Death Shall Have No Dominion,” Eleanor Roosevelt reading aloud the Universal Declaration of Human Rights, Maria Callas singing “O mio babbino caro,” Billie Holiday singing “My Man Don’t Love Me,” and Solomon Linda singing “Mbube”—experiences I could not have had for love or money just a few years ago.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Dominion\"} }\\n',\n",
       "  '{\"text\": \"Evidence-free pronouncements about the misery of mankind are an occupational hazard of the social critic. In the 1854 classic Walden, Henry David Thoreau famously wrote, “The mass of men lead lives of quiet desperation.\" , \"labels\":[\"positive\"], \"detected_ner\":\"Walden\", \"meta\":{\"book\":\"Walden\"} }\\n',\n",
       "  '{\"text\": \"Today we enjoy a world of personal freedom these characters could only fantasize about, a world in which people can marry, work, and live as they please. One can imagine a social critic of today warning Anna Karenina or Nora Helmer that a tolerant cosmopolitan society isn’t all it’s cracked up to be, that without the bonds of family and village they’ll have moments of anxiety and unhappiness.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Anna Karenina\"} }\\n',\n",
       "  '{\"text\": \"People have also lost their comforting faith in the goodness of their institutions. The historian William O’Neill entitled his history of the Baby Boomers’ childhood years American High: The Years of Confidence, 1945–1960.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"The Years\"} }\\n',\n",
       "  '{\"text\": \"The sentinels for the familiar horsemen tended to be romantics and Luddites. But those who warn of the higher-tech dangers are often scientists and technologists who have deployed their ingenuity to identify ever more ways in which the world will soon end. In 2003 the eminent astrophysicist Martin Rees published a book entitled Our Final Hour in which he warned that “humankind is potentially the maker of its own demise” and laid out some dozen ways in which we have “endangered the future of the entire universe.” For example, experiments in particle colliders could create a black hole that would annihilate the Earth, or a “strangelet” of compressed quarks that would cause all matter in the cosmos to bind to it and disappear. Rees tapped a rich vein of catastrophism. The book’s Amazon page notes, “Customers who viewed this item also viewed Global Catastrophic Risks; Our Final Invention: Artificial Intelligence and the End of the Human Era; The End: What Science and Religion Tell Us About the Apocalypse; and World War Z: An Oral History of the Zombie War.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"As the prospect of evil robots started to seem too kitschy to take seriously, a new digital apocalypse was spotted by the existential guardians. This storyline is based not on Frankenstein or the Golem but on the Genie granting us three wishes, the third of which is needed to undo the first two, and on King Midas ruing his ability to turn everything he touched into gold, including his food and his family.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Frankenstein\"} }\\n',\n",
       "  '{\"text\": \"This existentialism depends on a casual slide from nuisance to adversity to tragedy to disaster to annihilation. Suppose there was an episode of bioterror that killed a million people. Suppose a hacker did manage to take down the Internet. Would the country literally cease to exist? Would civilization collapse? Would the human species go extinct? A little proportion, please—even Hiroshima continues to exist! The assumption is that modern people are so helpless that if the Internet ever went down, farmers would stand by and watch their crops rot while dazed city-dwellers starved.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"survivors engaged in search and rescue, helped one another in whatever ways they could, and withdrew in controlled flight from burning areas. Within a day, apart from the planning undertaken by the government and military organizations that partly survived, other groups partially restored electric power to some areas, a steel company with 20 percent of workers attending began operations again, employees of the 12 banks in Hiroshima assembled in the Hiroshima branch in the city and began making payments, and trolley lines leading into the city were completely cleared with partial traffic restored the following day.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"In recent decades predictions of an imminent nuclear catastrophe have shifted from war to terrorism, such as when the American diplomat John Negroponte wrote in 2003, “There is a high probability that within two years al-Qaeda will attempt an attack using a nuclear or other weapon of mass destruction.”82 Though a probabilistic prediction of an event that fails to occur can never be gainsaid, the sheer number of false predictions (Mueller has more than seventy in his collection, with deadlines staggered over several decades) suggests that prognosticators are biased toward scaring people.83 (In 2004, four American political figures wrote an op-ed on the threat of nuclear terrorism entitled “Our Hair Is on Fire.”)84 The tactic is dubious. People are easily riled by actual attacks with guns and homemade bombs into supporting repressive measures like domestic surveillance or a ban on Muslim immigration. But predictions of a mushroom cloud on Main Street have aroused little interest in policies to combat nuclear terrorism, such as an international program to control fissile material.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Main Street\"} }\\n',\n",
       "  '{\"text\": \"While the engineering know-how required to build a basic fission device like the Hiroshima or Nagasaki bomb is readily available, highly enriched uranium and weapons-grade plutonium are not at all easily accessible, and to assemble and maintain—for a long period, out of sight of the huge intelligence and law enforcement resources that are now being devoted to this threat worldwide—the team of criminal operatives, scientists and engineers necessary to acquire the components of, build and deliver such a weapon would be a formidably difficult undertaking.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"Could nuclear weapons go the way of the Gustav Gun? In the late 1950s a movement arose to Ban the Bomb, and over the decades it escaped its founding circle of beatniks and eccentric professors and has gone mainstream. Global Zero, as the goal is now called, was broached in 1986 by Mikhail Gorbachev and Ronald Reagan, who famously mused, “A nuclear war cannot be won and must never be fought.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Ronald Reagan\"} }\\n',\n",
       "  '{\"text\": \"But the pathway has been laid out. If nuclear warheads continue to be dismantled faster than they are built, if they are taken off a hair trigger and guaranteed not to be used first, and if the trend away from interstate war continues, then by the second half of the century we could end up with small, secure arsenals kept only for mutual deterrence. After a few decades they might deter themselves out of a job. At that point they would seem ludicrous to our grandchildren, who will beat them into plowshares once and for all. During this climbdown we may never reach a point at which the chance of a catastrophe is zero. But each step down can lower the risk, until it is in the range of the other threats to our species’ immortality, like asteroids, supervolcanoes, or an Artificial Intelligence that turns us into paper clips.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"The answer to the first question is “false”; if it were true, your glass of Coke would overflow as the ice cubes melted. It’s icecaps on land, such as Greenland and Antarctica, that raise sea levels when they melt.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Antarctica\"} }\\n',\n",
       "  '{\"text\": \"The conspiracy theories of fervid hordes at a political rally represent an extreme case of self-expression trumping truth, but the Tragedy of the Belief Commons runs even deeper. Another paradox of rationality is that expertise, brainpower, and conscious reasoning do not, by themselves, guarantee that thinkers will approach the truth. On the contrary, they can be weapons for ever-more-ingenious rationalization. As Benjamin Franklin observed, “So convenient a thing is it to be a rational creature, since it enables us to find or make a reason for everything one has a mind to.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Benjamin Franklin\"} }\\n',\n",
       "  '{\"text\": \"Intellectual and political polarization feed each other. It’s harder to be a conservative intellectual when American conservative politics has become steadily more know-nothing, from Ronald Reagan to Dan Quayle to George W.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Ronald Reagan\"} }\\n',\n",
       "  '{\"text\": \"What can be done to improve standards of reasoning? Persuasion by facts and logic, the most direct strategy, is not always futile.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Persuasion\"} }\\n',\n",
       "  '{\"text\": \"The other reason that humanism needn’t be embarrassed by its overlap with utilitarianism is that this approach to ethics has an impressive track record of improving human welfare. The classical utilitarians—Cesare Beccaria, Jeremy Bentham, and John Stuart Mill—laid out arguments against slavery, sadistic punishment, cruelty to animals, the criminalization of homosexuality, and the subordination of women which carried the day.17 Even abstract rights like freedom of speech and religion were largely defended in terms of benefits and harms, as when Thomas Jefferson wrote, “The legitimate powers of government extend to such acts only as are injurious to others.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Thomas Jefferson\"} }\\n',\n",
       "  '{\"text\": \"But theistic morality has two fatal flaws. The first is that there is no good reason to believe that God exists. In a nonfiction appendix to her novel Thirty-Six Arguments for the Existence of God: A Work of Fiction, Rebecca Newberger Goldstein (drawing in part on Plato, Spinoza, Hume, Kant, and Russell) lays out refutations of every one of these arguments.32 The most common among them—faith, revelation, scripture, authority, tradition, and subjective appeal—are not arguments at all. It’s not just that reason says they cannot be trusted. It’s also that different religions, drawing on these sources, decree mutually incompatible beliefs about how many gods there are, which miracles they have wrought, and what they demand of their devotees. Historical scholarship has amply demonstrated that holy scriptures are all-too-human products of their historical eras, including internal contradictions, factual errors, plagiarism from neighboring civilizations, and scientific absurdities (such as God creating the sun three days after he distinguished day from night). The recondite arguments from sophisticated theologians are no sounder. The Cosmological and Ontological arguments for the existence of God are logically invalid, the Argument from Design was refuted by Darwin, and the others are either patently false (such as the theory that humans are endowed with an innate faculty for sensing the truth about God) or blatant escape hatches (such as the suggestion that the Resurrection was too cosmically important for God to have allowed it to be empirically verified).\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Resurrection\"} }\\n',\n",
       "  '{\"text\": \"Correlation is not causation, but if you combine the fact that much of Islamic doctrine is antihumanistic with the fact that many Muslims believe that Islamic doctrine is inerrant—and throw in the fact that the Muslims who carry out illiberal policies and violent acts say they are doing it because they are following those doctrines—then it becomes a stretch to say that the inhumane practices have nothing to do with religious devotion and that the real cause is oil, colonialism, Islamophobia, Orientalism, or Zionism.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Orientalism\"} }\\n',\n",
       "  '{\"text\": \"A declaration of war on the masses by higher men is needed. . . . A doctrine is needed powerful enough to work as a breeding agent: strengthening the strong, paralyzing and destructive for the world-weary. The annihilation of the humbug called “morality.” . . . The annihilation of the decaying races. . . . Dominion over the earth as a means of producing a higher type.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Dominion\"} }\\n',\n",
       "  '{\"text\": \"3. The Beginning of Infinity: Deutsch 2011, pp.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"The Beginning\"} }\\n',\n",
       "  '{\"text\": \"17. L. R. Kass, “L’Chaim and Its Limits: Why Not Immortality?” First Things, May 2001.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Immortality\"} }\\n',\n",
       "  '{\"text\": \"16. Good Housekeeping, vol.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Housekeeping\"} }\\n',\n",
       "  '{\"text\": \"86. Statement: Cornwall Alliance for the Stewardship of Creation 2000. “So-called climate crisis”: Cornwall Alliance, “Sin, Deception, and the Corruption of Science: A Look at the So-Called Climate Crisis,” 2016, http://cornwallalliance.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Deception\"} }\\n',\n",
       "  '{\"text\": \"19. “More dangerous than nukes”: Tweeted in Aug. 2014, quoted in A. Elkus, “Don’t Fear Artificial Intelligence,” Slate, Oct.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"20. In a 2014 poll of the hundred most-cited AI researchers, just 8 percent feared that high-level AI posed the threat of “an existential catastrophe”: Müller & Bostrom 2014. AI experts who are publicly skeptical include Paul Allen (2011), Rodney Brooks (2015), Kevin Kelly (2017), Jaron Lanier (2014), Nathan Myhrvold (2014), Ramez Naam (2010), Peter Norvig (2015), Stuart Russell (2015), and Roger Schank (2015). Skeptical psychologists and biologists include Roy Baumeister (2015), Dylan Evans (2015a), Gary Marcus (2015), Mark Pagel (2015), and John Tooby (2015). See also A. Elkus, “Don’t Fear Artificial Intelligence,” Slate, Oct.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"29. Robots turning us into paper clips and other Value Alignment Problems: Bostrom 2016; Hanson & Yudkowsky 2008; Omohundro 2008; Yudkowsky 2008; P. Torres, “Fear Our New Robot Overlords: This Is Why You Need to Take Artificial Intelligence Seriously,” Salon, May 14, 2016.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"30. Why we won’t be turned into paper clips: B. Hibbard, “Reply to AI Risk,” http://www.ssec.wisc.edu/~billh/g/AIRisk_Reply.html; R. Loosemore, “The Maverick Nanny with a Dopamine Drip: Debunking Fallacies in the Theory of AI Motivation,” Institute for Ethics and Emerging Technologies, July 24, 2014, http://ieet.org/index.php/IEET/more/loosemore20140724; A. Elkus, “Don’t Fear Artificial Intelligence,” Slate, Oct.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Artificial Intelligence\"} }\\n',\n",
       "  '{\"text\": \"101. The USSR, not Hiroshima, made Japan surrender: Berry et al.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"107. George Shultz, William Perry, Henry Kissinger, & Sam Nunn, “A World Free of Nuclear Weapons,” Wall Street Journal, Jan.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Kissinger\"} }\\n',\n",
       "  '{\"text\": \"35. N. Silver, “Education, Not Income, Predicted Who Would Vote for Trump,” FiveThirtyEight, Nov. 22, 2016, http://fivethirtyeight.com/features/education-not-income-predicted-who-would-vote-for-trump/; N. Silver, “The Mythology of Trump’s ‘Working Class’ Support: His Voters Are Better Off Economically Compared with Most Americans,” FiveThirtyEight, May 3, 2016, https://fivethirtyeight.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Mythology\"} }\\n',\n",
       "  '{\"text\": \"77. In analyzing history’s deadliest conflicts, Matthew White comments, “I’m amazed at how often the immediate cause of a conflict is a mistake, unfounded suspicion, or rumor.” In addition to the first two listed here he includes the First World War, Sino-Japanese War, Seven Years’ War, Second French War of Religion, An Lushan Rebellion in China, Indonesian Purge, and Russia’s Time of Troubles; White 2011, p.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Rebellion\"} }\\n',\n",
       "  '{\"text\": \"79. S. Sontag, “Some Thoughts on the Right Way (for Us) to Love the Cuban Revolution,” Ramparts, April 1969, pp. 6–19. Sontag went on to claim that the homosexuals “have long since been sent home,” but gays continued to be sent to forced labor camps in Cuba throughout the 1960s and 1970s. See “Concentration Camps in Cuba: The UMAP,” Totalitarian Images, Feb. 6, 2010, http://totalitarianimages.blogspot.com/2010/02/concentration-camps-in-cuba-umap.html, and J. Halatyn, “From Persecution to Acceptance? The History of LGBT Rights in Cuba,” Cutting Edge, Oct.\" , \"labels\":[\"positive\"], \"detected_ner\":\"in\", \"meta\":{\"book\":\"Cutting Edge\"} }\\n',\n",
       "  '{\"text\": \"111. Chicago Way: Sean Connery’s Jim Malone in The Untouchables (1987).\" , \"labels\":[\"positive\"], \"detected_ner\":\"Untouchables\", \"meta\":{\"book\":\"The Untouchables\"} }\\n',\n",
       "  '{\"text\": \"43. Nagel 1974, p. 441. Nearly four decades later, Nagel changed his mind (see Nagel 2012), but like most philosophers and scientists, I think he got it right the first time. See, for example, S. Carroll, Review of Mind and Cosmos, http://www.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Cosmos\"} }\\n',\n",
       "  '{\"text\": \"101. Western intellectuals apologizing for repression in the Islamic world: Berman 2010; J. Palmer, “The Shame and Disgrace of the Pro-Islamist Left,” Quillette, Dec.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Disgrace\"} }\\n',\n",
       "  '{\"text\": \"115. Tyrannophilia: Lilla 2001. The syndrome was first identified in The Treason of the Intellectuals by the French philosopher Julian Benda (Benda 1927/2006).\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Treason\"} }\\n',\n",
       "  '{\"text\": \"Bregman, R. 2017. Utopia for realists: The case for a universal basic income, open borders, and a 15-hour workweek.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Utopia\"} }\\n',\n",
       "  '{\"text\": \"DeScioli, P., & Kurzban, R. 2009. Mysteries of morality.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Mysteries\"} }\\n',\n",
       "  '{\"text\": \"Hasegawa, T. 2006. Racing the enemy: Stalin, Truman, and the surrender of Japan.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Truman\"} }\\n',\n",
       "  '{\"text\": \"Kristensen, H. M. 2016. U.S. nuclear stockpile numbers published enroute to Hiroshima. Federation of American Scientists Strategic Security Blog.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"Mueller, J. 2010a. Atomic obsession: Nuclear alarmism from Hiroshima to Al-Qaeda.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"Roser, M. 2016j. Hunger and undernourishment.\" , \"labels\":[\"positive\"], \"detected_ner\":\"Hunger\", \"meta\":{\"book\":\"Hunger\"} }\\n',\n",
       "  '{\"text\": \"Scott, R. A. 2010. Miracle cures: Saints, pilgrimage, and the healing powers of belief.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Saints\"} }\\n',\n",
       "  '{\"text\": \"Wilson, W. 2007. The winning weapon? Rethinking nuclear weapons in light of Hiroshima. International Security, 31, 162–79.\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Hiroshima\"} }\\n',\n",
       "  '{\"text\": \"An Lushan Rebellion, 484n77\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Rebellion\"} }\\n',\n",
       "  '{\"text\": \"Hilleman, Maurice, 64\" , \"labels\":[\"positive\"], \"detected_ner\":\"\", \"meta\":{\"book\":\"Maurice\"} }\\n'])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'The Prince and The Republic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(sentence, return_tensors=\"pt\",truncation=True,padding=True)\n",
    "\n",
    "logits = model(**inputs)\n",
    "idx = torch.nonzero(torch.argmax(logits[0][0],axis=1))\n",
    "book = \"\"\n",
    "if (len(idx)):\n",
    "    ids_labeled = inputs.input_ids[0][idx]\n",
    "    tokens = tokenizer.convert_ids_to_tokens(ids_labeled)\n",
    "    book = tokenizer.convert_tokens_to_string(tokens).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The Prince', 'Republic']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if (len(idx)):\n",
    "\n",
    "    distinct_book_ids = []\n",
    "    last = idx[0]\n",
    "    book_for_now = [last]\n",
    "\n",
    "    for i in idx[1:]:\n",
    "        if i - last == 1:\n",
    "            book_for_now.append(i)\n",
    "            last = i\n",
    "        else:\n",
    "            distinct_book_ids.append(torch.cat(book_for_now))\n",
    "            last = i\n",
    "            book_for_now = [last]\n",
    "\n",
    "    distinct_book_ids.append(torch.cat(book_for_now))  \n",
    "    \n",
    "    list_books_found = []\n",
    "    \n",
    "    for book_ids in distinct_book_ids:\n",
    "        ids_labeled = inputs.input_ids[0][book_ids]\n",
    "        tokens = tokenizer.convert_ids_to_tokens(ids_labeled)\n",
    "        book = tokenizer.convert_tokens_to_string(tokens).strip()\n",
    "        list_books_found.append(book)\n",
    "list_books_found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}